Saving/loading data

Author: Pete Luferenko

test/Microsoft.ML.Tests/Scenarios/Api/FileBasedSavingOfData.cs

@@ -44,5 +44,38 @@ void FileBasedSavingOfData()
                 DeleteOutputPath("i.idv");
             }
         }
+
+        /// <summary>
+        /// File-based saving of data: Come up with transform pipeline. Transform training and
+        /// test data, and save the featurized data to some file, using the .idv format.
+        /// Train and evaluate multiple models over that pre-featurized data. (Useful for
+        /// sweeping scenarios, where you are training many times on the same data,
+        /// and don't necessarily want to transform it every single time.)
+        /// </summary>
+        [Fact]
+        void New_FileBasedSavingOfData()
+        {
+            var dataPath = GetDataPath(SentimentDataPath);
+            var testDataPath = GetDataPath(SentimentTestPath);
+
+            using (var env = new TlcEnvironment(seed: 1, conc: 1))
+            {
+                // Pipeline.
+                var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
+                    .Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()));
+
+                var trainData = pipeline.Fit(new MultiFileSource(dataPath)).Read(new MultiFileSource(dataPath));
+
+                using (var file = env.CreateOutputFile("i.idv"))
+                    trainData.SaveAsBinary(env, file.CreateWriteStream());
+
+                var trainer = new MySdca(env, new LinearClassificationTrainer.Arguments { NumThreads = 1 }, "Features", "Label");
+                var loadedTrainData = new BinaryLoader(env, new BinaryLoader.Arguments(), new MultiFileSource("i.idv"));
+
+                // Train.
+                var model = trainer.Train(loadedTrainData);
+                DeleteOutputPath("i.idv");
+            }
+        }
     }
 }

test/Microsoft.ML.Tests/Scenarios/Api/Wrappers.cs

@@ -374,5 +374,13 @@ public BinaryClassificationMetrics Evaluate(IDataView data, string labelColumn,
         }
     }
 
-
+    public static class MyHelperExtensions
+    {
+        public static void SaveAsBinary(this IDataView data, IHostEnvironment env, Stream stream)
+        {
+            var saver = new BinarySaver(env, new BinarySaver.Arguments());
+            using (var ch = env.Start("SaveData"))
+                DataSaverUtils.SaveDataView(ch, saver, data, stream);
+        }
+    }
 }