CollectionDataSource (train on top of memory collection instead of loading data from file)

ZBaselines/Common/EntryPoints/core_ep-list.tsv

@@ -1,3 +1,4 @@
+Data.DataViewReference	Pass dataview from memory to experiment	Microsoft.ML.Runtime.EntryPoints.InMemoryDataView	ImportData	Microsoft.ML.Runtime.EntryPoints.InMemoryDataView+Input	Microsoft.ML.Runtime.EntryPoints.InMemoryDataView+Output
 Data.IDataViewArrayConverter	Create and array variable	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	MakeArray	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput
 Data.PredictorModelArrayConverter	Create and array variable	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	MakeArray	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput
 Data.TextLoader	Import a dataset from a text file	Microsoft.ML.Runtime.EntryPoints.ImportTextData	ImportText	Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input	Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output

ZBaselines/Common/EntryPoints/core_manifest.json

@@ -1,5 +1,31 @@
 {
   "EntryPoints": [
+    {
+      "Name": "Data.DataViewReference",
+      "Desc": "Pass dataview from memory to experiment",
+      "FriendlyName": null,
+      "ShortName": null,
+      "Inputs": [
+        {
+          "Name": "Data",
+          "Type": "DataView",
+          "Desc": "Pointer to IDataView in memory",
+          "Aliases": [
+            "data"
+          ],
+          "Required": true,
+          "SortOrder": 1.0,
+          "IsNullable": false
+        }
+      ],
+      "Outputs": [
+        {
+          "Name": "Data",
+          "Type": "DataView",
+          "Desc": "The resulting data view"
+        }
+      ]
+    },
     {
       "Name": "Data.IDataViewArrayConverter",
       "Desc": "Create and array variable",

src/Microsoft.ML/CSharpApi.cs

@@ -53,11 +53,22 @@ public Microsoft.ML.Data.TextLoader.Output Add(Microsoft.ML.Data.TextLoader inpu
                 return output;
             }
 
+            public Microsoft.ML.Data.DataViewReference.Output Add(Microsoft.ML.Data.DataViewReference input)
+            {
+                var output = new Microsoft.ML.Data.DataViewReference.Output();
+                Add(input, output);
+                return output;
+            }
+
             public void Add(Microsoft.ML.Data.TextLoader input, Microsoft.ML.Data.TextLoader.Output output)
             {
                 _jsonNodes.Add(Serialize("Data.TextLoader", input, output));
             }
 
+            public void Add(Microsoft.ML.Data.DataViewReference input, Microsoft.ML.Data.DataViewReference.Output output)
+            {
+                _jsonNodes.Add(Serialize("Data.DataViewReference", input, output));
+            }
             public Microsoft.ML.Models.AnomalyDetectionEvaluator.Output Add(Microsoft.ML.Models.AnomalyDetectionEvaluator input)
             {
                 var output = new Microsoft.ML.Models.AnomalyDetectionEvaluator.Output();
@@ -1311,6 +1322,23 @@ public sealed partial class TextLoader
             public string CustomSchema { get; set; }
 
 
+            public sealed class Output
+            {
+                /// <summary>
+                /// The resulting data view
+                /// </summary>
+                public Var<Microsoft.ML.Runtime.Data.IDataView> Data { get; set; } = new Var<Microsoft.ML.Runtime.Data.IDataView>();
+
+            }
+        }
+
+        public sealed partial class DataViewReference
+        {
+            /// <summary>
+            /// Location of the input file
+            /// </summary>
+            public Var<Microsoft.ML.Runtime.Data.IDataView> Data { get; set; } = new Var<Microsoft.ML.Runtime.Data.IDataView>();
+
             public sealed class Output
             {
                 /// <summary>

src/Microsoft.ML/MemoryCollection.cs

@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+using Microsoft.ML.Runtime.Internal.Utilities;
+
+namespace Microsoft.ML
+{
+    public class MemoryCollection<TInput> : ILearningPipelineLoader
+        where TInput : class
+    {
+        public ILearningPipelineStep ApplyStep(ILearningPipelineStep previousStep, Experiment experiment)
+        {
+            Contracts.Assert(previousStep == null);
+            _dataViewEntryPoint = new Data.DataViewReference();
+            var importOutput = experiment.Add(_dataViewEntryPoint);
+            return new MemoryCollectionPipelineStep(importOutput.Data);
+        }
+
+        private readonly IList<TInput> _listCollection;
+        private readonly IEnumerable<TInput> _enumerableCollection;
+
+        private Data.DataViewReference _dataViewEntryPoint;
+        private IDataView _dataView;
+
+        public MemoryCollection(IList<TInput> collection)
+        {
+            Contracts.CheckParamValue(Utils.Size(collection) > 0, collection, nameof(collection), "Must be non-empty");
+            _listCollection = collection;
+        }
+
+        public MemoryCollection(IEnumerable<TInput> collection)
+        {
+            Contracts.CheckParamValue(collection != null, collection, nameof(collection), "Must be non-null");
+            _enumerableCollection = collection;
+        }
+
+        public void SetInput(IHostEnvironment env, Experiment experiment)
+        {
+            if (_listCollection != null)
+                _dataView = ComponentCreation.CreateDataView(env, _listCollection);
+            if (_enumerableCollection != null)
+                _dataView = ComponentCreation.CreateStreamingDataView(env, _listCollection);
+            env.CheckValue(_dataView, nameof(_dataView));
+            experiment.SetInput(_dataViewEntryPoint.Data, _dataView);
+        }
+
+        private class MemoryCollectionPipelineStep : ILearningPipelineDataStep
+        {
+            public MemoryCollectionPipelineStep(Var<IDataView> data)
+            {
+                Data = data;
+                Model = null;
+            }
+
+            public Var<IDataView> Data { get; }
+            public Var<ITransformModel> Model { get; }
+        }
+    }
+}

src/Microsoft.ML/Runtime/EntryPoints/InMemoryDataView.cs

@@ -0,0 +1,37 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.CommandLine;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.Runtime.EntryPoints;
+
+[assembly: LoadableClass(typeof(void), typeof(InMemoryDataView), null, typeof(SignatureEntryPointModule), "InMemoryDataView")]
+namespace Microsoft.ML.Runtime.EntryPoints
+{
+    public class InMemoryDataView
+    {
+        public sealed class Input
+        {
+            [Argument(ArgumentType.Required, ShortName = "data", HelpText = "Pointer to IDataView in memory", SortOrder = 1)]
+            public IDataView Data;
+        }
+
+        public sealed class Output
+        {
+            [TlcModule.Output(Desc = "The resulting data view", SortOrder = 1)]
+            public IDataView Data;
+        }
+
+        [TlcModule.EntryPoint(Name = "Data.DataViewReference", Desc = "Pass dataview from memory to experiment")]
+        public static Output ImportData(IHostEnvironment env, Input input)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            var host = env.Register("DataViewReference");
+            env.CheckValue(input, nameof(input));
+            EntryPointUtils.CheckInputArgs(host, input);
+            return new Output { Data = input.Data };
+        }
+    }
+}

test/Microsoft.ML.Tests/MemoryCollectionTests.cs

@@ -0,0 +1,201 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Runtime.Data;
+using Microsoft.ML.TestFramework;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using System.Collections.Generic;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.ML.EntryPoints.Tests
+{
+    public class MemoryCollectionTests : BaseTestClass
+    {
+        public MemoryCollectionTests(ITestOutputHelper output)
+            : base(output)
+        {
+
+        }
+
+        [Fact]
+        public void CheckConstructor()
+        {
+            Assert.NotNull(new MemoryCollection<Input>(new List<Input>() { new Input { Number1 = 1, String1 = "1" } }));
+            Assert.NotNull(new MemoryCollection<Input>(new Input[1] { new Input { Number1 = 1, String1 = "1" } }));
+            bool thrown = false;
+            try
+            {
+                new MemoryCollection<Input>(null);
+            }
+            catch
+            {
+                thrown = true;
+            }
+            Assert.True(thrown);
+            thrown = false;
+            try
+            {
+                new MemoryCollection<Input>(new List<Input>());
+            }
+            catch
+            {
+                thrown = true;
+            }
+            Assert.True(thrown);
+
+            thrown = false;
+            try
+            {
+                new MemoryCollection<Input>(new Input[0]);
+            }
+            catch
+            {
+                thrown = true;
+            }
+            Assert.True(thrown);
+        }
+
+        [Fact]
+        public void CanSuccessfullyApplyATransform()
+        {
+            var collection = new MemoryCollection<Input>(new List<Input>() { new Input { Number1 = 1, String1 = "1" } });
+            using (var environment = new TlcEnvironment())
+            {
+                Experiment experiment = environment.CreateExperiment();
+                ILearningPipelineDataStep output = collection.ApplyStep(null, experiment) as ILearningPipelineDataStep;
+
+                Assert.NotNull(output.Data);
+                Assert.NotNull(output.Data.VarName);
+                Assert.Null(output.Model);
+            }
+        }
+
+        [Fact]
+        public void CanSuccessfullyEnumerated()
+        {
+            var collection = new MemoryCollection<Input>(new List<Input>() {
+                new Input { Number1 = 1, String1 = "1" },
+                new Input { Number1 = 2, String1 = "2" },
+                new Input { Number1 = 3, String1 = "3" }
+            });
+
+            using (var environment = new TlcEnvironment())
+            {
+                Experiment experiment = environment.CreateExperiment();
+                ILearningPipelineDataStep output = collection.ApplyStep(null, experiment) as ILearningPipelineDataStep;
+
+                experiment.Compile();
+                collection.SetInput(environment, experiment);
+                experiment.Run();
+
+                IDataView data = experiment.GetOutput(output.Data);
+                Assert.NotNull(data);
+
+                using (var cursor = data.GetRowCursor((a => true)))
+                {
+                    var IDGetter = cursor.GetGetter<float>(0);
+                    var TextGetter = cursor.GetGetter<DvText>(1);
+
+                    Assert.True(cursor.MoveNext());
+
+                    float ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(1, ID);
+
+                    DvText Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("1", Text.ToString());
+
+                    Assert.True(cursor.MoveNext());
+
+                    ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(2, ID);
+
+                    Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("2", Text.ToString());
+
+                    Assert.True(cursor.MoveNext());
+
+                    ID = 0;
+                    IDGetter(ref ID);
+                    Assert.Equal(3, ID);
+
+                    Text = new DvText();
+                    TextGetter(ref Text);
+                    Assert.Equal("3", Text.ToString());
+
+                    Assert.False(cursor.MoveNext());
+                }
+            }
+        }
+
+        [Fact]
+        public void CanTrain()
+        {
+            var pipeline = new LearningPipeline();
+            var collection = new MemoryCollection<IrisData>(new List<IrisData>() {
+                new IrisData { SepalLength = 1f, SepalWidth = 1f ,PetalLength=0.3f, PetalWidth=5.1f, Label=1},
+                new IrisData { SepalLength = 1f, SepalWidth = 1f ,PetalLength=0.3f, PetalWidth=5.1f, Label=1},
+                new IrisData { SepalLength = 1.2f, SepalWidth = 0.5f ,PetalLength=0.3f, PetalWidth=5.1f, Label=0}
+            });
+
+            pipeline.Add(collection);
+
+            pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
+                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
+
+            pipeline.Add(new StochasticDualCoordinateAscentClassifier());
+            PredictionModel<IrisData, IrisPrediction> model = pipeline.Train<IrisData, IrisPrediction>();
+
+            IrisPrediction prediction = model.Predict(new IrisData()
+            {
+                SepalLength = 3.3f,
+                SepalWidth = 1.6f,
+                PetalLength = 0.2f,
+                PetalWidth = 5.1f,
+            });
+
+        }
+
+        public class Input
+        {
+            [Column("0")]
+            public float Number1;
+
+            [Column("1")]
+            public string String1;
+        }
+
+        public class IrisData
+        {
+            [Column("0")]
+            public float Label;
+
+            [Column("1")]
+            public float SepalLength;
+
+            [Column("2")]
+            public float SepalWidth;
+
+            [Column("3")]
+            public float PetalLength;
+
+            [Column("4")]
+            public float PetalWidth;
+        }
+
+        public class IrisPrediction
+        {
+            [ColumnName("Score")]
+            public float[] PredictedLabels;
+        }
+
+    }
+}