Skip to content

Commit 2f9263c

Browse files
daholstedaholste
authored
migrate to private ML.NET nuget for latest bug fixes (dotnet#131)
1 parent c4090f3 commit 2f9263c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+199
-233
lines changed

‎NuGet.Config

+1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@
22
<configuration>
33
<packageSources>
44
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
5+
<add key="DotNetMyGet" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" protocolVersion="3" />
56
</packageSources>
67
</configuration>

‎src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using Microsoft.Data.DataView;
9-
using Microsoft.ML.Core.Data;
109
using Microsoft.ML.Data;
1110

1211
namespace Microsoft.ML.Auto

‎src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using Microsoft.Data.DataView;
9-
using Microsoft.ML.Core.Data;
109
using Microsoft.ML.Data;
1110

1211
namespace Microsoft.ML.Auto

‎src/Microsoft.ML.Auto/API/Pipeline.cs

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System.Collections.Generic;
6-
using Microsoft.ML.Core.Data;
76

87
namespace Microsoft.ML.Auto
98
{

‎src/Microsoft.ML.Auto/API/RegressionExperiment.cs

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using Microsoft.Data.DataView;
9-
using Microsoft.ML.Core.Data;
109
using Microsoft.ML.Data;
1110

1211
namespace Microsoft.ML.Auto

‎src/Microsoft.ML.Auto/API/RunResult.cs

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
using System;
66
using System.Linq;
7-
using Microsoft.ML.Core.Data;
87

98
namespace Microsoft.ML.Auto
109
{

‎src/Microsoft.ML.Auto/AutoMlUtils.cs

+5-13
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,6 @@ public static void Assert(bool boolVal, string message = null)
2323
}
2424
}
2525

26-
public static IDataView Take(this IDataView data, MLContext context, int count)
27-
{
28-
return TakeFilter.Create(context, data, count);
29-
}
30-
3126
public static IDataView DropLastColumn(this IDataView data, MLContext context)
3227
{
3328
return context.Transforms.DropColumns(data.Schema[data.Schema.Count - 1].Name).Fit(data).Transform(data);
@@ -37,23 +32,20 @@ public static (IDataView testData, IDataView validationData) TestValidateSplit(t
3732
MLContext context, IDataView trainData)
3833
{
3934
IDataView validationData;
40-
(trainData, validationData) = catalog.TrainTestSplit(trainData);
35+
var splitData = catalog.TrainTestSplit(trainData);
36+
trainData = splitData.TrainSet;
37+
validationData = splitData.TestSet;
4138
trainData = trainData.DropLastColumn(context);
4239
validationData = validationData.DropLastColumn(context);
4340
return (trainData, validationData);
4441
}
4542

46-
public static IDataView Skip(this IDataView data, MLContext context, int count)
47-
{
48-
return SkipFilter.Create(context, data, count);
49-
}
50-
51-
public static (string, ColumnType, ColumnPurpose, ColumnDimensions)[] GetColumnInfoTuples(MLContext context,
43+
public static (string, DataViewType, ColumnPurpose, ColumnDimensions)[] GetColumnInfoTuples(MLContext context,
5244
IDataView data, ColumnInformation columnInfo)
5345
{
5446
var purposes = PurposeInference.InferPurposes(context, data, columnInfo);
5547
var colDimensions = DatasetDimensionsApi.CalcColumnDimensions(context, data, purposes);
56-
var cols = new (string, ColumnType, ColumnPurpose, ColumnDimensions)[data.Schema.Count];
48+
var cols = new (string, DataViewType, ColumnPurpose, ColumnDimensions)[data.Schema.Count];
5749
for (var i = 0; i < cols.Length; i++)
5850
{
5951
var schemaCol = data.Schema[i];

‎src/Microsoft.ML.Auto/ColumnInference/ColumnInferenceApi.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public static ColumnInferenceResults InferColumns(MLContext context, string path
5252
var loaderColumns = ColumnTypeInference.GenerateLoaderColumns(typeInference.Columns);
5353
var typedLoaderArgs = new TextLoader.Arguments
5454
{
55-
Column = loaderColumns,
55+
Columns = loaderColumns,
5656
Separators = new[] { splitInference.Separator.Value },
5757
AllowSparse = splitInference.AllowSparse,
5858
AllowQuoting = splitInference.AllowQuote,
@@ -85,7 +85,7 @@ public static ColumnInferenceResults InferColumns(MLContext context, string path
8585

8686
var textLoaderArgs = new TextLoader.Arguments()
8787
{
88-
Column = columnResults.ToArray(),
88+
Columns = columnResults.ToArray(),
8989
AllowQuoting = splitInference.AllowQuote,
9090
AllowSparse = splitInference.AllowSparse,
9191
Separators = new char[] { splitInference.Separator.Value },

‎src/Microsoft.ML.Auto/ColumnInference/ColumnInformationUtil.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ internal static ColumnInformation BuildColumnInfo(IEnumerable<(string name, Colu
8282
return columnInfo;
8383
}
8484

85-
public static ColumnInformation BuildColumnInfo(IEnumerable<(string, ColumnType, ColumnPurpose, ColumnDimensions)> columns)
85+
public static ColumnInformation BuildColumnInfo(IEnumerable<(string, DataViewType, ColumnPurpose, ColumnDimensions)> columns)
8686
{
8787
return BuildColumnInfo(columns.Select(c => (c.Item1, c.Item3)));
8888
}

‎src/Microsoft.ML.Auto/ColumnInference/ColumnTypeInference.cs

+10-10
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,15 @@ private class IntermediateColumn
4343
{
4444
private readonly ReadOnlyMemory<char>[] _data;
4545
private readonly int _columnId;
46-
private PrimitiveType _suggestedType;
46+
private PrimitiveDataViewType _suggestedType;
4747
private bool? _hasHeader;
4848

4949
public int ColumnId
5050
{
5151
get { return _columnId; }
5252
}
5353

54-
public PrimitiveType SuggestedType
54+
public PrimitiveDataViewType SuggestedType
5555
{
5656
get { return _suggestedType; }
5757
set { _suggestedType = value; }
@@ -95,10 +95,10 @@ public class Column
9595
{
9696
public readonly int ColumnIndex;
9797

98-
public PrimitiveType ItemType;
98+
public PrimitiveDataViewType ItemType;
9999
public string SuggestedName;
100100

101-
public Column(int columnIndex, string suggestedName, PrimitiveType itemType)
101+
public Column(int columnIndex, string suggestedName, PrimitiveDataViewType itemType)
102102
{
103103
ColumnIndex = columnIndex;
104104
SuggestedName = suggestedName;
@@ -160,7 +160,7 @@ public void Apply(IntermediateColumn[] columns)
160160
continue;
161161
}
162162

163-
col.SuggestedType = BoolType.Instance;
163+
col.SuggestedType = BooleanDataViewType.Instance;
164164
bool first;
165165

166166
col.HasHeader = !Conversions.TryParse(in col.RawData[0], out first);
@@ -185,7 +185,7 @@ public void Apply(IntermediateColumn[] columns)
185185
continue;
186186
}
187187

188-
col.SuggestedType = NumberType.R4;
188+
col.SuggestedType = NumberDataViewType.Single;
189189

190190
var headerStr = col.RawData[0].ToString();
191191
col.HasHeader = !double.TryParse(headerStr, out var doubleVal);
@@ -202,7 +202,7 @@ public void Apply(IntermediateColumn[] columns)
202202
if (col.SuggestedType != null)
203203
continue;
204204

205-
col.SuggestedType = TextType.Instance;
205+
col.SuggestedType = TextDataViewType.Instance;
206206
col.HasHeader = IsLookLikeHeader(col.RawData[0]);
207207
}
208208
}
@@ -258,14 +258,14 @@ private static InferenceResult InferTextFileColumnTypesCore(MLContext context, I
258258
// read the file as the specified number of text columns
259259
var textLoaderArgs = new TextLoader.Arguments
260260
{
261-
Column = new[] { new TextLoader.Column("C", DataKind.TX, 0, args.ColumnCount - 1) },
261+
Columns = new[] { new TextLoader.Column("C", DataKind.TX, 0, args.ColumnCount - 1) },
262262
Separators = new[] { args.Separator },
263263
AllowSparse = args.AllowSparse,
264264
AllowQuoting = args.AllowQuote,
265265
};
266266
var textLoader = new TextLoader(context, textLoaderArgs);
267267
var idv = textLoader.Read(fileSource);
268-
idv = idv.Take(context, args.MaxRowsToRead);
268+
idv = context.Data.TakeRows(idv, args.MaxRowsToRead);
269269

270270
// read all the data into memory.
271271
// list items are rows of the dataset.
@@ -361,7 +361,7 @@ private static InferenceResult InferTextFileColumnTypesCore(MLContext context, I
361361
// if label column has all Boolean values, set its type as Boolean
362362
if (labelColumn.HasAllBooleanValues())
363363
{
364-
labelColumn.SuggestedType = BoolType.Instance;
364+
labelColumn.SuggestedType = BooleanDataViewType.Instance;
365365
}
366366

367367
var outCols = cols.Select(x => new Column(x.ColumnId, x.Name, x.SuggestedType)).ToArray();

‎src/Microsoft.ML.Auto/ColumnInference/PurposeInference.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ private class IntermediateColumn
4949
private readonly int _columnId;
5050
private bool _isPurposeSuggested;
5151
private ColumnPurpose _suggestedPurpose;
52-
private readonly Lazy<ColumnType> _type;
52+
private readonly Lazy<DataViewType> _type;
5353
private readonly Lazy<string> _columnName;
5454
private object _cachedData;
5555

@@ -65,15 +65,15 @@ public ColumnPurpose SuggestedPurpose
6565
}
6666
}
6767

68-
public ColumnType Type { get { return _type.Value; } }
68+
public DataViewType Type { get { return _type.Value; } }
6969

7070
public string ColumnName { get { return _columnName.Value; } }
7171

7272
public IntermediateColumn(IDataView data, int columnId, ColumnPurpose suggestedPurpose = ColumnPurpose.Ignore)
7373
{
7474
_data = data;
7575
_columnId = columnId;
76-
_type = new Lazy<ColumnType>(() => _data.Schema[_columnId].Type);
76+
_type = new Lazy<DataViewType>(() => _data.Schema[_columnId].Type);
7777
_columnName = new Lazy<string>(() => _data.Schema[_columnId].Name);
7878
_suggestedPurpose = suggestedPurpose;
7979
}
@@ -238,7 +238,7 @@ private static IEnumerable<IPurposeInferenceExpert> GetExperts()
238238
public static PurposeInference.Column[] InferPurposes(MLContext context, IDataView data,
239239
ColumnInformation columnInfo)
240240
{
241-
data = data.Take(context, MaxRowsToRead);
241+
data = context.Data.TakeRows(data, MaxRowsToRead);
242242

243243
var allColumns = new List<IntermediateColumn>();
244244
var columnsToInfer = new List<IntermediateColumn>();

‎src/Microsoft.ML.Auto/ColumnInference/TextFileContents.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ from _sep in separatorCandidates
5959
{
6060
var args = new TextLoader.Arguments
6161
{
62-
Column = new[] { new TextLoader.Column() {
62+
Columns = new[] { new TextLoader.Column() {
6363
Name = "C",
6464
Type = DataKind.TX,
6565
Source = new[] { new TextLoader.Range(0, null) }
@@ -87,7 +87,7 @@ private static bool TryParseFile(MLContext context, TextLoader.Arguments args, I
8787
{
8888

8989
var textLoader = new TextLoader(context, args, source);
90-
var idv = textLoader.Read(source).Take(context, 1000);
90+
var idv = context.Data.TakeRows(textLoader.Read(source), 1000);
9191
var columnCounts = new List<int>();
9292
var column = idv.Schema["C"];
9393
var columnIndex = column.Index;

‎src/Microsoft.ML.Auto/DatasetDimensions/DatasetDimensionsApi.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ internal class DatasetDimensionsApi
1212

1313
public static ColumnDimensions[] CalcColumnDimensions(MLContext context, IDataView data, PurposeInference.Column[] purposes)
1414
{
15-
data = data.Take(context, MaxRowsToRead);
15+
data = context.Data.TakeRows(data, MaxRowsToRead);
1616

1717
var colDimensions = new ColumnDimensions[data.Schema.Count];
1818

@@ -34,7 +34,7 @@ public static ColumnDimensions[] CalcColumnDimensions(MLContext context, IDataVi
3434
}
3535

3636
// If numeric feature, discover missing values
37-
if (itemType == NumberType.R4)
37+
if (itemType == NumberDataViewType.Single)
3838
{
3939
hasMissing = column.Type.IsVector() ?
4040
DatasetDimensionsUtil.HasMissingNumericVector(data, i) :

‎src/Microsoft.ML.Auto/EstimatorExtensions/EstimatorExtensions.cs

+4-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.ML.Core.Data;
65
using Microsoft.ML.Data;
76
using Microsoft.ML.Transforms;
87
using Microsoft.ML.Transforms.Categorical;
@@ -96,10 +95,10 @@ public static SuggestedTransform CreateSuggestedTransform(MLContext context, str
9695

9796
private static IEstimator<ITransformer> CreateInstance(MLContext context, string[] inColumns, string[] outColumns)
9897
{
99-
var pairs = new MissingValueReplacingTransformer.ColumnInfo[inColumns.Length];
98+
var pairs = new MissingValueReplacingEstimator.ColumnInfo[inColumns.Length];
10099
for (var i = 0; i < inColumns.Length; i++)
101100
{
102-
var pair = new MissingValueReplacingTransformer.ColumnInfo(outColumns[i], inColumns[i]);
101+
var pair = new MissingValueReplacingEstimator.ColumnInfo(outColumns[i], inColumns[i]);
103102
pairs[i] = pair;
104103
}
105104
return context.Transforms.ReplaceMissingValues(pairs);
@@ -222,10 +221,10 @@ public static SuggestedTransform CreateSuggestedTransform(MLContext context, str
222221

223222
private static IEstimator<ITransformer> CreateInstance(MLContext context, string[] inColumns, string[] outColumns)
224223
{
225-
var cols = new TypeConvertingTransformer.ColumnInfo[inColumns.Length];
224+
var cols = new TypeConvertingEstimator.ColumnInfo[inColumns.Length];
226225
for (var i = 0; i < cols.Length; i++)
227226
{
228-
cols[i] = new TypeConvertingTransformer.ColumnInfo(outColumns[i], DataKind.R4, inColumns[i]);
227+
cols[i] = new TypeConvertingEstimator.ColumnInfo(outColumns[i], DataKind.R4, inColumns[i]);
229228
}
230229
return context.Transforms.Conversion.ConvertType(cols);
231230
}

‎src/Microsoft.ML.Auto/EstimatorExtensions/IEstimatorExtension.cs

-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.ML.Core.Data;
6-
75
namespace Microsoft.ML.Auto
86
{
97
internal interface IEstimatorExtension

‎src/Microsoft.ML.Auto/Experiment/Experiment.cs

-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
using System.Diagnostics;
88
using System.Text;
99
using Microsoft.Data.DataView;
10-
using Microsoft.ML.Core.Data;
1110

1211
namespace Microsoft.ML.Auto
1312
{

‎src/Microsoft.ML.Auto/Experiment/SuggestedPipeline.cs

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using Microsoft.Data.DataView;
9-
using Microsoft.ML.Core.Data;
109
using Microsoft.ML.Data;
1110

1211
namespace Microsoft.ML.Auto

‎src/Microsoft.ML.Auto/Experiment/SuggestedPipelineResult.cs

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.ML.Core.Data;
76

87
namespace Microsoft.ML.Auto
98
{

‎src/Microsoft.ML.Auto/Microsoft.ML.Auto.csproj

+3-3
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66
</PropertyGroup>
77

88
<ItemGroup>
9-
<PackageReference Include="Microsoft.ML" Version="0.10.0" />
10-
<PackageReference Include="Microsoft.ML.HalLearners" Version="0.10.0" />
11-
<PackageReference Include="Microsoft.ML.LightGBM" Version="0.10.0" />
9+
<PackageReference Include="Microsoft.ML" Version="0.11.0-preview-27416-1" />
10+
<PackageReference Include="Microsoft.ML.HalLearners" Version="0.11.0-preview-27416-1" />
11+
<PackageReference Include="Microsoft.ML.LightGBM" Version="0.11.0-preview-27416-1" />
1212
</ItemGroup>
1313

1414
<PropertyGroup>

‎src/Microsoft.ML.Auto/PipelineSuggesters/PipelineSuggester.cs

+3-3
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ internal static class PipelineSuggester
1616

1717
public static Pipeline GetNextPipeline(MLContext context,
1818
IEnumerable<PipelineScore> history,
19-
(string, ColumnType, ColumnPurpose, ColumnDimensions)[] columns,
19+
(string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns,
2020
TaskKind task,
2121
bool isMaximizingMetric = true)
2222
{
@@ -27,7 +27,7 @@ public static Pipeline GetNextPipeline(MLContext context,
2727

2828
public static SuggestedPipeline GetNextInferredPipeline(MLContext context,
2929
IEnumerable<SuggestedPipelineResult> history,
30-
(string, ColumnType, ColumnPurpose, ColumnDimensions)[] columns,
30+
(string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns,
3131
TaskKind task,
3232
bool isMaximizingMetric,
3333
IEnumerable<TrainerName> trainerWhitelist = null)
@@ -214,7 +214,7 @@ private static bool SampleHyperparameters(MLContext context, SuggestedTrainer tr
214214

215215
private static IEnumerable<SuggestedTransform> CalculateTransforms(
216216
MLContext context,
217-
(string, ColumnType, ColumnPurpose, ColumnDimensions)[] columns,
217+
(string, DataViewType, ColumnPurpose, ColumnDimensions)[] columns,
218218
TaskKind task)
219219
{
220220
var transforms = TransformInferenceApi.InferTransforms(context, columns).ToList();

0 commit comments

Comments
 (0)