Skip to content

Commit 40eb3f4

Browse files
abgoswamabgoswam
committed
Merge branch 'master' into abgoswam/featurize_text
2 parents d72c344 + 3ad489e commit 40eb3f4

File tree

441 files changed

+289
-541
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

441 files changed

+289
-541
lines changed

‎.vsts-dotnet-ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
resources:
66
containers:
77
- container: CentosContainer
8-
image: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
8+
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
99

1010
- container: UbuntuContainer
11-
image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-10fcdcf-20190208200917
11+
image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303
1212

1313
phases:
1414
- template: /build/ci/phase-template.yml

‎Microsoft.ML.sln

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -250,12 +250,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "common", "common", "{A84717
250250
pkg\common\DnnImageFeaturizer.props = pkg\common\DnnImageFeaturizer.props
251251
EndProjectSection
252252
EndProject
253-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Data.DataView", "src\Microsoft.Data.DataView\Microsoft.Data.DataView.csproj", "{85D0CAFD-2FE8-496A-88C7-585D35B94243}"
253+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.DataView", "src\Microsoft.ML.DataView\Microsoft.ML.DataView.csproj", "{85D0CAFD-2FE8-496A-88C7-585D35B94243}"
254254
EndProject
255-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.Data.DataView", "Microsoft.Data.DataView", "{31D38B21-102B-41C0-9E0A-2FE0BF68D123}"
255+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.DataView", "Microsoft.ML.DataView", "{31D38B21-102B-41C0-9E0A-2FE0BF68D123}"
256256
ProjectSection(SolutionItems) = preProject
257-
pkg\Microsoft.Data.DataView\Microsoft.Data.DataView.nupkgproj = pkg\Microsoft.Data.DataView\Microsoft.Data.DataView.nupkgproj
258-
pkg\Microsoft.Data.DataView\Microsoft.Data.DataView.symbols.nupkgproj = pkg\Microsoft.Data.DataView\Microsoft.Data.DataView.symbols.nupkgproj
257+
pkg\Microsoft.ML.DataView\Microsoft.ML.DataView.nupkgproj = pkg\Microsoft.ML.DataView\Microsoft.ML.DataView.nupkgproj
258+
pkg\Microsoft.ML.DataView\Microsoft.ML.DataView.symbols.nupkgproj = pkg\Microsoft.ML.DataView\Microsoft.ML.DataView.symbols.nupkgproj
259259
EndProjectSection
260260
EndProject
261261
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RemoteExecutorConsoleApp", "test\RemoteExecutorConsoleApp\RemoteExecutorConsoleApp.csproj", "{5E920CAC-5A28-42FB-936E-49C472130953}"

‎build/Dependencies.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
<GoogleProtobufPackageVersion>3.5.1</GoogleProtobufPackageVersion>
1616
<LightGBMPackageVersion>2.2.3</LightGBMPackageVersion>
1717
<MicrosoftMLOnnxRuntimePackageVersion>0.2.1</MicrosoftMLOnnxRuntimePackageVersion>
18-
<MlNetMklDepsPackageVersion>0.0.0.7</MlNetMklDepsPackageVersion>
18+
<MlNetMklDepsPackageVersion>0.0.0.9</MlNetMklDepsPackageVersion>
1919
<ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
2020
<SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
2121
<SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>

‎build/ci/phase-template.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ phases:
3333
${{ insert }}: ${{ parameters.customMatrixes }}
3434
${{ insert }}: ${{ parameters.queue }}
3535
steps:
36+
- ${{ if eq(parameters.name, 'MacOS') }}:
37+
- script: brew update && brew install libomp && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force
38+
displayName: Install build dependencies
3639
- script: $(_buildScript) -$(_configuration) -buildArch=$(_arch)
3740
displayName: Build
3841
- script: $(_buildScript) -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
3942
displayName: Download Benchmark Data
40-
- ${{ if eq(parameters.name, 'MacOS') }}:
41-
- script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
42-
displayName: Install runtime dependencies
4343
- script: $(_buildScript) -$(_configuration) -runtests -coverage=$(_codeCoverage)
4444
displayName: Run Tests.
4545
- script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN)

‎build/vsts-ci.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
resources:
66
containers:
77
- container: CentosContainer
8-
image: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
8+
image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
99

1010
phases:
1111
################################################################################
@@ -48,6 +48,8 @@ phases:
4848
demands:
4949
- agent.os -equals Darwin
5050
steps:
51+
- script: brew update && brew install libomp && brew link libomp --force
52+
displayName: Install build dependencies
5153
# Only build native assets to avoid conflicts.
5254
- script: ./build.sh -buildNative -$(BuildConfig) -skipRIDAgnosticAssets
5355
displayName: Build

‎docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/Cache.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using System;
2-
using Microsoft.Data.DataView;
32
using Microsoft.ML.SamplesUtils;
43

54
namespace Microsoft.ML.Samples.Dynamic

‎docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/DataViewEnumerable.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.Data.DataView;
43
using Microsoft.ML.SamplesUtils;
54

65
namespace Microsoft.ML.Samples.Dynamic

‎docs/samples/Microsoft.ML.Samples/Dynamic/LogisticRegression.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using System;
2-
using Microsoft.Data.DataView;
32
using Microsoft.ML.Data;
43

54
namespace Microsoft.ML.Samples.Dynamic

‎docs/samples/Microsoft.ML.Samples/Dynamic/PermutationFeatureImportance/PFIHelper.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Linq;
3-
using Microsoft.Data.DataView;
43
using Microsoft.ML.Trainers;
54
using Microsoft.ML.SamplesUtils;
65

‎docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/Regression/FastTree.cs

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,48 @@ public static void Example()
2020
var examples = GenerateRandomDataPoints(1000);
2121

2222
// Convert the examples list to an IDataView object, which is consumable by ML.NET API.
23-
var data = mlContext.Data.LoadFromEnumerable(examples);
23+
var trainingData = mlContext.Data.LoadFromEnumerable(examples);
2424

2525
// Define the trainer.
26-
var pipeline = mlContext.BinaryClassification.Trainers.FastTree();
26+
var pipeline = mlContext.Regression.Trainers.FastTree();
2727

2828
// Train the model.
29-
var model = pipeline.Fit(data);
29+
var model = pipeline.Fit(trainingData);
30+
31+
// Create testing examples. Use different random seed to make it different from training data.
32+
var testData = mlContext.Data.LoadFromEnumerable(GenerateRandomDataPoints(500, seed:123));
33+
34+
// Run the model on test data set.
35+
var transformedTestData = model.Transform(testData);
36+
37+
// Convert IDataView object to a list.
38+
var predictions = mlContext.Data.CreateEnumerable<Prediction>(transformedTestData, reuseRowObject: false).ToList();
39+
40+
// Look at 5 predictions
41+
foreach (var p in predictions.Take(5))
42+
Console.WriteLine($"Label: {p.Label:F3}, Prediction: {p.Score:F3}");
43+
44+
// Expected output:
45+
// Label: 0.985, Prediction: 0.938
46+
// Label: 0.155, Prediction: 0.131
47+
// Label: 0.515, Prediction: 0.517
48+
// Label: 0.566, Prediction: 0.519
49+
// Label: 0.096, Prediction: 0.089
50+
51+
// Evaluate the overall metrics
52+
var metrics = mlContext.Regression.Evaluate(transformedTestData);
53+
SamplesUtils.ConsoleUtils.PrintMetrics(metrics);
54+
55+
// Expected output:
56+
// Mean Absolute Error: 0.05
57+
// Mean Squared Error: 0.00
58+
// Root Mean Squared Error: 0.06
59+
// RSquared: 0.95
3060
}
3161

32-
private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count)
62+
private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count, int seed=0)
3363
{
34-
var random = new Random(0);
64+
var random = new Random(seed);
3565
float randomFloat() => (float)random.NextDouble();
3666
for (int i = 0; i < count; i++)
3767
{
@@ -45,11 +75,21 @@ private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count)
4575
}
4676
}
4777

78+
// Example with label and 50 feature values. A data set is a collection of such examples.
4879
private class DataPoint
4980
{
5081
public float Label { get; set; }
5182
[VectorType(50)]
5283
public float[] Features { get; set; }
5384
}
85+
86+
// Class used to capture predictions.
87+
private class Prediction
88+
{
89+
// Original label.
90+
public float Label { get; set; }
91+
// Predicted score from the trainer.
92+
public float Score { get; set; }
93+
}
5494
}
5595
}

‎docs/samples/Microsoft.ML.Samples/Dynamic/ValueMapping.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.Data.DataView;
43
namespace Microsoft.ML.Samples.Dynamic
54
{
65
public static partial class ValueMapping

‎docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingFloatToString.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.Data.DataView;
43
namespace Microsoft.ML.Samples.Dynamic
54
{
65
public static class ValueMappingFloatToString

‎docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToArray.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.Data.DataView;
43
namespace Microsoft.ML.Samples.Dynamic
54
{
65
public static class ValueMappingStringToArray

‎docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToKeyType.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using System;
22
using System.Collections.Generic;
3-
using Microsoft.Data.DataView;
43
namespace Microsoft.ML.Samples.Dynamic
54
{
65
public static class ValueMappingStringToKeyType

‎pkg/Microsoft.Data.DataView/Microsoft.Data.DataView.symbols.nupkgproj

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎pkg/Microsoft.Data.DataView/Microsoft.Data.DataView.nupkgproj renamed to ‎pkg/Microsoft.ML.DataView/Microsoft.ML.DataView.nupkgproj

File renamed without changes.

‎pkg/Microsoft.ML.DataView/Microsoft.ML.DataView.symbols.nupkgproj

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<Project DefaultTargets="Pack">
2+
3+
<Import Project="Microsoft.ML.DataView.nupkgproj" />
4+
5+
</Project>

‎pkg/Microsoft.ML/Microsoft.ML.nupkgproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
</PropertyGroup>
77

88
<ItemGroup>
9-
<ProjectReference Include="../Microsoft.Data.DataView/Microsoft.Data.DataView.nupkgproj" />
9+
<ProjectReference Include="../Microsoft.ML.DataView/Microsoft.ML.DataView.nupkgproj" />
1010
<ProjectReference Include="../Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.nupkgproj" />
1111

1212
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonPackageVersion)" />

‎src/Microsoft.ML.Console/Microsoft.ML.Console.csproj

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,11 @@
3131
<NativeAssemblyReference Include="FastTreeNative" />
3232
<NativeAssemblyReference Include="CpuMathNative" />
3333
<NativeAssemblyReference Include="FactorizationMachineNative" />
34+
<NativeAssemblyReference Include="MatrixFactorizationNative" />
3435
<NativeAssemblyReference Include="LdaNative" />
35-
<NativeAssemblyReference Include="SymSgdNative" />
36+
<NativeAssemblyReference Include="SymSgdNative"/>
37+
<NativeAssemblyReference Include="MklImports"/>
38+
<NativeAssemblyReference Condition="'$(OS)' == 'Windows_NT'" Include="libiomp5md"/>
3639
</ItemGroup>
3740

3841
</Project>

‎src/Microsoft.ML.Core/Data/AnnotationBuilderExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.Data.DataView;
76

87
namespace Microsoft.ML.Data
98
{

‎src/Microsoft.ML.Core/Data/AnnotationUtils.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using System.Threading;
9-
using Microsoft.Data.DataView;
109
using Microsoft.ML.Internal.Utilities;
1110
using Microsoft.ML.Runtime;
1211

‎src/Microsoft.ML.Core/Data/ColumnTypeExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.Data.DataView;
76
using Microsoft.ML.Runtime;
87

98
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/DataKind.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.Data.DataView;
76
using Microsoft.ML.Runtime;
87
namespace Microsoft.ML.Data
98
{

‎src/Microsoft.ML.Core/Data/IEstimator.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
using System.Collections;
66
using System.Collections.Generic;
77
using System.Linq;
8-
using Microsoft.Data.DataView;
98
using Microsoft.ML.Data;
109
using Microsoft.ML.Runtime;
1110

‎src/Microsoft.ML.Core/Data/IRowToRowMapper.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
using System;
66
using System.Collections.Generic;
7-
using Microsoft.Data.DataView;
87

98
namespace Microsoft.ML.Data
109
{

‎src/Microsoft.ML.Core/Data/ISchemaBindableMapper.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
using System;
66
using System.Collections.Generic;
7-
using Microsoft.Data.DataView;
87
using Microsoft.ML.Runtime;
98

109
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/IValueMapper.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
6-
75
namespace Microsoft.ML.Data
86
{
97
/// <summary>

‎src/Microsoft.ML.Core/Data/KeyType.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.Data.DataView;
76
using Microsoft.ML.Internal.Utilities;
87
using Microsoft.ML.Runtime;
98

‎src/Microsoft.ML.Core/Data/LinkedRootCursorBase.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
65
using Microsoft.ML.Runtime;
76

87
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/LinkedRowFilterCursorBase.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
65
using Microsoft.ML.Runtime;
76

87
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/LinkedRowRootCursorBase.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
65
using Microsoft.ML.Runtime;
76

87
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/RoleMappedSchema.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System.Collections.Generic;
6-
using Microsoft.Data.DataView;
76
using Microsoft.ML.Internal.Utilities;
87
using Microsoft.ML.Runtime;
98

‎src/Microsoft.ML.Core/Data/RootCursorBase.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
65
using Microsoft.ML.Runtime;
76

87
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/SchemaExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System.Collections.Generic;
6-
using Microsoft.Data.DataView;
76

87
namespace Microsoft.ML.Data
98
{

‎src/Microsoft.ML.Core/Data/SynchronizedCursorBase.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5-
using Microsoft.Data.DataView;
65
using Microsoft.ML.Runtime;
76

87
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/Data/VectorType.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Immutable;
77
using System.Linq;
88
using System.Text;
9-
using Microsoft.Data.DataView;
109
using Microsoft.ML.Internal.Utilities;
1110
using Microsoft.ML.Runtime;
1211

‎src/Microsoft.ML.Core/Data/WrappingRow.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
using Microsoft.Data.DataView;
76
using Microsoft.ML.Runtime;
87

98
namespace Microsoft.ML.Data

‎src/Microsoft.ML.Core/EntryPoints/EntryPointUtils.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
using System;
66
using System.Linq;
77
using System.Reflection;
8-
using Microsoft.Data.DataView;
98
using Microsoft.ML.CommandLine;
109
using Microsoft.ML.Internal.Utilities;
1110
using Microsoft.ML.Runtime;

‎src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
using System.Collections.Generic;
77
using System.Linq;
88
using System.Text;
9-
using Microsoft.Data.DataView;
109
using Microsoft.ML.Data;
1110
using Microsoft.ML.Runtime;
1211

0 commit comments

Comments
 (0)