Skip to content

Commit 62cdfa0

Browse files
frank-dong-ms-zzantoniovs1029eerhardt
authored
merge master to arcade branch (#5424)
* Update to Onnxruntime 1.5.1 (#5406) * Added variables to tests to control Gpu settings * Added dependency to prerelease * Updated to 1.5.1 * Remove prerelease feed * Nit on GPU variables * Change the _maxCalibrationExamples default on CalibratorUtils (#5415) * Change the _maxCalibrationExamples default * Improving comments * Fix perf regression in ShuffleRows (#5417) RowShufflingTransformer is using ChannelReader incorrectly. It needs to block waiting for items to read and was Thread.Sleeping in order to wait, but not spin the current core. This caused a major perf regression. The fix is to block synchronously correctly - by calling AsTask() on the ValueTask that is returned from the ChannelReader and block on the Task. Fix #5416 Co-authored-by: Antonio Velázquez <[email protected]> Co-authored-by: Eric Erhardt <[email protected]>
1 parent 91cf1a8 commit 62cdfa0

File tree

8 files changed

+128
-49
lines changed

8 files changed

+128
-49
lines changed

‎build/Dependencies.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
<GoogleProtobufPackageVersion>3.10.1</GoogleProtobufPackageVersion>
1717
<LightGBMPackageVersion>2.2.3</LightGBMPackageVersion>
1818
<MicrosoftExtensionsPackageVersion>2.1.0</MicrosoftExtensionsPackageVersion>
19-
<MicrosoftMLOnnxRuntimePackageVersion>1.3.0</MicrosoftMLOnnxRuntimePackageVersion>
19+
<MicrosoftMLOnnxRuntimePackageVersion>1.5.1</MicrosoftMLOnnxRuntimePackageVersion>
2020
<MlNetMklDepsPackageVersion>0.0.0.9</MlNetMklDepsPackageVersion>
2121
<ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
2222
<SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>

‎src/Microsoft.ML.Data/Prediction/Calibrator.cs

+7-2
Original file line numberDiff line numberDiff line change
@@ -837,8 +837,9 @@ internal static object Create(IHostEnvironment env, ModelLoadContext ctx, object
837837
[BestFriend]
838838
internal static class CalibratorUtils
839839
{
840-
// maximum number of rows passed to the calibrator.
841-
private const int _maxCalibrationExamples = 1000000;
840+
// Maximum number of rows to process when training the Calibrator.
841+
// If 0, we'll actually process the whole dataset.
842+
private const int _maxCalibrationExamples = 0;
842843

843844
private static bool NeedCalibration(IHostEnvironment env, IChannel ch, ICalibratorTrainer calibrator,
844845
ITrainer trainer, IPredictor predictor, RoleMappedSchema schema)
@@ -988,6 +989,10 @@ public static ICalibrator TrainCalibrator(IHostEnvironment env, IChannel ch, ICa
988989
caliTrainer.ProcessTrainingExample(score, label > 0, weight);
989990

990991
if (maxRows > 0 && ++num >= maxRows)
992+
// If maxRows was 0, we'll process all of the rows in the dataset
993+
// Notice that depending on the calibrator, "processing" might mean
994+
// randomly choosing some of the "processed" rows
995+
// to actually train the calibrator.
991996
break;
992997
}
993998
}

‎src/Microsoft.ML.Data/Transforms/RowShufflingTransformer.cs

+17-18
Original file line numberDiff line numberDiff line change
@@ -634,26 +634,25 @@ protected override bool MoveNextCore()
634634
while (_liveCount < _poolRows && !_doneConsuming)
635635
{
636636
// We are under capacity. Try to get some more.
637-
var hasReadItem = _toConsumeChannel.Reader.TryRead(out int got);
638-
if (hasReadItem)
637+
ValueTask<int> readTask = _toConsumeChannel.Reader.ReadAsync();
638+
639+
// Note you can't wait synchronously on a ValueTask. So if it
640+
// hasn't been completed yet, need to call AsTask() to get a Task
641+
// which can be waited on synchronously.
642+
int got = readTask.IsCompletedSuccessfully ?
643+
readTask.Result :
644+
readTask.AsTask().GetAwaiter().GetResult();
645+
if (got == 0)
639646
{
640-
if (got == 0)
641-
{
642-
// We've reached the end of the Channel. There's no reason
643-
// to attempt further communication with the producer.
644-
// Check whether something horrible happened.
645-
if (_producerTaskException != null)
646-
throw Ch.Except(_producerTaskException, "Shuffle input cursor reader failed with an exception");
647-
_doneConsuming = true;
648-
break;
649-
}
650-
_liveCount += got;
651-
}
652-
else
653-
{
654-
// Sleeping for one millisecond to stop the thread from spinning while waiting for the producer.
655-
Thread.Sleep(1);
647+
// We've reached the end of the Channel. There's no reason
648+
// to attempt further communication with the producer.
649+
// Check whether something horrible happened.
650+
if (_producerTaskException != null)
651+
throw Ch.Except(_producerTaskException, "Shuffle input cursor reader failed with an exception");
652+
_doneConsuming = true;
653+
break;
656654
}
655+
_liveCount += got;
657656
}
658657
if (_liveCount == 0)
659658
return false;

‎test/Microsoft.ML.Benchmarks/ShuffleRowsBench.cs

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using BenchmarkDotNet.Attributes;
6+
using Microsoft.ML.Benchmarks.Harness;
7+
using Microsoft.ML.Data;
8+
9+
namespace Microsoft.ML.Benchmarks
10+
{
11+
[CIBenchmark]
12+
public class ShuffleRowsBench : BenchmarkBase
13+
{
14+
private TrainRow[] _rows;
15+
private MLContext _context;
16+
17+
[GlobalSetup]
18+
public void Setup()
19+
{
20+
_rows = new TrainRow[10_000];
21+
for (var i = 0; i < _rows.Length; i++)
22+
{
23+
_rows[i] = new TrainRow() { Sample = i.ToString(), Week = i, Label = i / 2 };
24+
}
25+
26+
_context = new MLContext();
27+
}
28+
29+
[Benchmark]
30+
public void ShuffleRows()
31+
{
32+
IDataView data = _context.Data.LoadFromEnumerable(_rows);
33+
34+
IDataView shuffledData = _context.Data.ShuffleRows(data, seed: 0);
35+
36+
foreach (string sample in shuffledData.GetColumn<string>("Sample"))
37+
{
38+
}
39+
}
40+
41+
private class TrainRow
42+
{
43+
[ColumnName("Sample")]
44+
public string Sample;
45+
46+
[ColumnName("Week")]
47+
public float Week;
48+
49+
[ColumnName("Label")]
50+
public float Label;
51+
}
52+
}
53+
}

‎test/Microsoft.ML.Functional.Tests/ONNX.cs

+8-3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ namespace Microsoft.ML.Functional.Tests
1616
{
1717
public class ONNX : FunctionalTestBaseClass
1818
{
19+
// These two members are meant to be changed
20+
// Only when manually testing the Onnx GPU nuggets
21+
private const bool _fallbackToCpu = true;
22+
private static int? _gpuDeviceId = null;
23+
1924
public ONNX(ITestOutputHelper output) : base(output)
2025
{
2126
}
@@ -52,7 +57,7 @@ public void SaveOnnxModelLoadAndScoreFastTree()
5257
// Therefore the VectorScoreColumn class (which contains a float [] field called Score) is used for the return
5358
// type on the Prediction engine.
5459
// See #2980 and #2981 for more information.
55-
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath);
60+
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
5661
var onnxModel = onnxEstimator.Fit(data);
5762

5863
// Create prediction engine and test predictions.
@@ -98,7 +103,7 @@ public void SaveOnnxModelLoadAndScoreKMeans()
98103
mlContext.Model.ConvertToOnnx(model, data, file);
99104

100105
// Load the model as a transform.
101-
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath);
106+
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
102107
var onnxModel = onnxEstimator.Fit(data);
103108

104109
// TODO #2980: ONNX outputs don't match the outputs of the model, so we must hand-correct this for now.
@@ -150,7 +155,7 @@ public void SaveOnnxModelLoadAndScoreSDCA()
150155
mlContext.Model.ConvertToOnnx(model, data, file);
151156

152157
// Load the model as a transform.
153-
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath);
158+
var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(modelPath, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
154159
var onnxModel = onnxEstimator.Fit(data);
155160

156161
// Create prediction engine and test predictions.

‎test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs

+21-16
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ namespace Microsoft.ML.Tests
2323
{
2424
public class OnnxTransformTests : TestDataPipeBase
2525
{
26+
// These two members are meant to be changed
27+
// Only when manually testing the Onnx GPU nuggets
28+
private const bool _fallbackToCpu = true;
29+
private static int? _gpuDeviceId = null;
30+
2631
private const int InputSize = 150528;
2732

2833
private class TestData
@@ -134,7 +139,7 @@ public void TestSimpleCase()
134139
var xyData = new List<TestDataXY> { new TestDataXY() { A = new float[InputSize] } };
135140
var stringData = new List<TestDataDifferntType> { new TestDataDifferntType() { data_0 = new string[InputSize] } };
136141
var sizeData = new List<TestDataSize> { new TestDataSize() { data_0 = new float[2] } };
137-
var pipe = ML.Transforms.ApplyOnnxModel(new[] { "softmaxout_1" }, new[] { "data_0" }, modelFile);
142+
var pipe = ML.Transforms.ApplyOnnxModel(new[] { "softmaxout_1" }, new[] { "data_0" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
138143

139144
var invalidDataWrongNames = ML.Data.LoadFromEnumerable(xyData);
140145
var invalidDataWrongTypes = ML.Data.LoadFromEnumerable(stringData);
@@ -231,7 +236,7 @@ public void OnnxWorkout()
231236
var pipe = ML.Transforms.LoadImages("data_0", imageFolder, "imagePath")
232237
.Append(ML.Transforms.ResizeImages("data_0", imageHeight, imageWidth))
233238
.Append(ML.Transforms.ExtractPixels("data_0", interleavePixelColors: true))
234-
.Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile));
239+
.Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu));
235240

236241
TestEstimatorCore(pipe, data);
237242

@@ -292,7 +297,7 @@ public void OnnxModelScenario()
292297
}
293298
});
294299

295-
var pipeline = ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile);
300+
var pipeline = ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
296301
var onnxTransformer = pipeline.Fit(dataView);
297302
var onnx = onnxTransformer.Transform(dataView);
298303
var scoreCol = onnx.Schema["softmaxout_1"];
@@ -325,7 +330,7 @@ public void OnnxModelMultiInput()
325330
inb = new float[] {1,2,3,4,5}
326331
}
327332
});
328-
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outa", "outb" }, new[] { "ina", "inb" }, modelFile);
333+
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outa", "outb" }, new[] { "ina", "inb" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
329334
var onnxTransformer = pipeline.Fit(dataView);
330335
var onnx = onnxTransformer.Transform(dataView);
331336

@@ -365,7 +370,7 @@ public void OnnxModelOutputDifferentOrder()
365370
}
366371
});
367372
// The model returns the output columns in the order outa, outb. We are doing the opposite here, making sure the name mapping is correct.
368-
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outb", "outa" }, new[] { "ina", "inb" }, modelFile);
373+
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outb", "outa" }, new[] { "ina", "inb" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
369374
var onnxTransformer = pipeline.Fit(dataView);
370375
var onnx = onnxTransformer.Transform(dataView);
371376

@@ -391,7 +396,7 @@ public void OnnxModelOutputDifferentOrder()
391396
(onnxTransformer as IDisposable)?.Dispose();
392397

393398
// The model returns the output columns in the order outa, outb. We are doing only a subset, outb, to make sure the mapping works.
394-
pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outb" }, new[] { "ina", "inb" }, modelFile);
399+
pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outb" }, new[] { "ina", "inb" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
395400
onnxTransformer = pipeline.Fit(dataView);
396401
onnx = onnxTransformer.Transform(dataView);
397402

@@ -425,7 +430,7 @@ public void TestUnknownDimensions()
425430
new TestDataUnknownDimensions(){input = new float[] {-1.1f, -1.3f, 1.2f }},
426431
};
427432
var idv = mlContext.Data.LoadFromEnumerable(data);
428-
var pipeline = ML.Transforms.ApplyOnnxModel(modelFile);
433+
var pipeline = ML.Transforms.ApplyOnnxModel(modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
429434
var onnxTransformer = pipeline.Fit(idv);
430435
var transformedValues = onnxTransformer.Transform(idv);
431436
var predictions = mlContext.Data.CreateEnumerable<PredictionUnknownDimensions>(transformedValues, reuseRowObject: false).ToArray();
@@ -451,7 +456,7 @@ public void TestOnnxNoneDimValue()
451456
new TestDataNoneDimension(){features = new float[] { 6.3f, 3.3f, 6.0f, 2.5f }},
452457
};
453458
var idv = mlContext.Data.LoadFromEnumerable(data);
454-
var pipeline = ML.Transforms.ApplyOnnxModel(modelFile);
459+
var pipeline = ML.Transforms.ApplyOnnxModel(modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
455460
var onnxTransformer = pipeline.Fit(idv);
456461
var transformedValues = onnxTransformer.Transform(idv);
457462
var predictions = mlContext.Data.CreateEnumerable<PredictionNoneDimension>(transformedValues, reuseRowObject: false).ToArray();
@@ -526,7 +531,7 @@ public void OnnxModelInMemoryImage()
526531
// "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to
527532
// get the right input and output column names.
528533
var pipeline = ML.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0"
529-
.Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile)); // Map column "data_0" to column "softmaxout_1"
534+
.Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu)); // Map column "data_0" to column "softmaxout_1"
530535
var model = pipeline.Fit(dataView);
531536
var onnx = model.Transform(dataView);
532537

@@ -576,7 +581,7 @@ public void TestOnnxZipMapWithInt64Keys()
576581
};
577582

578583
var dataView = ML.Data.LoadFromEnumerable(dataPoints);
579-
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile);
584+
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
580585
var onnxTransformer = pipeline.Fit(dataView);
581586
var transformedDataView = onnxTransformer.Transform(dataView);
582587

@@ -629,7 +634,7 @@ public void TestOnnxZipMapWithStringKeys()
629634
};
630635

631636
var dataView = ML.Data.LoadFromEnumerable(dataPoints);
632-
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile);
637+
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "output" }, new[] { "input" }, modelFile, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
633638
var onnxTransformer = pipeline.Fit(dataView);
634639
var transformedDataView = onnxTransformer.Transform(dataView);
635640

@@ -794,19 +799,19 @@ public void TestOnnxTransformWithCustomShapes()
794799
// Test 1.
795800
pipeline[0] = ML.Transforms.ApplyOnnxModel(
796801
new[] { nameof(PredictionWithCustomShape.argmax) }, new[] { nameof(InputWithCustomShape.input) },
797-
modelFile, shapeDictionary);
802+
modelFile, shapeDictionary, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
798803
onnxTransformer[0] = pipeline[0].Fit(dataView);
799804
transformedDataViews[0] = onnxTransformer[0].Transform(dataView);
800805

801806
// Test 2.
802807
pipeline[1] = ML.Transforms.ApplyOnnxModel(
803808
nameof(PredictionWithCustomShape.argmax), nameof(InputWithCustomShape.input),
804-
modelFile, shapeDictionary);
809+
modelFile, shapeDictionary, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
805810
onnxTransformer[1] = pipeline[1].Fit(dataView);
806811
transformedDataViews[1] = onnxTransformer[1].Transform(dataView);
807812

808813
// Test 3.
809-
pipeline[2] = ML.Transforms.ApplyOnnxModel(modelFile, shapeDictionary);
814+
pipeline[2] = ML.Transforms.ApplyOnnxModel(modelFile, shapeDictionary, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
810815
onnxTransformer[2] = pipeline[2].Fit(dataView);
811816
transformedDataViews[2] = onnxTransformer[2].Transform(dataView);
812817

@@ -856,7 +861,7 @@ private void TryModelWithCustomShapesHelper(IDictionary<string, int[]> shapeDict
856861

857862
// Define a ONNX transform, trains it, and apply it to the input data.
858863
var pipeline = ML.Transforms.ApplyOnnxModel(new[] { "outa", "outb" }, new[] { "ina", "inb" },
859-
modelFile, shapeDictionary);
864+
modelFile, shapeDictionary, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
860865
}
861866

862867
/// <summary>
@@ -956,7 +961,7 @@ public void TestOnnxTransformSaveAndLoadWithCustomShapes()
956961
var dataView = ML.Data.LoadFromEnumerable(dataPoints);
957962

958963
var pipeline = ML.Transforms.ApplyOnnxModel(nameof(PredictionWithCustomShape.argmax),
959-
nameof(InputWithCustomShape.input), modelFile, shapeDictionary);
964+
nameof(InputWithCustomShape.input), modelFile, shapeDictionary, gpuDeviceId: _gpuDeviceId, fallbackToCpu: _fallbackToCpu);
960965

961966
var model = pipeline.Fit(dataView);
962967

0 commit comments

Comments
 (0)