Skip to content

Commit 8806dac

Browse files
author
Ivan Matantsev
committed
merge with master
2 parents 9496ca2 + 3b1358e commit 8806dac

File tree

126 files changed

+1097
-956
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+1097
-956
lines changed

docs/samples/Microsoft.ML.Samples/Dynamic/OnnxTransform.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public static void OnnxTransformSample()
3535
var mlContext = new MLContext();
3636
var data = GetTensorData();
3737
var idv = mlContext.Data.ReadFromEnumerable(data);
38-
var pipeline = new OnnxScoringEstimator(mlContext, new[] { outputInfo.Key }, new[] { inputInfo.Key }, modelPath);
38+
var pipeline = mlContext.Transforms.ApplyOnnxModel(modelPath, new[] { outputInfo.Key }, new[] { inputInfo.Key });
3939

4040
// Run the pipeline and get the transformed values
4141
var transformedValues = pipeline.Fit(idv).Transform(idv);

docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToArray.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public static void Run()
5252
};
5353

5454
// Constructs the ValueMappingEstimator making the ML.net pipeline
55-
var pipeline = new ValueMappingEstimator<string, int>(mlContext, educationKeys, educationValues, ("EducationFeature", "Education"));
55+
var pipeline = mlContext.Transforms.Conversion.ValueMap<string, int>(educationKeys, educationValues, ("EducationFeature", "Education"));
5656

5757
// Fits the ValueMappingEstimator and transforms the data adding the EducationFeature column.
5858
IDataView transformedData = pipeline.Fit(trainData).Transform(trainData);

docs/samples/Microsoft.ML.Samples/Dynamic/ValueMappingStringToKeyType.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public static void Run()
5858
// Generate the ValueMappingEstimator that will output KeyTypes even though our values are strings.
5959
// The KeyToValueMappingEstimator is added to provide a reverse lookup of the KeyType, converting the KeyType value back
6060
// to the original value.
61-
var pipeline = new ValueMappingEstimator<string, string>(mlContext, educationKeys, educationValues, true, ("EducationKeyType", "Education"))
61+
var pipeline = mlContext.Transforms.Conversion.ValueMap<string, string>(educationKeys, educationValues, true, ("EducationKeyType", "Education"))
6262
.Append(mlContext.Transforms.Conversion.MapKeyToValue(("EducationCategory", "EducationKeyType")));
6363

6464
// Fits the ValueMappingEstimator and transforms the data adding the EducationKeyType column.

src/Microsoft.ML.Core/Utilities/BitUtils.cs

+1-4
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,7 @@ public static ulong MakeUlong(uint uHi, uint uLo)
2121
[MethodImpl(MethodImplOptions.AggressiveInlining)]
2222
public static uint GetLo(ulong uu)
2323
{
24-
// REVIEW: Work around Dev10 Bug 884217: JIT64 -Silent bad codegen for accessing 4-byte parts of 8-byte locals
25-
// http://vstfdevdiv:8080/WorkItemTracking/WorkItem.aspx?artifactMoniker=884217
26-
// return (uint)uu;
27-
return (uint)(uu & 0xFFFFFFFF);
24+
return (uint)uu;
2825
}
2926

3027
[MethodImpl(MethodImplOptions.AggressiveInlining)]

src/Microsoft.ML.Data/Commands/CrossValidationCommand.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ private void RunCore(IChannel ch, string cmd)
168168
"", ComponentFactoryUtils.CreateFromFunction<IDataView, IDataTransform>(
169169
(env, input) =>
170170
{
171-
var args = new GenerateNumberTransform.Arguments();
171+
var args = new GenerateNumberTransform.Options();
172172
args.Columns = new[] { new GenerateNumberTransform.Column() { Name = DefaultColumnNames.Name }, };
173173
args.UseCounter = true;
174174
return new GenerateNumberTransform(env, args, input);
@@ -313,7 +313,7 @@ private string GetSplitColumn(IChannel ch, IDataView input, ref IDataView output
313313
int inc = 0;
314314
while (input.Schema.TryGetColumnIndex(stratificationColumn, out tmp))
315315
stratificationColumn = string.Format("StratificationColumn_{0:000}", ++inc);
316-
var keyGenArgs = new GenerateNumberTransform.Arguments();
316+
var keyGenArgs = new GenerateNumberTransform.Options();
317317
var col = new GenerateNumberTransform.Column();
318318
col.Name = stratificationColumn;
319319
keyGenArgs.Columns = new[] { col };
@@ -514,7 +514,7 @@ private FoldResult RunFold(int fold)
514514
ITrainer trainer = _trainer.CreateComponent(host);
515515

516516
// Train pipe.
517-
var trainFilter = new RangeFilter.Arguments();
517+
var trainFilter = new RangeFilter.Options();
518518
trainFilter.Column = _splitColumn;
519519
trainFilter.Min = (Double)fold / _numFolds;
520520
trainFilter.Max = (Double)(fold + 1) / _numFolds;
@@ -524,7 +524,7 @@ private FoldResult RunFold(int fold)
524524
var trainData = _createExamples(host, ch, trainPipe, trainer);
525525

526526
// Test pipe.
527-
var testFilter = new RangeFilter.Arguments();
527+
var testFilter = new RangeFilter.Options();
528528
testFilter.Column = trainFilter.Column;
529529
testFilter.Min = trainFilter.Min;
530530
testFilter.Max = trainFilter.Max;

src/Microsoft.ML.Data/Commands/SaveDataCommand.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ private void RunCore(IChannel ch)
158158
// Send the first N lines to console.
159159
if (Args.Rows > 0)
160160
{
161-
var args = new SkipTakeFilter.TakeArguments() { Count = Args.Rows };
161+
var args = new SkipTakeFilter.TakeOptions() { Count = Args.Rows };
162162
data = SkipTakeFilter.Create(Host, args, data);
163163
}
164164
var textSaver = saver as TextSaver;

src/Microsoft.ML.Data/Commands/ScoreCommand.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ internal interface IDataScorerTransform : IDataTransform, ITransformTemplate
3636
[BestFriend]
3737
internal delegate void SignatureDataScorer(IDataView data, ISchemaBoundMapper mapper, RoleMappedSchema trainSchema);
3838

39-
public delegate void SignatureBindableMapper(IPredictor predictor);
39+
[BestFriend]
40+
internal delegate void SignatureBindableMapper(IPredictor predictor);
4041

4142
internal sealed class ScoreCommand : DataCommand.ImplBase<ScoreCommand.Arguments>
4243
{

src/Microsoft.ML.Data/DataLoadSave/Binary/CompressionKind.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ namespace Microsoft.ML.Data.IO
1212
/// A code indicating the kind of compression. It is supposed that each kind of compression is totally
1313
/// sufficient to describe how a given stream should be decompressed.
1414
/// </summary>
15-
public enum CompressionKind : byte
15+
internal enum CompressionKind : byte
1616
{
1717
None = 0, // No compression at all.
1818
Deflate = 1, // DEFLATE algorithm as in zlib's headerless/tailless compression.
1919
Default = Deflate
2020
}
2121

22-
public static class CompressionCodecExtension
22+
internal static class CompressionCodecExtension
2323
{
2424
/// <summary>
2525
/// Generate an appropriate wrapping compressing stream for the codec. This

src/Microsoft.ML.Data/DataLoadSave/Binary/Header.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
namespace Microsoft.ML.Data.IO
88
{
99
[StructLayout(LayoutKind.Explicit, Size = HeaderSize)]
10-
public struct Header
10+
internal struct Header
1111
{
1212
/// <summary>
1313
/// The fixed header size. This should not be changed even in future versions of the format.

src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ public IDataView SkipRows(IDataView input, long count)
242242
Environment.CheckValue(input, nameof(input));
243243
Environment.CheckUserArg(count > 0, nameof(count), "Must be greater than zero.");
244244

245-
var options = new SkipTakeFilter.SkipArguments()
245+
var options = new SkipTakeFilter.SkipOptions()
246246
{
247247
Count = count
248248
};
@@ -270,7 +270,7 @@ public IDataView TakeRows(IDataView input, long count)
270270
Environment.CheckValue(input, nameof(input));
271271
Environment.CheckUserArg(count > 0, nameof(count), "Must be greater than zero.");
272272

273-
var options = new SkipTakeFilter.TakeArguments()
273+
var options = new SkipTakeFilter.TakeOptions()
274274
{
275275
Count = count
276276
};

src/Microsoft.ML.Data/DataLoadSave/Transpose/TransposeSaver.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,10 @@ private void SaveTransposedData(IChannel ch, Stream stream, ITransposeDataView d
142142

143143
// First write out the no-row data, limited to these columns.
144144
IDataView subdata = new ChooseColumnsByIndexTransform(_host,
145-
new ChooseColumnsByIndexTransform.Arguments() { Indices = cols }, data);
145+
new ChooseColumnsByIndexTransform.Options() { Indices = cols }, data);
146146
// If we want the "dual mode" row-wise and slot-wise file, don't filter out anything.
147147
if (!_writeRowData)
148-
subdata = SkipTakeFilter.Create(_host, new SkipTakeFilter.TakeArguments() { Count = 0 }, subdata);
148+
subdata = SkipTakeFilter.Create(_host, new SkipTakeFilter.TakeOptions() { Count = 0 }, subdata);
149149

150150
string msg = _writeRowData ? "row-wise data, schema, and metadata" : "schema and metadata";
151151
viewAction(msg, subdata);

src/Microsoft.ML.Data/DataView/CacheDataView.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ private static IDataView SelectCachableColumns(IDataView data, IHostEnvironment
165165
return data;
166166

167167
// REVIEW: This can potentially cause hidden columns to become unhidden. See task 3739.
168-
var args = new ChooseColumnsByIndexTransform.Arguments();
168+
var args = new ChooseColumnsByIndexTransform.Options();
169169
args.Drop = true;
170170
args.Indices = columnsToDrop.ToArray();
171171
return new ChooseColumnsByIndexTransform(env, args, data);

src/Microsoft.ML.Data/DebuggerExtensions.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public static DataDebuggerPreview Preview(this IEstimator<ITransformer> estimato
4848

4949
using (var env = new LocalEnvironment(conc: 1))
5050
{
51-
var trainData = SkipTakeFilter.Create(env, new SkipTakeFilter.TakeArguments { Count = maxTrainingRows }, data);
51+
var trainData = SkipTakeFilter.Create(env, new SkipTakeFilter.TakeOptions { Count = maxTrainingRows }, data);
5252
return new DataDebuggerPreview(estimator.Fit(trainData).Transform(data), maxRows);
5353
}
5454
}

src/Microsoft.ML.Data/Depricated/Vector/VBufferMathUtils.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ namespace Microsoft.ML.Numeric
1313
// parameters, parameter order, etc.
1414
using Float = System.Single;
1515

16-
public static partial class VectorUtils
16+
internal static partial class VectorUtils
1717
{
1818
/// <summary>
1919
/// Returns the L2 norm squared of the vector (sum of squares of the components).

src/Microsoft.ML.Data/Depricated/Vector/VectorUtils.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ namespace Microsoft.ML.Numeric
1717
/// as a constant, it might have the name <c>a</c> or <c>b</c> or <c>src</c>, but in a situation
1818
/// where the vector structure might be changed the parameter might have the name <c>dst</c>.
1919
/// </summary>
20-
public static partial class VectorUtils
20+
[BestFriend]
21+
internal static partial class VectorUtils
2122
{
2223
public static Float DotProduct(Float[] a, Float[] b)
2324
{

src/Microsoft.ML.Data/Dirty/ChooseColumnsByIndexTransform.cs

+11-10
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,18 @@
1212
using Microsoft.ML.Internal.Utilities;
1313
using Microsoft.ML.Model;
1414

15-
[assembly: LoadableClass(typeof(ChooseColumnsByIndexTransform), typeof(ChooseColumnsByIndexTransform.Arguments), typeof(SignatureDataTransform),
15+
[assembly: LoadableClass(typeof(ChooseColumnsByIndexTransform), typeof(ChooseColumnsByIndexTransform.Options), typeof(SignatureDataTransform),
1616
"", "ChooseColumnsByIndexTransform", "ChooseColumnsByIndex")]
1717

1818
[assembly: LoadableClass(typeof(ChooseColumnsByIndexTransform), null, typeof(SignatureLoadDataTransform),
1919
"", ChooseColumnsByIndexTransform.LoaderSignature, ChooseColumnsByIndexTransform.LoaderSignatureOld)]
2020

2121
namespace Microsoft.ML.Data
2222
{
23-
public sealed class ChooseColumnsByIndexTransform : RowToRowTransformBase
23+
[BestFriend]
24+
internal sealed class ChooseColumnsByIndexTransform : RowToRowTransformBase
2425
{
25-
public sealed class Arguments
26+
public sealed class Options
2627
{
2728
[Argument(ArgumentType.Multiple | ArgumentType.Required, HelpText = "Column indices to select", Name = "Index", ShortName = "ind")]
2829
public int[] Indices;
@@ -59,17 +60,17 @@ private sealed class Bindings
5960
// This transform's output schema.
6061
internal Schema OutputSchema { get; }
6162

62-
internal Bindings(Arguments args, Schema sourceSchema)
63+
internal Bindings(Options options, Schema sourceSchema)
6364
{
64-
Contracts.AssertValue(args);
65+
Contracts.AssertValue(options);
6566
Contracts.AssertValue(sourceSchema);
6667

6768
_sourceSchema = sourceSchema;
6869

6970
// Store user-specified arguments as the major state of this transform. Only the major states will
7071
// be saved and all other attributes can be reconstructed from them.
71-
_drop = args.Drop;
72-
_selectedColumnIndexes = args.Indices;
72+
_drop = options.Drop;
73+
_selectedColumnIndexes = options.Indices;
7374

7475
// Compute actually used attributes in runtime from those major states.
7576
ComputeSources(_drop, _selectedColumnIndexes, _sourceSchema, out _sources);
@@ -194,12 +195,12 @@ private static VersionInfo GetVersionInfo()
194195
/// <summary>
195196
/// Public constructor corresponding to SignatureDataTransform.
196197
/// </summary>
197-
public ChooseColumnsByIndexTransform(IHostEnvironment env, Arguments args, IDataView input)
198+
public ChooseColumnsByIndexTransform(IHostEnvironment env, Options options, IDataView input)
198199
: base(env, RegistrationName, input)
199200
{
200-
Host.CheckValue(args, nameof(args));
201+
Host.CheckValue(options, nameof(options));
201202

202-
_bindings = new Bindings(args, Source.Schema);
203+
_bindings = new Bindings(options, Source.Schema);
203204
}
204205

205206
private ChooseColumnsByIndexTransform(IHost host, ModelLoadContext ctx, IDataView input)

src/Microsoft.ML.Data/EntryPoints/SchemaManipulation.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public static CommonOutputs.TransformOutput ConcatColumns(IHostEnvironment env,
2626
}
2727

2828
[TlcModule.EntryPoint(Name = "Transforms.ColumnSelector", Desc = "Selects a set of columns, dropping all others", UserName = "Select Columns")]
29-
public static CommonOutputs.TransformOutput SelectColumns(IHostEnvironment env, ColumnSelectingTransformer.Arguments input)
29+
public static CommonOutputs.TransformOutput SelectColumns(IHostEnvironment env, ColumnSelectingTransformer.Options input)
3030
{
3131
Contracts.CheckValue(env, nameof(env));
3232
var host = env.Register("SelectColumns");

src/Microsoft.ML.Data/EntryPoints/SelectRows.cs

+4-4
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ namespace Microsoft.ML.EntryPoints
1515
internal static class SelectRows
1616
{
1717
[TlcModule.EntryPoint(Name = "Transforms.RowRangeFilter", Desc = RangeFilter.Summary, UserName = RangeFilter.UserName, ShortName = RangeFilter.LoaderSignature)]
18-
public static CommonOutputs.TransformOutput FilterByRange(IHostEnvironment env, RangeFilter.Arguments input)
18+
public static CommonOutputs.TransformOutput FilterByRange(IHostEnvironment env, RangeFilter.Options input)
1919
{
2020
Contracts.CheckValue(env, nameof(env));
2121
var host = env.Register(RangeFilter.LoaderSignature);
@@ -28,7 +28,7 @@ public static CommonOutputs.TransformOutput FilterByRange(IHostEnvironment env,
2828

2929
[TlcModule.EntryPoint(Name = "Transforms.RowSkipFilter", Desc = SkipTakeFilter.SkipFilterSummary, UserName = SkipTakeFilter.SkipFilterUserName,
3030
ShortName = SkipTakeFilter.SkipFilterShortName)]
31-
public static CommonOutputs.TransformOutput SkipFilter(IHostEnvironment env, SkipTakeFilter.SkipArguments input)
31+
public static CommonOutputs.TransformOutput SkipFilter(IHostEnvironment env, SkipTakeFilter.SkipOptions input)
3232
{
3333
Contracts.CheckValue(env, nameof(env));
3434
var host = env.Register("SkipFilter");
@@ -40,7 +40,7 @@ public static CommonOutputs.TransformOutput SkipFilter(IHostEnvironment env, Ski
4040

4141
[TlcModule.EntryPoint(Name = "Transforms.RowTakeFilter", Desc = SkipTakeFilter.TakeFilterSummary, UserName = SkipTakeFilter.TakeFilterUserName,
4242
ShortName = SkipTakeFilter.TakeFilterShortName)]
43-
public static CommonOutputs.TransformOutput TakeFilter(IHostEnvironment env, SkipTakeFilter.TakeArguments input)
43+
public static CommonOutputs.TransformOutput TakeFilter(IHostEnvironment env, SkipTakeFilter.TakeOptions input)
4444
{
4545
Contracts.CheckValue(env, nameof(env));
4646
var host = env.Register("TakeFilter");
@@ -52,7 +52,7 @@ public static CommonOutputs.TransformOutput TakeFilter(IHostEnvironment env, Ski
5252

5353
[TlcModule.EntryPoint(Name = "Transforms.RowSkipAndTakeFilter", Desc = SkipTakeFilter.SkipTakeFilterSummary,
5454
UserName = SkipTakeFilter.SkipTakeFilterUserName, ShortName = SkipTakeFilter.SkipTakeFilterShortName)]
55-
public static CommonOutputs.TransformOutput SkipAndTakeFilter(IHostEnvironment env, SkipTakeFilter.Arguments input)
55+
public static CommonOutputs.TransformOutput SkipAndTakeFilter(IHostEnvironment env, SkipTakeFilter.Options input)
5656
{
5757
Contracts.CheckValue(env, nameof(env));
5858
var host = env.Register("SkipTakeFilter");

src/Microsoft.ML.Data/Evaluators/AnomalyDetectionEvaluator.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222

2323
namespace Microsoft.ML.Data
2424
{
25-
public sealed class AnomalyDetectionEvaluator : EvaluatorBase<AnomalyDetectionEvaluator.Aggregator>
25+
[BestFriend]
26+
internal sealed class AnomalyDetectionEvaluator : EvaluatorBase<AnomalyDetectionEvaluator.Aggregator>
2627
{
2728
public sealed class Arguments
2829
{
@@ -577,7 +578,8 @@ public void Finish()
577578
}
578579
}
579580

580-
public sealed class AnomalyDetectionMamlEvaluator : MamlEvaluatorBase
581+
[BestFriend]
582+
internal sealed class AnomalyDetectionMamlEvaluator : MamlEvaluatorBase
581583
{
582584
public sealed class Arguments : ArgumentsBase
583585
{

src/Microsoft.ML.Data/Evaluators/AucAggregator.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
namespace Microsoft.ML.Data
1212
{
13-
public abstract partial class EvaluatorBase<TAgg>
13+
internal abstract partial class EvaluatorBase<TAgg>
1414
{
1515
internal abstract class AucAggregatorBase
1616
{

src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs

+5-3
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@
2828

2929
namespace Microsoft.ML.Data
3030
{
31-
public sealed class BinaryClassifierEvaluator : RowToRowEvaluatorBase<BinaryClassifierEvaluator.Aggregator>
31+
[BestFriend]
32+
internal sealed class BinaryClassifierEvaluator : RowToRowEvaluatorBase<BinaryClassifierEvaluator.Aggregator>
3233
{
3334
public sealed class Arguments
3435
{
@@ -866,7 +867,7 @@ public BinaryClassificationMetrics Evaluate(IDataView data, string label, string
866867
}
867868
}
868869

869-
public sealed class BinaryPerInstanceEvaluator : PerInstanceEvaluatorBase
870+
internal sealed class BinaryPerInstanceEvaluator : PerInstanceEvaluatorBase
870871
{
871872
public const string LoaderSignature = "BinaryPerInstance";
872873
private static VersionInfo GetVersionInfo()
@@ -1116,7 +1117,8 @@ private void CheckInputColumnTypes(Schema schema)
11161117
}
11171118
}
11181119

1119-
public sealed class BinaryClassifierMamlEvaluator : MamlEvaluatorBase
1120+
[BestFriend]
1121+
internal sealed class BinaryClassifierMamlEvaluator : MamlEvaluatorBase
11201122
{
11211123
public class Arguments : ArgumentsBase
11221124
{

src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs

+5-3
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ namespace Microsoft.ML.Data
2929
{
3030
using Conditional = System.Diagnostics.ConditionalAttribute;
3131

32-
public sealed class ClusteringEvaluator : RowToRowEvaluatorBase<ClusteringEvaluator.Aggregator>
32+
[BestFriend]
33+
internal sealed class ClusteringEvaluator : RowToRowEvaluatorBase<ClusteringEvaluator.Aggregator>
3334
{
3435
public sealed class Arguments
3536
{
@@ -561,7 +562,7 @@ private void AssertValid(bool assertGetters)
561562
}
562563
}
563564

564-
public sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase
565+
internal sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase
565566
{
566567
public const string LoaderSignature = "ClusteringPerInstance";
567568
private static VersionInfo GetVersionInfo()
@@ -758,7 +759,8 @@ private void CheckInputColumnTypes(Schema schema)
758759
}
759760
}
760761

761-
public sealed class ClusteringMamlEvaluator : MamlEvaluatorBase
762+
[BestFriend]
763+
internal sealed class ClusteringMamlEvaluator : MamlEvaluatorBase
762764
{
763765
public class Arguments : ArgumentsBase
764766
{

0 commit comments

Comments
 (0)