Skip to content

Commit aaa7820

Browse files
TomFinleyTomFinley
committed
PR comments
1 parent f19592d commit aaa7820

File tree

14 files changed

+189
-226
lines changed

14 files changed

+189
-226
lines changed

‎Microsoft.ML.sln

+1
Original file line numberDiff line numberDiff line change
@@ -464,6 +464,7 @@ Global
464464
{00E38F77-1E61-4CDF-8F97-1417D4E85053} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
465465
{A7222F41-1CF0-47D9-B80C-B4D77B027A61} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
466466
{570A0B8A-5463-44D2-8521-54C0CA4CACA9} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
467+
{6DEF0F40-3853-47B3-8165-5F24BA5E14DF} = {7F13E156-3EBA-4021-84A5-CD56BA72F99E}
467468
{8B38BF24-35F4-4787-A9C5-22D35987106E} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
468469
EndGlobalSection
469470
GlobalSection(ExtensibilityGlobals) = postSolution

‎src/Microsoft.ML.Data/StaticPipe/BlockMaker.cs

-58
This file was deleted.

‎src/Microsoft.ML.Data/StaticPipe/DataReader.cs

+6-7
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,22 @@
88

99
namespace Microsoft.ML.Data.StaticPipe
1010
{
11-
public sealed class DataReader<TIn, TTupleShape>
12-
: BlockMaker<TTupleShape>
11+
public sealed class DataReader<TIn, TTupleShape> : SchemaBearing<TTupleShape>
1312
{
14-
public IDataReader<TIn> Wrapped { get; }
13+
public IDataReader<TIn> AsDynamic { get; }
1514

1615
public DataReader(IHostEnvironment env, IDataReader<TIn> reader)
1716
: base(env)
1817
{
19-
Wrapped = reader;
18+
AsDynamic = reader;
2019
}
2120

2221
public DataReaderEstimator<TIn, TNewOut, IDataReader<TIn>> Append<TNewOut, TTrans>(Estimator<TTupleShape, TNewOut, TTrans> estimator)
2322
where TTrans : class, ITransformer
2423
{
2524
Contracts.Assert(nameof(Append) == nameof(CompositeReaderEstimator<TIn, ITransformer>.Append));
2625

27-
var readerEst = Wrapped.Append(estimator.Wrapped);
26+
var readerEst = AsDynamic.Append(estimator.AsDynamic);
2827
return new DataReaderEstimator<TIn, TNewOut, IDataReader<TIn>>(Env, readerEst);
2928
}
3029

@@ -34,7 +33,7 @@ public DataReader<TIn, TNewTupleShape> Append<TNewTupleShape, TTransformer>(Tran
3433
Env.CheckValue(transformer, nameof(transformer));
3534
Env.Assert(nameof(Append) == nameof(CompositeReaderEstimator<TIn, ITransformer>.Append));
3635

37-
var reader = Wrapped.Append(transformer.Wrapped);
36+
var reader = AsDynamic.Append(transformer.AsDynamic);
3837
return new DataReader<TIn, TNewTupleShape>(Env, reader);
3938
}
4039

@@ -46,7 +45,7 @@ public DataView<TTupleShape> Read(TIn input)
4645
// that determination.
4746
Env.Assert(nameof(Read) == nameof(IDataReader<TIn>.Read));
4847

49-
var data = Wrapped.Read(input);
48+
var data = AsDynamic.Read(input);
5049
return new DataView<TTupleShape>(Env, data);
5150
}
5251
}

‎src/Microsoft.ML.Data/StaticPipe/DataReaderEstimator.cs

+5-5
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,23 @@
88

99
namespace Microsoft.ML.Data.StaticPipe
1010
{
11-
public sealed class DataReaderEstimator<TIn, TTupleShape, TDataReader> : BlockMaker<TTupleShape>
11+
public sealed class DataReaderEstimator<TIn, TTupleShape, TDataReader> : SchemaBearing<TTupleShape>
1212
where TDataReader : class, IDataReader<TIn>
1313
{
14-
public IDataReaderEstimator<TIn, TDataReader> Wrapped { get; }
14+
public IDataReaderEstimator<TIn, TDataReader> AsDynamic { get; }
1515

1616
public DataReaderEstimator(IHostEnvironment env, IDataReaderEstimator<TIn, TDataReader> estimator)
1717
: base(env)
1818
{
1919
Env.CheckValue(estimator, nameof(estimator));
20-
Wrapped = estimator;
20+
AsDynamic = estimator;
2121
}
2222

2323
public DataReader<TIn, TTupleShape> Fit(TIn input)
2424
{
2525
Contracts.Assert(nameof(Fit) == nameof(IDataReaderEstimator<TIn, TDataReader>.Fit));
2626

27-
var reader = Wrapped.Fit(input);
27+
var reader = AsDynamic.Fit(input);
2828
return new DataReader<TIn, TTupleShape>(Env, reader);
2929
}
3030

@@ -33,7 +33,7 @@ public DataReaderEstimator<TIn, TNewOut, IDataReader<TIn>> Append<TNewOut, TTran
3333
{
3434
Contracts.Assert(nameof(Append) == nameof(CompositeReaderEstimator<TIn, ITransformer>.Append));
3535

36-
var readerEst = Wrapped.Append(est.Wrapped);
36+
var readerEst = AsDynamic.Append(est.AsDynamic);
3737
return new DataReaderEstimator<TIn, TNewOut, IDataReader<TIn>>(Env, readerEst);
3838
}
3939
}

‎src/Microsoft.ML.Data/StaticPipe/DataView.cs

+4-5
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,17 @@
77

88
namespace Microsoft.ML.Data.StaticPipe
99
{
10-
public class DataView<TTupleShape>
10+
public class DataView<TTupleShape> : SchemaBearing<TTupleShape>
1111
{
12-
private readonly IHostEnvironment _env;
13-
public IDataView Wrapped { get; }
12+
public IDataView AsDynamic { get; }
1413

1514
public DataView(IHostEnvironment env, IDataView view)
15+
: base(env)
1616
{
1717
Contracts.CheckValue(env, nameof(env));
1818
env.CheckValue(view, nameof(view));
1919

20-
_env = env;
21-
Wrapped = view;
20+
AsDynamic = view;
2221
}
2322
}
2423
}

‎src/Microsoft.ML.Data/StaticPipe/Estimator.cs

+64-5
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,90 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using System;
6+
using System.Collections.Generic;
57
using Microsoft.ML.Core.Data;
8+
using Microsoft.ML.Data.StaticPipe.Runtime;
69
using Microsoft.ML.Runtime;
710
using Microsoft.ML.Runtime.Data;
811

912
namespace Microsoft.ML.Data.StaticPipe
1013
{
11-
public sealed class Estimator<TTupleInShape, TTupleOutShape, TTransformer>
12-
: BlockMaker<TTupleOutShape>
14+
public sealed class Estimator<TTupleInShape, TTupleOutShape, TTransformer> : SchemaBearing<TTupleOutShape>
1315
where TTransformer : class, ITransformer
1416
{
15-
public IEstimator<TTransformer> Wrapped { get; }
17+
public IEstimator<TTransformer> AsDynamic { get; }
1618

1719
public Estimator(IHostEnvironment env, IEstimator<TTransformer> estimator)
1820
: base(env)
1921
{
2022
Env.CheckValue(estimator, nameof(estimator));
21-
Wrapped = estimator;
23+
AsDynamic = estimator;
2224
}
2325

2426
public Transformer<TTupleInShape, TTupleOutShape, TTransformer> Fit(DataView<TTupleInShape> view)
2527
{
2628
Contracts.Assert(nameof(Fit) == nameof(IEstimator<TTransformer>.Fit));
2729

28-
var trans = Wrapped.Fit(view.Wrapped);
30+
var trans = AsDynamic.Fit(view.AsDynamic);
2931
return new Transformer<TTupleInShape, TTupleOutShape, TTransformer>(Env, trans);
3032
}
33+
34+
public Estimator<TTupleInShape, TTupleNewOutShape, ITransformer> Append<TTupleNewOutShape>(Estimator<TTupleOutShape, TTupleNewOutShape, ITransformer> estimator)
35+
{
36+
Env.CheckValue(estimator, nameof(estimator));
37+
38+
var est = AsDynamic.Append(estimator.AsDynamic);
39+
return new Estimator<TTupleInShape, TTupleNewOutShape, ITransformer>(Env, est);
40+
}
41+
42+
public Estimator<TTupleInShape, TTupleNewOutShape, ITransformer> Append<TTupleNewOutShape>(Func<TTupleOutShape, TTupleNewOutShape> mapper)
43+
{
44+
Contracts.CheckValue(mapper, nameof(mapper));
45+
46+
using (var ch = Env.Start(nameof(Append)))
47+
{
48+
var method = mapper.Method;
49+
50+
// Construct the dummy column structure, then apply the mapping.
51+
var input = PipelineColumnAnalyzer.MakeAnalysisInstance<TTupleOutShape>(out var fakeReconciler);
52+
KeyValuePair<string, PipelineColumn>[] inPairs = PipelineColumnAnalyzer.GetNames(input, method.GetParameters()[0]);
53+
54+
// Initially we suppose we've only assigned names to the inputs.
55+
var inputColToName = new Dictionary<PipelineColumn, string>();
56+
foreach (var p in inPairs)
57+
inputColToName[p.Value] = p.Key;
58+
string NameMap(PipelineColumn col)
59+
{
60+
inputColToName.TryGetValue(col, out var val);
61+
return val;
62+
}
63+
64+
var readerEst = StaticPipeUtils.GeneralFunctionAnalyzer(Env, ch, input, fakeReconciler, mapper, out var estTail, NameMap);
65+
ch.Assert(readerEst == null);
66+
ch.AssertValue(estTail);
67+
68+
var est = AsDynamic.Append(estTail);
69+
var toReturn = new Estimator<TTupleInShape, TTupleNewOutShape, ITransformer>(Env, est);
70+
ch.Done();
71+
return toReturn;
72+
}
73+
}
74+
}
75+
76+
public static class Estimator
77+
{
78+
/// <summary>
79+
/// Create an object that can be used as the start of a new pipeline, that assumes it uses
80+
/// something with the sahape of <typeparamref name="TTupleShape"/> as its input schema shape.
81+
/// The returned object is an empty estimator.
82+
/// </summary>
83+
/// <param name="fromSchema"></param>
84+
/// <returns></returns>
85+
public static Estimator<TTupleShape, TTupleShape, ITransformer> MakeNew<TTupleShape>(SchemaBearing<TTupleShape> fromSchema)
86+
{
87+
Contracts.CheckValue(fromSchema, nameof(fromSchema));
88+
return fromSchema.MakeNewEstimator();
89+
}
3190
}
3291
}

‎src/Microsoft.ML.Data/StaticPipe/PipelineColumnAnalyzer.cs

+19-31
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,21 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33
// See the LICENSE file in the project root for more information.
44

5+
using System;
56
using System.Collections.Generic;
67
using System.Linq;
78
using System.Reflection;
89
using System.Runtime.CompilerServices;
10+
using Microsoft.ML.Core.Data;
911
using Microsoft.ML.Data.StaticPipe.Runtime;
1012
using Microsoft.ML.Runtime;
11-
using System;
1213
using Microsoft.ML.Runtime.Internal.Utilities;
13-
using Microsoft.ML.Core.Data;
1414

1515
namespace Microsoft.ML.Data.StaticPipe
1616
{
1717
internal static class PipelineColumnAnalyzer
1818
{
19-
public static void Analyze<TIn, TOut>(Func<TIn, TOut> func)
20-
{
21-
bool singletonIn = typeof(PipelineColumn).IsAssignableFrom(typeof(TIn));
22-
bool singletonOut = typeof(PipelineColumn).IsAssignableFrom(typeof(TOut));
23-
24-
var analysis = CreateAnalysisInstance<TIn>(out var reconciler);
25-
var analysisOut = func(analysis);
26-
27-
var inNames = GetNames<TIn, PipelineColumn>(analysis, func.Method.GetParameters()[0]);
28-
var outNames = GetNames<TOut, PipelineColumn>(analysisOut, func.Method.ReturnParameter);
29-
}
30-
31-
public interface IIsAnalysisColumn { }
19+
private interface IIsAnalysisColumn { }
3220

3321
/// <summary>
3422
/// Given a type which is a <see cref="ValueTuple"/> tree with <see cref="PipelineColumn"/> leaves, return an instance of that
@@ -40,11 +28,11 @@ public interface IIsAnalysisColumn { }
4028
/// (e.g., <see cref="Scalar{T}"/>, <see cref="Vector{T}"/>, etc.)</typeparam>
4129
/// <returns>An instance of <typeparamref name="T"/> where all <see cref="PipelineColumn"/> fields have instances implementing
4230
/// <see cref="IIsAnalysisColumn"/> filled in</returns>
43-
public static T CreateAnalysisInstance<T>(out ReaderReconciler<int> fakeReconciler)
31+
public static T MakeAnalysisInstance<T>(out ReaderReconciler<int> fakeReconciler)
4432
{
4533
var rec = new AnalyzeUtil.Rec();
4634
fakeReconciler = rec;
47-
return (T)AnalyzeUtil.CreateAnalysisInstanceCore<T>(rec);
35+
return (T)AnalyzeUtil.MakeAnalysisInstanceCore<T>(rec);
4836
}
4937

5038
private static class AnalyzeUtil
@@ -72,12 +60,12 @@ private sealed class AKey<T> : Key<T>, IIsAnalysisColumn { public AKey(Rec rec)
7260
private sealed class AKey<T, TV> : Key<T, TV>, IIsAnalysisColumn { public AKey(Rec rec) : base(rec, null) { } }
7361
private sealed class AVarKey<T> : VarKey<T>, IIsAnalysisColumn { public AVarKey(Rec rec) : base(rec, null) { } }
7462

75-
private static PipelineColumn CreateScalar<T>(Rec rec) => new AScalar<T>(rec);
76-
private static PipelineColumn CreateVector<T>(Rec rec) => new AVector<T>(rec);
77-
private static PipelineColumn CreateVarVector<T>(Rec rec) => new AVarVector<T>(rec);
78-
private static PipelineColumn CreateKey<T>(Rec rec) => new AKey<T>(rec);
79-
private static Key<T, TV> CreateKey<T, TV>(Rec rec) => new AKey<T, TV>(rec);
80-
private static PipelineColumn CreateVarKey<T>(Rec rec) => new AVarKey<T>(rec);
63+
private static PipelineColumn MakeScalar<T>(Rec rec) => new AScalar<T>(rec);
64+
private static PipelineColumn MakeVector<T>(Rec rec) => new AVector<T>(rec);
65+
private static PipelineColumn MakeVarVector<T>(Rec rec) => new AVarVector<T>(rec);
66+
private static PipelineColumn MakeKey<T>(Rec rec) => new AKey<T>(rec);
67+
private static Key<T, TV> MakeKey<T, TV>(Rec rec) => new AKey<T, TV>(rec);
68+
private static PipelineColumn MakeVarKey<T>(Rec rec) => new AVarKey<T>(rec);
8169

8270
private static MethodInfo[] _valueTupleCreateMethod = InitValueTupleCreateMethods();
8371

@@ -105,7 +93,7 @@ public static ValueTuple<T1, T2, T3, T4, T5, T6, T7, TRest>
10593
return new ValueTuple<T1, T2, T3, T4, T5, T6, T7, TRest>(v1, v2, v3, v4, v5, v6, v7, restTuple);
10694
}
10795

108-
public static object CreateAnalysisInstanceCore<T>(Rec rec)
96+
public static object MakeAnalysisInstanceCore<T>(Rec rec)
10997
{
11098
var t = typeof(T);
11199
if (typeof(PipelineColumn).IsAssignableFrom(t))
@@ -116,20 +104,20 @@ public static object CreateAnalysisInstanceCore<T>(Rec rec)
116104
var genT = t.GetGenericTypeDefinition();
117105

118106
if (genT == typeof(Scalar<>))
119-
return Utils.MarshalInvoke(CreateScalar<int>, genP[0], rec);
107+
return Utils.MarshalInvoke(MakeScalar<int>, genP[0], rec);
120108
if (genT == typeof(Vector<>))
121-
return Utils.MarshalInvoke(CreateVector<int>, genP[0], rec);
109+
return Utils.MarshalInvoke(MakeVector<int>, genP[0], rec);
122110
if (genT == typeof(VarVector<>))
123-
return Utils.MarshalInvoke(CreateVarVector<int>, genP[0], rec);
111+
return Utils.MarshalInvoke(MakeVarVector<int>, genP[0], rec);
124112
if (genT == typeof(Key<>))
125-
return Utils.MarshalInvoke(CreateKey<uint>, genP[0], rec);
113+
return Utils.MarshalInvoke(MakeKey<uint>, genP[0], rec);
126114
if (genT == typeof(Key<,>))
127115
{
128-
Func<Rec, PipelineColumn> f = CreateKey<uint, int>;
116+
Func<Rec, PipelineColumn> f = MakeKey<uint, int>;
129117
return f.Method.GetGenericMethodDefinition().MakeGenericMethod(genP).Invoke(null, new object[] { rec });
130118
}
131119
if (genT == typeof(VarKey<>))
132-
return Utils.MarshalInvoke(CreateVector<int>, genP[0], rec);
120+
return Utils.MarshalInvoke(MakeVector<int>, genP[0], rec);
133121
}
134122
throw Contracts.Except($"Type {t} is a {nameof(PipelineColumn)} yet does not appear to be directly one of " +
135123
$"the official types. This is commonly due to a mistake by the component author and can be addressed by " +
@@ -144,7 +132,7 @@ public static object CreateAnalysisInstanceCore<T>(Rec rec)
144132
if (1 <= genP.Length && genP.Length <= 8)
145133
{
146134
// First recursively create the sub-analysis objects.
147-
object[] subArgs = genP.Select(subType => Utils.MarshalInvoke(CreateAnalysisInstanceCore<int>, subType, rec)).ToArray();
135+
object[] subArgs = genP.Select(subType => Utils.MarshalInvoke(MakeAnalysisInstanceCore<int>, subType, rec)).ToArray();
148136
// Next create the tuple.
149137
return _valueTupleCreateMethod[subArgs.Length - 1].MakeGenericMethod(genP).Invoke(null, subArgs);
150138
}

0 commit comments

Comments
 (0)