Skip to content

Commit 1e7b8be

Browse files
codemzscodemzs
authored
Replace DV data type system with .NET standard type system. (#863)
1 parent 7b1c7d7 commit 1e7b8be

File tree

264 files changed

+4038
-8134
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

264 files changed

+4038
-8134
lines changed

‎build/Dependencies.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
<BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
1414
<TensorFlowVersion>1.10.0</TensorFlowVersion>
1515
<SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
16-
16+
<SystemMemoryVersion>4.5.1</SystemMemoryVersion>
1717
<MicrosoftCodeAnalysisCSharpVersion>2.9.0</MicrosoftCodeAnalysisCSharpVersion>
1818
<MicrosoftCSharpVersion>4.5.0</MicrosoftCSharpVersion>
1919
<SystemCompositionVersion>1.2.0</SystemCompositionVersion>

‎pkg/Microsoft.ML/Microsoft.ML.nupkgproj

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPackageVersion)" />
1313
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="$(SystemThreadingTasksDataflowPackageVersion)" />
1414
<PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
15+
<PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
1516
</ItemGroup>
1617

1718
<ItemGroup>

‎src/Microsoft.ML.Api/ApiUtils.cs

+2-3
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,10 @@ private static OpCode GetAssignmentOpCode(Type t)
1919
{
2020
// REVIEW: This should be a Dictionary<Type, OpCode> based solution.
2121
// DvTypes, strings, arrays, all nullable types, VBuffers and UInt128.
22-
if (t == typeof(DvInt8) || t == typeof(DvInt4) || t == typeof(DvInt2) || t == typeof(DvInt1) ||
23-
t == typeof(DvBool) || t == typeof(DvText) || t == typeof(string) || t.IsArray ||
22+
if (t == typeof(ReadOnlyMemory<char>) || t == typeof(string) || t.IsArray ||
2423
(t.IsGenericType && t.GetGenericTypeDefinition() == typeof(VBuffer<>)) ||
2524
(t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Nullable<>)) ||
26-
t == typeof(DvDateTime) || t == typeof(DvDateTimeZone) || t == typeof(DvTimeSpan) || t == typeof(UInt128))
25+
t == typeof(DateTime) || t == typeof(DateTimeOffset) || t == typeof(TimeSpan) || t == typeof(UInt128))
2726
{
2827
return OpCodes.Stobj;
2928
}

‎src/Microsoft.ML.Api/DataViewConstructionUtils.cs

+17-126
Original file line numberDiff line numberDiff line change
@@ -125,61 +125,11 @@ private Delegate CreateGetter(int index)
125125
if (outputType.IsArray)
126126
{
127127
Ch.Assert(colType.IsVector);
128-
// String[] -> VBuffer<DvText>
128+
// String[] -> ReadOnlyMemory<char>
129129
if (outputType.GetElementType() == typeof(string))
130130
{
131131
Ch.Assert(colType.ItemType.IsText);
132-
return CreateConvertingArrayGetterDelegate<String, DvText>(index, x => x == null ? DvText.NA : new DvText(x));
133-
}
134-
else if (outputType.GetElementType() == typeof(int))
135-
{
136-
Ch.Assert(colType.ItemType == NumberType.I4);
137-
return CreateConvertingArrayGetterDelegate<int, DvInt4>(index, x => x);
138-
}
139-
else if (outputType.GetElementType() == typeof(int?))
140-
{
141-
Ch.Assert(colType.ItemType == NumberType.I4);
142-
return CreateConvertingArrayGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
143-
}
144-
else if (outputType.GetElementType() == typeof(long))
145-
{
146-
Ch.Assert(colType.ItemType == NumberType.I8);
147-
return CreateConvertingArrayGetterDelegate<long, DvInt8>(index, x => x);
148-
}
149-
else if (outputType.GetElementType() == typeof(long?))
150-
{
151-
Ch.Assert(colType.ItemType == NumberType.I8);
152-
return CreateConvertingArrayGetterDelegate<long?, DvInt8>(index, x => x ?? DvInt8.NA);
153-
}
154-
else if (outputType.GetElementType() == typeof(short))
155-
{
156-
Ch.Assert(colType.ItemType == NumberType.I2);
157-
return CreateConvertingArrayGetterDelegate<short, DvInt2>(index, x => x);
158-
}
159-
else if (outputType.GetElementType() == typeof(short?))
160-
{
161-
Ch.Assert(colType.ItemType == NumberType.I2);
162-
return CreateConvertingArrayGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
163-
}
164-
else if (outputType.GetElementType() == typeof(sbyte))
165-
{
166-
Ch.Assert(colType.ItemType == NumberType.I1);
167-
return CreateConvertingArrayGetterDelegate<sbyte, DvInt1>(index, x => x);
168-
}
169-
else if (outputType.GetElementType() == typeof(sbyte?))
170-
{
171-
Ch.Assert(colType.ItemType == NumberType.I1);
172-
return CreateConvertingArrayGetterDelegate<sbyte?, DvInt1>(index, x => x ?? DvInt1.NA);
173-
}
174-
else if (outputType.GetElementType() == typeof(bool))
175-
{
176-
Ch.Assert(colType.ItemType.IsBool);
177-
return CreateConvertingArrayGetterDelegate<bool, DvBool>(index, x => x);
178-
}
179-
else if (outputType.GetElementType() == typeof(bool?))
180-
{
181-
Ch.Assert(colType.ItemType.IsBool);
182-
return CreateConvertingArrayGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
132+
return CreateConvertingArrayGetterDelegate<string, ReadOnlyMemory<char>>(index, x => x != null ? x.AsMemory() : ReadOnlyMemory<char>.Empty);
183133
}
184134

185135
// T[] -> VBuffer<T>
@@ -193,7 +143,7 @@ private Delegate CreateGetter(int index)
193143
else if (colType.IsVector)
194144
{
195145
// VBuffer<T> -> VBuffer<T>
196-
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<DvText>?
146+
// REVIEW: Do we care about accomodating VBuffer<string> -> ReadOnlyMemory<char>?
197147
Ch.Assert(outputType.IsGenericType);
198148
Ch.Assert(outputType.GetGenericTypeDefinition() == typeof(VBuffer<>));
199149
Ch.Assert(outputType.GetGenericArguments()[0] == colType.ItemType.RawType);
@@ -204,70 +154,11 @@ private Delegate CreateGetter(int index)
204154
{
205155
if (outputType == typeof(string))
206156
{
207-
// String -> DvText
157+
// String -> ReadOnlyMemory<char>
208158
Ch.Assert(colType.IsText);
209-
return CreateConvertingGetterDelegate<String, DvText>(index, x => x == null ? DvText.NA : new DvText(x));
210-
}
211-
else if (outputType == typeof(bool))
212-
{
213-
// Bool -> DvBool
214-
Ch.Assert(colType.IsBool);
215-
return CreateConvertingGetterDelegate<bool, DvBool>(index, x => x);
216-
}
217-
else if (outputType == typeof(bool?))
218-
{
219-
// Bool? -> DvBool
220-
Ch.Assert(colType.IsBool);
221-
return CreateConvertingGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
222-
}
223-
else if (outputType == typeof(int))
224-
{
225-
// int -> DvInt4
226-
Ch.Assert(colType == NumberType.I4);
227-
return CreateConvertingGetterDelegate<int, DvInt4>(index, x => x);
228-
}
229-
else if (outputType == typeof(int?))
230-
{
231-
// int? -> DvInt4
232-
Ch.Assert(colType == NumberType.I4);
233-
return CreateConvertingGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
234-
}
235-
else if (outputType == typeof(short))
236-
{
237-
// short -> DvInt2
238-
Ch.Assert(colType == NumberType.I2);
239-
return CreateConvertingGetterDelegate<short, DvInt2>(index, x => x);
240-
}
241-
else if (outputType == typeof(short?))
242-
{
243-
// short? -> DvInt2
244-
Ch.Assert(colType == NumberType.I2);
245-
return CreateConvertingGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
246-
}
247-
else if (outputType == typeof(long))
248-
{
249-
// long -> DvInt8
250-
Ch.Assert(colType == NumberType.I8);
251-
return CreateConvertingGetterDelegate<long, DvInt8>(index, x => x);
252-
}
253-
else if (outputType == typeof(long?))
254-
{
255-
// long? -> DvInt8
256-
Ch.Assert(colType == NumberType.I8);
257-
return CreateConvertingGetterDelegate<long?, DvInt8>(index, x => x ?? DvInt8.NA);
258-
}
259-
else if (outputType == typeof(sbyte))
260-
{
261-
// sbyte -> DvInt1
262-
Ch.Assert(colType == NumberType.I1);
263-
return CreateConvertingGetterDelegate<sbyte, DvInt1>(index, x => x);
264-
}
265-
else if (outputType == typeof(sbyte?))
266-
{
267-
// sbyte? -> DvInt1
268-
Ch.Assert(colType == NumberType.I1);
269-
return CreateConvertingGetterDelegate<sbyte?, DvInt1>(index, x => x ?? DvInt1.NA);
159+
return CreateConvertingGetterDelegate<String, ReadOnlyMemory<char>>(index, x => x != null ? x.AsMemory() : ReadOnlyMemory<char>.Empty);
270160
}
161+
271162
// T -> T
272163
if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(Nullable<>))
273164
Ch.Assert(colType.RawType == Nullable.GetUnderlyingType(outputType));
@@ -805,12 +696,12 @@ public override ValueGetter<TDst> GetGetter<TDst>()
805696
var itemType = typeT.GetElementType();
806697
var dstItemType = typeof(TDst).GetGenericArguments()[0];
807698

808-
// String[] -> VBuffer<DvText>
699+
// String[] -> VBuffer<ReadOnlyMemory<char>>
809700
if (itemType == typeof(string))
810701
{
811-
Contracts.Check(dstItemType == typeof(DvText));
702+
Contracts.Check(dstItemType == typeof(ReadOnlyMemory<char>));
812703

813-
ValueGetter<VBuffer<DvText>> method = GetStringArray;
704+
ValueGetter<VBuffer<ReadOnlyMemory<char>>> method = GetStringArray;
814705
return method as ValueGetter<TDst>;
815706
}
816707

@@ -825,7 +716,7 @@ public override ValueGetter<TDst> GetGetter<TDst>()
825716
if (MetadataType.IsVector)
826717
{
827718
// VBuffer<T> -> VBuffer<T>
828-
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<DvText>?
719+
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
829720

830721
Contracts.Assert(typeT.IsGenericType);
831722
Contracts.Check(typeof(TDst).IsGenericType);
@@ -845,9 +736,9 @@ public override ValueGetter<TDst> GetGetter<TDst>()
845736
{
846737
if (typeT == typeof(string))
847738
{
848-
// String -> DvText
739+
// String -> ReadOnlyMemory<char>
849740
Contracts.Assert(MetadataType.IsText);
850-
ValueGetter<DvText> m = GetString;
741+
ValueGetter<ReadOnlyMemory<char>> m = GetString;
851742
return m as ValueGetter<TDst>;
852743
}
853744
// T -> T
@@ -861,14 +752,14 @@ public class TElement
861752
{
862753
}
863754

864-
private void GetStringArray(ref VBuffer<DvText> dst)
755+
private void GetStringArray(ref VBuffer<ReadOnlyMemory<char>> dst)
865756
{
866757
var value = (string[])(object)Value;
867758
var n = Utils.Size(value);
868-
dst = new VBuffer<DvText>(n, Utils.Size(dst.Values) < n ? new DvText[n] : dst.Values, dst.Indices);
759+
dst = new VBuffer<ReadOnlyMemory<char>>(n, Utils.Size(dst.Values) < n ? new ReadOnlyMemory<char>[n] : dst.Values, dst.Indices);
869760

870761
for (int i = 0; i < n; i++)
871-
dst.Values[i] = new DvText(value[i]);
762+
dst.Values[i] = value[i].AsMemory();
872763

873764
}
874765

@@ -890,9 +781,9 @@ private ValueGetter<VBuffer<TDst>> GetVBufferGetter<TDst>()
890781
return (ref VBuffer<TDst> dst) => castValue.CopyTo(ref dst);
891782
}
892783

893-
private void GetString(ref DvText dst)
784+
private void GetString(ref ReadOnlyMemory<char> dst)
894785
{
895-
dst = new DvText((string)(object)Value);
786+
dst = ((string)(object)Value).AsMemory();
896787
}
897788

898789
private void GetDirectValue<TDst>(ref TDst dst)

0 commit comments

Comments
 (0)