Skip to content

Commit f494e3b

Browse files
eerhardteerhardt
committed
Load and register all assemblies in the Maml directory.
Ensure all loaded assemblies are registered in Experiment to maintain compability. Fix tests to not use ComponentCatalog but direct instantiation instead.
1 parent 7cd4e42 commit f494e3b

File tree

7 files changed

+186
-16
lines changed

7 files changed

+186
-16
lines changed

‎src/Common/AssemblyLoadingUtils.cs

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,28 @@ public static void LoadAndRegister(IHostEnvironment env, string[] assemblies)
7575
}
7676
}
7777

78+
public static IDisposable CreateAssemblyRegistrar(IHostEnvironment env, string loadAssembliesPath = null)
79+
{
80+
Contracts.CheckValue(env, nameof(env));
81+
env.CheckValueOrNull(loadAssembliesPath);
82+
83+
return new AssemblyRegistrar(env, loadAssembliesPath);
84+
}
85+
86+
public static void RegisterCurrentLoadedAssemblies(IHostEnvironment env)
87+
{
88+
Contracts.CheckValue(env, nameof(env));
89+
90+
foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
91+
{
92+
// Ignore dynamic assemblies.
93+
if (a.IsDynamic)
94+
continue;
95+
96+
env.ComponentCatalog.RegisterAssembly(a);
97+
}
98+
}
99+
78100
private static string CreateTempDirectory()
79101
{
80102
string dir = GetTempPath();
@@ -117,5 +139,40 @@ private static Assembly LoadAssembly(IHostEnvironment env, string path)
117139
return null;
118140
}
119141
}
142+
143+
private sealed class AssemblyRegistrar : IDisposable
144+
{
145+
private readonly IHostEnvironment _env;
146+
147+
public AssemblyRegistrar(IHostEnvironment env, string path)
148+
{
149+
_env = env;
150+
151+
RegisterCurrentLoadedAssemblies(_env);
152+
153+
if (!string.IsNullOrEmpty(path))
154+
{
155+
LoadAssembliesInDir(_env, path);
156+
path = Path.Combine(path, "AutoLoad");
157+
LoadAssembliesInDir(_env, path);
158+
}
159+
160+
AppDomain.CurrentDomain.AssemblyLoad += CurrentDomainAssemblyLoad;
161+
}
162+
163+
public void Dispose()
164+
{
165+
AppDomain.CurrentDomain.AssemblyLoad -= CurrentDomainAssemblyLoad;
166+
}
167+
168+
private void CurrentDomainAssemblyLoad(object sender, AssemblyLoadEventArgs args)
169+
{
170+
// Don't try to index dynamic generated assembly
171+
if (args.LoadedAssembly.IsDynamic)
172+
return;
173+
174+
_env.ComponentCatalog.RegisterAssembly(args.LoadedAssembly);
175+
}
176+
}
120177
}
121178
}

‎src/Microsoft.ML.Legacy/Microsoft.ML.Legacy.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
<DefineConstants>CORECLR</DefineConstants>
77
</PropertyGroup>
88

9+
<ItemGroup>
10+
<Compile Include="..\Common\AssemblyLoadingUtils.cs" Link="Common\AssemblyLoadingUtils.cs" />
11+
</ItemGroup>
12+
913
<ItemGroup>
1014
<PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
1115
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonPackageVersion)" />

‎src/Microsoft.ML.Legacy/PredictionModel.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ public static Task<PredictionModel<TInput, TOutput>> ReadAsync<TInput, TOutput>(
126126
throw new ArgumentNullException(nameof(stream));
127127

128128
using (var environment = new ConsoleEnvironment())
129+
using (AssemblyLoadingUtils.CreateAssemblyRegistrar(environment))
129130
{
130131
BatchPredictionEngine<TInput, TOutput> predictor =
131132
environment.CreateBatchPredictionEngine<TInput, TOutput>(stream);

‎src/Microsoft.ML.Legacy/Runtime/Experiment/Experiment.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ private sealed class SerializationHelper
3737
public Experiment(Runtime.IHostEnvironment env)
3838
{
3939
_env = env;
40+
AssemblyLoadingUtils.RegisterCurrentLoadedAssemblies(_env);
41+
4042
_catalog = ModuleCatalog.CreateInstance(_env);
4143
_jsonNodes = new List<string>();
4244
_serializer = new JsonSerializer();

‎src/Microsoft.ML.Maml/MAML.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,10 @@ public static int Main()
5555

5656
private static int MainWithProgress(string args)
5757
{
58+
string currentDirectory = Path.GetDirectoryName(typeof(Maml).Module.FullyQualifiedName);
59+
5860
using (var env = CreateEnvironment())
61+
using (AssemblyLoadingUtils.CreateAssemblyRegistrar(env, currentDirectory))
5962
using (var progressCancel = new CancellationTokenSource())
6063
{
6164
var progressTrackerTask = Task.Run(() => TrackProgress(env, progressCancel.Token));

‎test/Microsoft.ML.Tests/ImagesTests.cs

Lines changed: 103 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,21 @@ public void TestEstimatorChain()
2929
{
3030
var dataFile = GetDataPath("images/images.tsv");
3131
var imageFolder = Path.GetDirectoryName(dataFile);
32-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
33-
var invalidData = env.CreateLoader("Text{col=ImagePath:R4:0}", new MultiFileSource(dataFile));
32+
var data = TextLoader.Create(env, new TextLoader.Arguments()
33+
{
34+
Column = new[]
35+
{
36+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
37+
new TextLoader.Column("Name", DataKind.TX, 1),
38+
}
39+
}, new MultiFileSource(dataFile));
40+
var invalidData = TextLoader.Create(env, new TextLoader.Arguments()
41+
{
42+
Column = new[]
43+
{
44+
new TextLoader.Column("ImagePath", DataKind.R4, 0),
45+
}
46+
}, new MultiFileSource(dataFile));
3447

3548
var pipe = new ImageLoaderEstimator(env, imageFolder, ("ImagePath", "ImageReal"))
3649
.Append(new ImageResizerEstimator(env, "ImageReal", "ImageReal", 100, 100))
@@ -49,7 +62,14 @@ public void TestEstimatorSaveLoad()
4962
{
5063
var dataFile = GetDataPath("images/images.tsv");
5164
var imageFolder = Path.GetDirectoryName(dataFile);
52-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
65+
var data = TextLoader.Create(env, new TextLoader.Arguments()
66+
{
67+
Column = new[]
68+
{
69+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
70+
new TextLoader.Column("Name", DataKind.TX, 1),
71+
}
72+
}, new MultiFileSource(dataFile));
5373

5474
var pipe = new ImageLoaderEstimator(env, imageFolder, ("ImagePath", "ImageReal"))
5575
.Append(new ImageResizerEstimator(env, "ImageReal", "ImageReal", 100, 100))
@@ -82,7 +102,14 @@ public void TestSaveImages()
82102
{
83103
var dataFile = GetDataPath("images/images.tsv");
84104
var imageFolder = Path.GetDirectoryName(dataFile);
85-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
105+
var data = TextLoader.Create(env, new TextLoader.Arguments()
106+
{
107+
Column = new[]
108+
{
109+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
110+
new TextLoader.Column("Name", DataKind.TX, 1),
111+
}
112+
}, new MultiFileSource(dataFile));
86113
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
87114
{
88115
Column = new ImageLoaderTransform.Column[1]
@@ -129,7 +156,14 @@ public void TestGreyscaleTransformImages()
129156
var imageWidth = 100;
130157
var dataFile = GetDataPath("images/images.tsv");
131158
var imageFolder = Path.GetDirectoryName(dataFile);
132-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
159+
var data = TextLoader.Create(env, new TextLoader.Arguments()
160+
{
161+
Column = new[]
162+
{
163+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
164+
new TextLoader.Column("Name", DataKind.TX, 1),
165+
}
166+
}, new MultiFileSource(dataFile));
133167
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
134168
{
135169
Column = new ImageLoaderTransform.Column[1]
@@ -192,7 +226,14 @@ public void TestBackAndForthConversionWithAlphaInterleave()
192226
var imageWidth = 130;
193227
var dataFile = GetDataPath("images/images.tsv");
194228
var imageFolder = Path.GetDirectoryName(dataFile);
195-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
229+
var data = TextLoader.Create(env, new TextLoader.Arguments()
230+
{
231+
Column = new[]
232+
{
233+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
234+
new TextLoader.Column("Name", DataKind.TX, 1),
235+
}
236+
}, new MultiFileSource(dataFile));
196237
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
197238
{
198239
Column = new ImageLoaderTransform.Column[1]
@@ -275,7 +316,14 @@ public void TestBackAndForthConversionWithoutAlphaInterleave()
275316
var imageWidth = 130;
276317
var dataFile = GetDataPath("images/images.tsv");
277318
var imageFolder = Path.GetDirectoryName(dataFile);
278-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
319+
var data = TextLoader.Create(env, new TextLoader.Arguments()
320+
{
321+
Column = new[]
322+
{
323+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
324+
new TextLoader.Column("Name", DataKind.TX, 1),
325+
}
326+
}, new MultiFileSource(dataFile));
279327
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
280328
{
281329
Column = new ImageLoaderTransform.Column[1]
@@ -358,7 +406,14 @@ public void TestBackAndForthConversionWithAlphaNoInterleave()
358406
var imageWidth = 130;
359407
var dataFile = GetDataPath("images/images.tsv");
360408
var imageFolder = Path.GetDirectoryName(dataFile);
361-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
409+
var data = TextLoader.Create(env, new TextLoader.Arguments()
410+
{
411+
Column = new[]
412+
{
413+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
414+
new TextLoader.Column("Name", DataKind.TX, 1),
415+
}
416+
}, new MultiFileSource(dataFile));
362417
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
363418
{
364419
Column = new ImageLoaderTransform.Column[1]
@@ -441,7 +496,14 @@ public void TestBackAndForthConversionWithoutAlphaNoInterleave()
441496
var imageWidth = 130;
442497
var dataFile = GetDataPath("images/images.tsv");
443498
var imageFolder = Path.GetDirectoryName(dataFile);
444-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
499+
var data = TextLoader.Create(env, new TextLoader.Arguments()
500+
{
501+
Column = new[]
502+
{
503+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
504+
new TextLoader.Column("Name", DataKind.TX, 1),
505+
}
506+
}, new MultiFileSource(dataFile));
445507
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
446508
{
447509
Column = new ImageLoaderTransform.Column[1]
@@ -524,7 +586,14 @@ public void TestBackAndForthConversionWithAlphaInterleaveNoOffset()
524586
var imageWidth = 130;
525587
var dataFile = GetDataPath("images/images.tsv");
526588
var imageFolder = Path.GetDirectoryName(dataFile);
527-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
589+
var data = TextLoader.Create(env, new TextLoader.Arguments()
590+
{
591+
Column = new[]
592+
{
593+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
594+
new TextLoader.Column("Name", DataKind.TX, 1),
595+
}
596+
}, new MultiFileSource(dataFile));
528597
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
529598
{
530599
Column = new ImageLoaderTransform.Column[1]
@@ -603,7 +672,14 @@ public void TestBackAndForthConversionWithoutAlphaInterleaveNoOffset()
603672
var imageWidth = 130;
604673
var dataFile = GetDataPath("images/images.tsv");
605674
var imageFolder = Path.GetDirectoryName(dataFile);
606-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
675+
var data = TextLoader.Create(env, new TextLoader.Arguments()
676+
{
677+
Column = new[]
678+
{
679+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
680+
new TextLoader.Column("Name", DataKind.TX, 1),
681+
}
682+
}, new MultiFileSource(dataFile));
607683
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
608684
{
609685
Column = new ImageLoaderTransform.Column[1]
@@ -682,7 +758,14 @@ public void TestBackAndForthConversionWithAlphaNoInterleaveNoOffset()
682758
var imageWidth = 130;
683759
var dataFile = GetDataPath("images/images.tsv");
684760
var imageFolder = Path.GetDirectoryName(dataFile);
685-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
761+
var data = TextLoader.Create(env, new TextLoader.Arguments()
762+
{
763+
Column = new[]
764+
{
765+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
766+
new TextLoader.Column("Name", DataKind.TX, 1),
767+
}
768+
}, new MultiFileSource(dataFile));
686769
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
687770
{
688771
Column = new ImageLoaderTransform.Column[1]
@@ -761,7 +844,14 @@ public void TestBackAndForthConversionWithoutAlphaNoInterleaveNoOffset()
761844
var imageWidth = 130;
762845
var dataFile = GetDataPath("images/images.tsv");
763846
var imageFolder = Path.GetDirectoryName(dataFile);
764-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
847+
var data = TextLoader.Create(env, new TextLoader.Arguments()
848+
{
849+
Column = new[]
850+
{
851+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
852+
new TextLoader.Column("Name", DataKind.TX, 1),
853+
}
854+
}, new MultiFileSource(dataFile));
765855
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
766856
{
767857
Column = new ImageLoaderTransform.Column[1]

‎test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,14 @@ public void TensorFlowTransformCifar()
420420
var imageWidth = 32;
421421
var dataFile = GetDataPath("images/images.tsv");
422422
var imageFolder = Path.GetDirectoryName(dataFile);
423-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
423+
var data = TextLoader.Create(env, new TextLoader.Arguments()
424+
{
425+
Column = new[]
426+
{
427+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
428+
new TextLoader.Column("Name", DataKind.TX, 1),
429+
}
430+
}, new MultiFileSource(dataFile));
424431
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
425432
{
426433
Column = new ImageLoaderTransform.Column[1]
@@ -474,8 +481,14 @@ public void TensorFlowTransformCifarInvalidShape()
474481
var imageWidth = 28;
475482
var dataFile = GetDataPath("images/images.tsv");
476483
var imageFolder = Path.GetDirectoryName(dataFile);
477-
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
478-
484+
var data = TextLoader.Create(env, new TextLoader.Arguments()
485+
{
486+
Column = new[]
487+
{
488+
new TextLoader.Column("ImagePath", DataKind.TX, 0),
489+
new TextLoader.Column("Name", DataKind.TX, 1),
490+
}
491+
}, new MultiFileSource(dataFile));
479492
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
480493
{
481494
Column = new ImageLoaderTransform.Column[1]

0 commit comments

Comments
 (0)