review comments - 16; include words in LdaSummary (this also resolves dotnet#1411)

Author: abgoswam

src/Microsoft.ML.Transforms/Text/LdaStaticExtensions.cs

@@ -24,8 +24,8 @@ public sealed class LdaFitResult
         /// <param name="result"></param>
         public delegate void OnFit(LdaFitResult result);
 
-        public LatentDirichletAllocationTransformer.LdaTopicSummary LdaTopicSummary;
-        public LdaFitResult(LatentDirichletAllocationTransformer.LdaTopicSummary ldaTopicSummary)
+        public LatentDirichletAllocationTransformer.LdaSummary LdaTopicSummary;
+        public LdaFitResult(LatentDirichletAllocationTransformer.LdaSummary ldaTopicSummary)
         {
             LdaTopicSummary = ldaTopicSummary;
         }
@@ -47,11 +47,11 @@ private struct Config
             public readonly int NumBurninIter;
             public readonly bool ResetRandomGenerator;
 
-            public readonly Action<LatentDirichletAllocationTransformer.LdaTopicSummary> OnFit;
+            public readonly Action<LatentDirichletAllocationTransformer.LdaSummary> OnFit;
 
             public Config(int numTopic, Single alphaSum, Single beta, int mhStep, int numIter, int likelihoodInterval,
                 int numThread, int numMaxDocToken, int numSummaryTermPerTopic, int numBurninIter, bool resetRandomGenerator,
-                Action<LatentDirichletAllocationTransformer.LdaTopicSummary> onFit)
+                Action<LatentDirichletAllocationTransformer.LdaSummary> onFit)
             {
                 NumTopic = numTopic;
                 AlphaSum = alphaSum;
@@ -69,7 +69,7 @@ public Config(int numTopic, Single alphaSum, Single beta, int mhStep, int numIte
             }
         }
 
-        private static Action<LatentDirichletAllocationTransformer.LdaTopicSummary> Wrap(LdaFitResult.OnFit onFit)
+        private static Action<LatentDirichletAllocationTransformer.LdaSummary> Wrap(LdaFitResult.OnFit onFit)
         {
             if (onFit == null)
                 return null;
@@ -126,7 +126,7 @@ public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
                     if (tcol.Config.OnFit != null)
                     {
                         int ii = i; // Necessary because if we capture i that will change to toOutput.Length on call.
-                        onFit += tt => tcol.Config.OnFit(tt.GetLdaTopicSummary(ii));
+                        onFit += tt => tcol.Config.OnFit(tt.GetLdaDetails(ii));
                     }
                 }
 

src/Microsoft.ML.Transforms/Text/LdaTransform.cs

@@ -343,17 +343,35 @@ internal void Save(ModelSaveContext ctx)
         /// <summary>
         /// Provide details about the topics discovered by <a href="https://arxiv.org/abs/1412.1576">LightLDA.</a>
         /// </summary>
-        public sealed class LdaTopicSummary
+        public sealed class LdaSummary
         {
-            // For each topic, provide information about the set of words in the topic and their corresponding scores.
-            public readonly Dictionary<int, KeyValuePair<int, float>[]> WordScoresPerTopic;
+            // For each topic, provide information about the (item, score) pairs.
+            public readonly Dictionary<int, List<Tuple<int, float>>> ItemScoresPerTopic;
 
-            internal LdaTopicSummary(Dictionary<int, KeyValuePair<int, float>[]> wordScoresPerTopic)
+            // For each topic, provide information about the (item, word, score) tuple.
+            public readonly Dictionary<int, List<Tuple<int, string, float>>> WordScoresPerTopic;
+
+            internal LdaSummary(Dictionary<int, List<Tuple<int, float>>> itemScoresPerTopic)
             {
-                WordScoresPerTopic = wordScoresPerTopic;
+                ItemScoresPerTopic = itemScoresPerTopic;
+            }
+
+            internal LdaSummary(Dictionary<int, List<Tuple<int, string, float>>> wordScoresExPerTopic)
+            {
+                WordScoresPerTopic = wordScoresExPerTopic;
             }
         }
 
+        internal LdaSummary GetLdaDetails(int iinfo)
+        {
+            Contracts.Assert(0 <= iinfo && iinfo < _ldas.Length);
+
+            var ldaState = _ldas[iinfo];
+            var mapping = _columnMappings[iinfo];
+
+            return ldaState.GetLdaSummary(mapping);
+        }
+
         private sealed class LdaState : IDisposable
         {
             internal readonly ColumnInfo InfoEx;
@@ -463,16 +481,43 @@ internal LdaState(IExceptionContext ectx, ModelLoadContext ctx)
                 }
             }
 
-            internal LdaTopicSummary GetTopicSummary()
+            internal LdaSummary GetLdaSummary(VBuffer<ReadOnlyMemory<char>> mapping)
             {
-                var wordScoresPerTopic = new Dictionary<int, KeyValuePair<int, float>[]>();
-                for (int i = 0; i < _ldaTrainer.NumTopic; i++)
+                if (mapping.Length == 0)
                 {
-                    var wordScores = _ldaTrainer.GetTopicSummary(i);
-                    wordScoresPerTopic.Add(i, wordScores);
+                    var itemScoresPerTopic = new Dictionary<int, List<Tuple<int, float>>>();
+
+                    for (int i = 0; i < _ldaTrainer.NumTopic; i++)
+                    {
+                        var scores = _ldaTrainer.GetTopicSummary(i);
+                        var itemScores = new List<Tuple<int, float>>();
+                        foreach (KeyValuePair<int, float> p in scores)
+                        {
+                            itemScores.Add(new Tuple<int, float>(p.Key, p.Value));
+                        }
+                        itemScoresPerTopic.Add(i, itemScores);
+                    }
+                    return new LdaSummary(itemScoresPerTopic);
                 }
+                else
+                {
+                    ReadOnlyMemory<char> slotName = default;
+                    var wordScoresPerTopic = new Dictionary<int, List<Tuple<int, string, float>>>();
+
+                    for (int i = 0; i < _ldaTrainer.NumTopic; i++)
+                    {
+                        var scores = _ldaTrainer.GetTopicSummary(i);
+                        var wordScores = new List<Tuple<int, string, float>>();
+                        foreach (KeyValuePair<int, float> p in scores)
+                        {
+                            mapping.GetItemOrDefault(p.Key, ref slotName);
+                            wordScores.Add(new Tuple<int, string, float>(p.Key, slotName.ToString(), p.Value));
+                        }
+                        wordScoresPerTopic.Add(i, wordScores);
+                    }
 
-                return new LdaTopicSummary(wordScoresPerTopic);
+                    return new LdaSummary(wordScoresPerTopic);
+                }
             }
 
             public void Save(ModelSaveContext ctx)
@@ -739,6 +784,7 @@ private static VersionInfo GetVersionInfo()
 
         private readonly ColumnInfo[] _columns;
         private readonly LdaState[] _ldas;
+        private readonly List<VBuffer<ReadOnlyMemory<char>>> _columnMappings;
 
         private const string RegistrationName = "LightLda";
         private const string WordTopicModelFilename = "word_topic_summary.txt";
@@ -757,13 +803,18 @@ private static (string input, string output)[] GetColumnPairs(ColumnInfo[] colum
         /// </summary>
         /// <param name="env">Host Environment.</param>
         /// <param name="ldas">An array of LdaState objects, where ldas[i] is learnt from the i-th element of <paramref name="columns"/>.</param>
+        /// <param name="columnMappings">A list of mappings, where columnMapping[i] is a map of slot names for the i-th element of <paramref name="columns"/>.</param>
         /// <param name="columns">Describes the parameters of the LDA process for each column pair.</param>
-        private LatentDirichletAllocationTransformer(IHostEnvironment env, LdaState[] ldas, params ColumnInfo[] columns)
+        private LatentDirichletAllocationTransformer(IHostEnvironment env,
+            LdaState[] ldas,
+            List<VBuffer<ReadOnlyMemory<char>>> columnMappings,
+            params ColumnInfo[] columns)
             : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(LatentDirichletAllocationTransformer)), GetColumnPairs(columns))
         {
             Host.AssertNonEmpty(ColumnPairs);
-            _columns = columns;
             _ldas = ldas;
+            _columnMappings = columnMappings;
+            _columns = columns;
         }
 
         private LatentDirichletAllocationTransformer(IHost host, ModelLoadContext ctx) : base(host, ctx)
@@ -789,12 +840,14 @@ private LatentDirichletAllocationTransformer(IHost host, ModelLoadContext ctx) :
         internal static LatentDirichletAllocationTransformer TrainLdaTransformer(IHostEnvironment env, IDataView inputData, params ColumnInfo[] columns)
         {
             var ldas = new LdaState[columns.Length];
+
+            List<VBuffer<ReadOnlyMemory<char>>> columnMappings;
             using (var ch = env.Start("Train"))
             {
-                Train(env, ch, inputData, ldas, columns);
+                columnMappings = Train(env, ch, inputData, ldas, columns);
             }
 
-            return new LatentDirichletAllocationTransformer(env, ldas, columns);
+            return new LatentDirichletAllocationTransformer(env, ldas, columnMappings, columns);
         }
 
         private void Dispose(bool disposing)
@@ -818,14 +871,6 @@ public void Dispose()
             Dispose(false);
         }
 
-        internal LdaTopicSummary GetLdaTopicSummary(int iinfo)
-        {
-            Contracts.Assert(0 <= iinfo && iinfo < _ldas.Length);
-
-            var ldaState = _ldas[iinfo];
-            return ldaState.GetTopicSummary();
-        }
-
         // Factory method for SignatureLoadDataTransform.
         private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
             => Create(env, ctx).MakeDataTransform(input);
@@ -895,7 +940,7 @@ private static int GetFrequency(double value)
             return result;
         }
 
-        private static void Train(IHostEnvironment env, IChannel ch, IDataView inputData, LdaState[] states, params ColumnInfo[] columns)
+        private static List<VBuffer<ReadOnlyMemory<char>>> Train(IHostEnvironment env, IChannel ch, IDataView inputData, LdaState[] states, params ColumnInfo[] columns)
         {
             env.AssertValue(ch);
             ch.AssertValue(inputData);
@@ -906,6 +951,8 @@ private static void Train(IHostEnvironment env, IChannel ch, IDataView inputData
             int[] numVocabs = new int[columns.Length];
             int[] srcCols = new int[columns.Length];
 
+            var columnMappings = new List<VBuffer<ReadOnlyMemory<char>>>();
+
             var inputSchema = inputData.Schema;
             for (int i = 0; i < columns.Length; i++)
             {
@@ -919,6 +966,13 @@ private static void Train(IHostEnvironment env, IChannel ch, IDataView inputData
                 srcCols[i] = srcCol;
                 activeColumns[srcCol] = true;
                 numVocabs[i] = 0;
+
+                VBuffer<ReadOnlyMemory<char>> dst = default;
+                if (inputSchema.HasSlotNames(srcCol, srcColType.ValueCount))
+                    inputSchema.GetMetadata(MetadataUtils.Kinds.SlotNames, srcCol, ref dst);
+                else
+                    dst = default(VBuffer<ReadOnlyMemory<char>>);
+                columnMappings.Add(dst);
             }
 
             //the current lda needs the memory allocation before feedin data, so needs two sweeping of the data,
@@ -979,7 +1033,7 @@ private static void Train(IHostEnvironment env, IChannel ch, IDataView inputData
 
                 // No data to train on, just return
                 if (rowCount == 0)
-                    return;
+                    return columnMappings;
 
                 for (int i = 0; i < columns.Length; ++i)
                 {
@@ -1032,6 +1086,8 @@ private static void Train(IHostEnvironment env, IChannel ch, IDataView inputData
                     states[i].CompleteTrain();
                 }
             }
+
+            return columnMappings;
         }
 
         protected override IRowMapper MakeRowMapper(Schema schema)

test/Microsoft.ML.StaticPipelineTesting/StaticPipeTests.cs

@@ -679,14 +679,15 @@ public void LdaTopicModel()
             var data = reader.Read(dataSource);
 
             // This will be populated once we call fit.
-            LdaTopicSummary ldaTopicSummary;
+            LdaSummary ldaSummary;
 
             var est = data.MakeNewEstimator()
                 .Append(r => (
                     r.label,
-                    topics: r.text.ToBagofWords().ToLdaTopicVector(numTopic: 3, numSummaryTermPerTopic:5, alphaSum: 10, onFit: m => ldaTopicSummary = m.LdaTopicSummary)));
+                    topics: r.text.ToBagofWords().ToLdaTopicVector(numTopic: 3, numSummaryTermPerTopic:5, alphaSum: 10, onFit: m => ldaSummary = m.LdaTopicSummary)));
 
-            var tdata = est.Fit(data).Transform(data);
+            var transformer = est.Fit(data);
+            var tdata = transformer.Transform(data);
 
             var schema = tdata.AsDynamic.Schema;
             Assert.True(schema.TryGetColumnIndex("topics", out int topicsCol));