|
| 1 | +using Microsoft.ML; |
| 2 | +using Microsoft.ML.Data; |
| 3 | +using WebRanking.DataStructures; |
| 4 | +using System; |
| 5 | +using System.Collections.Generic; |
| 6 | +using System.Linq; |
| 7 | + |
| 8 | +namespace WebRanking.Common |
| 9 | +{ |
| 10 | + public class ConsoleHelper |
| 11 | + { |
| 12 | + // To evaluate the accuracy of the model's predicted rankings, prints out the Discounted Cumulative Gain and Normalized Discounted Cumulative Gain for search queries. |
| 13 | + public static void EvaluateMetrics(MLContext mlContext, IDataView predictions) |
| 14 | + { |
| 15 | + // Evaluate the metrics for the data using NDCG; by default, metrics for the up to 3 search results in the query are reported (e.g. NDCG@3). |
| 16 | + RankingMetrics metrics = mlContext.Ranking.Evaluate(predictions); |
| 17 | + |
| 18 | + Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}"); |
| 19 | + |
| 20 | + Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}\n"); |
| 21 | + } |
| 22 | + |
| 23 | + // Performs evaluation with the truncation level set up to 10 search results within a query. |
| 24 | + // This is a temporary workaround for this issue: https://github.com/dotnet/machinelearning/issues/2728. |
| 25 | + public static void EvaluateMetrics(MLContext mlContext, IDataView predictions, int truncationLevel) |
| 26 | + { |
| 27 | + if (truncationLevel < 1 || truncationLevel > 10) |
| 28 | + { |
| 29 | + throw new InvalidOperationException("Currently metrics are only supported for 1 to 10 truncation levels."); |
| 30 | + } |
| 31 | + |
| 32 | + // Uses reflection to set the truncation level before calling evaluate. |
| 33 | + var mlAssembly = typeof(TextLoader).Assembly; |
| 34 | + var rankEvalType = mlAssembly.DefinedTypes.Where(t => t.Name.Contains("RankingEvaluator")).First(); |
| 35 | + |
| 36 | + var evalArgsType = rankEvalType.GetNestedType("Arguments"); |
| 37 | + var evalArgs = Activator.CreateInstance(rankEvalType.GetNestedType("Arguments")); |
| 38 | + |
| 39 | + var dcgLevel = evalArgsType.GetField("DcgTruncationLevel"); |
| 40 | + dcgLevel.SetValue(evalArgs, truncationLevel); |
| 41 | + |
| 42 | + var ctor = rankEvalType.GetConstructors().First(); |
| 43 | + var evaluator = ctor.Invoke(new object[] { mlContext, evalArgs }); |
| 44 | + |
| 45 | + var evaluateMethod = rankEvalType.GetMethod("Evaluate"); |
| 46 | + RankingMetrics metrics = (RankingMetrics)evaluateMethod.Invoke(evaluator, new object[] { predictions, "Label", "GroupId", "Score" }); |
| 47 | + |
| 48 | + Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}"); |
| 49 | + |
| 50 | + Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}\n"); |
| 51 | + } |
| 52 | + |
| 53 | + // Prints out the the individual scores used to determine the relative ranking. |
| 54 | + public static void PrintScores(IEnumerable<SearchResultPrediction> predictions) |
| 55 | + { |
| 56 | + foreach (var prediction in predictions) |
| 57 | + { |
| 58 | + Console.WriteLine($"GroupId: {prediction.GroupId}, Score: {prediction.Score}"); |
| 59 | + } |
| 60 | + } |
| 61 | + } |
| 62 | +} |
0 commit comments