Ranking Sample - Hotel Search Results (changed to Bing Search Engine result ranking)

samples/csharp/getting-started/Ranking_PersonalizedSort/PersonalizedRanking.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Remove="PersonalizedRanking\bin\**" />
+    <Compile Remove="PersonalizedRanking\obj\**" />
+    <EmbeddedResource Remove="PersonalizedRanking\bin\**" />
+    <EmbeddedResource Remove="PersonalizedRanking\obj\**" />
+    <None Remove="PersonalizedRanking\bin\**" />
+    <None Remove="PersonalizedRanking\obj\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Remove="README.md" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML" Version="1.1.0" />
+    <PackageReference Include="Microsoft.ML.LightGbm" Version="1.1.0" />
+  </ItemGroup>
+
+</Project>

samples/csharp/getting-started/Ranking_PersonalizedSort/PersonalizedRanking.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.28307.705
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersonalizedRanking", "PersonalizedRanking\PersonalizedRanking.csproj", "{F71F24D8-F174-461F-B375-508EFB827A33}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F71F24D8-F174-461F-B375-508EFB827A33}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F71F24D8-F174-461F-B375-508EFB827A33}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F71F24D8-F174-461F-B375-508EFB827A33}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F71F24D8-F174-461F-B375-508EFB827A33}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {92FA42B0-28BF-4531-B744-F3125DAAC91A}
+	EndGlobalSection
+EndGlobal

samples/csharp/getting-started/Ranking_PersonalizedSort/PersonalizedRanking/Common/ConsoleHelper.cs

@@ -0,0 +1,62 @@
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using PersonalizedRanking.DataStructures;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace PersonalizedRanking.Common
+{
+    public class ConsoleHelper
+    {
+        // To evaluate the accuracy of the model's predicted rankings, prints out the Discounted Cumulative Gain and Normalized Discounted Cumulative Gain for hotel search queries.
+        public static void EvaluateMetrics(MLContext mlContext, IDataView scoredData)
+        {
+            // Evaluate the metrics for the data using NDCG; by default, metrics for the up to 3 search results in the query are reported (e.g. NDCG@3).
+            RankingMetrics metrics = mlContext.Ranking.Evaluate(scoredData);
+
+            Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");
+
+            Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");
+        }
+
+        // Performs evaluation with the truncation level set up to 10 hotel search results within a query.
+        // This is a temporary workaround for this issue: https://github.com/dotnet/machinelearning/issues/2728.
+        public static void EvaluateMetrics(MLContext mlContext, IDataView scoredData, int truncationLevel)
+        {
+            if (truncationLevel < 1 || truncationLevel > 10)
+            {
+                throw new InvalidOperationException("Currently metrics are only supported for 1 to 10 truncation levels.");
+            }
+
+            //  Uses reflection to set the truncation level before calling evaluate.
+            var mlAssembly = AppDomain.CurrentDomain.GetAssemblies().Where(a => a.FullName.Contains("Microsoft.ML.Data")).First();
+            var rankEvalType = mlAssembly.DefinedTypes.Where(t => t.Name.Contains("RankingEvaluator")).First();
+
+            var evalArgsType = rankEvalType.GetNestedType("Arguments");
+            var evalArgs = Activator.CreateInstance(rankEvalType.GetNestedType("Arguments"));
+
+            var dcgLevel = evalArgsType.GetField("DcgTruncationLevel");
+            dcgLevel.SetValue(evalArgs, truncationLevel);
+
+            var ctor = rankEvalType.GetConstructors().First();
+            var evaluator = ctor.Invoke(new object[] { mlContext, evalArgs });
+
+            var evaluateMethod = rankEvalType.GetMethod("Evaluate");
+            RankingMetrics metrics = (RankingMetrics)evaluateMethod.Invoke(evaluator, new object[] { scoredData, "Label", "GroupId", "Score" });
+
+            Console.WriteLine($"DCG: {string.Join(", ", metrics.DiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");
+
+            Console.WriteLine($"NDCG: {string.Join(", ", metrics.NormalizedDiscountedCumulativeGains.Select((d, i) => $"@{i + 1}:{d:F4}").ToArray())}");
+        }
+
+        // Prints out the the individual scores used to determine the relative ranking.
+        public static void PrintScores(IEnumerable<HotelPrediction> predictions)
+        {
+            foreach (var prediction in predictions)
+            {
+                Console.WriteLine($"GroupId: {prediction.GroupId}, Score: {prediction.PredictedRank}");
+            }
+        }
+    }
+}