Skip to content

Commit c186b92

Browse files
elbrunoelbruno
authored
New C# projects and implement chat functionality using OnnxRuntime and Semantic Kernel (#262)
* New C# projects and implement chat functionality using OnnxRuntime and Semantic Kernel * Update links in documentation for Intel NPU and Apple MLX frameworks
1 parent d50eaa9 commit c186b92

26 files changed

+917
-5
lines changed

‎md/01.Introduction/01/01.Edgeandcloud.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,6 @@
1313
|Phi-3-mini-4k-instruct|Text|4k|[Playground & Deployment](https://aka.ms/phi3-mini-4k-azure-ml)|[Playground, Deployment](https://aka.ms/phi3-mini-4k-azure-ml) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx), [Web](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx)|[Playground & Download](https://huggingface.co/chat/models/microsoft/Phi-3-mini-4k-instruct)|[GGUF](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf)|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-mini-4k)|
1414
|Phi-3-mini-128k-instruct|Text|128k|[Playground & Deployment](https://ai.azure.com/explore/models/Phi-3-mini-128k-instruct/version/9/registry/azureml)|[Playground, Deployment](https://ai.azure.com/explore/models/Phi-3-mini-128k-instruct/version/9/registry/azureml) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct-onnx)|[Download](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct-onnx)|-NA-|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-mini)|
1515
|Phi-3-small-8k-instruct|Text|8k|[Playground & Deployment](https://ml.azure.com/registries/azureml/models/Phi-3-small-8k-instruct/version/2)|[Playground, Deployment](https://ai.azure.com/explore/models/Phi-3-small-8k-instruct/version/2/registry/azureml) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-small-8k-instruct-onnx-cuda)|[Download](https://huggingface.co/microsoft/Phi-3-small-8k-instruct-onnx-cuda)|-NA-|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-small-8k-instruct?docker=false)|
16-
|Phi-3-small-128k-instruct|Text|128k|[Playground & Deployment](https://ai.azure.com/explore/models/Phi-3-small-128k-instruct/version/2/registry/azureml)|[Playground, Deployment](https://ml.azure.com/registries/azureml/models/Phi-3-small-128k-instruct/version/2) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-small-128k-instruct-onnx-cuda)|[Download](https://huggingface.co/microsoft/Phi-3-small-128k-instruct)|-NA-|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-small-128k-instruct?docker=false)|
16+
|Phi-3-small-128k-instruct|Text|128k|[Playground & Deployment](https://ai.azure.com/explore/models/Phi-3-small-128k-instruct/version/2/registry/azureml)|[Playground, Deployment](https://ml.azure.com/registries/azureml/models/Phi-3-small-128k-instruct/version/2) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-cuda)|[Download](https://huggingface.co/microsoft/Phi-3-small-128k-instruct)|-NA-|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-small-128k-instruct?docker=false)|
1717
|Phi-3-medium-4k-instruct|Text|4k|[Playground & Deployment](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct)|[Playground, Deployment](https://ml.azure.com/registries/azureml/models/Phi-3-medium-4k-instruct/version/2) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-cuda/tree/main), [CPU](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-cpu/tree/main), [DirectML](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct-onnx-directml/tree/main)|[Download](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct)|-NA-|[NIM APIs](https://build.nvidia.com/microsoft/phi-3-medium-4k-instruct?docker=false)|
1818
|Phi-3-medium-128k-instruct|Text|128k|[Playground & Deployment](https://ai.azure.com/explore/models/Phi-3-medium-128k-instruct/version/2)|[Playground, Deployment](https://ml.azure.com/registries/azureml/models/Phi-3-medium-128k-instruct/version/2) & Finetuning|[CUDA](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-cuda/tree/main), [CPU](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-cpu/tree/main), [DirectML](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct-onnx-directml/tree/main)|[Download](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)|-NA-|-NA-|

‎md/02.Application/02.Code/Phi3/VSCodeExt/HOL/AIPC/01.Installations.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pf --version
4646

4747
### **4. Intel NPU Acceleration Library**
4848

49-
Intel's new generation processors support NPU. If you want to use NPU to run LLMs / SLMs locally, you can use ***Intel NPU Acceleration Library*** . If you want to know more, you can read [https://github.com/microsoft/Phi-3CookBook/blob/main/md/03.Inference/AIPC_Inference.md](https://github.com/microsoft/Phi-3CookBook/blob/main/md/03.Inference/AIPC_Inference.md).
49+
Intel's new generation processors support NPU. If you want to use NPU to run LLMs / SLMs locally, you can use ***Intel NPU Acceleration Library*** . If you want to know more, you can read [https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/03/AIPC_Inference.md](https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/03/AIPC_Inference.md).
5050

5151
Install Intel NPU Acceleration Library in bash
5252

‎md/02.Application/02.Code/Phi3/VSCodeExt/HOL/Apple/01.Installations.md

+1-2
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,9 @@ pf --version
4343

4444
![pf](../../../../../../../imgs/02/vscodeext/pf_ext.png)
4545

46-
4746
### **4. Apple's MLX Framework**
4847

49-
MLX is an array framework for machine learning research on Apple silicon, brought to you by Apple machine learning research. You can use **Apple MLX framework** to accelerate LLM / SLM with Apple Silicon. If you want to know more, you can read [https://github.com/microsoft/Phi-3CookBook/blob/main/md/03.Inference/MLX_Inference.md](https://github.com/microsoft/Phi-3CookBook/blob/main/md/03.Inference/MLX_Inference.md).
48+
MLX is an array framework for machine learning research on Apple silicon, brought to you by Apple machine learning research. You can use **Apple MLX framework** to accelerate LLM / SLM with Apple Silicon. If you want to know more, you can read [https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/03/MLX_Inference.md](https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/03/MLX_Inference.md).
5049

5150
Install MLX framework Library in bash
5251

‎md/02.Application/02.Code/Phi3/VSCodeExt/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ Have you used the workspace agent in GitHub Copilot Chat? Do you want to build y
88

99
Phi-3 is a family series, including phi-3-mini, phi-3-small, and phi-3-medium based on different training parameters for text generation, dialogue completion, and code generation. There is also phi-3-vision based on Vision. It is suitable for enterprises or different teams to create offline generative AI solutions.
1010

11-
Recommended to read this link [https://github.com/microsoft/Phi-3CookBook/blob/main/md/01.Introduce/Phi3Family.md](https://github.com/microsoft/Phi-3CookBook/blob/main/md/01.Introduce/Phi3Family.md)
11+
Recommended to read this link [https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/01/01.PhiFamily.md](https://github.com/microsoft/PhiCookBook/blob/main/md/01.Introduction/01/01.PhiFamily.md)
1212

1313
### **Microsoft GitHub Copilot Chat**
1414

‎md/04.HOL/dotnet/csharplabs.md

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
## Welcome to the Phi labs using C#
2+
3+
There is a selection of labs that showcases how to integrate the powerful different versions of Phi-3 models in a .NET environment.
4+
5+
## Prerequisites
6+
7+
Before running the sample, ensure you have the following installed:
8+
9+
**.NET 9:** Make sure you have the [latest version of .NET](https://dotnet.microsoft.com/download/dotnet/) installed on your machine.
10+
11+
**(Optional) Visual Studio or Visual Studio Code:** You will need an IDE or code editor capable of running .NET projects. [Visual Studio](https://visualstudio.microsoft.com/) or [Visual Studio Code](https://code.visualstudio.com/) are recommended.
12+
13+
**Using git** clone locally one of the available Phi-3, Phi3.5 or Phi-4 versions from [Hugging Face](https://huggingface.co).
14+
15+
**Download Phi-4 onnx models** to your local machine:
16+
17+
### navigate to the folder to store the models
18+
19+
```bash
20+
cd c:\phi\models
21+
```
22+
23+
### add support for lfs
24+
25+
```bash
26+
git lfs install
27+
```
28+
29+
### clone and download Phi-4 mini instruct model and the Phi-4 multi modal model
30+
31+
```bash
32+
git clone https://huggingface.co/microsoft/Phi-4-mini-instruct-onnx
33+
34+
git clone https://huggingface.co/microsoft/Phi-4-multimodal-instruct-onnx
35+
```
36+
37+
**Download the phi3-mini-4k-instruct-onnx model** to your local machine:
38+
39+
### clone and download mini 4K instruct model
40+
41+
```bash
42+
git clone https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-onnx
43+
```
44+
45+
### clone and download vision 128K model
46+
47+
```bash
48+
git clone https://huggingface.co/microsoft/Phi-3-vision-128k-instruct-onnx-cpu
49+
```
50+
51+
**Important:** The current demos are designed to use the ONNX versions of the model. The previous steps clone the following models.
52+
53+
## About the Labs
54+
55+
The main solution have several sample Labs that demonstrates the capabilities of the Phi-3 models using C#.
56+
57+
| Project | Description | Location |
58+
| ------------ | ----------- | -------- |
59+
| LabsPhi301 | This is a sample project that uses a local phi3 model to ask a question. The project load a local ONNX Phi-3 model using the `Microsoft.ML.OnnxRuntime` libraries. | .\src\LabsPhi301\ |
60+
| LabsPhi302 | This is a sample project that implement a Console chat using Semantic Kernel. | .\src\LabsPhi302\ |
61+
| LabsPhi303 | This is a sample project that uses a local phi3 vision model to analyze images.. The project load a local ONNX Phi-3 Vision model using the `Microsoft.ML.OnnxRuntime` libraries. | .\src\LabsPhi303\ |
62+
| LabsPhi304 | This is a sample project that uses a local phi3 vision model to analyze images.. The project load a local ONNX Phi-3 Vision model using the `Microsoft.ML.OnnxRuntime` libraries. The project also presents a menu with different options to interacti with the user. | .\src\LabsPhi304\ |
63+
| LabsPhi4-Chat-01OnnxRuntime | This is a sample project that uses a local Phi-4 model to work in a chat in the console. The project load a local ONNX Phi-4 model using the `Microsoft.ML.OnnxRuntime` libraries. | \src\LabsPhi4-Chat-01OnnxRuntime\ |
64+
| LabsPhi4-Chat-02SK | This is a sample project that uses a local Phi-4 model to work in a chat in the console. The project load a local ONNX Phi-4 model using the `Semantic Kernel` libraries. | \src\LabsPhi4-Chat-02SK\ |
65+
66+
## How to Run the Projects
67+
68+
To run the projects, follow these steps:
69+
70+
1. Clone the repository to your local machine.
71+
72+
1. Open a terminal and navigate to the desired project. In example, let's run `LabsPhi4-Chat-01OnnxRuntime`.
73+
74+
```bash
75+
cd .\src\LabsPhi4-Chat-01OnnxRuntime \
76+
```
77+
78+
1. Run the project with the command
79+
80+
```bash
81+
dotnet run
82+
```
83+
84+
1. The sample project ask for a user input and replies using the local mode.
85+
86+
The running demo is similar to this one:
87+
88+
```bash
89+
PS D:\phi\PhiCookBook\md\04.HOL\dotnet\src\LabsPhi4-Chat-01OnnxRuntime> dotnet run
90+
Ask your question. Type an empty string to Exit.
91+
Q: 2+2
92+
Phi4: The sum of 2 and 2 is 4.
93+
Q:
94+
```

‎md/04.HOL/dotnet/src/LabsPhi.sln

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.10.34928.147
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LabsPhi301", "LabsPhi301\LabsPhi301.csproj", "{22131B1B-1289-41DF-882F-A2E16A63BE9E}"
7+
EndProject
8+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LabsPhi302", "LabsPhi302\LabsPhi302.csproj", "{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F}"
9+
EndProject
10+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LabsPhi303", "LabsPhi303\LabsPhi303.csproj", "{1254AB34-B99A-4E4C-BD95-18BB22BF478E}"
11+
EndProject
12+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LabsPhi304", "LabsPhi304\LabsPhi304.csproj", "{4EB2FBF6-75D3-4B10-B5B5-675469C24780}"
13+
EndProject
14+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Phi3", "Phi3", "{02EA681E-C7D8-13C7-8484-4AC65E1B71E8}"
15+
EndProject
16+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Phi4", "Phi4", "{9DF367B9-E0EA-4ABB-A144-5E13A9508A69}"
17+
EndProject
18+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LabsPhi4-Chat-01OnnxRuntime", "LabsPhi4-Chat-01OnnxRuntime\LabsPhi4-Chat-01OnnxRuntime.csproj", "{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2}"
19+
EndProject
20+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LabsPhi4-Chat-02SK", "LabsPhi4-Chat-02SK\LabsPhi4-Chat-02SK.csproj", "{66045465-B4A8-D929-3D15-926FD376FE2E}"
21+
EndProject
22+
Global
23+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
24+
Debug|Any CPU = Debug|Any CPU
25+
Release|Any CPU = Release|Any CPU
26+
EndGlobalSection
27+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
28+
{22131B1B-1289-41DF-882F-A2E16A63BE9E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
29+
{22131B1B-1289-41DF-882F-A2E16A63BE9E}.Debug|Any CPU.Build.0 = Debug|Any CPU
30+
{22131B1B-1289-41DF-882F-A2E16A63BE9E}.Release|Any CPU.ActiveCfg = Release|Any CPU
31+
{22131B1B-1289-41DF-882F-A2E16A63BE9E}.Release|Any CPU.Build.0 = Release|Any CPU
32+
{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
33+
{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F}.Debug|Any CPU.Build.0 = Debug|Any CPU
34+
{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F}.Release|Any CPU.ActiveCfg = Release|Any CPU
35+
{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F}.Release|Any CPU.Build.0 = Release|Any CPU
36+
{1254AB34-B99A-4E4C-BD95-18BB22BF478E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
37+
{1254AB34-B99A-4E4C-BD95-18BB22BF478E}.Debug|Any CPU.Build.0 = Debug|Any CPU
38+
{1254AB34-B99A-4E4C-BD95-18BB22BF478E}.Release|Any CPU.ActiveCfg = Release|Any CPU
39+
{1254AB34-B99A-4E4C-BD95-18BB22BF478E}.Release|Any CPU.Build.0 = Release|Any CPU
40+
{4EB2FBF6-75D3-4B10-B5B5-675469C24780}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
41+
{4EB2FBF6-75D3-4B10-B5B5-675469C24780}.Debug|Any CPU.Build.0 = Debug|Any CPU
42+
{4EB2FBF6-75D3-4B10-B5B5-675469C24780}.Release|Any CPU.ActiveCfg = Release|Any CPU
43+
{4EB2FBF6-75D3-4B10-B5B5-675469C24780}.Release|Any CPU.Build.0 = Release|Any CPU
44+
{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
45+
{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2}.Debug|Any CPU.Build.0 = Debug|Any CPU
46+
{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2}.Release|Any CPU.ActiveCfg = Release|Any CPU
47+
{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2}.Release|Any CPU.Build.0 = Release|Any CPU
48+
{66045465-B4A8-D929-3D15-926FD376FE2E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
49+
{66045465-B4A8-D929-3D15-926FD376FE2E}.Debug|Any CPU.Build.0 = Debug|Any CPU
50+
{66045465-B4A8-D929-3D15-926FD376FE2E}.Release|Any CPU.ActiveCfg = Release|Any CPU
51+
{66045465-B4A8-D929-3D15-926FD376FE2E}.Release|Any CPU.Build.0 = Release|Any CPU
52+
EndGlobalSection
53+
GlobalSection(SolutionProperties) = preSolution
54+
HideSolutionNode = FALSE
55+
EndGlobalSection
56+
GlobalSection(NestedProjects) = preSolution
57+
{22131B1B-1289-41DF-882F-A2E16A63BE9E} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
58+
{1373D0EA-81B1-43BE-A8CA-0DD7A162FC3F} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
59+
{1254AB34-B99A-4E4C-BD95-18BB22BF478E} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
60+
{4EB2FBF6-75D3-4B10-B5B5-675469C24780} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
61+
{C6986362-DDF7-6EF2-75EE-B042B4F4B4D2} = {9DF367B9-E0EA-4ABB-A144-5E13A9508A69}
62+
{66045465-B4A8-D929-3D15-926FD376FE2E} = {9DF367B9-E0EA-4ABB-A144-5E13A9508A69}
63+
EndGlobalSection
64+
GlobalSection(ExtensibilityGlobals) = postSolution
65+
SolutionGuid = {EAA25EC1-C5F2-40DC-8080-4DAF13311AEE}
66+
EndGlobalSection
67+
EndGlobal

‎md/04.HOL/dotnet/src/LabsPhi301/LabsPhi301.csproj

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.18.0" />
12+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.3.0-rc2" />
13+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" Version="0.3.0-rc2" />
14+
</ItemGroup>
15+
16+
</Project>

‎md/04.HOL/dotnet/src/LabsPhi301/Program.cs

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright (c) 2024
2+
// Author : Bruno Capuano
3+
// Change Log :
4+
//
5+
// The MIT License (MIT)
6+
//
7+
// Permission is hereby granted, free of charge, to any person obtaining a copy
8+
// of this software and associated documentation files (the "Software"), to deal
9+
// in the Software without restriction, including without limitation the rights
10+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
// copies of the Software, and to permit persons to whom the Software is
12+
// furnished to do so, subject to the following conditions:
13+
//
14+
// The above copyright notice and this permission notice shall be included in
15+
// all copies or substantial portions of the Software.
16+
//
17+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23+
// THE SOFTWARE.
24+
25+
using Microsoft.ML.OnnxRuntimeGenAI;
26+
27+
28+
var modelPath = @"D:\phi3\models\Phi-3-mini-4k-instruct-onnx\cpu_and_mobile\cpu-int4-rtn-block-32";
29+
var model = new Model(modelPath);
30+
var tokenizer = new Tokenizer(model);
31+
32+
var systemPrompt = "You are an AI assistant that helps people find information. Answer questions using a direct style. Do not share more information that the requested by the users.";
33+
34+
// chat start
35+
Console.WriteLine(@"Ask your question. Type an empty string to Exit.");
36+
37+
38+
// chat loop
39+
while (true)
40+
{
41+
// Get user question
42+
Console.WriteLine();
43+
Console.Write(@"Q: ");
44+
var userQ = Console.ReadLine();
45+
if (string.IsNullOrEmpty(userQ))
46+
{
47+
break;
48+
}
49+
50+
// show phi3 response
51+
Console.Write("Phi3: ");
52+
var fullPrompt = $"<|system|>{systemPrompt}<|end|><|user|>{userQ}<|end|><|assistant|>";
53+
var tokens = tokenizer.Encode(fullPrompt);
54+
55+
var generatorParams = new GeneratorParams(model);
56+
generatorParams.SetSearchOption("max_length", 2048);
57+
generatorParams.SetSearchOption("past_present_share_buffer", false);
58+
generatorParams.SetInputSequences(tokens);
59+
60+
var generator = new Generator(model, generatorParams);
61+
while (!generator.IsDone())
62+
{
63+
generator.ComputeLogits();
64+
generator.GenerateNextToken();
65+
var outputTokens = generator.GetSequence(0);
66+
var newToken = outputTokens.Slice(outputTokens.Length - 1, 1);
67+
var output = tokenizer.Decode(newToken);
68+
Console.Write(output);
69+
}
70+
Console.WriteLine();
71+
}

‎md/04.HOL/dotnet/src/LabsPhi302/LabsPhi302.csproj

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="feiyun0112.SemanticKernel.Connectors.OnnxRuntimeGenAI.CPU" Version="1.0.0" />
12+
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.18.0" />
13+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" Version="0.3.0-rc2" />
14+
<PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" Version="0.3.0-rc2" />
15+
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="9.0.0-preview.4.24266.19" />
16+
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0-preview.4.24266.19" />
17+
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0-preview.4.24266.19" />
18+
<PackageReference Include="Microsoft.SemanticKernel" Version="1.13.0" />
19+
<PackageReference Include="Microsoft.SemanticKernel.Connectors.Onnx" Version="1.13.0-alpha" />
20+
</ItemGroup>
21+
22+
</Project>

0 commit comments

Comments
 (0)