Skip to content

Commit 0de2e66

Browse files
YUNQIUGUOrachguoskottmckay
authored
Add basic iOS phi-3 sample (#433)
* initial ios phi-3 app * update to remove tests etc. * update brief read me * minor update * update * address pr comments * minor update * minor updates * remove the unnecessary delay step * Update mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md Co-authored-by: Scott McKay <[email protected]> * address pr comments * pr comments * minor update * pr comments * update * Update mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md Co-authored-by: Scott McKay <[email protected]> --------- Co-authored-by: rachguo <[email protected]> Co-authored-by: Scott McKay <[email protected]>
1 parent 8dc4650 commit 0de2e66

File tree

19 files changed

+6067
-0
lines changed

19 files changed

+6067
-0
lines changed

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM.xcodeproj/project.pbxproj

+477
Large diffs are not rendered by default.

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/AccentColor.colorset/Contents.json

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"colors" : [
3+
{
4+
"idiom" : "universal"
5+
}
6+
],
7+
"info" : {
8+
"author" : "xcode",
9+
"version" : 1
10+
}
11+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/AppIcon.appiconset/Contents.json

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"images" : [
3+
{
4+
"idiom" : "universal",
5+
"platform" : "ios",
6+
"size" : "1024x1024"
7+
}
8+
],
9+
"info" : {
10+
"author" : "xcode",
11+
"version" : 1
12+
}
13+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Assets.xcassets/Contents.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"info" : {
3+
"author" : "xcode",
4+
"version" : 1
5+
}
6+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/ContentView.swift

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import SwiftUI
5+
6+
struct ContentView: View {
7+
@ObservedObject var tokenUpdater = SharedTokenUpdater.shared
8+
9+
var body: some View {
10+
VStack {
11+
ScrollView {
12+
VStack(alignment: .leading) {
13+
ForEach(tokenUpdater.decodedTokens, id: \.self) { token in
14+
Text(token)
15+
.padding(.horizontal, 5)
16+
}
17+
}
18+
.padding()
19+
}
20+
Button("Generate Tokens") {
21+
DispatchQueue.global(qos: .background).async {
22+
// TODO: add user prompt question UI
23+
GenAIGenerator.generate("Who is the current US president?");
24+
}
25+
}
26+
}
27+
}
28+
}
29+
30+
struct ContentView_Previews: PreviewProvider {
31+
static var previews: some View {
32+
ContentView()
33+
}
34+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/GenAIGenerator.h

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
#ifndef GenAIGenerator_h
5+
#define GenAIGenerator_h
6+
7+
#import <Foundation/Foundation.h>
8+
#import <UIKit/UIKit.h>
9+
10+
NS_ASSUME_NONNULL_BEGIN
11+
12+
@interface GenAIGenerator : NSObject
13+
14+
+ (void)generate:(NSString *)input_user_question;
15+
16+
@end
17+
18+
NS_ASSUME_NONNULL_END
19+
20+
#endif /* GenAIGenerator_h */

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/GenAIGenerator.mm

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
#import "GenAIGenerator.h"
5+
#include "LocalLLM-Swift.h"
6+
#include "ort_genai.h"
7+
#include "ort_genai_c.h"
8+
9+
10+
@implementation GenAIGenerator
11+
12+
+ (void)generate:(nonnull NSString*)input_user_question {
13+
NSString* llmPath = [[NSBundle mainBundle] resourcePath];
14+
const char* modelPath = llmPath.cString;
15+
16+
auto model = OgaModel::Create(modelPath);
17+
auto tokenizer = OgaTokenizer::Create(*model);
18+
19+
NSString* promptString = [NSString stringWithFormat:@"<|user|>\n%@<|end|>\n<|assistant|>", input_user_question];
20+
const char* prompt = [promptString UTF8String];
21+
22+
auto sequences = OgaSequences::Create();
23+
tokenizer->Encode(prompt, *sequences);
24+
25+
auto params = OgaGeneratorParams::Create(*model);
26+
params->SetSearchOption("max_length", 200);
27+
params->SetInputSequences(*sequences);
28+
29+
// Streaming Output to generate token by token
30+
auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer);
31+
32+
auto generator = OgaGenerator::Create(*model, *params);
33+
34+
while (!generator->IsDone()) {
35+
generator->ComputeLogits();
36+
generator->GenerateNextToken();
37+
38+
const int32_t* seq = generator->GetSequenceData(0);
39+
size_t seq_len = generator->GetSequenceCount(0);
40+
const char* decode_tokens = tokenizer_stream->Decode(seq[seq_len - 1]);
41+
42+
NSLog(@"Decoded tokens: %s", decode_tokens);
43+
44+
// Add decoded token to SharedTokenUpdater
45+
NSString* decodedTokenString = [NSString stringWithUTF8String:decode_tokens];
46+
[SharedTokenUpdater.shared addDecodedToken:decodedTokenString];
47+
}
48+
}
49+
@end

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/LocalLLM-Bridging-Header.h

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
#ifndef LocalLLM_Bridging_Header_h
5+
#define LocalLLM_Bridging_Header_h
6+
7+
#import "GenAIGenerator.h"
8+
9+
#endif /* LocalLLM_Bridging_Header_h */

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/LocalLLMApp.swift

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
5+
import SwiftUI
6+
7+
@main
8+
struct LocalLLMApp: App {
9+
var body: some Scene {
10+
WindowGroup {
11+
ContentView()
12+
}
13+
}
14+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/Preview Content/Preview Assets.xcassets/Contents.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"info" : {
3+
"author" : "xcode",
4+
"version" : 1
5+
}
6+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/README.md

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# **Local LLM sample application running Phi3-mini on iOS**
2+
3+
## **Steps**
4+
5+
### General prerequisites
6+
7+
See the general prerequisites [here](../../../../../README.md#General-Prerequisites).
8+
9+
For this application, the following prerequisites are preferred:
10+
11+
1. macOS 14+
12+
13+
2. Xcode 15+ (latest Xcode version perferred.)
14+
15+
3. iOS SDK 16.x + (iPhone 14 or iPhone 15 powered by a A16 or A17 preferred)
16+
17+
**Note**:
18+
The current Xcode project contains a built .dylib for ORT and ORT GenAI. The following steps `A, B, C` under `step 1.` for building from source for the libraries are optional.
19+
However if you want to build from source to include the latest updates, please use the `step 1.` as a reference.
20+
21+
### 1. Steps to build from source for ONNX Runtime and Generative AI libraries [Optional]
22+
23+
#### **A. Preparation**
24+
25+
- Install Python 3.10+
26+
27+
- Install flatbuffers
28+
```
29+
pip3 install flatbuffers
30+
```
31+
32+
- Install [CMake](https://cmake.org/download/)
33+
34+
#### **B. Compiling ONNX Runtime for iOS**
35+
36+
```bash
37+
38+
git clone https://github.com/microsoft/onnxruntime.git
39+
40+
cd onnxruntime
41+
42+
./build.sh --build_shared_lib --skip_tests --parallel --build_dir ./build_ios --ios --apple_sysroot iphoneos --osx_arch arm64 --apple_deploy_target 16.6 --cmake_generator Xcode --config Release
43+
44+
```
45+
46+
***Notice***
47+
48+
1. Before compiling, you must ensure that Xcode is configured correctly and set it on the terminal
49+
50+
```bash
51+
52+
sudo xcode-select -switch /Applications/Xcode.app/Contents/Developer
53+
54+
```
55+
56+
2. ONNX Runtime needs to be compiled based on different platforms. For iOS, you can compile for arm64 or x86_64 based on needs. If you are running an iOS simulator on an Intel mac, compile for x86_64. Use arm64 for an ARM based mac to run the simulator, and to run on an iPhone.
57+
58+
3. It is recommended to directly use the latest iOS SDK for compilation. Of course, you can also lower the version to be compatible with past SDKs.
59+
60+
#### **C. Compiling Generative AI with ONNX Runtime for iOS**
61+
62+
```bash
63+
64+
git clone https://github.com/microsoft/onnxruntime-genai
65+
66+
cd onnxruntime-genai
67+
68+
python3 build.py --parallel --build_dir ./build_iphoneos --ios --ios_sysroot iphoneos --ios_arch arm64 --ios_deployment_target 16.6 --cmake_generator Xcode
69+
70+
```
71+
72+
#### **D. Copy over latest header files and required .dylibs built from source**
73+
74+
If you build from source and get the latest .dylibs for ORT and ORT GenAI, please copy the .dylibs over to `mobile\examples\phi-3\ios\LocalLLM\LocalLLM\lib` and copy the latest header files over to `mobile\examples\phi-3\ios\LocalLLM\LocalLLM\header`
75+
76+
The build output path for libonnxruntime.dylib is `<ORT_PROJECT_ROOT>/build/intermediates/<platform>_<arch>/<build_config>/<build_config-platform>/libonnxruntime.dylib`
77+
The build output path for libonnxruntime-genai.dylib is `<ORT_GENAI_PROJECT_ROOT>/build/<build_config-platform>/libonnxruntime-genai.dylib`.
78+
79+
For example:
80+
- `onnxruntime/build/intermediates/iphoneos_arm64/Release/Release-iphoneos/libonnxruntime.1.19.0.dylib`
81+
- `onnxruntime-genai/build/Release/Release-iphoneos/libonnxruntime-genai.dylib`.
82+
83+
Note that you will need to build and copy the correct dylib for the target architecture you wish to run the app on.
84+
e.g.
85+
if you want to run on the iOS simulator on an Intel mac, you must build both onnxruntime and onnxruntime-genai for x86_64 and copy the dylibs to the app's `lib` directory.
86+
if you want to run on an iPhone, you must build both onnxruntime and onnxruntime-genai for arm64 and copy the dylibs to the app's `lib` directory.
87+
88+
The header files to copy are:
89+
`<ORT_MAIN_SOURCE_REPO>/onnxruntime/core/session/onnxruntime_c_api.h`,
90+
`<ORT_GENAI_MAIN_SOURCE_REPO>/src/ort_genai.h`,
91+
`<ORT_GENAI_MAIN_SOURCE_REPO>/src/ort_genai_c.h`.
92+
93+
### 2. Create/Open the iOS application in Xcode
94+
95+
The app uses Objective-C/C++ since using Generative AI with ONNX Runtime C++ API, Objective-C has better compatiblility.
96+
97+
### 3. Copy the ONNX quantized INT4 model to the App application project
98+
99+
Download from hf repo: <https://huggingface.co/microsoft/Phi-3-mini-128k-instruct-onnx/tree/main/cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4>
100+
101+
After downloading completes, you need to copy files over to the `Resources` directory in the `Destination` column of `Target-LocalLLM`->`Build Phases`-> `New Copy File Phases` -> `Copy Files`.
102+
103+
Upon app launching, Xcode will automatically copy and install the model files from Resources folder and directly download to the iOS device.
104+
105+
### 4. Run the app and checkout the streaming output token results
106+
107+
**Note**: The current app only sets up with a simple initial prompt question, you can adjust/try your own or refine the UI based on requirements.
108+
109+
***Notice:*** The current Xcode project runs on iOS 16.6, feel free to adjust latest iOS/build for lates iOS versions accordingly.

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/SharedTokenUpdater.swift

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import Combine
5+
import Foundation
6+
7+
@objc class SharedTokenUpdater: NSObject, ObservableObject {
8+
@Published var decodedTokens: [String] = []
9+
10+
@objc static let shared = SharedTokenUpdater()
11+
12+
@objc func addDecodedToken(_ token: String) {
13+
DispatchQueue.main.async {
14+
self.decodedTokens.append(token)
15+
}
16+
}
17+
}

Diff for: ‎mobile/examples/phi-3/ios/LocalLLM/LocalLLM/header/README.md

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
**Note**:
2+
This folder contains the latest C++ headers that's required for the project. Copied over from the ORT and ORT GenAI repo matches the latest build.

0 commit comments

Comments
 (0)