Skip to content

Commit ef1ff14

Browse files
author
fmz
committed
Minor fixes
1 parent 739b01a commit ef1ff14

File tree

6 files changed

+79
-200
lines changed

6 files changed

+79
-200
lines changed

‎CMakePresets.json

+62-194
Original file line numberDiff line numberDiff line change
@@ -1,197 +1,65 @@
11
{
2-
"version": 4,
3-
"configurePresets": [
4-
{
5-
"name": "base",
6-
"hidden": true,
7-
"generator": "Ninja",
8-
"binaryDir": "${sourceDir}/build-${presetName}",
9-
"cacheVariables": {
10-
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
11-
"CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.."
12-
}
13-
},
14-
{
15-
"name": "sycl-base",
16-
"hidden": true,
17-
"generator": "Ninja",
18-
"binaryDir": "${sourceDir}/build-${presetName}",
19-
"cacheVariables": {
20-
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
21-
"CMAKE_CXX_COMPILER": "icx",
22-
"CMAKE_C_COMPILER": "cl",
23-
"GGML_SYCL": "ON",
24-
"CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.."
25-
}
26-
},
27-
{
28-
"name": "debug",
29-
"hidden": true,
30-
"cacheVariables": {
31-
"CMAKE_BUILD_TYPE": "Debug"
32-
}
33-
},
34-
{
35-
"name": "release",
36-
"hidden": true,
37-
"cacheVariables": {
38-
"CMAKE_BUILD_TYPE": "Release"
39-
}
40-
},
41-
{
42-
"name": "reldbg",
43-
"hidden": true,
44-
"cacheVariables": {
45-
"CMAKE_BUILD_TYPE": "RelWithDebInfo"
46-
}
47-
},
48-
{
49-
"name": "static",
50-
"hidden": true,
51-
"cacheVariables": {
52-
"GGML_STATIC": "ON"
53-
}
54-
},
55-
{
56-
"name": "arm64-windows-msvc",
57-
"hidden": true,
58-
"architecture": {
59-
"value": "arm64",
60-
"strategy": "external"
61-
},
62-
"toolset": {
63-
"value": "host=x86_64",
64-
"strategy": "external"
65-
},
66-
"cacheVariables": {
67-
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-msvc.cmake"
68-
}
69-
},
70-
{
71-
"name": "arm64-windows-llvm",
72-
"hidden": true,
73-
"architecture": {
74-
"value": "arm64",
75-
"strategy": "external"
76-
},
77-
"toolset": {
78-
"value": "host=x86_64",
79-
"strategy": "external"
80-
},
81-
"cacheVariables": {
82-
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-llvm.cmake"
83-
}
84-
},
85-
{
86-
"name": "arm64-windows-llvm-debug",
87-
"inherits": [
88-
"base",
89-
"arm64-windows-llvm",
90-
"debug"
91-
]
92-
},
93-
{
94-
"name": "arm64-windows-llvm-release",
95-
"inherits": [
96-
"base",
97-
"arm64-windows-llvm",
98-
"reldbg"
99-
]
100-
},
101-
{
102-
"name": "arm64-windows-llvm+static-release",
103-
"inherits": [
104-
"base",
105-
"arm64-windows-llvm",
106-
"reldbg",
107-
"static"
108-
]
109-
},
110-
{
111-
"name": "arm64-windows-msvc-debug",
112-
"inherits": [
113-
"base",
114-
"arm64-windows-msvc",
115-
"debug"
116-
]
117-
},
118-
{
119-
"name": "arm64-windows-msvc-release",
120-
"inherits": [
121-
"base",
122-
"arm64-windows-msvc",
123-
"reldbg"
124-
]
125-
},
126-
{
127-
"name": "arm64-windows-msvc+static-release",
128-
"inherits": [
129-
"base",
130-
"arm64-windows-msvc",
131-
"reldbg",
132-
"static"
133-
]
134-
},
135-
{
136-
"name": "x64-windows-msvc-debug",
137-
"inherits": [
138-
"base",
139-
"debug"
140-
]
141-
},
142-
{
143-
"name": "x64-windows-msvc-release",
144-
"inherits": [
145-
"base",
146-
"reldbg"
147-
]
148-
},
149-
{
150-
"name": "x64-windows-msvc+static-release",
151-
"inherits": [
152-
"base",
153-
"reldbg",
154-
"static"
155-
]
156-
},
157-
{
158-
"name": "x64-windows-sycl-debug",
159-
"inherits": [
160-
"sycl-base",
161-
"debug"
162-
]
163-
},
164-
{
165-
"name": "x64-windows-sycl-release",
166-
"inherits": [
167-
"sycl-base",
168-
"release"
169-
]
170-
},
171-
{
172-
"name": "clang10",
173-
"displayName": "Clang 10.0.0 x86_64-pc-linux-gnu",
174-
"description": "Using compilers: C = /usr/bin/clang, CXX = /usr/bin/clang++",
175-
"binaryDir": "${sourceDir}/out/build/${presetName}",
176-
"cacheVariables": {
177-
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
178-
"CMAKE_C_COMPILER": "/usr/bin/clang",
179-
"CMAKE_CXX_COMPILER": "/usr/bin/clang++",
180-
"CMAKE_RC_COMPILER": "/usr/bin/llvm-rc-10",
181-
"CMAKE_BUILD_TYPE": "Debug"
182-
}
183-
},
184-
{
185-
"name": "gcc8.4",
186-
"displayName": "GCC 8.4.0 x86_64-linux-gnu",
187-
"description": "Using compilers: C = /usr/bin/gcc, CXX = /usr/bin/g++",
188-
"binaryDir": "${sourceDir}/out/build/${presetName}",
189-
"cacheVariables": {
190-
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
191-
"CMAKE_C_COMPILER": "/usr/bin/gcc",
192-
"CMAKE_CXX_COMPILER": "/usr/bin/g++",
193-
"CMAKE_BUILD_TYPE": "Debug"
194-
}
2+
"version": 4,
3+
"configurePresets": [
4+
{
5+
"name": "base",
6+
"hidden": true,
7+
"generator": "Ninja",
8+
"binaryDir": "${sourceDir}/build-${presetName}",
9+
"cacheVariables": {
10+
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
11+
"CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.."
19512
}
196-
]
13+
},
14+
{
15+
"name": "sycl-base",
16+
"hidden": true,
17+
"generator": "Ninja",
18+
"binaryDir": "${sourceDir}/build-${presetName}",
19+
"cacheVariables": {
20+
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
21+
"CMAKE_CXX_COMPILER": "icx",
22+
"CMAKE_C_COMPILER": "cl",
23+
"GGML_SYCL": "ON",
24+
"CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.."
25+
}
26+
},
27+
{ "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } },
28+
{ "name": "release", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } },
29+
{ "name": "reldbg", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
30+
{ "name": "static", "hidden": true, "cacheVariables": { "GGML_STATIC": "ON" } },
31+
32+
{
33+
"name": "arm64-windows-msvc", "hidden": true,
34+
"architecture": { "value": "arm64", "strategy": "external" },
35+
"toolset": { "value": "host=x86_64", "strategy": "external" },
36+
"cacheVariables": {
37+
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-msvc.cmake"
38+
}
39+
},
40+
41+
{
42+
"name": "arm64-windows-llvm", "hidden": true,
43+
"architecture": { "value": "arm64", "strategy": "external" },
44+
"toolset": { "value": "host=x86_64", "strategy": "external" },
45+
"cacheVariables": {
46+
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-llvm.cmake"
47+
}
48+
},
49+
50+
{ "name": "arm64-windows-llvm-debug" , "inherits": [ "base", "arm64-windows-llvm", "debug" ] },
51+
{ "name": "arm64-windows-llvm-release", "inherits": [ "base", "arm64-windows-llvm", "reldbg" ] },
52+
{ "name": "arm64-windows-llvm+static-release", "inherits": [ "base", "arm64-windows-llvm", "reldbg", "static" ] },
53+
54+
{ "name": "arm64-windows-msvc-debug" , "inherits": [ "base", "arm64-windows-msvc", "debug" ] },
55+
{ "name": "arm64-windows-msvc-release", "inherits": [ "base", "arm64-windows-msvc", "reldbg" ] },
56+
{ "name": "arm64-windows-msvc+static-release", "inherits": [ "base", "arm64-windows-msvc", "reldbg", "static" ] },
57+
58+
{ "name": "x64-windows-msvc-debug" , "inherits": [ "base", "debug" ] },
59+
{ "name": "x64-windows-msvc-release", "inherits": [ "base", "reldbg" ] },
60+
{ "name": "x64-windows-msvc+static-release", "inherits": [ "base", "reldbg", "static" ] },
61+
62+
{ "name": "x64-windows-sycl-debug" , "inherits": [ "sycl-base", "debug" ] },
63+
{ "name": "x64-windows-sycl-release", "inherits": [ "sycl-base", "release" ] }
64+
]
19765
}

‎examples/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,6 @@ else()
5050
endif()
5151
add_subdirectory(save-load-state)
5252
add_subdirectory(simple)
53-
#add_subdirectory(speculative)
53+
add_subdirectory(speculative)
5454
add_subdirectory(tokenize)
5555
endif()

‎examples/speculative/speculative.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,10 @@ int main(int argc, char ** argv) {
7171
// load the draft model
7272
params.model = params.model_draft;
7373
params.n_gpu_layers = params.n_gpu_layers_draft;
74-
if (params.n_threads_draft > 0) {
75-
params.n_threads = params.n_threads_draft;
74+
if (params.draft_cpuparams.n_threads > 0) {
75+
params.cpuparams.n_threads = params.draft_cpuparams.n_threads;
7676
}
77-
params.n_threads_batch = params.n_threads_batch_draft;
77+
params.cpuparams_batch.n_threads = params.draft_cpuparams_batch.n_threads;
7878
std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params);
7979

8080
const bool vocab_type_tgt = llama_vocab_type(model_tgt);

‎ggml/src/ggml.c

+2-2
Original file line numberDiff line numberDiff line change
@@ -18751,7 +18751,7 @@ static bool __thread_affinity(const bool * mask) {
1875118751

1875218752
for (uint32_t i = 0; i < GGML_MAX_N_THREADS; i++) {
1875318753
if (mask[i]) {
18754-
printf("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
18754+
GGML_PRINT_DEBUG("Thread %lx: adding %d to cpuset\n", pthread_self(), i);
1875518755
CPU_SET(i, &cpuset);
1875618756
}
1875718757
}
@@ -19144,7 +19144,7 @@ static bool ggml_graph_compute_check_for_work(struct ggml_compute_state * state)
1914419144
if (threadpool->poll) {
1914519145
while (!threadpool->new_work && !threadpool->stop && !threadpool->pause) {
1914619146
// No new work. Yield and keep polling.
19147-
//__cpu_relax();
19147+
__cpu_relax();
1914819148
}
1914919149
} else {
1915019150
ggml_mutex_lock_shared(&threadpool->mutex);

‎include/llama.h

+2
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,8 @@ extern "C" {
441441
LLAMA_API void llama_detach_batch_threadpool(struct llama_context * ctx);
442442
LLAMA_API void llama_detach_threadpools(struct llama_context * ctx);
443443

444+
// Pauses all attached threadpools
445+
LLAMA_API void llama_pause_threadpools(struct llama_context * ctx);
444446

445447
// Call once at the end of the program - currently only used for MPI
446448
LLAMA_API void llama_backend_free(void);

‎src/llama.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -16435,6 +16435,15 @@ void llama_detach_threadpools(struct llama_context * ctx) {
1643516435
llama_detach_batch_threadpool(ctx);
1643616436
}
1643716437

16438+
void llama_pause_threadpools(struct llama_context * ctx) {
16439+
if (ctx->threadpool) {
16440+
ggml_pause_threadpool(ctx->threadpool);
16441+
}
16442+
if (ctx->threadpool_batch) {
16443+
ggml_pause_threadpool(ctx->threadpool_batch);
16444+
}
16445+
}
16446+
1643816447
void llama_backend_free(void) {
1643916448
ggml_quantize_free();
1644016449
}

0 commit comments

Comments
 (0)