1010#include " llama.h"
1111
1212#include " ggml.h"
13- #ifdef GGML_USE_CUBLAS
13+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
1414#include " ggml-cuda.h"
1515#elif defined(GGML_USE_CLBLAST)
1616#include " ggml-opencl.h"
@@ -80,7 +80,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH0()
8080 { MODEL_3B, 256ull * MB },
8181 { MODEL_7B, 512ull * MB },
8282 { MODEL_13B, 512ull * MB },
83- { MODEL_30B, 512ull * MB },
83+ { MODEL_30B, 640ull * MB },
8484 { MODEL_65B, 1024ull * MB },
8585 };
8686 return k_sizes;
@@ -92,7 +92,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
9292 { MODEL_3B, 256ull * MB },
9393 { MODEL_7B, 512ull * MB },
9494 { MODEL_13B, 512ull * MB },
95- { MODEL_30B, 512ull * MB },
95+ { MODEL_30B, 640ull * MB },
9696 { MODEL_65B, 1024ull * MB },
9797 };
9898 return k_sizes;
@@ -105,7 +105,7 @@ static const std::map<e_model, size_t> & MEM_REQ_KV_SELF()
105105 { MODEL_3B, 682ull * MB },
106106 { MODEL_7B, 1026ull * MB },
107107 { MODEL_13B, 1608ull * MB },
108- { MODEL_30B, 3124ull * MB },
108+ { MODEL_30B, 3224ull * MB },
109109 { MODEL_65B, 5120ull * MB },
110110 };
111111 return k_sizes;
@@ -117,9 +117,9 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
117117{
118118 static std::map<e_model, size_t > k_sizes = {
119119 { MODEL_3B, 512ull * MB },
120- { MODEL_7B, 768ull * MB },
120+ { MODEL_7B, 800ull * MB },
121121 { MODEL_13B, 1024ull * MB },
122- { MODEL_30B, 1280ull * MB },
122+ { MODEL_30B, 1380ull * MB },
123123 { MODEL_65B, 1536ull * MB },
124124 };
125125 return k_sizes;
@@ -175,7 +175,7 @@ struct llama_kv_cache {
175175 ggml_free (ctx);
176176 }
177177
178- #ifdef GGML_USE_CUBLAS
178+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
179179 ggml_cuda_free_data (k);
180180 ggml_cuda_free_data (v);
181181#endif // GGML_USE_CUBLAS
@@ -234,7 +234,7 @@ struct llama_model {
234234 ggml_free (ctx);
235235 }
236236
237- #ifdef GGML_USE_CUBLAS
237+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
238238 for (size_t i = 0 ; i < tensors_by_name.size (); ++i) {
239239 ggml_cuda_free_data (tensors_by_name[i].second );
240240 }
@@ -800,7 +800,7 @@ struct llama_model_loader {
800800 lmlock->grow_to (lock_size);
801801 }
802802 break ;
803- #if defined(GGML_USE_CUBLAS)
803+ #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_HIPBLAS)
804804 case GGML_BACKEND_GPU:
805805 case GGML_BACKEND_GPU_SPLIT:
806806 ggml_cuda_transform_tensor (lt.data , lt.ggml_tensor );
@@ -920,7 +920,7 @@ static bool kv_cache_init(
920920 ggml_set_name (cache.v , " cache_v"
921921
922922 (void ) n_gpu_layers;
923- #ifdef GGML_USE_CUBLAS
923+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
924924 if (n_gpu_layers > n_layer + 1 ) {
925925 ggml_cuda_assign_buffers_no_scratch (cache.v );
926926 }
@@ -1106,15 +1106,15 @@ static void llama_model_load_internal(
11061106 if (hparams.ftype != LLAMA_FTYPE_ALL_F32 &&
11071107 hparams.ftype != LLAMA_FTYPE_MOSTLY_F16 &&
11081108 hparams.ftype != LLAMA_FTYPE_MOSTLY_Q8_0) {
1109- throw std::runtime_error ( format ( " this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1405)" ) );
1109+ printf ( " \n this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1405)"
11101110 }
11111111 }
11121112
11131113 if (file_version < LLAMA_FILE_VERSION_GGJT_V3) {
11141114 if (hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||
11151115 hparams.ftype == LLAMA_FTYPE_MOSTLY_Q4_1 ||
11161116 hparams.ftype == LLAMA_FTYPE_MOSTLY_Q8_0) {
1117- throw std::runtime_error ( format ( " this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)" ) );
1117+ printf ( " \n this format is no longer supported (see https://github.com/ggerganov/llama.cpp/pull/1508)"
11181118 }
11191119 }
11201120
@@ -1150,7 +1150,7 @@ static void llama_model_load_internal(
11501150 }
11511151
11521152 (void ) main_gpu;
1153- #if defined(GGML_USE_CUBLAS)
1153+ #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_HIPBLAS)
11541154 fprintf (stderr, " %s: using CUDA for GPU acceleration\n "
11551155 ggml_cuda_set_main_device (main_gpu);
11561156#define LLAMA_BACKEND_OFFLOAD GGML_BACKEND_GPU
@@ -1261,7 +1261,7 @@ static void llama_model_load_internal(
12611261
12621262 (void ) vram_scratch;
12631263 (void ) n_batch;
1264- #ifdef GGML_USE_CUBLAS
1264+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
12651265 if (low_vram) {
12661266 fprintf (stderr, " %s: not allocating a VRAM scratch buffer due to low VRAM option\n "
12671267 ggml_cuda_set_scratch_size (0 ); // disable scratch
@@ -1274,7 +1274,7 @@ static void llama_model_load_internal(
12741274 }
12751275 }
12761276#endif // GGML_USE_CUBLAS
1277- #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)
1277+ #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_HIPBLAS) || defined( GGML_USE_CLBLAST)
12781278 const int n_gpu = std::min (n_gpu_layers, int (hparams.n_layer ));
12791279
12801280 fprintf (stderr, " %s: offloading %d repeating layers to GPU\n "
@@ -1314,7 +1314,7 @@ static void llama_model_load_internal(
13141314 }
13151315
13161316 (void ) tensor_split;
1317- #if defined(GGML_USE_CUBLAS)
1317+ #if defined(GGML_USE_CUBLAS) || defined(GGML_USE_HIPBLAS)
13181318 {
13191319 ggml_cuda_set_tensor_split (tensor_split);
13201320 }
@@ -1375,11 +1375,11 @@ static bool llama_eval_internal(
13751375 const int n_threads,
13761376 const char * cgraph_fname) {
13771377
1378- // enforce that the first token is BOS
1379- if (n_past == 0 && tokens[0 ] != llama_token_bos ()) {
1380- fprintf (stderr, " %s: first token must be BOS\n "
1381- return false ;
1382- }
1378+ // // enforce that the first token is BOS
1379+ // if (n_past == 0 && tokens[0] != llama_token_bos()) {
1380+ // fprintf(stderr, "%s: first token must be BOS\n", __func__);
1381+ // return false;
1382+ // }
13831383
13841384 const int64_t t_start_us = ggml_time_us ();
13851385
@@ -1435,7 +1435,7 @@ static bool llama_eval_internal(
14351435 offload_func_t offload_func_kq = llama_nop;
14361436 offload_func_t offload_func_v = llama_nop;
14371437
1438- #ifdef GGML_USE_CUBLAS
1438+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
14391439 if (n_gpu_layers > n_layer) {
14401440 offload_func_nr = ggml_cuda_assign_buffers;
14411441 }
@@ -1450,7 +1450,7 @@ static bool llama_eval_internal(
14501450 for (int il = 0 ; il < n_layer; ++il) {
14511451 offload_func_t offload_func = llama_nop;
14521452
1453- #ifdef GGML_USE_CUBLAS
1453+ #if defined GGML_USE_CUBLAS || defined GGML_USE_HIPBLAS
14541454 if (il >= i_gpu_start) {
14551455 offload_func = ggml_cuda_assign_buffers;
14561456 }
0 commit comments