Skip to content

Commit 2da6686

Browse files
authored
Fix stale tensor-split params for draft models (ggml-org#24814)
* meta: fix tensor split metadata for GQA attention * Tidied the code a bit to match existing style * Revert "Tidied the code a bit to match existing style" This reverts commit b90c6c6. * Reverted the ggml-backend-meta asset hack.
1 parent 3e5036f commit 2da6686

1 file changed

Lines changed: 8 additions & 0 deletions

File tree

src/llama-model.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,9 +1012,17 @@ struct llama_model::impl {
10121012
std::vector<layer_dev> dev_layer;
10131013

10141014
bool has_tensor_overrides;
1015+
1016+
std::vector<float> tensor_split_owned;
10151017
};
10161018

10171019
llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {
1020+
if (params.tensor_split != nullptr) {
1021+
// llama_model_params stores tensor_split as a borrowed pointer, but the model
1022+
// may need it later for tensor-parallel KV-cache split metadata.
1023+
pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());
1024+
this->params.tensor_split = pimpl->tensor_split_owned.data();
1025+
}
10181026
pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;
10191027
}
10201028

0 commit comments

Comments
 (0)