Skip to content

[ggml-quants] Add memsets and other fixes for IQ quants #19861

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ggerganov merged 4 commits into from
Mar 6, 2026
+23 −4
Merged

[ggml-quants] Add memsets and other fixes for IQ quants #19861

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
abfca17
Add memsets and other fixes for IQ quants
bartowski1182 Feb 24, 2026
9354ce0
Make memset unconditional, change Laux back to L
bartowski1182 Feb 24, 2026
5da083c
Move another memset
bartowski1182 Feb 24, 2026
163093c
Merge branch 'ggml-org:master' into master
bartowski1182 Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 23 additions & 4 deletions ggml/src/ggml-quants.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3104,6 +3104,11 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
}
float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight);
float eff_max = scale*kMaxQ;
if (eff_max <= 0) {
scales[ib] = 0;
memset(L, 0, 32);
continue;
}
float best = 0;
for (int is = -6; is <= 6; ++is) {
float id = (2*kMaxQ-1+is*0.1f)/eff_max;
Expand Down Expand Up @@ -3273,9 +3278,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
}
float max = xval[0];
for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);
memset(L, 0, 16);
if (max < GROUP_MAX_EPS) {
scales[ib] = 0;
memset(L, 0, 16);
continue;
}
float best = 0;
Expand Down Expand Up @@ -3714,9 +3719,9 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * GGML_RESTRICT
}
float max = xval[0];
for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]);
memset(L, 0, 32);
if (max < GROUP_MAX_EPS_IQ3_XXS) {
scales[ib] = 0;
memset(L, 0, 32);
continue;
}
float best = 0;
Expand Down Expand Up @@ -3922,6 +3927,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * GGML_RESTRICT
}
float max = xval[0];
for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i]);
memset(L, 0, block_size);
if (!max) {
scales[ib] = 0;
continue;
Expand Down Expand Up @@ -4245,6 +4251,7 @@ static void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_R
for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));
if (max < GROUP_MAX_EPS_IQ1_S) {
scales[ib] = 0;
shifts[ib] = 1;
memset(L, 1, block_size);
continue;
}
Expand Down Expand Up @@ -4285,7 +4292,12 @@ static void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_R
}
}
}
GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_shift != 0);
if (besti1 < 0 || besti2 < 0 || best_shift == 0) {
scales[ib] = 0;
shifts[ib] = 1;
memset(L, 1, block_size);
continue;
}
for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;
for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;
for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;
Expand Down Expand Up @@ -4429,6 +4441,7 @@ static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_R
for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]));
if (max < GROUP_MAX_EPS_IQ1_M) {
scales[ib] = 0;
shifts[ib] = 0;
memset(L, 1, block_size);
continue;
}
Expand Down Expand Up @@ -4527,7 +4540,12 @@ static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_R
}
}
}
GGML_ASSERT(besti1 >= 0 && besti2 >= 0 && best_k >= 0);
if (besti1 < 0 || besti2 < 0 || best_k < 0) {
scales[ib] = 0;
shifts[ib] = 0;
memset(L, 1, block_size);
continue;
}
for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0;
for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1;
for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2;
Expand Down Expand Up @@ -4874,6 +4892,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
}
float max = xval[0];
for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]);
memset(L, 0, 16);
if (max < GROUP_MAX_EPS_IQ2_S) {
scales[ib] = 0;
continue;
Expand Down
Loading