BUG FIX: HQQ quantization would error out if torch.dtype (dataType) was set to auto, it now force-sets to torch.bfloat16

Author: abgulati

web_app/hf_waitress.py

@@ -568,7 +568,8 @@ def initialize_model():
                 quantization_config  = QuantoConfig(weights="int4")
                 model_params["quantization_config"] = quantization_config
         elif quantize == "hqq":
-            print("HQQ-Quantizing")
+            print("HQQ-Quantizing - Force-setting torch_dtype to torch.bfloat16")
+            model_params["torch_dtype"] = torch.bfloat16
             quant_level = quant_level.lower().strip()
 
             if quant_level == "int8":