Fix uqff dummy layer ISQ application (#1351)

EricLBuehler · web-flow · commit a9a4c990ab44 · 2025-05-19T14:50:55.000-04:00
diff --git a/mistralrs-quant/src/dummy/mod.rs b/mistralrs-quant/src/dummy/mod.rs
@@ -29,10 +29,11 @@ impl QuantMethod for DummyLayer {
         _imatrix_weight: Option<Vec<f32>>,
         _guard: QuantizeOntoGuard,
     ) -> candle_core::Result<std::sync::Arc<dyn QuantMethod>> {
-        candle_core::bail!("DummyLayer should not ever be present in forward pass!")
+        // This is necessary for the immediate ISQ
+        Ok(self)
     }
     fn dtype_and_device(&self) -> (candle_core::DType, candle_core::Device) {
-        (candle_core::DType::F64, candle_core::Device::Cpu)
+        (candle_core::DType::F32, candle_core::Device::Cpu)
     }
     fn forward(&self, _a: &candle_core::Tensor) -> candle_core::Result<candle_core::Tensor> {
         candle_core::bail!("DummyLayer should not ever be present in forward pass!")

Original file line number	Diff line number	Diff line change
`@@ -29,10 +29,11 @@ impl QuantMethod for DummyLayer {`
`29`	`29`	`_imatrix_weight: Option<Vec<f32>>,`
`30`	`30`	`_guard: QuantizeOntoGuard,`
`31`	`31`	`) -> candle_core::Result<std::sync::Arc<dyn QuantMethod>> {`
`32`		`- candle_core::bail!("DummyLayer should not ever be present in forward pass!")`
	`32`	`+ // This is necessary for the immediate ISQ`
	`33`	`+ Ok(self)`
`33`	`34`	`}`
`34`	`35`	`fn dtype_and_device(&self) -> (candle_core::DType, candle_core::Device) {`
`35`		`- (candle_core::DType::F64, candle_core::Device::Cpu)`
	`36`	`+ (candle_core::DType::F32, candle_core::Device::Cpu)`
`36`	`37`	`}`
`37`	`38`	`fn forward(&self, _a: &candle_core::Tensor) -> candle_core::Result<candle_core::Tensor> {`
`38`	`39`	`candle_core::bail!("DummyLayer should not ever be present in forward pass!")`