Fix output being sent to legacy in AutoTokenizer

CodeWithKyrian · CodeWithKyrian · commit 05e55888d9ad · 2024-03-23T11:59:37.000+01:00
diff --git a/src/Pipelines/Pipeline.php b/src/Pipelines/Pipeline.php
@@ -78,7 +78,7 @@ function pipeline(
 
     $model = $task->pretrainedModel($modelName, $quantized, $config, $cacheDir, $revision, $modelFilename, $output);
 
-    $tokenizer = AutoTokenizer::fromPretrained($modelName, $quantized, $config, $cacheDir, $revision, $output);
+    $tokenizer = AutoTokenizer::fromPretrained($modelName, $quantized, $config, $cacheDir, $revision, null, $output);
 
     return $task->getPipeline($model, $tokenizer);
 }

Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ function pipeline(`
`78`	`78`
`79`	`79`	`$model = $task->pretrainedModel($modelName, $quantized, $config, $cacheDir, $revision, $modelFilename, $output);`
`80`	`80`
`81`		`- $tokenizer = AutoTokenizer::fromPretrained($modelName, $quantized, $config, $cacheDir, $revision, $output);`
	`81`	`+ $tokenizer = AutoTokenizer::fromPretrained($modelName, $quantized, $config, $cacheDir, $revision, null, $output);`
`82`	`82`
`83`	`83`	`return $task->getPipeline($model, $tokenizer);`
`84`	`84`	`}`