From 23ee689f1c7dcb2948eb396d00e9aa5f34996d12 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:10 -0600 Subject: [PATCH 01/15] Initial test of Sherpa, example works, so will attempt full backend from here --- team_b/yappy/android/app/build.gradle | 2 +- team_b/yappy/lib/online_model.dart | 82 ++++++++ team_b/yappy/lib/streaming_asr.dart | 241 ++++++++++++++++++++++++ team_b/yappy/lib/transcription_box.dart | 1 + team_b/yappy/lib/utils.dart | 39 ++++ team_b/yappy/pubspec.yaml | 4 + 6 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 team_b/yappy/lib/online_model.dart create mode 100644 team_b/yappy/lib/streaming_asr.dart create mode 100644 team_b/yappy/lib/utils.dart diff --git a/team_b/yappy/android/app/build.gradle b/team_b/yappy/android/app/build.gradle index 709e79eb..55d6c13f 100644 --- a/team_b/yappy/android/app/build.gradle +++ b/team_b/yappy/android/app/build.gradle @@ -21,7 +21,7 @@ android { defaultConfig { applicationId = "com.spring2025.yappy" - minSdkVersion 21 + minSdkVersion 23 targetSdkVersion 34 // Latest stable version versionCode = project.hasProperty('flutterVersionCode') ? flutterVersionCode.toInteger() : 1 versionName = project.hasProperty('flutterVersionName') ? flutterVersionName : "1.0.0" diff --git a/team_b/yappy/lib/online_model.dart b/team_b/yappy/lib/online_model.dart new file mode 100644 index 00000000..d0703d64 --- /dev/null +++ b/team_b/yappy/lib/online_model.dart @@ -0,0 +1,82 @@ +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; +import './utils.dart'; + +// Remember to change `assets` in ../pubspec.yaml +// and download files to ../assets +Future getOnlineModelConfig( + {required int type}) async { + switch (type) { + case 0: + final modelDir = + 'assets/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: + await copyAssetFile('$modelDir/encoder-epoch-99-avg-1.int8.onnx'), + decoder: await copyAssetFile('$modelDir/decoder-epoch-99-avg-1.onnx'), + joiner: await copyAssetFile('$modelDir/joiner-epoch-99-avg-1.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer', + ); + case 1: + final modelDir = 'assets/sherpa-onnx-streaming-zipformer-en-2023-06-26'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile( + '$modelDir/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx'), + decoder: await copyAssetFile( + '$modelDir/decoder-epoch-99-avg-1-chunk-16-left-128.onnx'), + joiner: await copyAssetFile( + '$modelDir/joiner-epoch-99-avg-1-chunk-16-left-128.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer2', + ); + case 2: + final modelDir = + 'assets/icefall-asr-zipformer-streaming-wenetspeech-20230615'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile( + '$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx'), + decoder: await copyAssetFile( + '$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx'), + joiner: await copyAssetFile( + '$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx'), + ), + tokens: await copyAssetFile('$modelDir/data/lang_char/tokens.txt'), + modelType: 'zipformer2', + ); + case 3: + final modelDir = 'assets/sherpa-onnx-streaming-zipformer-fr-2023-04-14'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile( + '$modelDir/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx'), + decoder: await copyAssetFile( + '$modelDir/decoder-epoch-29-avg-9-with-averaged-model.onnx'), + joiner: await copyAssetFile( + '$modelDir/joiner-epoch-29-avg-9-with-averaged-model.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer', + ); + case 4: + final modelDir = 'assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile'; + return sherpa_onnx.OnlineModelConfig( + transducer: sherpa_onnx.OnlineTransducerModelConfig( + encoder: await copyAssetFile( + '$modelDir/encoder-epoch-99-avg-1.int8.onnx'), + decoder: await copyAssetFile( + '$modelDir/decoder-epoch-99-avg-1.onnx'), + joiner: await copyAssetFile( + '$modelDir/joiner-epoch-99-avg-1.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tokens.txt'), + modelType: 'zipformer', + ); + default: + throw ArgumentError('Unsupported type: $type'); + } +} diff --git a/team_b/yappy/lib/streaming_asr.dart b/team_b/yappy/lib/streaming_asr.dart new file mode 100644 index 00000000..a12cf5da --- /dev/null +++ b/team_b/yappy/lib/streaming_asr.dart @@ -0,0 +1,241 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; +import 'package:record/record.dart'; + +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './utils.dart'; +import './online_model.dart'; + +Future createOnlineRecognizer() async { + final type = 4; + + final modelConfig = await getOnlineModelConfig(type: type); + final config = sherpa_onnx.OnlineRecognizerConfig( + model: modelConfig, + ruleFsts: '', + ); + + return sherpa_onnx.OnlineRecognizer(config); +} + +class StreamingAsrScreen extends StatefulWidget { + const StreamingAsrScreen({super.key}); + + @override + State createState() => _StreamingAsrScreenState(); +} + +class _StreamingAsrScreenState extends State { + late final TextEditingController _controller; + late final AudioRecorder _audioRecorder; + + String _title = 'Real-time speech recognition'; + String _last = ''; + int _index = 0; + bool _isInitialized = false; + + sherpa_onnx.OnlineRecognizer? _recognizer; + sherpa_onnx.OnlineStream? _stream; + int _sampleRate = 16000; + + StreamSubscription? _recordSub; + RecordState _recordState = RecordState.stop; + + @override + void initState() { + _audioRecorder = AudioRecorder(); + _controller = TextEditingController(); + + _recordSub = _audioRecorder.onStateChanged().listen((recordState) { + _updateRecordState(recordState); + }); + + super.initState(); + } + + Future _start() async { + if (!_isInitialized) { + sherpa_onnx.initBindings(); + _recognizer = await createOnlineRecognizer(); + _stream = _recognizer?.createStream(); + + _isInitialized = true; + } + + try { + if (await _audioRecorder.hasPermission()) { + const encoder = AudioEncoder.pcm16bits; + + if (!await _isEncoderSupported(encoder)) { + return; + } + + final devs = await _audioRecorder.listInputDevices(); + debugPrint(devs.toString()); + + const config = RecordConfig( + encoder: encoder, + sampleRate: 16000, + numChannels: 1, + ); + + final stream = await _audioRecorder.startStream(config); + + stream.listen( + (data) { + final samplesFloat32 = + convertBytesToFloat32(Uint8List.fromList(data)); + + _stream!.acceptWaveform( + samples: samplesFloat32, sampleRate: _sampleRate); + while (_recognizer!.isReady(_stream!)) { + _recognizer!.decode(_stream!); + } + final text = _recognizer!.getResult(_stream!).text; + String textToDisplay = _last; + if (text != '') { + if (_last == '') { + textToDisplay = '$_index: $text'; + } else { + textToDisplay = '$_index: $text\n$_last'; + } + } + + if (_recognizer!.isEndpoint(_stream!)) { + _recognizer!.reset(_stream!); + if (text != '') { + _last = textToDisplay; + _index += 1; + } + } + // print('text: $textToDisplay'); + + _controller.value = TextEditingValue( + text: textToDisplay, + selection: TextSelection.collapsed(offset: textToDisplay.length), + ); + }, + onDone: () { + print('stream stopped.'); + }, + ); + } + } catch (e) { + print(e); + } + } + + Future _stop() async { + _stream!.free(); + _stream = _recognizer!.createStream(); + + await _audioRecorder.stop(); + } + + Future _pause() => _audioRecorder.pause(); + + Future _resume() => _audioRecorder.resume(); + + void _updateRecordState(RecordState recordState) { + setState(() => _recordState = recordState); + } + + Future _isEncoderSupported(AudioEncoder encoder) async { + final isSupported = await _audioRecorder.isEncoderSupported( + encoder, + ); + + if (!isSupported) { + debugPrint('${encoder.name} is not supported on this platform.'); + debugPrint('Supported encoders are:'); + + for (final e in AudioEncoder.values) { + if (await _audioRecorder.isEncoderSupported(e)) { + debugPrint('- ${encoder.name}'); + } + } + } + + return isSupported; + } + + @override + Widget build(BuildContext context) { + return MaterialApp( + home: Scaffold( + appBar: AppBar( + title: Text(_title), + ), + body: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + const SizedBox(height: 50), + TextField( + maxLines: 5, + controller: _controller, + readOnly: true, + ), + const SizedBox(height: 50), + Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + _buildRecordStopControl(), + const SizedBox(width: 20), + _buildText(), + ], + ), + ], + ), + ), + ); + } + + @override + void dispose() { + _recordSub?.cancel(); + _audioRecorder.dispose(); + _stream?.free(); + _recognizer?.free(); + super.dispose(); + } + + Widget _buildRecordStopControl() { + late Icon icon; + late Color color; + + if (_recordState != RecordState.stop) { + icon = const Icon(Icons.stop, color: Colors.red, size: 30); + color = Colors.red.withOpacity(0.1); + } else { + final theme = Theme.of(context); + icon = Icon(Icons.mic, color: theme.primaryColor, size: 30); + color = theme.primaryColor.withOpacity(0.1); + } + + return ClipOval( + child: Material( + color: color, + child: InkWell( + child: SizedBox(width: 56, height: 56, child: icon), + onTap: () { + (_recordState != RecordState.stop) ? _stop() : _start(); + }, + ), + ), + ); + } + + Widget _buildText() { + if (_recordState == RecordState.stop) { + return const Text("Start"); + } else { + return const Text("Stop"); + } + } +} diff --git a/team_b/yappy/lib/transcription_box.dart b/team_b/yappy/lib/transcription_box.dart index a3b47ed7..89a44add 100644 --- a/team_b/yappy/lib/transcription_box.dart +++ b/team_b/yappy/lib/transcription_box.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:yappy/streaming_asr.dart'; void main() { runApp(TranscriptionBox()); diff --git a/team_b/yappy/lib/utils.dart b/team_b/yappy/lib/utils.dart new file mode 100644 index 00000000..e93618c7 --- /dev/null +++ b/team_b/yappy/lib/utils.dart @@ -0,0 +1,39 @@ +// Copyright (c) 2024 Xiaomi Corporation +import 'package:path/path.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:flutter/services.dart' show rootBundle; +import 'dart:typed_data'; +import "dart:io"; + +// Copy the asset file from src to dst +Future copyAssetFile(String src, [String? dst]) async { + final Directory directory = await getApplicationDocumentsDirectory(); + if (dst == null) { + dst = basename(src); + } + final target = join(directory.path, dst); + bool exists = await new File(target).exists(); + + final data = await rootBundle.load(src); + + if (!exists || File(target).lengthSync() != data.lengthInBytes) { + final List bytes = + data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes); + await File(target).writeAsBytes(bytes); + } + + return target; +} + +Float32List convertBytesToFloat32(Uint8List bytes, [endian = Endian.little]) { + final values = Float32List(bytes.length ~/ 2); + + final data = ByteData.view(bytes.buffer); + + for (var i = 0; i < bytes.length; i += 2) { + int short = data.getInt16(i, endian); + values[i ~/ 2] = short / 32678.0; + } + + return values; +} diff --git a/team_b/yappy/pubspec.yaml b/team_b/yappy/pubspec.yaml index 0583fe72..56f2b6ce 100644 --- a/team_b/yappy/pubspec.yaml +++ b/team_b/yappy/pubspec.yaml @@ -43,6 +43,8 @@ dependencies: path: ^1.8.0 envied: ^1.1.1 path_provider: ^2.0.11 + sherpa_onnx: ^1.10.45 + record: ^5.2.1 dev_dependencies: flutter_test: @@ -77,6 +79,8 @@ flutter: - assets/icon/app_icon.png - assets/yappy_database.db - assets/test_document.txt + - assets/ + - assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/ # To add assets to your application, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg From ab4eb735ca2c3a83edf077246c8c3957987e6c38 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:10 -0600 Subject: [PATCH 02/15] gitignores --- team_b/yappy/linux/.gitignore | 2 ++ .../flutter/generated_plugin_registrant.cc | 11 --------- .../flutter/generated_plugin_registrant.h | 15 ------------ .../linux/flutter/generated_plugins.cmake | 23 ------------------- team_b/yappy/macos/.gitignore | 1 + .../Flutter/GeneratedPluginRegistrant.swift | 12 ---------- 6 files changed, 3 insertions(+), 61 deletions(-) delete mode 100644 team_b/yappy/linux/flutter/generated_plugin_registrant.cc delete mode 100644 team_b/yappy/linux/flutter/generated_plugin_registrant.h delete mode 100644 team_b/yappy/linux/flutter/generated_plugins.cmake delete mode 100644 team_b/yappy/macos/Flutter/GeneratedPluginRegistrant.swift diff --git a/team_b/yappy/linux/.gitignore b/team_b/yappy/linux/.gitignore index d3896c98..24ce4374 100644 --- a/team_b/yappy/linux/.gitignore +++ b/team_b/yappy/linux/.gitignore @@ -1 +1,3 @@ flutter/ephemeral +flutter/generated* +**/flutter/generated* \ No newline at end of file diff --git a/team_b/yappy/linux/flutter/generated_plugin_registrant.cc b/team_b/yappy/linux/flutter/generated_plugin_registrant.cc deleted file mode 100644 index e71a16d2..00000000 --- a/team_b/yappy/linux/flutter/generated_plugin_registrant.cc +++ /dev/null @@ -1,11 +0,0 @@ -// -// Generated file. Do not edit. -// - -// clang-format off - -#include "generated_plugin_registrant.h" - - -void fl_register_plugins(FlPluginRegistry* registry) { -} diff --git a/team_b/yappy/linux/flutter/generated_plugin_registrant.h b/team_b/yappy/linux/flutter/generated_plugin_registrant.h deleted file mode 100644 index e0f0a47b..00000000 --- a/team_b/yappy/linux/flutter/generated_plugin_registrant.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// Generated file. Do not edit. -// - -// clang-format off - -#ifndef GENERATED_PLUGIN_REGISTRANT_ -#define GENERATED_PLUGIN_REGISTRANT_ - -#include - -// Registers Flutter plugins. -void fl_register_plugins(FlPluginRegistry* registry); - -#endif // GENERATED_PLUGIN_REGISTRANT_ diff --git a/team_b/yappy/linux/flutter/generated_plugins.cmake b/team_b/yappy/linux/flutter/generated_plugins.cmake deleted file mode 100644 index 2e1de87a..00000000 --- a/team_b/yappy/linux/flutter/generated_plugins.cmake +++ /dev/null @@ -1,23 +0,0 @@ -# -# Generated file, do not edit. -# - -list(APPEND FLUTTER_PLUGIN_LIST -) - -list(APPEND FLUTTER_FFI_PLUGIN_LIST -) - -set(PLUGIN_BUNDLED_LIBRARIES) - -foreach(plugin ${FLUTTER_PLUGIN_LIST}) - add_subdirectory(flutter/ephemeral/.plugin_symlinks/${plugin}/linux plugins/${plugin}) - target_link_libraries(${BINARY_NAME} PRIVATE ${plugin}_plugin) - list(APPEND PLUGIN_BUNDLED_LIBRARIES $) - list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${plugin}_bundled_libraries}) -endforeach(plugin) - -foreach(ffi_plugin ${FLUTTER_FFI_PLUGIN_LIST}) - add_subdirectory(flutter/ephemeral/.plugin_symlinks/${ffi_plugin}/linux plugins/${ffi_plugin}) - list(APPEND PLUGIN_BUNDLED_LIBRARIES ${${ffi_plugin}_bundled_libraries}) -endforeach(ffi_plugin) diff --git a/team_b/yappy/macos/.gitignore b/team_b/yappy/macos/.gitignore index 7f1506c5..55610ecf 100644 --- a/team_b/yappy/macos/.gitignore +++ b/team_b/yappy/macos/.gitignore @@ -2,6 +2,7 @@ **/Flutter/ephemeral/ **/Pods/ **/Flutter/GeneratedPluginRegistrant.* +Flutter/Generated* # Xcode-related **/dgph diff --git a/team_b/yappy/macos/Flutter/GeneratedPluginRegistrant.swift b/team_b/yappy/macos/Flutter/GeneratedPluginRegistrant.swift deleted file mode 100644 index 29d4e7f0..00000000 --- a/team_b/yappy/macos/Flutter/GeneratedPluginRegistrant.swift +++ /dev/null @@ -1,12 +0,0 @@ -// -// Generated file. Do not edit. -// - -import FlutterMacOS -import Foundation - -import sqflite_darwin - -func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { - SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin")) -} From b5b6e4eba103fca39f677bb59fdc0f1bddd61ecc Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:10 -0600 Subject: [PATCH 03/15] integrate sherpa into yappy (just the simple streaming asr) will implement more of the required ai stuff in later commits --- team_b/yappy/lib/industry_menu.dart | 17 +-- team_b/yappy/lib/mechanic.dart | 35 ++++-- team_b/yappy/lib/medical_doctor.dart | 35 ++++-- team_b/yappy/lib/medical_patient.dart | 35 ++++-- team_b/yappy/lib/restaurant.dart | 35 ++++-- team_b/yappy/lib/speech_state.dart | 136 ++++++++++++++++++++++++ team_b/yappy/lib/streaming_asr.dart | 22 ++-- team_b/yappy/lib/transcription_box.dart | 35 ++---- team_b/yappy/lib/utils.dart | 6 +- 9 files changed, 263 insertions(+), 93 deletions(-) create mode 100644 team_b/yappy/lib/speech_state.dart diff --git a/team_b/yappy/lib/industry_menu.dart b/team_b/yappy/lib/industry_menu.dart index 6052f3e1..86fcfc5d 100644 --- a/team_b/yappy/lib/industry_menu.dart +++ b/team_b/yappy/lib/industry_menu.dart @@ -1,12 +1,15 @@ import 'package:flutter/material.dart'; +import 'package:record/record.dart'; +import 'package:yappy/speech_state.dart'; import 'package:yappy/services/database_helper.dart'; import 'package:share_plus/share_plus.dart'; class IndustryMenu extends StatelessWidget { final String title; final IconData icon; + final SpeechState speechState; - const IndustryMenu({required this.title, required this.icon, super.key}); + const IndustryMenu({required this.title, required this.icon, required this.speechState, super.key}); Widget generateTranscript(BuildContext context, String title, String content) { return AlertDialog( title: Text(title), @@ -97,18 +100,18 @@ class IndustryMenu extends StatelessWidget { children: [ // Creates the chat button for each menu Container( - decoration: - BoxDecoration(shape: BoxShape.circle, color: Colors.grey), + decoration: BoxDecoration( + shape: BoxShape.circle, + color: speechState.recordState == RecordState.stop ? Colors.grey : Colors.red + ), padding: EdgeInsets.all(5), child: IconButton( icon: Icon( - Icons.chat, + speechState.recordState == RecordState.stop ? Icons.mic : Icons.stop, color: Colors.white, size: screenHeight * .05, ), - onPressed: () { - //add Bernhards code here - }, + onPressed: () => speechState.toggleRecording(), ), ), SizedBox(width: screenWidth * .06), diff --git a/team_b/yappy/lib/mechanic.dart b/team_b/yappy/lib/mechanic.dart index 0345af40..bfbcbc27 100644 --- a/team_b/yappy/lib/mechanic.dart +++ b/team_b/yappy/lib/mechanic.dart @@ -2,6 +2,7 @@ import 'package:flutter/material.dart'; import 'package:yappy/industry_menu.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/transcription_box.dart'; +import 'package:yappy/speech_state.dart'; void main() { runApp(MechanicalAidApp()); @@ -20,7 +21,8 @@ class MechanicalAidApp extends StatelessWidget { //Creates a page for the Mechanical Aid industry //The page will contain the industry menu and the transcription box class MechanicalAidPage extends StatelessWidget { - const MechanicalAidPage({super.key}); + MechanicalAidPage({super.key}); + final speechState = SpeechState(); @override Widget build(BuildContext context) { @@ -31,16 +33,27 @@ class MechanicalAidPage extends StatelessWidget { child: ToolBar(), ), drawer: HamburgerDrawer(), - body: Column( - children: [ - IndustryMenu(title: "Vehicle Maintenance", icon: Icons.directions_car), - Expanded( - child: Padding( - padding: const EdgeInsets.all(16.0), - child: TranscriptionBox(), - ), - ), - ], + body: ListenableBuilder( + listenable: speechState, + builder: (context, child) { + return Column( + children: [ + IndustryMenu( + title: "Vehicle Maintenance", + icon: Icons.directions_car, + speechState: speechState, + ), + Expanded( + child: Padding( + padding: const EdgeInsets.all(16.0), + child: TranscriptionBox( + controller: speechState.controller, + ), + ), + ), + ], + ); + } ), ); } diff --git a/team_b/yappy/lib/medical_doctor.dart b/team_b/yappy/lib/medical_doctor.dart index 8837701d..1d02842d 100644 --- a/team_b/yappy/lib/medical_doctor.dart +++ b/team_b/yappy/lib/medical_doctor.dart @@ -1,5 +1,6 @@ import 'package:flutter/material.dart'; import 'package:yappy/industry_menu.dart'; +import 'package:yappy/speech_state.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/transcription_box.dart'; @@ -17,7 +18,8 @@ class MedicalDoctorApp extends StatelessWidget { //Creates a page for the Medical Doctor industry //The page will contain the industry menu and the transcription box class MedicalDoctorPage extends StatelessWidget { - const MedicalDoctorPage({super.key}); + MedicalDoctorPage({super.key}); + final speechState = SpeechState(); @override Widget build(BuildContext context) { @@ -28,16 +30,27 @@ class MedicalDoctorPage extends StatelessWidget { child: ToolBar() ), drawer: HamburgerDrawer(), - body: Column( - children: [ - IndustryMenu(title: "Medical Doctor", icon: Icons.medical_services), - Expanded( - child: Padding( - padding: const EdgeInsets.all(16.0), - child: TranscriptionBox(), - ), - ), - ], + body: ListenableBuilder( + listenable: speechState, + builder: (context, child) { + return Column( + children: [ + IndustryMenu( + title: "Medical Doctor", + icon: Icons.medical_services, + speechState: speechState, + ), + Expanded( + child: Padding( + padding: const EdgeInsets.all(16.0), + child: TranscriptionBox( + controller: speechState.controller, + ), + ), + ), + ], + ); + } ), ); } diff --git a/team_b/yappy/lib/medical_patient.dart b/team_b/yappy/lib/medical_patient.dart index aed3396b..a521a1e7 100644 --- a/team_b/yappy/lib/medical_patient.dart +++ b/team_b/yappy/lib/medical_patient.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:yappy/speech_state.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/industry_menu.dart'; import 'package:yappy/transcription_box.dart'; @@ -17,7 +18,8 @@ class MedicalPatientApp extends StatelessWidget { //Creates a page for the Medical Patient industry //The page will contain the industry menu and the transcription box class MedicalPatientPage extends StatelessWidget { - const MedicalPatientPage({super.key}); + MedicalPatientPage({super.key}); + final speechState = SpeechState(); @override Widget build(BuildContext context) { @@ -28,16 +30,27 @@ class MedicalPatientPage extends StatelessWidget { child: ToolBar() ), drawer: HamburgerDrawer(), - body: Column( - children: [ - IndustryMenu(title: "Medical Patient", icon: Icons.local_pharmacy), - Expanded( - child: Padding( - padding: const EdgeInsets.all(16.0), - child: TranscriptionBox(), - ), - ), - ], + body: ListenableBuilder( + listenable: speechState, + builder: (context, child) { + return Column( + children: [ + IndustryMenu( + title: "Medical Patient", + icon: Icons.local_pharmacy, + speechState: speechState, + ), + Expanded( + child: Padding( + padding: const EdgeInsets.all(16.0), + child: TranscriptionBox( + controller: speechState.controller, + ), + ), + ), + ], + ); + } ), ); } diff --git a/team_b/yappy/lib/restaurant.dart b/team_b/yappy/lib/restaurant.dart index 994eed5c..a1bd3c2c 100644 --- a/team_b/yappy/lib/restaurant.dart +++ b/team_b/yappy/lib/restaurant.dart @@ -2,6 +2,7 @@ import 'package:flutter/material.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/industry_menu.dart'; import 'package:yappy/transcription_box.dart'; +import 'package:yappy/speech_state.dart'; class RestaurantApp extends StatelessWidget { const RestaurantApp({super.key}); @@ -16,7 +17,8 @@ class RestaurantApp extends StatelessWidget { //Creates a page for the Restaurant industry //The page will contain the industry menu and the transcription box class RestaurantPage extends StatelessWidget { - const RestaurantPage({super.key}); + RestaurantPage({super.key}); + final speechState = SpeechState(); @override Widget build(BuildContext context) { @@ -27,16 +29,27 @@ class RestaurantPage extends StatelessWidget { child: ToolBar() ), drawer: HamburgerDrawer(), - body: Column( - children: [ - IndustryMenu(title: "Restaurant", icon: Icons.restaurant_menu), - Expanded( - child: Padding( - padding: const EdgeInsets.all(16.0), - child: TranscriptionBox(), - ), - ), - ], + body: ListenableBuilder( + listenable: speechState, + builder: (context, child) { + return Column( + children: [ + IndustryMenu( + title: "Restaurant", + icon: Icons.restaurant_menu, + speechState: speechState, + ), + Expanded( + child: Padding( + padding: const EdgeInsets.all(16.0), + child: TranscriptionBox( + controller: speechState.controller, + ), + ), + ), + ], + ); + } ), ); } diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart new file mode 100644 index 00000000..27b71d91 --- /dev/null +++ b/team_b/yappy/lib/speech_state.dart @@ -0,0 +1,136 @@ +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:record/record.dart'; + +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; + +import './utils.dart'; +import './online_model.dart'; + +Future createOnlineRecognizer() async { + final type = 4; + + final modelConfig = await getOnlineModelConfig(type: type); + final config = sherpa_onnx.OnlineRecognizerConfig( + model: modelConfig, + ruleFsts: '', + ); + + return sherpa_onnx.OnlineRecognizer(config); +} + +class SpeechState extends ChangeNotifier { + final TextEditingController controller = TextEditingController(); + final AudioRecorder audioRecorder = AudioRecorder(); + + RecordState recordState = RecordState.stop; + bool isInitialized = false; + String last = ''; + int index = 0; + + sherpa_onnx.OnlineRecognizer? recognizer; + sherpa_onnx.OnlineStream? stream; + final int sampleRate = 16000; + + Future initialize() async { + if (!isInitialized) { + sherpa_onnx.initBindings(); + recognizer = await createOnlineRecognizer(); + stream = recognizer?.createStream(); + isInitialized = true; + notifyListeners(); + } + } + + Future toggleRecording() async { + if (recordState == RecordState.stop) { + await startRecording(); + } else { + await stopRecording(); + } + } + + Future startRecording() async { + if (!isInitialized) { + await initialize(); + } + + try { + if (await audioRecorder.hasPermission()) { + const encoder = AudioEncoder.pcm16bits; + + const config = RecordConfig( + encoder: encoder, + sampleRate: 16000, + numChannels: 1, + ); + + final recordStream = await audioRecorder.startStream(config); + recordState = RecordState.record; + notifyListeners(); + + recordStream.listen( + (data) { + final samplesFloat32 = convertBytesToFloat32(Uint8List.fromList(data)); + + stream!.acceptWaveform( + samples: samplesFloat32, + sampleRate: sampleRate + ); + + while (recognizer!.isReady(stream!)) { + recognizer!.decode(stream!); + } + + final text = recognizer!.getResult(stream!).text; + String textToDisplay = last; + + if (text.isNotEmpty) { + if (last.isEmpty) { + textToDisplay = '$index: $text'; + } else { + textToDisplay = '$index: $text\n$last'; + } + } + + if (recognizer!.isEndpoint(stream!)) { + recognizer!.reset(stream!); + if (text.isNotEmpty) { + last = textToDisplay; + index += 1; + } + } + + controller.value = TextEditingValue( + text: textToDisplay, + selection: TextSelection.collapsed(offset: textToDisplay.length), + ); + }, + ); + } + } catch (e) { + if (kDebugMode) { + print('Error starting recording: $e'); + } + } + } + + Future stopRecording() async { + stream!.free(); + stream = recognizer?.createStream(); + await audioRecorder.stop(); + recordState = RecordState.stop; + notifyListeners(); + } + + @override + void dispose() { + controller.dispose(); + audioRecorder.dispose(); + stream?.free(); + recognizer?.free(); + super.dispose(); + } +} \ No newline at end of file diff --git a/team_b/yappy/lib/streaming_asr.dart b/team_b/yappy/lib/streaming_asr.dart index a12cf5da..f14d300c 100644 --- a/team_b/yappy/lib/streaming_asr.dart +++ b/team_b/yappy/lib/streaming_asr.dart @@ -3,8 +3,6 @@ import 'dart:async'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; -import 'package:path/path.dart' as p; -import 'package:path_provider/path_provider.dart'; import 'package:record/record.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; @@ -35,14 +33,14 @@ class _StreamingAsrScreenState extends State { late final TextEditingController _controller; late final AudioRecorder _audioRecorder; - String _title = 'Real-time speech recognition'; + final String _title = 'Real-time speech recognition'; String _last = ''; int _index = 0; bool _isInitialized = false; sherpa_onnx.OnlineRecognizer? _recognizer; sherpa_onnx.OnlineStream? _stream; - int _sampleRate = 16000; + final int _sampleRate = 16000; StreamSubscription? _recordSub; RecordState _recordState = RecordState.stop; @@ -122,12 +120,16 @@ class _StreamingAsrScreenState extends State { ); }, onDone: () { - print('stream stopped.'); + if (kDebugMode) { + print('stream stopped.'); + } }, ); } } catch (e) { - print(e); + if (kDebugMode) { + print(e); + } } } @@ -138,9 +140,9 @@ class _StreamingAsrScreenState extends State { await _audioRecorder.stop(); } - Future _pause() => _audioRecorder.pause(); + // Future _pause() => _audioRecorder.pause(); - Future _resume() => _audioRecorder.resume(); + // Future _resume() => _audioRecorder.resume(); void _updateRecordState(RecordState recordState) { setState(() => _recordState = recordState); @@ -211,11 +213,11 @@ class _StreamingAsrScreenState extends State { if (_recordState != RecordState.stop) { icon = const Icon(Icons.stop, color: Colors.red, size: 30); - color = Colors.red.withOpacity(0.1); + color = Colors.red.withValues(alpha: 0.1); } else { final theme = Theme.of(context); icon = Icon(Icons.mic, color: theme.primaryColor, size: 30); - color = theme.primaryColor.withOpacity(0.1); + color = theme.primaryColor.withValues(alpha: 0.1); } return ClipOval( diff --git a/team_b/yappy/lib/transcription_box.dart b/team_b/yappy/lib/transcription_box.dart index 89a44add..130817c1 100644 --- a/team_b/yappy/lib/transcription_box.dart +++ b/team_b/yappy/lib/transcription_box.dart @@ -1,35 +1,12 @@ import 'package:flutter/material.dart'; -import 'package:yappy/streaming_asr.dart'; -void main() { - runApp(TranscriptionBox()); -} -//Creates a transcription box that will display the AI transcript -//The box will be a text field that will display the AI transcript class TranscriptionBox extends StatelessWidget { - const TranscriptionBox({super.key}); + final TextEditingController controller; - @override - Widget build(BuildContext context) { - return MaterialApp( - debugShowCheckedModeBanner: false, - home: Scaffold( - backgroundColor: Colors.black, - body: Column( - children: [ - Expanded( - child: ChatBox(), - ), - ], - ), - ), - ); - } -} -//Creates a chat box that will display the AI transcript -//The box will be a text field that will display the AI transcript -class ChatBox extends StatelessWidget { - const ChatBox({super.key}); + const TranscriptionBox({ + required this.controller, + super.key + }); @override Widget build(BuildContext context) { @@ -39,7 +16,9 @@ class ChatBox extends StatelessWidget { color: const Color.fromARGB(255, 67, 67, 67) ), child: TextField( + controller: controller, maxLines: null, + readOnly: true, decoration: InputDecoration( hintText: 'AI Transcript will go here', hintStyle: TextStyle( diff --git a/team_b/yappy/lib/utils.dart b/team_b/yappy/lib/utils.dart index e93618c7..b502321d 100644 --- a/team_b/yappy/lib/utils.dart +++ b/team_b/yappy/lib/utils.dart @@ -8,11 +8,9 @@ import "dart:io"; // Copy the asset file from src to dst Future copyAssetFile(String src, [String? dst]) async { final Directory directory = await getApplicationDocumentsDirectory(); - if (dst == null) { - dst = basename(src); - } + dst ??= basename(src); final target = join(directory.path, dst); - bool exists = await new File(target).exists(); + bool exists = await File(target).exists(); final data = await rootBundle.load(src); From 06a8715609f3d434eb53aa8b5f60e1acccfddfb8 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:10 -0600 Subject: [PATCH 04/15] implemented two-pass speach recognition, first is quick & bad, second is slow & good --- team_b/yappy/lib/offline_model.dart | 24 ++++ team_b/yappy/lib/speech_state.dart | 184 +++++++++++++++++++++++----- 2 files changed, 175 insertions(+), 33 deletions(-) create mode 100644 team_b/yappy/lib/offline_model.dart diff --git a/team_b/yappy/lib/offline_model.dart b/team_b/yappy/lib/offline_model.dart new file mode 100644 index 00000000..7ffd59a8 --- /dev/null +++ b/team_b/yappy/lib/offline_model.dart @@ -0,0 +1,24 @@ +import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; +import './utils.dart'; + +// Remember to change `assets` in ../pubspec.yaml +// and download files to ../assets +Future getOfflineModelConfig( + {required int type}) async { + switch (type) { + case 0: + final modelDir = 'assets/sherpa-onnx-whisper-tiny.en'; + return sherpa_onnx.OfflineModelConfig( + whisper: sherpa_onnx.OfflineWhisperModelConfig( + encoder: await copyAssetFile('$modelDir/tiny.en-encoder.int8.onnx'), + decoder: await copyAssetFile('$modelDir/tiny.en-decoder.int8.onnx'), + ), + tokens: await copyAssetFile('$modelDir/tiny.en-tokens.txt'), + modelType: 'whisper', + debug: false, + numThreads: 1 + ); + default: + throw ArgumentError('Unsupported type: $type'); + } +} diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart index 27b71d91..8dc3a9d8 100644 --- a/team_b/yappy/lib/speech_state.dart +++ b/team_b/yappy/lib/speech_state.dart @@ -8,6 +8,7 @@ import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; import './utils.dart'; import './online_model.dart'; +import './offline_model.dart'; Future createOnlineRecognizer() async { final type = 4; @@ -21,29 +22,106 @@ Future createOnlineRecognizer() async { return sherpa_onnx.OnlineRecognizer(config); } +Future createOfflineRecognizer() async { + final type = 0; + + final modelConfig = await getOfflineModelConfig(type: type); + final config = sherpa_onnx.OfflineRecognizerConfig( + model: modelConfig + ); + + return sherpa_onnx.OfflineRecognizer(config); +} + +class AudioSegment { + final Float32List samples; + final int sampleRate; + final String streamingText; + + AudioSegment({ + required this.samples, + required this.sampleRate, + required this.streamingText, + }); +} + class SpeechState extends ChangeNotifier { final TextEditingController controller = TextEditingController(); final AudioRecorder audioRecorder = AudioRecorder(); RecordState recordState = RecordState.stop; bool isInitialized = false; - String last = ''; + String streamingText = ''; int index = 0; - sherpa_onnx.OnlineRecognizer? recognizer; - sherpa_onnx.OnlineStream? stream; + // First pass - streaming recognition + sherpa_onnx.OnlineRecognizer? onlineRecognizer; + sherpa_onnx.OnlineStream? onlineStream; + + // Second pass - Whisper offline recognition + sherpa_onnx.OfflineRecognizer? offlineRecognizer; + + // Buffer for collecting samples between endpoints + List currentSegmentSamples = []; final int sampleRate = 16000; + // Store segments that need offline processing + List pendingSegments = []; + bool isProcessingOffline = false; + Future initialize() async { if (!isInitialized) { + // init online recognizer sherpa_onnx.initBindings(); - recognizer = await createOnlineRecognizer(); - stream = recognizer?.createStream(); + onlineRecognizer = await createOnlineRecognizer(); + onlineStream = onlineRecognizer?.createStream(); + // init offline recognizer + offlineRecognizer = await createOfflineRecognizer(); + isInitialized = true; notifyListeners(); } } +Future processSegmentOffline(AudioSegment segment) async { + final offlineStream = offlineRecognizer!.createStream(); + + offlineStream.acceptWaveform( + samples: segment.samples, + sampleRate: segment.sampleRate + ); + + offlineRecognizer!.decode(offlineStream); + final result = offlineRecognizer!.getResult(offlineStream); + + // Replace the streaming result with the offline result + final oldText = segment.streamingText; + final newText = result.text; + + final currentText = controller.text; + final updatedText = currentText.replaceAll(oldText, newText); + + controller.value = TextEditingValue( + text: updatedText, + selection: TextSelection.collapsed(offset: updatedText.length), + ); + + offlineStream.free(); + } + + Future processPendingSegments() async { + if (pendingSegments.isEmpty || isProcessingOffline) return; + + isProcessingOffline = true; + + for (final segment in pendingSegments) { + await processSegmentOffline(segment); + } + + pendingSegments.clear(); + isProcessingOffline = false; + } + Future toggleRecording() async { if (recordState == RecordState.stop) { await startRecording(); @@ -69,59 +147,98 @@ class SpeechState extends ChangeNotifier { final recordStream = await audioRecorder.startStream(config); recordState = RecordState.record; + currentSegmentSamples.clear(); notifyListeners(); recordStream.listen( (data) { final samplesFloat32 = convertBytesToFloat32(Uint8List.fromList(data)); - stream!.acceptWaveform( + // Add samples to current segment buffer + currentSegmentSamples.add(samplesFloat32); + + onlineStream!.acceptWaveform( samples: samplesFloat32, sampleRate: sampleRate ); - while (recognizer!.isReady(stream!)) { - recognizer!.decode(stream!); + while (onlineRecognizer!.isReady(onlineStream!)) { + onlineRecognizer!.decode(onlineStream!); } - final text = recognizer!.getResult(stream!).text; - String textToDisplay = last; - + final text = onlineRecognizer!.getResult(onlineStream!).text; + if (text.isNotEmpty) { - if (last.isEmpty) { - textToDisplay = '$index: $text'; - } else { - textToDisplay = '$index: $text\n$last'; - } + streamingText = '$index: $text'; + controller.value = TextEditingValue( + text: streamingText, + selection: TextSelection.collapsed(offset: streamingText.length), + ); } - if (recognizer!.isEndpoint(stream!)) { - recognizer!.reset(stream!); - if (text.isNotEmpty) { - last = textToDisplay; - index += 1; + if (onlineRecognizer!.isEndpoint(onlineStream!)) { + // Store the current segment for offline processing + if (currentSegmentSamples.isNotEmpty && streamingText.isNotEmpty) { + // Combine all Float32Lists into a single one + final combinedSamples = Float32List(currentSegmentSamples.fold( + 0, (sum, list) => sum + list.length)); + var offset = 0; + for (var samples in currentSegmentSamples) { + combinedSamples.setRange(offset, offset + samples.length, samples); + offset += samples.length; + } + + pendingSegments.add(AudioSegment( + samples: combinedSamples, + sampleRate: sampleRate, + streamingText: streamingText, + )); + + // Process with Whisper in the background + processPendingSegments(); } + + // Reset for next segment + onlineRecognizer!.reset(onlineStream!); + currentSegmentSamples.clear(); + index += 1; } - - controller.value = TextEditingValue( - text: textToDisplay, - selection: TextSelection.collapsed(offset: textToDisplay.length), - ); }, ); } } catch (e) { - if (kDebugMode) { - print('Error starting recording: $e'); - } + debugPrint('Error starting recording: $e'); } } Future stopRecording() async { - stream!.free(); - stream = recognizer?.createStream(); + // Process any remaining audio with Whisper + if (currentSegmentSamples.isNotEmpty && streamingText.isNotEmpty) { + // Combine all Float32Lists into a single one + final combinedSamples = Float32List(currentSegmentSamples.fold( + 0, (sum, list) => sum + list.length)); + var offset = 0; + for (var samples in currentSegmentSamples) { + combinedSamples.setRange(offset, offset + samples.length, samples); + offset += samples.length; + } + + pendingSegments.add(AudioSegment( + samples: combinedSamples, + sampleRate: sampleRate, + streamingText: streamingText, + )); + } + + onlineStream!.free(); + onlineStream = onlineRecognizer?.createStream(); await audioRecorder.stop(); recordState = RecordState.stop; + + // Process final segments + await processPendingSegments(); + + currentSegmentSamples.clear(); notifyListeners(); } @@ -129,8 +246,9 @@ class SpeechState extends ChangeNotifier { void dispose() { controller.dispose(); audioRecorder.dispose(); - stream?.free(); - recognizer?.free(); + onlineStream?.free(); + onlineRecognizer?.free(); + offlineRecognizer?.free(); super.dispose(); } } \ No newline at end of file From 95e746dd923bbcc5781d5471569a73c78cc0151e Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 05/15] fixed to actually show the transcript --- team_b/yappy/lib/speech_state.dart | 101 ++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart index 8dc3a9d8..6003e4a3 100644 --- a/team_b/yappy/lib/speech_state.dart +++ b/team_b/yappy/lib/speech_state.dart @@ -33,15 +33,27 @@ Future createOfflineRecognizer() async { return sherpa_onnx.OfflineRecognizer(config); } +class RecognizedSegment { + final int index; + String text; + bool isProcessed; + + RecognizedSegment({ + required this.index, + required this.text, + this.isProcessed = false, + }); +} + class AudioSegment { final Float32List samples; final int sampleRate; - final String streamingText; + final int index; AudioSegment({ required this.samples, required this.sampleRate, - required this.streamingText, + required this.index, }); } @@ -51,14 +63,16 @@ class SpeechState extends ChangeNotifier { RecordState recordState = RecordState.stop; bool isInitialized = false; - String streamingText = ''; - int index = 0; + int currentIndex = 0; + // Store all recognized segments + final List recognizedSegments = []; + // First pass - streaming recognition sherpa_onnx.OnlineRecognizer? onlineRecognizer; sherpa_onnx.OnlineStream? onlineStream; - // Second pass - Whisper offline recognition + // Second pass - offline recognition sherpa_onnx.OfflineRecognizer? offlineRecognizer; // Buffer for collecting samples between endpoints @@ -83,7 +97,44 @@ class SpeechState extends ChangeNotifier { } } -Future processSegmentOffline(AudioSegment segment) async { + // Helper method to update the displayed text + void _updateDisplayText() { + final buffer = StringBuffer(); + for (final segment in recognizedSegments) { + if (segment.text.isNotEmpty) { + if (buffer.isNotEmpty) { + buffer.write('\n'); + } + buffer.write('${segment.index}: ${segment.text}'); + } + } + + controller.value = TextEditingValue( + text: buffer.toString(), + selection: TextSelection.collapsed(offset: buffer.length), + ); + } + + // Add a new segment of recognized text + void _addRecognizedSegment(String text) { + recognizedSegments.add(RecognizedSegment( + index: currentIndex, + text: text, + )); + _updateDisplayText(); + } + + // Update an existing segment with improved recognition + void _updateRecognizedSegment(int index, String newText) { + final segmentIndex = recognizedSegments.indexWhere((s) => s.index == index); + if (segmentIndex != -1) { + recognizedSegments[segmentIndex].text = newText; + recognizedSegments[segmentIndex].isProcessed = true; + _updateDisplayText(); + } + } + + Future processSegmentOffline(AudioSegment segment) async { final offlineStream = offlineRecognizer!.createStream(); offlineStream.acceptWaveform( @@ -95,16 +146,7 @@ Future processSegmentOffline(AudioSegment segment) async { final result = offlineRecognizer!.getResult(offlineStream); // Replace the streaming result with the offline result - final oldText = segment.streamingText; - final newText = result.text; - - final currentText = controller.text; - final updatedText = currentText.replaceAll(oldText, newText); - - controller.value = TextEditingValue( - text: updatedText, - selection: TextSelection.collapsed(offset: updatedText.length), - ); + _updateRecognizedSegment(segment.index, result.text); offlineStream.free(); } @@ -137,10 +179,8 @@ Future processSegmentOffline(AudioSegment segment) async { try { if (await audioRecorder.hasPermission()) { - const encoder = AudioEncoder.pcm16bits; - const config = RecordConfig( - encoder: encoder, + encoder: AudioEncoder.pcm16bits, sampleRate: 16000, numChannels: 1, ); @@ -169,16 +209,19 @@ Future processSegmentOffline(AudioSegment segment) async { final text = onlineRecognizer!.getResult(onlineStream!).text; if (text.isNotEmpty) { - streamingText = '$index: $text'; - controller.value = TextEditingValue( - text: streamingText, - selection: TextSelection.collapsed(offset: streamingText.length), - ); + // Update or add the current segment + final existingSegment = recognizedSegments.lastOrNull; + if (existingSegment?.index == currentIndex) { + existingSegment!.text = text; + _updateDisplayText(); + } else { + _addRecognizedSegment(text); + } } if (onlineRecognizer!.isEndpoint(onlineStream!)) { // Store the current segment for offline processing - if (currentSegmentSamples.isNotEmpty && streamingText.isNotEmpty) { + if (currentSegmentSamples.isNotEmpty) { // Combine all Float32Lists into a single one final combinedSamples = Float32List(currentSegmentSamples.fold( 0, (sum, list) => sum + list.length)); @@ -191,7 +234,7 @@ Future processSegmentOffline(AudioSegment segment) async { pendingSegments.add(AudioSegment( samples: combinedSamples, sampleRate: sampleRate, - streamingText: streamingText, + index: currentIndex, )); // Process with Whisper in the background @@ -201,7 +244,7 @@ Future processSegmentOffline(AudioSegment segment) async { // Reset for next segment onlineRecognizer!.reset(onlineStream!); currentSegmentSamples.clear(); - index += 1; + currentIndex += 1; } }, ); @@ -213,7 +256,7 @@ Future processSegmentOffline(AudioSegment segment) async { Future stopRecording() async { // Process any remaining audio with Whisper - if (currentSegmentSamples.isNotEmpty && streamingText.isNotEmpty) { + if (currentSegmentSamples.isNotEmpty) { // Combine all Float32Lists into a single one final combinedSamples = Float32List(currentSegmentSamples.fold( 0, (sum, list) => sum + list.length)); @@ -226,7 +269,7 @@ Future processSegmentOffline(AudioSegment segment) async { pendingSegments.add(AudioSegment( samples: combinedSamples, sampleRate: sampleRate, - streamingText: streamingText, + index: currentIndex, )); } From ba45cf634f95ffb84f43c36ea3a70044fe1d15c4 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 06/15] pubspec and other minor change --- team_b/yappy/lib/utils.dart | 2 +- team_b/yappy/pubspec.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/team_b/yappy/lib/utils.dart b/team_b/yappy/lib/utils.dart index b502321d..12a044c8 100644 --- a/team_b/yappy/lib/utils.dart +++ b/team_b/yappy/lib/utils.dart @@ -7,7 +7,7 @@ import "dart:io"; // Copy the asset file from src to dst Future copyAssetFile(String src, [String? dst]) async { - final Directory directory = await getApplicationDocumentsDirectory(); + final Directory directory = await getApplicationCacheDirectory(); dst ??= basename(src); final target = join(directory.path, dst); bool exists = await File(target).exists(); diff --git a/team_b/yappy/pubspec.yaml b/team_b/yappy/pubspec.yaml index 56f2b6ce..10442866 100644 --- a/team_b/yappy/pubspec.yaml +++ b/team_b/yappy/pubspec.yaml @@ -81,6 +81,9 @@ flutter: - assets/test_document.txt - assets/ - assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/ + - assets/sherpa-onnx-whisper-tiny.en/ + - assets/sherpa-onnx-pyannote-segmentation-3-0/ + - assets/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx # To add assets to your application, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg From 71d46ff8d053747bdf4d04bdd6e24988b9d9eba7 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 07/15] progress, now works to identify speakers --- team_b/yappy/lib/speaker_model.dart | 14 + team_b/yappy/lib/speech_state.dart | 524 +++++++++++++++++++++++++--- 2 files changed, 485 insertions(+), 53 deletions(-) create mode 100644 team_b/yappy/lib/speaker_model.dart diff --git a/team_b/yappy/lib/speaker_model.dart b/team_b/yappy/lib/speaker_model.dart new file mode 100644 index 00000000..1aff23b6 --- /dev/null +++ b/team_b/yappy/lib/speaker_model.dart @@ -0,0 +1,14 @@ +import './utils.dart'; + +// Remember to change `assets` in ../pubspec.yaml +// and download files to ../assets +Future getSpeakerModel( + {required int type}) async { + final modelDir = 'assets'; + switch (type) { + case 0: + return await copyAssetFile('$modelDir/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx'); + default: + throw ArgumentError('Unsupported type: $type'); + } + } \ No newline at end of file diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart index 6003e4a3..5f9b5c46 100644 --- a/team_b/yappy/lib/speech_state.dart +++ b/team_b/yappy/lib/speech_state.dart @@ -1,14 +1,18 @@ import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:record/record.dart'; +import 'package:path_provider/path_provider.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; import './utils.dart'; import './online_model.dart'; import './offline_model.dart'; +import './speaker_model.dart'; Future createOnlineRecognizer() async { final type = 4; @@ -33,27 +37,125 @@ Future createOfflineRecognizer() async { return sherpa_onnx.OfflineRecognizer(config); } +Future createSpeakerExtractor() async { + final type = 0; + + final model = await getSpeakerModel(type: type); + final config = sherpa_onnx.SpeakerEmbeddingExtractorConfig( + model: model, + numThreads: 2, + debug: false, + provider: 'cpu', + ); + + return sherpa_onnx.SpeakerEmbeddingExtractor(config: config); +} + +class Conversation { + final List segments; + final String audioFilePath; + + Conversation({ + required this.segments, + required this.audioFilePath, + }); + + // Convert to JSON for persistence + Map toJson() => { + 'segments': segments.map((s) => s.toJson()).toList(), + 'audioFilePath': audioFilePath, + }; + + factory Conversation.fromJson(Map json) => Conversation( + segments: (json['segments'] as List) + .map((s) => RecognizedSegment.fromJson(s)) + .toList(), + audioFilePath: json['audioFilePath'], + ); + + // Generate a transcript from the conversation + String getTranscript({bool includeSpeakerTags = true}) { + final buffer = StringBuffer(); + RecognizedSegment? lastSegment; + + for (final segment in segments) { + if (segment.text.isEmpty) continue; + + if (buffer.isNotEmpty) { + // Add a newline if the speaker changes or if this is a new thought + if (lastSegment == null || + lastSegment.speakerId != segment.speakerId) { + buffer.write('\n\n'); + } else { + buffer.write(' '); + } + } + + // Add speaker tag if requested and available + if (includeSpeakerTags && segment.speakerId != null) { + buffer.write('${segment.speakerId}: '); + } + + buffer.write(segment.text); + lastSegment = segment; + } + + return buffer.toString(); + } +} + class RecognizedSegment { final int index; String text; + String? speakerId; bool isProcessed; + double start; + double end; + Float32List? speakerEmbedding; RecognizedSegment({ required this.index, required this.text, + this.speakerId, this.isProcessed = false, + required this.start, + required this.end, + this.speakerEmbedding, }); + + // Add a method to convert to/from JSON for persistence + Map toJson() => { + 'index': index, + 'text': text, + 'speakerId': speakerId, + 'isProcessed': isProcessed, + 'start': start, + 'end': end, + }; + + factory RecognizedSegment.fromJson(Map json) => RecognizedSegment( + index: json['index'], + text: json['text'], + speakerId: json['speakerId'], + isProcessed: json['isProcessed'], + start: json['start'], + end: json['end'], + ); } class AudioSegment { final Float32List samples; final int sampleRate; final int index; + final double start; + final double end; AudioSegment({ required this.samples, required this.sampleRate, required this.index, + required this.start, + required this.end, }); } @@ -64,6 +166,8 @@ class SpeechState extends ChangeNotifier { RecordState recordState = RecordState.stop; bool isInitialized = false; int currentIndex = 0; + double currentTimestamp = 0.0; + int currentSpeakerCount = 0; // Store all recognized segments final List recognizedSegments = []; @@ -75,6 +179,11 @@ class SpeechState extends ChangeNotifier { // Second pass - offline recognition sherpa_onnx.OfflineRecognizer? offlineRecognizer; + // Speaker identification + sherpa_onnx.SpeakerEmbeddingExtractor? speakerExtractor; + sherpa_onnx.SpeakerEmbeddingManager? speakerManager; + + List allAudioSamples = []; // Buffer for collecting samples between endpoints List currentSegmentSamples = []; final int sampleRate = 16000; @@ -83,15 +192,26 @@ class SpeechState extends ChangeNotifier { List pendingSegments = []; bool isProcessingOffline = false; + // Path to save the complete audio recording + String? recordingFilePath; + + // Create a Conversation object when recording is stopped + Conversation? lastConversation; + Future initialize() async { if (!isInitialized) { // init online recognizer sherpa_onnx.initBindings(); onlineRecognizer = await createOnlineRecognizer(); onlineStream = onlineRecognizer?.createStream(); + // init offline recognizer offlineRecognizer = await createOfflineRecognizer(); + // init speaker identification components + speakerExtractor = await createSpeakerExtractor(); + speakerManager = sherpa_onnx.SpeakerEmbeddingManager(speakerExtractor!.dim); + isInitialized = true; notifyListeners(); } @@ -100,78 +220,198 @@ class SpeechState extends ChangeNotifier { // Helper method to update the displayed text void _updateDisplayText() { final buffer = StringBuffer(); + + // Debug log + // debugPrint('Updating display text with ${recognizedSegments.length} segments'); + for (final segment in recognizedSegments) { + // Debug log + // debugPrint('Segment ${segment.index}: "${segment.text}" (Speaker: ${segment.speakerId ?? "Unknown"})'); + if (segment.text.isNotEmpty) { if (buffer.isNotEmpty) { buffer.write('\n'); } - buffer.write('${segment.index}: ${segment.text}'); + final prefix = segment.speakerId ?? 'Speaker Unknown'; + buffer.write('$prefix: ${segment.text}'); } } + + // final displayText = buffer.toString(); + // debugPrint('Setting display text: $displayText'); controller.value = TextEditingValue( text: buffer.toString(), selection: TextSelection.collapsed(offset: buffer.length), ); + + notifyListeners(); // Make sure UI gets updated } // Add a new segment of recognized text - void _addRecognizedSegment(String text) { + void _addRecognizedSegment(String text, double start) { recognizedSegments.add(RecognizedSegment( index: currentIndex, text: text, + start: start, + end: start, // Will be updated when segment ends )); - _updateDisplayText(); } // Update an existing segment with improved recognition - void _updateRecognizedSegment(int index, String newText) { + void _updateRecognizedSegment(int index, String newText, {String? speakerId, Float32List? embedding}) { final segmentIndex = recognizedSegments.indexWhere((s) => s.index == index); if (segmentIndex != -1) { recognizedSegments[segmentIndex].text = newText; recognizedSegments[segmentIndex].isProcessed = true; - _updateDisplayText(); + + if (speakerId != null) { + recognizedSegments[segmentIndex].speakerId = speakerId; + } + + if (embedding != null) { + recognizedSegments[segmentIndex].speakerEmbedding = embedding; + } + + // _updateDisplayText(); } } Future processSegmentOffline(AudioSegment segment) async { - final offlineStream = offlineRecognizer!.createStream(); - - offlineStream.acceptWaveform( - samples: segment.samples, - sampleRate: segment.sampleRate - ); + debugPrint('Processing segment ${segment.index} offline (${segment.samples.length} samples)'); - offlineRecognizer!.decode(offlineStream); - final result = offlineRecognizer!.getResult(offlineStream); - - // Replace the streaming result with the offline result - _updateRecognizedSegment(segment.index, result.text); - - offlineStream.free(); + if (segment.samples.isEmpty) { + debugPrint('Empty samples for segment ${segment.index}, skipping'); + return; + } + + try { + // // Check if the segment exists in recognizedSegments + // final segmentIndex = recognizedSegments.indexWhere((s) => s.index == segment.index); + // if (segmentIndex == -1) { + // debugPrint('Segment ${segment.index} not found in recognizedSegments, creating placeholder'); + // // Create a placeholder if it doesn't exist + // recognizedSegments.add(RecognizedSegment( + // index: segment.index, + // text: "", + // start: segment.start, + // end: segment.end, + // )); + // } + + // Perform offline speech recognition + final offlineStream = offlineRecognizer!.createStream(); + + debugPrint('Running offline recognition for segment ${segment.index}'); + offlineStream.acceptWaveform( + samples: segment.samples, + sampleRate: segment.sampleRate + ); + + offlineRecognizer!.decode(offlineStream); + final result = offlineRecognizer!.getResult(offlineStream); + + debugPrint('Offline recognition result for segment ${segment.index}: "${result.text}"'); + + // Perform speaker identification + final speakerStream = speakerExtractor!.createStream(); + + speakerStream.acceptWaveform( + samples: segment.samples, + sampleRate: segment.sampleRate, + ); + + speakerStream.inputFinished(); + + final embedding = speakerExtractor!.compute(speakerStream); + + // Search for matching speaker + final threshold = 0.6; // Adjust threshold as needed + var speakerId = speakerManager!.search(embedding: embedding, threshold: threshold); + + // If no match, register a new speaker + if (speakerId.isEmpty) { + currentSpeakerCount++; + speakerId = 'Speaker $currentSpeakerCount'; + debugPrint('New speaker detected: $speakerId for segment ${segment.index}'); + speakerManager!.add(name: speakerId, embedding: embedding); + } else { + debugPrint('Matched existing speaker: $speakerId for segment ${segment.index}'); + } + + // Ignore empty results + if (result.text.trim().isNotEmpty) { + // Update the recognized segment with both improved text and speaker ID + _updateRecognizedSegment( + segment.index, + result.text, + speakerId: speakerId, + embedding: embedding, + ); + } else { + // debugPrint('Empty text result for segment ${segment.index}, only updating speaker ID'); + // _updateRecognizedSegment( + // segment.index, + // result.text, + // speakerId: speakerId, + // embedding: embedding, + // ); + } + + // Free resources + offlineStream.free(); + speakerStream.free(); + } catch (e) { + debugPrint('Error processing segment ${segment.index} offline: $e'); + } } Future processPendingSegments() async { - if (pendingSegments.isEmpty || isProcessingOffline) return; + if (pendingSegments.isEmpty || isProcessingOffline) { + debugPrint('No pending segments to process or already processing'); + return; + } + debugPrint('Processing ${pendingSegments.length} pending segments'); isProcessingOffline = true; - for (final segment in pendingSegments) { - await processSegmentOffline(segment); + try { + // Create a local copy to process to avoid modification during iteration + final segmentsToProcess = List.from(pendingSegments); + pendingSegments.clear(); + + for (final segment in segmentsToProcess) { + debugPrint('Processing pending segment ${segment.index}'); + await processSegmentOffline(segment); + // Update display after each segment is processed + _updateDisplayText(); + } + } catch (e) { + debugPrint('Error processing pending segments: $e'); + } finally { + isProcessingOffline = false; + debugPrint('Finished processing pending segments'); } - - pendingSegments.clear(); - isProcessingOffline = false; } Future toggleRecording() async { if (recordState == RecordState.stop) { + controller.value = TextEditingValue( + text: "Initializing Yappy!, just a moment..." + ); await startRecording(); } else { await stopRecording(); } } + Future _createRecordingFilePath() async { + // final dir = await getApplicationDocumentsDirectory(); + final timestamp = DateTime.now().millisecondsSinceEpoch; + // return '${dir.path}/recording_$timestamp.wav'; + return '/storage/emulated/0/Documents/recording_$timestamp.wav'; + } + Future startRecording() async { if (!isInitialized) { await initialize(); @@ -179,6 +419,15 @@ class SpeechState extends ChangeNotifier { try { if (await audioRecorder.hasPermission()) { + // Reset speakers for new recording + // speakerManager?.free(); + // speakerManager = sherpa_onnx.SpeakerEmbeddingManager(speakerExtractor!.dim); + currentSpeakerCount = 0; + // recognizedSegments.clear(); + + // Create a path for saving the recording + recordingFilePath = await _createRecordingFilePath(); + const config = RecordConfig( encoder: AudioEncoder.pcm16bits, sampleRate: 16000, @@ -186,8 +435,15 @@ class SpeechState extends ChangeNotifier { ); final recordStream = await audioRecorder.startStream(config); - recordState = RecordState.record; currentSegmentSamples.clear(); + allAudioSamples.clear(); + currentTimestamp = 0.0; + currentIndex = 0; + + recordState = RecordState.record; + controller.value = TextEditingValue( + text: "Ready to record..." + ); notifyListeners(); recordStream.listen( @@ -196,6 +452,10 @@ class SpeechState extends ChangeNotifier { // Add samples to current segment buffer currentSegmentSamples.add(samplesFloat32); + allAudioSamples.add(samplesFloat32); + + // Update current timestamp based on number of samples + currentTimestamp += samplesFloat32.length / sampleRate; onlineStream!.acceptWaveform( samples: samplesFloat32, @@ -210,18 +470,28 @@ class SpeechState extends ChangeNotifier { if (text.isNotEmpty) { // Update or add the current segment - final existingSegment = recognizedSegments.lastOrNull; - if (existingSegment?.index == currentIndex) { - existingSegment!.text = text; - _updateDisplayText(); + final existingSegmentIndex = recognizedSegments.indexWhere((s) => s.index == currentIndex); + + if (existingSegmentIndex != -1) { + // Update existing segment + recognizedSegments[existingSegmentIndex].text = text; + // debugPrint('Updated segment $currentIndex with text: "$text"'); } else { - _addRecognizedSegment(text); + // Add new segment + debugPrint('Adding new segment $currentIndex with text: "$text"'); + _addRecognizedSegment(text, currentTimestamp); } + + // Always update display when we have new text + _updateDisplayText(); } if (onlineRecognizer!.isEndpoint(onlineStream!)) { // Store the current segment for offline processing - if (currentSegmentSamples.isNotEmpty) { + + //ISSUE IS HERE, need to use recognizedSegments.LastOrNull like before, or integrate VAD + if (currentSegmentSamples.isNotEmpty && recognizedSegments.lastOrNull != null + ) { // Combine all Float32Lists into a single one final combinedSamples = Float32List(currentSegmentSamples.fold( 0, (sum, list) => sum + list.length)); @@ -231,13 +501,17 @@ class SpeechState extends ChangeNotifier { offset += samples.length; } + final segmentStart = recognizedSegments.lastOrNull?.start ?? 0.0; + pendingSegments.add(AudioSegment( samples: combinedSamples, sampleRate: sampleRate, index: currentIndex, + start: segmentStart, + end: currentTimestamp, )); - // Process with Whisper in the background + // Process with online recognizer in the background processPendingSegments(); } @@ -254,35 +528,177 @@ class SpeechState extends ChangeNotifier { } } - Future stopRecording() async { - // Process any remaining audio with Whisper - if (currentSegmentSamples.isNotEmpty) { - // Combine all Float32Lists into a single one - final combinedSamples = Float32List(currentSegmentSamples.fold( - 0, (sum, list) => sum + list.length)); + Future saveWavFile() async { + if (allAudioSamples.isEmpty || recordingFilePath == null) return; + + try { + // Combine all audio samples + final totalSamples = allAudioSamples.fold(0, (sum, list) => sum + list.length); + final combinedSamples = Float32List(totalSamples); + var offset = 0; - for (var samples in currentSegmentSamples) { + for (var samples in allAudioSamples) { combinedSamples.setRange(offset, offset + samples.length, samples); offset += samples.length; } - - pendingSegments.add(AudioSegment( - samples: combinedSamples, - sampleRate: sampleRate, - index: currentIndex, - )); + + // Convert Float32List to Int16List for WAV format + final int16Samples = Int16List(combinedSamples.length); + for (var i = 0; i < combinedSamples.length; i++) { + // Scale and clamp to Int16 range + final scaledSample = combinedSamples[i] * 32767; + int16Samples[i] = scaledSample.clamp(-32768, 32767).toInt(); + } + + // Create WAV file + final file = File(recordingFilePath!); + final sink = file.openWrite(); + + // WAV header (44 bytes) + final header = ByteData(44); + + // RIFF chunk descriptor + header.setUint8(0, 0x52); // 'R' + header.setUint8(1, 0x49); // 'I' + header.setUint8(2, 0x46); // 'F' + header.setUint8(3, 0x46); // 'F' + + // Chunk size (file size - 8) + final dataSize = int16Samples.length * 2; // 2 bytes per sample + final fileSize = 36 + dataSize; + header.setUint32(4, fileSize, Endian.little); + + // Format ('WAVE') + header.setUint8(8, 0x57); // 'W' + header.setUint8(9, 0x41); // 'A' + header.setUint8(10, 0x56); // 'V' + header.setUint8(11, 0x45); // 'E' + + // 'fmt ' subchunk + header.setUint8(12, 0x66); // 'f' + header.setUint8(13, 0x6D); // 'm' + header.setUint8(14, 0x74); // 't' + header.setUint8(15, 0x20); // ' ' + + // Subchunk1 size (16 for PCM) + header.setUint32(16, 16, Endian.little); + + // Audio format (1 for PCM) + header.setUint16(20, 1, Endian.little); + + // Number of channels (1 for mono) + header.setUint16(22, 1, Endian.little); + + // Sample rate + header.setUint32(24, sampleRate, Endian.little); + + // Byte rate (SampleRate * NumChannels * BitsPerSample/8) + header.setUint32(28, sampleRate * 1 * 16 ~/ 8, Endian.little); + + // Block align (NumChannels * BitsPerSample/8) + header.setUint16(32, 1 * 16 ~/ 8, Endian.little); + + // Bits per sample + header.setUint16(34, 16, Endian.little); + + // 'data' subchunk + header.setUint8(36, 0x64); // 'd' + header.setUint8(37, 0x61); // 'a' + header.setUint8(38, 0x74); // 't' + header.setUint8(39, 0x61); // 'a' + + // Subchunk2 size (data size) + header.setUint32(40, dataSize, Endian.little); + + // Write header + sink.add(header.buffer.asUint8List()); + + // Write data + final dataBytes = int16Samples.buffer.asUint8List(); + sink.add(dataBytes); + + await sink.close(); + + debugPrint('WAV file saved to: $recordingFilePath'); + } catch (e) { + debugPrint('Error saving WAV file: $e'); } + } + + Future createConversation() async { + // Ensure WAV file is saved + await saveWavFile(); + + // Create conversation object + return Conversation( + segments: List.from(recognizedSegments), // Make a copy + audioFilePath: recordingFilePath ?? '', + ); + } + + Future stopRecording() async { + debugPrint('Stopping recording'); + + try { + // Update UI immediately to show we're stopping + recordState = RecordState.stop; + notifyListeners(); - onlineStream!.free(); - onlineStream = onlineRecognizer?.createStream(); - await audioRecorder.stop(); - recordState = RecordState.stop; + // Process any remaining audio + if (currentSegmentSamples.isNotEmpty) { + debugPrint('Processing final segment $currentIndex'); + + // Combine all Float32Lists into a single one + final combinedSamples = Float32List(currentSegmentSamples.fold( + 0, (sum, list) => sum + list.length)); + var offset = 0; + for (var samples in currentSegmentSamples) { + combinedSamples.setRange(offset, offset + samples.length, samples); + offset += samples.length; + } + + final segmentStart = recognizedSegments.lastOrNull?.start ?? 0.0; + + pendingSegments.add(AudioSegment( + samples: combinedSamples, + sampleRate: sampleRate, + index: currentIndex, + start: segmentStart, + end: currentTimestamp, + )); + } + + // Clean up resources + if (onlineStream != null) { + onlineStream!.free(); + onlineStream = onlineRecognizer?.createStream(); + } + + await audioRecorder.stop(); - // Process final segments - await processPendingSegments(); + // Process final segments + debugPrint('Processing final segments with offline recognizer'); + await processPendingSegments(); - currentSegmentSamples.clear(); - notifyListeners(); + // Save the recording as a WAV file + debugPrint('Saving WAV file'); + await saveWavFile(); + + // Create conversation object + debugPrint('Creating conversation object'); + lastConversation = await createConversation(); + + currentSegmentSamples.clear(); + // allAudioSamples.clear(); + // Final update to display text + _updateDisplayText(); + + debugPrint('Recording stopped successfully'); + } catch (e) { + debugPrint('Error stopping recording: $e'); + } finally { + notifyListeners(); + } } @override @@ -292,6 +708,8 @@ class SpeechState extends ChangeNotifier { onlineStream?.free(); onlineRecognizer?.free(); offlineRecognizer?.free(); + speakerExtractor?.free(); + speakerManager?.free(); super.dispose(); } } \ No newline at end of file From bb994af5a52255f3d5afef8d24c184781f354785 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 08/15] fix issues from rebase --- team_b/yappy/lib/speech_state.dart | 2 +- team_b/yappy/pubspec.yaml | 5 ++--- team_b/yappy/test/industry_menu_test.dart | 7 +++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart index 5f9b5c46..1edb19c8 100644 --- a/team_b/yappy/lib/speech_state.dart +++ b/team_b/yappy/lib/speech_state.dart @@ -5,7 +5,7 @@ import 'dart:typed_data'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:record/record.dart'; -import 'package:path_provider/path_provider.dart'; +// import 'package:path_provider/path_provider.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; diff --git a/team_b/yappy/pubspec.yaml b/team_b/yappy/pubspec.yaml index 10442866..e2e74cef 100644 --- a/team_b/yappy/pubspec.yaml +++ b/team_b/yappy/pubspec.yaml @@ -31,9 +31,8 @@ dependencies: flutter: sdk: flutter -# Allows sharing with the phones default apps. - share_plus: ^10.1.4 - + # Allows sharing with the phones default apps. + share_plus: ^10.1.4 # The following adds the Cupertino Icons font to your application. # Use with the CupertinoIcons class for iOS style icons. diff --git a/team_b/yappy/test/industry_menu_test.dart b/team_b/yappy/test/industry_menu_test.dart index ed189361..08c35101 100644 --- a/team_b/yappy/test/industry_menu_test.dart +++ b/team_b/yappy/test/industry_menu_test.dart @@ -1,13 +1,16 @@ import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:yappy/industry_menu.dart'; +import 'package:yappy/speech_state.dart'; void main() { + + final SpeechState speechState = SpeechState(); testWidgets('IndustryMenu displays title and icons', (WidgetTester tester) async { // Build the IndustryMenu widget. await tester.pumpWidget(MaterialApp( home: Scaffold( - body: IndustryMenu(title: 'Industry Title', icon: Icons.business), + body: IndustryMenu(title: 'Industry Title', icon: Icons.business, speechState: speechState), ), )); @@ -28,7 +31,7 @@ void main() { // Build the IndustryMenu widget. await tester.pumpWidget(MaterialApp( home: Scaffold( - body: IndustryMenu(title: 'Industry Title', icon: Icons.business), + body: IndustryMenu(title: 'Industry Title', icon: Icons.business, speechState: speechState), ), )); From 9e03e114fa1edeb0d7c2e30b28169f5bdaf95e77 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 09/15] Some debugging and error catching improvements --- team_b/yappy/lib/speech_state.dart | 70 +++++++++++++----------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/team_b/yappy/lib/speech_state.dart b/team_b/yappy/lib/speech_state.dart index 1edb19c8..b90f2165 100644 --- a/team_b/yappy/lib/speech_state.dart +++ b/team_b/yappy/lib/speech_state.dart @@ -5,7 +5,6 @@ import 'dart:typed_data'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:record/record.dart'; -// import 'package:path_provider/path_provider.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; @@ -200,20 +199,24 @@ class SpeechState extends ChangeNotifier { Future initialize() async { if (!isInitialized) { - // init online recognizer - sherpa_onnx.initBindings(); - onlineRecognizer = await createOnlineRecognizer(); - onlineStream = onlineRecognizer?.createStream(); + try { + // init online recognizer + sherpa_onnx.initBindings(); + onlineRecognizer = await createOnlineRecognizer(); + onlineStream = onlineRecognizer?.createStream(); - // init offline recognizer - offlineRecognizer = await createOfflineRecognizer(); + // init offline recognizer + offlineRecognizer = await createOfflineRecognizer(); - // init speaker identification components - speakerExtractor = await createSpeakerExtractor(); - speakerManager = sherpa_onnx.SpeakerEmbeddingManager(speakerExtractor!.dim); + // init speaker identification components + speakerExtractor = await createSpeakerExtractor(); + speakerManager = sherpa_onnx.SpeakerEmbeddingManager(speakerExtractor!.dim); - isInitialized = true; - notifyListeners(); + isInitialized = true; + notifyListeners(); + } catch (e) { + debugPrint('Sherpa initialization failed: $e'); + } } } @@ -256,13 +259,17 @@ class SpeechState extends ChangeNotifier { start: start, end: start, // Will be updated when segment ends )); + + debugPrint('Added new segment $currentIndex: "$text" (${start.toStringAsFixed(2)}s - ${start.toStringAsFixed(2)}s)'); } // Update an existing segment with improved recognition void _updateRecognizedSegment(int index, String newText, {String? speakerId, Float32List? embedding}) { final segmentIndex = recognizedSegments.indexWhere((s) => s.index == index); if (segmentIndex != -1) { - recognizedSegments[segmentIndex].text = newText; + if (newText.trim().isNotEmpty) { + recognizedSegments[segmentIndex].text = newText; + } recognizedSegments[segmentIndex].isProcessed = true; if (speakerId != null) { @@ -286,19 +293,6 @@ class SpeechState extends ChangeNotifier { } try { - // // Check if the segment exists in recognizedSegments - // final segmentIndex = recognizedSegments.indexWhere((s) => s.index == segment.index); - // if (segmentIndex == -1) { - // debugPrint('Segment ${segment.index} not found in recognizedSegments, creating placeholder'); - // // Create a placeholder if it doesn't exist - // recognizedSegments.add(RecognizedSegment( - // index: segment.index, - // text: "", - // start: segment.start, - // end: segment.end, - // )); - // } - // Perform offline speech recognition final offlineStream = offlineRecognizer!.createStream(); @@ -348,15 +342,7 @@ class SpeechState extends ChangeNotifier { speakerId: speakerId, embedding: embedding, ); - } else { - // debugPrint('Empty text result for segment ${segment.index}, only updating speaker ID'); - // _updateRecognizedSegment( - // segment.index, - // result.text, - // speakerId: speakerId, - // embedding: embedding, - // ); - } + } // Free resources offlineStream.free(); @@ -420,10 +406,8 @@ class SpeechState extends ChangeNotifier { try { if (await audioRecorder.hasPermission()) { // Reset speakers for new recording - // speakerManager?.free(); - // speakerManager = sherpa_onnx.SpeakerEmbeddingManager(speakerExtractor!.dim); currentSpeakerCount = 0; - // recognizedSegments.clear(); + recognizedSegments.clear(); // Create a path for saving the recording recordingFilePath = await _createRecordingFilePath(); @@ -442,7 +426,7 @@ class SpeechState extends ChangeNotifier { recordState = RecordState.record; controller.value = TextEditingValue( - text: "Ready to record..." + text: "Listening..." ); notifyListeners(); @@ -521,6 +505,12 @@ class SpeechState extends ChangeNotifier { currentIndex += 1; } }, + onError: (error) { + debugPrint('Error from audio stream: $error'); + }, + onDone: () { + debugPrint('Audio stream done'); + }, ); } } catch (e) { @@ -712,4 +702,4 @@ class SpeechState extends ChangeNotifier { speakerManager?.free(); super.dispose(); } -} \ No newline at end of file +} From b240abaa4f0220ece9395ccfde716146446912e7 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 10/15] initial model download function, needs cleanup --- team_b/yappy/lib/services/model_manager.dart | 500 +++++++++++++++++++ team_b/yappy/lib/settings_page.dart | 244 +++++++-- team_b/yappy/pubspec.yaml | 27 +- 3 files changed, 721 insertions(+), 50 deletions(-) create mode 100644 team_b/yappy/lib/services/model_manager.dart diff --git a/team_b/yappy/lib/services/model_manager.dart b/team_b/yappy/lib/services/model_manager.dart new file mode 100644 index 00000000..5e2e281b --- /dev/null +++ b/team_b/yappy/lib/services/model_manager.dart @@ -0,0 +1,500 @@ +import 'dart:async'; +import 'dart:io'; +import 'package:flutter/material.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:http/http.dart' as http; +import 'package:archive/archive.dart'; +import 'package:path/path.dart' as path; +import 'package:connectivity_plus/connectivity_plus.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +class ModelManager { + // Base directory for storing models + late final Future _modelDirPath; + + // Model information with URLs and extraction details + final List _models = [ + ModelInfo( + name: 'Speaker Recognition Model', + url: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx', + isCompressed: false, + size: 25.3, // Size in MB + ), + ModelInfo( + name: 'Offline Whisper Model (Tiny)', + url: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.en.tar.bz2', + isCompressed: true, + keepPaths: ['sherpa-onnx-whisper-tiny.en/tiny.en-tokens.txt', 'sherpa-onnx-whisper-tiny.en/tiny.en-decoder.int8.onnx', 'sherpa-onnx-whisper-tiny.en/tiny.en-encoder.int8.onnx'], + size: 113, // Size in MB + ), + ModelInfo( + name: 'Online Zipformer Model', + url: 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile.tar.bz2', + isCompressed: true, + keepPaths: ['sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/tokens.txt', 'sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/decoder-epoch-99-avg-1.onnx', 'sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/encoder-epoch-99-avg-1.int8.onnx', 'sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/joiner-epoch-99-avg-1.int8.onnx'], + size: 103, // Size in MB + ), + ]; + + // Variables to store dialog information + String _progressMessage = 'Starting download...'; + double _progressValue = 0.0; + StateSetter? _progressSetState; + + ModelManager() { + _modelDirPath = _initModelDir(); + } + + // Initialize the models directory + Future _initModelDir() async { + final appDir = await getApplicationCacheDirectory(); + final modelDir = Directory('${appDir.path}/models'); + if (!await modelDir.exists()) { + await modelDir.create(recursive: true); + } + return modelDir.path; + } + + // Check if all required models exist + Future modelsExist() async { + try { + final modelDir = await _modelDirPath; + + // Check for a marker file indicating successful model installation + final markerFile = File('$modelDir/models_installed.txt'); + if (await markerFile.exists()) { + return true; + } + + // Check individual models if marker doesn't exist + for (var model in _models) { + final modelFile = File('$modelDir/${model.getFilename()}'); + if (!await modelFile.exists()) { + return false; + } + } + + // If all models exist but marker doesn't, create the marker + await markerFile.writeAsString('Models installed on ${DateTime.now()}'); + return true; + } catch (e) { + debugPrint('Error checking models: $e'); + return false; + } + } + + // Check if we should only download on Wi-Fi + Future _getWifiOnlySetting() async { + final prefs = await SharedPreferences.getInstance(); + return prefs.getBool('wifi_only_downloads') ?? true; // Default to true + } + + // Save Wi-Fi only setting + Future saveWifiOnlySetting(bool wifiOnly) async { + final prefs = await SharedPreferences.getInstance(); + await prefs.setBool('wifi_only_downloads', wifiOnly); + } + + // Check network connectivity + Future _checkConnectivity() async { + final wifiOnly = await _getWifiOnlySetting(); + + if (!wifiOnly) { + return true; // Download allowed on any connection + } + + // Check for WiFi connection + final connectivityResult = await Connectivity().checkConnectivity(); + return connectivityResult.contains(ConnectivityResult.wifi); + } + + // Show download dialog to user + Future showDownloadDialog(BuildContext context) async { + // Calculate total download size + final totalSize = _models.fold(0.0, (sum, model) => sum + model.size); + + return await showDialog( + context: context, + barrierDismissible: false, + builder: (BuildContext dialogContext) { + return AlertDialog( + title: const Text('Download AI Models'), + content: SingleChildScrollView( + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'This app requires downloading speech recognition models ' + 'to function properly (${totalSize.toStringAsFixed(1)} MB total).', + ), + const SizedBox(height: 12), + const Text( + 'Models will be downloaded when connected to WiFi. ' + 'You can change this in Settings later.', + ), + const SizedBox(height: 16), + const Text('Models to download:'), + ...List.generate(_models.length, (index) { + final model = _models[index]; + return Padding( + padding: const EdgeInsets.only(top: 8), + child: Text( + '• ${model.name} (${model.size.toStringAsFixed(1)} MB)', + style: const TextStyle(fontSize: 14), + ), + ); + }), + ], + ), + ), + actions: [ + TextButton( + child: const Text('Later'), + onPressed: () { + Navigator.of(dialogContext).pop(false); + }, + ), + TextButton( + child: const Text('Download Now'), + onPressed: () { + Navigator.of(dialogContext).pop(true); + }, + ), + ], + ); + }, + ) ?? false; // Default to false if dialog is dismissed + } + + // Show connectivity warning dialog + Future _showConnectivityWarning(BuildContext context) async { + return await showDialog( + context: context, + barrierDismissible: false, + builder: (BuildContext dialogContext) { + return AlertDialog( + title: const Text('Wi-Fi Required'), + content: const Text( + 'You have selected to download models only on Wi-Fi. ' + 'Please connect to a Wi-Fi network or change your settings to continue.' + ), + actions: [ + TextButton( + child: const Text('Cancel'), + onPressed: () { + Navigator.of(dialogContext).pop(false); + }, + ), + TextButton( + child: const Text('Change Setting'), + onPressed: () { + Navigator.of(dialogContext).pop(true); + }, + ), + ], + ); + }, + ) ?? false; // Default to false if dialog is dismissed + } + + // Download all models + Future downloadModels(BuildContext context) async { + // Store context for later use since we'll be crossing async gaps + final scaffoldContext = context; + + // Check connectivity first + final canDownload = await _checkConnectivity(); + if (!canDownload) { + final changeSettings = await _showConnectivityWarning(scaffoldContext); + if (changeSettings) { + // User wants to change settings + await saveWifiOnlySetting(false); + } else { + // User canceled download + return false; + } + } + + // Create a completer for the progress dialog + late Completer progressDialogCompleter; + + try { + // Show the download progress dialog and get the completer + progressDialogCompleter = _showDownloadProgressDialog(scaffoldContext); + + final modelDir = await _modelDirPath; + int completedModels = 0; + + // Process each model + for (var model in _models) { + // Update progress dialog + _updateDownloadProgress( + message: 'Downloading ${model.name}...', + progress: completedModels / _models.length, + ); + + // Download file + final response = await http.get(Uri.parse(model.url)); + if (response.statusCode != 200) { + throw Exception('Failed to download ${model.name}'); + } + + // Process based on file type + if (model.isCompressed) { + // Update progress dialog + _updateDownloadProgress( + message: 'Extracting ${model.name}...', + progress: completedModels / _models.length, + ); + + // Handle compressed .bz2 file + await _processCompressedFile( + response.bodyBytes, + modelDir, + model.keepPaths ?? [], + ); + } else { + // Handle direct .onnx file + final file = File('$modelDir/${model.getFilename()}'); + await file.writeAsBytes(response.bodyBytes); + } + + completedModels++; + _updateDownloadProgress( + progress: completedModels / _models.length, + ); + } + + // Create marker file to indicate successful installation + final markerFile = File('$modelDir/models_installed.txt'); + await markerFile.writeAsString('Models installed on ${DateTime.now()}'); + + // Complete the progress dialog + progressDialogCompleter.complete(); + + // Show success message + await _showDownloadCompleteDialog(scaffoldContext); + + return true; + } catch (e) { + debugPrint('Error downloading models: $e'); + + // Complete the progress dialog to close it + progressDialogCompleter.complete(); + + // Show error dialog + await showDialog( + context: scaffoldContext, + builder: (dialogContext) => AlertDialog( + title: const Text('Download Failed'), + content: Text('Failed to download models: $e'), + actions: [ + TextButton( + child: const Text('OK'), + onPressed: () => Navigator.of(dialogContext).pop(), + ), + ], + ), + ); + + return false; + } + } + + // Show download progress dialog + Completer _showDownloadProgressDialog(BuildContext context) { + final completer = Completer(); + + // Reset progress values + _progressMessage = 'Starting download...'; + _progressValue = 0.0; + + showDialog( + context: context, + barrierDismissible: false, + builder: (BuildContext dialogContext) { + return WillPopScope( + onWillPop: () async => false, // Prevent back button + child: AlertDialog( + title: const Text('Downloading Models'), + content: StatefulBuilder( + builder: (context, setState) { + // Store setState for later use + _progressSetState = setState; + + return Column( + mainAxisSize: MainAxisSize.min, + children: [ + const CircularProgressIndicator(), + const SizedBox(height: 16), + Text(_progressMessage), + const SizedBox(height: 8), + LinearProgressIndicator(value: _progressValue), + ], + ); + }, + ), + ), + ); + }, + ); + + // When download is complete + completer.future.then((_) { + // Check if dialog is still showing (context is valid) + if (_progressSetState != null) { + Navigator.of(context, rootNavigator: true).pop(); + _progressSetState = null; // Clear reference + } + }); + + return completer; + } + + // Update download progress dialog + void _updateDownloadProgress({ + String? message, + required double progress, + }) { + // Update progress values + if (message != null) { + _progressMessage = message; + } + _progressValue = progress; + + // Update dialog if setState is available + if (_progressSetState != null) { + _progressSetState!(() {}); + } + } + + // Show download complete dialog + Future _showDownloadCompleteDialog(BuildContext context) { + return showDialog( + context: context, + builder: (BuildContext dialogContext) { + return AlertDialog( + title: const Text('Download Complete'), + content: const Text('All models have been downloaded successfully.'), + actions: [ + TextButton( + child: const Text('Continue'), + onPressed: () { + Navigator.of(dialogContext).pop(); + }, + ), + ], + ); + }, + ); + } + + // Process compressed .bz2 file + Future _processCompressedFile( + List fileBytes, + String modelDir, + List keepPaths, + ) async { + // Decompress bz2 + final archive = BZip2Decoder().decodeBytes(fileBytes); + + // Extract tar archive + final tarArchive = TarDecoder().decodeBytes(archive); + + // Process each file in the archive + for (final file in tarArchive) { + // Check if this file/directory should be kept + bool shouldKeep = keepPaths.isEmpty; + for (final keepPath in keepPaths) { + if (file.name.startsWith(keepPath)) { + shouldKeep = true; + break; + } + } + + if (shouldKeep) { + final filePath = '$modelDir/${file.name}'; + + if (file.isFile) { + // Create parent directories if needed + final parentDir = Directory(path.dirname(filePath)); + if (!await parentDir.exists()) { + await parentDir.create(recursive: true); + } + + // Write file + await File(filePath).writeAsBytes(file.content as List); + } else { + // Create directory + await Directory(filePath).create(recursive: true); + } + } + } + } + + // Delete all downloaded models + Future deleteModels() async { + try { + final modelDir = await _modelDirPath; + final directory = Directory(modelDir); + + if (await directory.exists()) { + // Delete all contents of the directory + await for (var entity in directory.list(recursive: true)) { + if (entity is File) { + await entity.delete(); + } else if (entity is Directory) { + // Only delete subdirectories, not the base model directory + if (entity.path != modelDir) { + await entity.delete(recursive: true); + } + } + } + } + + // Delete the installation marker file + final markerFile = File('$modelDir/models_installed.txt'); + if (await markerFile.exists()) { + await markerFile.delete(); + } + + return true; + } catch (e) { + debugPrint('Error deleting models: $e'); + return false; + } + } + + // Trigger model download from settings page + Future downloadModelsFromSettings(BuildContext context) async { + // Store context for later use + final scaffoldContext = context; + + final shouldDownload = await showDownloadDialog(scaffoldContext); + if (shouldDownload) { + return await downloadModels(scaffoldContext); + } + return false; + } +} + +class ModelInfo { + final String name; + final String url; + final bool isCompressed; + final List? keepPaths; + final double size; // Size in MB + + ModelInfo({ + required this.name, + required this.url, + required this.isCompressed, + this.keepPaths, + required this.size, + }); + + String getFilename() { + return path.basename(url); + } +} \ No newline at end of file diff --git a/team_b/yappy/lib/settings_page.dart b/team_b/yappy/lib/settings_page.dart index 2975f409..00987191 100644 --- a/team_b/yappy/lib/settings_page.dart +++ b/team_b/yappy/lib/settings_page.dart @@ -1,29 +1,75 @@ import 'package:flutter/material.dart'; +import './services/model_manager.dart'; +import 'package:shared_preferences/shared_preferences.dart'; -class SettingsPage extends StatelessWidget { +class SettingsPage extends StatefulWidget { const SettingsPage({super.key}); + @override + State createState() => _SettingsPageState(); +} + +class _SettingsPageState extends State { + bool _modelsDownloaded = false; + bool _wifiOnlyDownloads = true; + bool _isLoading = true; + final ModelManager _modelManager = ModelManager(); + + @override + void initState() { + super.initState(); + _loadSettings(); + } + + Future _loadSettings() async { + try { + // Check models status + final exist = await _modelManager.modelsExist(); + + // Load Wi-Fi only setting + final prefs = await SharedPreferences.getInstance(); + final wifiOnly = prefs.getBool('wifi_only_downloads') ?? true; + + if (mounted) { + setState(() { + _modelsDownloaded = exist; + _wifiOnlyDownloads = wifiOnly; + _isLoading = false; + }); + } + } catch (e) { + if (mounted) { + setState(() { + _isLoading = false; + }); + } + } + } + @override Widget build(BuildContext context) { return Scaffold( appBar: AppBar( title: Text('Settings'), ), - body: ListView( + body: _isLoading + ? const Center(child: CircularProgressIndicator()) + : ListView( children: [ + // Original settings items ListTile( - leading: Icon(Icons.account_circle), - title: Text('Account'), + leading: const Icon(Icons.account_circle), + title: const Text('Account'), onTap: () { showDialog( context: context, builder: (BuildContext context) { return AlertDialog( - title: Text('Version'), - content: Text('Version 1.0.0'), + title: const Text('Version'), + content: const Text('Version 1.0.0'), actions: [ TextButton( - child: Text('OK'), + child: const Text('OK'), onPressed: () { Navigator.of(context).pop(); }, @@ -35,43 +81,167 @@ class SettingsPage extends StatelessWidget { }, ), ListTile( - leading: Icon(Icons.notifications), - title: Text('API Keys'), + leading: const Icon(Icons.key), + title: const Text('API Keys'), onTap: () { showDialog( - context: context, - builder: (BuildContext context) { - TextEditingController apiKeyController = TextEditingController(); - return AlertDialog( - title: Text('Enter API Key'), - content: TextField( - controller: apiKeyController, - decoration: InputDecoration(hintText: "API Key"), - ), - actions: [ - TextButton( - child: Text('Cancel'), - onPressed: () { - Navigator.of(context).pop(); - }, - ), - TextButton( - child: Text('Save'), - onPressed: () { - // Save the API key to env.dart file - // String apiKey = apiKeyController.text; - // Add your logic to save the API key here - Navigator.of(context).pop(); - }, - ), - ], - ); - }, + context: context, + builder: (BuildContext context) { + TextEditingController apiKeyController = TextEditingController(); + return AlertDialog( + title: const Text('Enter API Key'), + content: TextField( + controller: apiKeyController, + decoration: const InputDecoration(hintText: "API Key"), + ), + actions: [ + TextButton( + child: const Text('Cancel'), + onPressed: () { + Navigator.of(context).pop(); + }, + ), + TextButton( + child: const Text('Save'), + onPressed: () { + // Save the API key to env.dart file + // String apiKey = apiKeyController.text; + // Add your logic to save the API key here + Navigator.of(context).pop(); + }, + ), + ], + ); + }, ); }, ), + + // Divider to separate original and new settings + const Divider(), + + // New model management settings + const Padding( + padding: EdgeInsets.fromLTRB(16, 16, 16, 8), + child: Text( + 'AI Speech Models', + style: TextStyle( + fontSize: 16, + fontWeight: FontWeight.bold, + color: Colors.grey, + ), + ), + ), + + ListTile( + leading: const Icon(Icons.cloud_download), + title: const Text('Speech Recognition Models'), + subtitle: Text(_modelsDownloaded + ? 'Downloaded and ready to use' + : 'Not downloaded'), + trailing: _modelsDownloaded + ? const Icon(Icons.check_circle, color: Colors.green) + : ElevatedButton( + onPressed: () => _handleModelDownload(context), + child: const Text('Download'), + ), + ), + + SwitchListTile( + secondary: const Icon(Icons.wifi), + title: const Text('Download on Wi-Fi only'), + subtitle: const Text( + 'When enabled, models will only download when connected to Wi-Fi'), + value: _wifiOnlyDownloads, + onChanged: (value) async { + setState(() { + _wifiOnlyDownloads = value; + }); + // Save preference to shared preferences + await _modelManager.saveWifiOnlySetting(value); + }, + ), + + if (_modelsDownloaded) + ListTile( + leading: const Icon(Icons.delete_outline), + title: const Text('Delete Models'), + subtitle: const Text('Free up space by removing downloaded models'), + onTap: () => _handleDeleteModels(context), + ), ], ), ); } + + Future _handleModelDownload(BuildContext context) async { + final result = await _modelManager.downloadModelsFromSettings(context); + + if (!mounted) return; + + // Refresh the model status + final exist = await _modelManager.modelsExist(); + + if (!mounted) return; + + setState(() { + _modelsDownloaded = exist; + }); + } + + Future _handleDeleteModels(BuildContext context) async { + final shouldDelete = await showDialog( + context: context, + builder: (dialogContext) => AlertDialog( + title: const Text('Delete Models?'), + content: const Text( + 'This will delete all downloaded speech models. ' + 'You will need to download them again to use the app\'s ' + 'speech recognition features.' + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(dialogContext).pop(false), + child: const Text('Cancel'), + ), + TextButton( + onPressed: () => Navigator.of(dialogContext).pop(true), + child: const Text('Delete'), + ), + ], + ), + ) ?? false; + + if (!shouldDelete || !mounted) return; + + final success = await _modelManager.deleteModels(); + + if (!mounted) return; + + setState(() { + _modelsDownloaded = !success; + }); + + _showResultSnackBar(success); + } + + // Separate method for showing the snackbar to avoid context across async gap + void _showResultSnackBar(bool success) { + if (success) { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Models deleted successfully'), + duration: Duration(seconds: 2), + ), + ); + } else { + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('Failed to delete models'), + backgroundColor: Colors.red, + duration: Duration(seconds: 2), + ), + ); + } + } } \ No newline at end of file diff --git a/team_b/yappy/pubspec.yaml b/team_b/yappy/pubspec.yaml index e2e74cef..8ad7b9c1 100644 --- a/team_b/yappy/pubspec.yaml +++ b/team_b/yappy/pubspec.yaml @@ -31,19 +31,20 @@ dependencies: flutter: sdk: flutter - # Allows sharing with the phones default apps. - share_plus: ^10.1.4 - - # The following adds the Cupertino Icons font to your application. - # Use with the CupertinoIcons class for iOS style icons. cupertino_icons: ^1.0.8 flutter_launcher_icons: ^0.14.3 - sqflite: ^2.0.3+1 - path: ^1.8.0 + sqflite: ^2.4.2 + path: ^1.9.1 + path_provider: ^2.1.5 envied: ^1.1.1 - path_provider: ^2.0.11 - sherpa_onnx: ^1.10.45 + sherpa_onnx: ^1.10.46 record: ^5.2.1 + connectivity_plus: ^6.1.3 + shared_preferences: ^2.5.2 + share_plus: ^10.1.4 # Allows sharing with the phones default apps. + http: ^1.3.0 # For network requests + permission_handler: ^11.4.0 # For storage permissions + archive: ^4.0.4 # For BZ2 decompression dev_dependencies: flutter_test: @@ -79,10 +80,10 @@ flutter: - assets/yappy_database.db - assets/test_document.txt - assets/ - - assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/ - - assets/sherpa-onnx-whisper-tiny.en/ - - assets/sherpa-onnx-pyannote-segmentation-3-0/ - - assets/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx + # - assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/ + # - assets/sherpa-onnx-whisper-tiny.en/ + # - assets/sherpa-onnx-pyannote-segmentation-3-0/ + # - assets/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx # To add assets to your application, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg From feee6deb2a5a8467dcead7510220c2264b9526e8 Mon Sep 17 00:00:00 2001 From: Bernhard Zwahlen Date: Wed, 5 Mar 2025 11:17:11 -0600 Subject: [PATCH 11/15] downloaded models hooked into sherpa (still buggy) --- team_b/yappy/lib/home_page.dart | 40 +++++++++++++-- team_b/yappy/lib/offline_model.dart | 13 +++-- team_b/yappy/lib/online_model.dart | 76 ++++------------------------- team_b/yappy/lib/speaker_model.dart | 10 ++-- team_b/yappy/pubspec.yaml | 4 -- 5 files changed, 57 insertions(+), 86 deletions(-) diff --git a/team_b/yappy/lib/home_page.dart b/team_b/yappy/lib/home_page.dart index 02abc8b0..4a190b57 100644 --- a/team_b/yappy/lib/home_page.dart +++ b/team_b/yappy/lib/home_page.dart @@ -7,11 +7,42 @@ import 'package:yappy/medical_patient.dart'; import 'package:yappy/restaurant.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/settings_page.dart'; +import './services/model_manager.dart'; -class HomePage extends StatelessWidget { +class HomePage extends StatefulWidget { const HomePage({super.key}); -//Creates the home page of the app -//The page will contain buttons that will navigate to different industries + + @override + State createState() => _HomePageState(); +} + +class _HomePageState extends State { + final ModelManager _modelManager = ModelManager(); + + @override + void initState() { + super.initState(); + // Check if models exist after the first frame is rendered + WidgetsBinding.instance.addPostFrameCallback((_) { + _checkModelsExist(); + }); + } + + Future _checkModelsExist() async { + try { + final modelsExist = await _modelManager.modelsExist(); + if (!modelsExist && mounted) { + // Models don't exist, prompt for download + final shouldDownload = await _modelManager.showDownloadDialog(context); + if (shouldDownload && mounted) { + await _modelManager.downloadModels(context); + } + } + } catch (e) { + debugPrint('Error checking models: $e'); + } + } + @override Widget build(BuildContext context) { return Scaffold( @@ -59,5 +90,4 @@ class HomePage extends StatelessWidget { ), ); } -} - +} \ No newline at end of file diff --git a/team_b/yappy/lib/offline_model.dart b/team_b/yappy/lib/offline_model.dart index 7ffd59a8..4bb68027 100644 --- a/team_b/yappy/lib/offline_model.dart +++ b/team_b/yappy/lib/offline_model.dart @@ -1,19 +1,22 @@ +import 'dart:io'; +import 'package:path_provider/path_provider.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import './utils.dart'; // Remember to change `assets` in ../pubspec.yaml // and download files to ../assets Future getOfflineModelConfig( {required int type}) async { + final appDir = await getApplicationCacheDirectory(); + final modelDir = Directory('${appDir.path}/models'); switch (type) { case 0: - final modelDir = 'assets/sherpa-onnx-whisper-tiny.en'; + final fullDir = '${modelDir.path}/sherpa-onnx-whisper-tiny.en'; return sherpa_onnx.OfflineModelConfig( whisper: sherpa_onnx.OfflineWhisperModelConfig( - encoder: await copyAssetFile('$modelDir/tiny.en-encoder.int8.onnx'), - decoder: await copyAssetFile('$modelDir/tiny.en-decoder.int8.onnx'), + encoder: '$fullDir/tiny.en-encoder.int8.onnx', + decoder: '$fullDir/tiny.en-decoder.int8.onnx', ), - tokens: await copyAssetFile('$modelDir/tiny.en-tokens.txt'), + tokens: '$fullDir/tiny.en-tokens.txt', modelType: 'whisper', debug: false, numThreads: 1 diff --git a/team_b/yappy/lib/online_model.dart b/team_b/yappy/lib/online_model.dart index d0703d64..1e2d7332 100644 --- a/team_b/yappy/lib/online_model.dart +++ b/team_b/yappy/lib/online_model.dart @@ -1,79 +1,21 @@ +import 'dart:io'; +import 'package:path_provider/path_provider.dart'; import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx; -import './utils.dart'; -// Remember to change `assets` in ../pubspec.yaml -// and download files to ../assets Future getOnlineModelConfig( {required int type}) async { + final appDir = await getApplicationCacheDirectory(); + final modelDir = Directory('${appDir.path}/models'); switch (type) { - case 0: - final modelDir = - 'assets/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20'; - return sherpa_onnx.OnlineModelConfig( - transducer: sherpa_onnx.OnlineTransducerModelConfig( - encoder: - await copyAssetFile('$modelDir/encoder-epoch-99-avg-1.int8.onnx'), - decoder: await copyAssetFile('$modelDir/decoder-epoch-99-avg-1.onnx'), - joiner: await copyAssetFile('$modelDir/joiner-epoch-99-avg-1.onnx'), - ), - tokens: await copyAssetFile('$modelDir/tokens.txt'), - modelType: 'zipformer', - ); - case 1: - final modelDir = 'assets/sherpa-onnx-streaming-zipformer-en-2023-06-26'; - return sherpa_onnx.OnlineModelConfig( - transducer: sherpa_onnx.OnlineTransducerModelConfig( - encoder: await copyAssetFile( - '$modelDir/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx'), - decoder: await copyAssetFile( - '$modelDir/decoder-epoch-99-avg-1-chunk-16-left-128.onnx'), - joiner: await copyAssetFile( - '$modelDir/joiner-epoch-99-avg-1-chunk-16-left-128.onnx'), - ), - tokens: await copyAssetFile('$modelDir/tokens.txt'), - modelType: 'zipformer2', - ); - case 2: - final modelDir = - 'assets/icefall-asr-zipformer-streaming-wenetspeech-20230615'; - return sherpa_onnx.OnlineModelConfig( - transducer: sherpa_onnx.OnlineTransducerModelConfig( - encoder: await copyAssetFile( - '$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx'), - decoder: await copyAssetFile( - '$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx'), - joiner: await copyAssetFile( - '$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx'), - ), - tokens: await copyAssetFile('$modelDir/data/lang_char/tokens.txt'), - modelType: 'zipformer2', - ); - case 3: - final modelDir = 'assets/sherpa-onnx-streaming-zipformer-fr-2023-04-14'; - return sherpa_onnx.OnlineModelConfig( - transducer: sherpa_onnx.OnlineTransducerModelConfig( - encoder: await copyAssetFile( - '$modelDir/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx'), - decoder: await copyAssetFile( - '$modelDir/decoder-epoch-29-avg-9-with-averaged-model.onnx'), - joiner: await copyAssetFile( - '$modelDir/joiner-epoch-29-avg-9-with-averaged-model.onnx'), - ), - tokens: await copyAssetFile('$modelDir/tokens.txt'), - modelType: 'zipformer', - ); case 4: - final modelDir = 'assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile'; + final fullDir = '${modelDir.path}/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile'; return sherpa_onnx.OnlineModelConfig( transducer: sherpa_onnx.OnlineTransducerModelConfig( - encoder: await copyAssetFile( - '$modelDir/encoder-epoch-99-avg-1.int8.onnx'), - decoder: await copyAssetFile( - '$modelDir/decoder-epoch-99-avg-1.onnx'), - joiner: await copyAssetFile( - '$modelDir/joiner-epoch-99-avg-1.int8.onnx'), + encoder: '$fullDir/encoder-epoch-99-avg-1.int8.onnx', + decoder: '$fullDir/decoder-epoch-99-avg-1.onnx', + joiner: '$fullDir/joiner-epoch-99-avg-1.int8.onnx', ), - tokens: await copyAssetFile('$modelDir/tokens.txt'), + tokens: '$fullDir/tokens.txt', modelType: 'zipformer', ); default: diff --git a/team_b/yappy/lib/speaker_model.dart b/team_b/yappy/lib/speaker_model.dart index 1aff23b6..5cb8e38a 100644 --- a/team_b/yappy/lib/speaker_model.dart +++ b/team_b/yappy/lib/speaker_model.dart @@ -1,13 +1,13 @@ -import './utils.dart'; +import 'dart:io'; +import 'package:path_provider/path_provider.dart'; -// Remember to change `assets` in ../pubspec.yaml -// and download files to ../assets Future getSpeakerModel( {required int type}) async { - final modelDir = 'assets'; + final appDir = await getApplicationCacheDirectory(); + final modelDir = Directory('${appDir.path}/models'); switch (type) { case 0: - return await copyAssetFile('$modelDir/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx'); + return '${modelDir.path}/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx'; default: throw ArgumentError('Unsupported type: $type'); } diff --git a/team_b/yappy/pubspec.yaml b/team_b/yappy/pubspec.yaml index 8ad7b9c1..b8410829 100644 --- a/team_b/yappy/pubspec.yaml +++ b/team_b/yappy/pubspec.yaml @@ -80,10 +80,6 @@ flutter: - assets/yappy_database.db - assets/test_document.txt - assets/ - # - assets/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17-mobile/ - # - assets/sherpa-onnx-whisper-tiny.en/ - # - assets/sherpa-onnx-pyannote-segmentation-3-0/ - # - assets/3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx # To add assets to your application, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg From dde359bc637006102248c0c69249602f98c726a3 Mon Sep 17 00:00:00 2001 From: FlyingWaffle Date: Wed, 5 Mar 2025 11:59:49 -0600 Subject: [PATCH 12/15] swapped from dialog during download to a toast --- team_b/yappy/lib/services/model_manager.dart | 124 ++++++------------- team_b/yappy/lib/services/toast_manager.dart | 98 +++++++++++++++ 2 files changed, 133 insertions(+), 89 deletions(-) create mode 100644 team_b/yappy/lib/services/toast_manager.dart diff --git a/team_b/yappy/lib/services/model_manager.dart b/team_b/yappy/lib/services/model_manager.dart index 5e2e281b..494e7ef8 100644 --- a/team_b/yappy/lib/services/model_manager.dart +++ b/team_b/yappy/lib/services/model_manager.dart @@ -7,6 +7,7 @@ import 'package:archive/archive.dart'; import 'package:path/path.dart' as path; import 'package:connectivity_plus/connectivity_plus.dart'; import 'package:shared_preferences/shared_preferences.dart'; +import './toast_manager.dart'; class ModelManager { // Base directory for storing models @@ -36,10 +37,9 @@ class ModelManager { ), ]; - // Variables to store dialog information + // Variables to store progress information String _progressMessage = 'Starting download...'; double _progressValue = 0.0; - StateSetter? _progressSetState; ModelManager() { _modelDirPath = _initModelDir(); @@ -216,20 +216,22 @@ class ModelManager { } } - // Create a completer for the progress dialog - late Completer progressDialogCompleter; - try { - // Show the download progress dialog and get the completer - progressDialogCompleter = _showDownloadProgressDialog(scaffoldContext); + // Show initial toast notification + _updateDownloadProgress( + context: scaffoldContext, + message: 'Starting downloads...', + progress: 0.0, + ); final modelDir = await _modelDirPath; int completedModels = 0; // Process each model for (var model in _models) { - // Update progress dialog + // Update progress _updateDownloadProgress( + context: scaffoldContext, message: 'Downloading ${model.name}...', progress: completedModels / _models.length, ); @@ -242,8 +244,9 @@ class ModelManager { // Process based on file type if (model.isCompressed) { - // Update progress dialog + // Update progress _updateDownloadProgress( + context: scaffoldContext, message: 'Extracting ${model.name}...', progress: completedModels / _models.length, ); @@ -262,6 +265,7 @@ class ModelManager { completedModels++; _updateDownloadProgress( + context: scaffoldContext, progress: completedModels / _models.length, ); } @@ -270,18 +274,16 @@ class ModelManager { final markerFile = File('$modelDir/models_installed.txt'); await markerFile.writeAsString('Models installed on ${DateTime.now()}'); - // Complete the progress dialog - progressDialogCompleter.complete(); - - // Show success message - await _showDownloadCompleteDialog(scaffoldContext); + // Hide toast and show success toast + ToastManager.hideToast(); + _showDownloadCompleteToast(scaffoldContext); return true; } catch (e) { debugPrint('Error downloading models: $e'); - // Complete the progress dialog to close it - progressDialogCompleter.complete(); + // Hide progress toast + ToastManager.hideToast(); // Show error dialog await showDialog( @@ -302,58 +304,9 @@ class ModelManager { } } - // Show download progress dialog - Completer _showDownloadProgressDialog(BuildContext context) { - final completer = Completer(); - - // Reset progress values - _progressMessage = 'Starting download...'; - _progressValue = 0.0; - - showDialog( - context: context, - barrierDismissible: false, - builder: (BuildContext dialogContext) { - return WillPopScope( - onWillPop: () async => false, // Prevent back button - child: AlertDialog( - title: const Text('Downloading Models'), - content: StatefulBuilder( - builder: (context, setState) { - // Store setState for later use - _progressSetState = setState; - - return Column( - mainAxisSize: MainAxisSize.min, - children: [ - const CircularProgressIndicator(), - const SizedBox(height: 16), - Text(_progressMessage), - const SizedBox(height: 8), - LinearProgressIndicator(value: _progressValue), - ], - ); - }, - ), - ), - ); - }, - ); - - // When download is complete - completer.future.then((_) { - // Check if dialog is still showing (context is valid) - if (_progressSetState != null) { - Navigator.of(context, rootNavigator: true).pop(); - _progressSetState = null; // Clear reference - } - }); - - return completer; - } - - // Update download progress dialog + // Update download progress toast void _updateDownloadProgress({ + required BuildContext context, String? message, required double progress, }) { @@ -363,30 +316,23 @@ class ModelManager { } _progressValue = progress; - // Update dialog if setState is available - if (_progressSetState != null) { - _progressSetState!(() {}); - } + // Show or update toast + ToastManager.showPersistentToast( + context: context, + message: _progressMessage, + progress: _progressValue, + ); } - // Show download complete dialog - Future _showDownloadCompleteDialog(BuildContext context) { - return showDialog( - context: context, - builder: (BuildContext dialogContext) { - return AlertDialog( - title: const Text('Download Complete'), - content: const Text('All models have been downloaded successfully.'), - actions: [ - TextButton( - child: const Text('Continue'), - onPressed: () { - Navigator.of(dialogContext).pop(); - }, - ), - ], - ); - }, + // Show temporary download complete toast + void _showDownloadCompleteToast(BuildContext context) { + // Show a success toast + ScaffoldMessenger.of(context).showSnackBar( + const SnackBar( + content: Text('All models have been downloaded successfully.'), + backgroundColor: Colors.green, + duration: Duration(seconds: 3), + ), ); } diff --git a/team_b/yappy/lib/services/toast_manager.dart b/team_b/yappy/lib/services/toast_manager.dart new file mode 100644 index 00000000..9f207343 --- /dev/null +++ b/team_b/yappy/lib/services/toast_manager.dart @@ -0,0 +1,98 @@ +import 'package:flutter/material.dart'; + +/// A service for showing persistent toast notifications +class ToastManager { + static OverlayEntry? _currentToast; + static bool _isVisible = false; + + /// Shows a persistent toast with progress at the bottom of the screen + static void showPersistentToast({ + required BuildContext context, + required String message, + double progress = 0.0, + }) { + // Hide any existing toast first + if (_isVisible) { + hideToast(); + } + + // Create overlay entry + final overlay = Overlay.of(context); + _currentToast = OverlayEntry( + builder: (context) => Positioned( + bottom: 32.0, + left: 16.0, + right: 16.0, + child: Material( + elevation: 4.0, + borderRadius: BorderRadius.circular(8.0), + color: Colors.black87, + child: Padding( + padding: const EdgeInsets.symmetric( + horizontal: 16.0, + vertical: 12.0, + ), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + const SizedBox( + height: 24, + width: 24, + child: CircularProgressIndicator( + strokeWidth: 2.0, + valueColor: AlwaysStoppedAnimation(Colors.white), + ), + ), + const SizedBox(width: 12.0), + Expanded( + child: Text( + message, + style: const TextStyle( + color: Colors.white, + fontSize: 14.0, + ), + ), + ), + ], + ), + const SizedBox(height: 8.0), + LinearProgressIndicator( + value: progress, + backgroundColor: Colors.white24, + valueColor: const AlwaysStoppedAnimation(Colors.white), + ), + ], + ), + ), + ), + ), + ); + + // Show the toast + overlay.insert(_currentToast!); + _isVisible = true; + } + + /// Updates the existing toast message and progress + static void updateToast({ + String? message, + double? progress, + }) { + if (!_isVisible || _currentToast == null) return; + + // Force rebuild of the overlay + _currentToast!.markNeedsBuild(); + } + + /// Hides the toast + static void hideToast() { + if (_currentToast != null) { + _currentToast!.remove(); + _currentToast = null; + _isVisible = false; + } + } +} \ No newline at end of file From 4504caec98dae46ea0ee3f58854827e50c581e7d Mon Sep 17 00:00:00 2001 From: FlyingWaffle Date: Wed, 5 Mar 2025 12:27:49 -0600 Subject: [PATCH 13/15] improvements to toasts --- team_b/yappy/lib/services/model_manager.dart | 108 ++++---------- team_b/yappy/lib/services/toast_manager.dart | 98 ------------- team_b/yappy/lib/services/toast_service.dart | 59 ++++++++ team_b/yappy/lib/toast_widget.dart | 146 +++++++++++++++++++ 4 files changed, 232 insertions(+), 179 deletions(-) delete mode 100644 team_b/yappy/lib/services/toast_manager.dart create mode 100644 team_b/yappy/lib/services/toast_service.dart create mode 100644 team_b/yappy/lib/toast_widget.dart diff --git a/team_b/yappy/lib/services/model_manager.dart b/team_b/yappy/lib/services/model_manager.dart index 494e7ef8..fd410a9a 100644 --- a/team_b/yappy/lib/services/model_manager.dart +++ b/team_b/yappy/lib/services/model_manager.dart @@ -7,7 +7,7 @@ import 'package:archive/archive.dart'; import 'package:path/path.dart' as path; import 'package:connectivity_plus/connectivity_plus.dart'; import 'package:shared_preferences/shared_preferences.dart'; -import './toast_manager.dart'; +import 'toast_service.dart'; class ModelManager { // Base directory for storing models @@ -37,9 +37,8 @@ class ModelManager { ), ]; - // Variables to store progress information - String _progressMessage = 'Starting download...'; - double _progressValue = 0.0; + // Toast service + final _toastService = ToastService(); ModelManager() { _modelDirPath = _initModelDir(); @@ -200,13 +199,10 @@ class ModelManager { // Download all models Future downloadModels(BuildContext context) async { - // Store context for later use since we'll be crossing async gaps - final scaffoldContext = context; - - // Check connectivity first + // Check connectivity first - still need context for initial dialogs final canDownload = await _checkConnectivity(); if (!canDownload) { - final changeSettings = await _showConnectivityWarning(scaffoldContext); + final changeSettings = await _showConnectivityWarning(context); if (changeSettings) { // User wants to change settings await saveWifiOnlySetting(false); @@ -216,13 +212,18 @@ class ModelManager { } } + // Start async download process + _startDownloadProcess(); + + // Return true to indicate the download has started + return true; + } + + // Handle the download process independently of any specific context + Future _startDownloadProcess() async { try { // Show initial toast notification - _updateDownloadProgress( - context: scaffoldContext, - message: 'Starting downloads...', - progress: 0.0, - ); + _toastService.showToast('Starting downloads...', progress: 0.0); final modelDir = await _modelDirPath; int completedModels = 0; @@ -230,9 +231,8 @@ class ModelManager { // Process each model for (var model in _models) { // Update progress - _updateDownloadProgress( - context: scaffoldContext, - message: 'Downloading ${model.name}...', + _toastService.showToast( + 'Downloading ${model.name}...', progress: completedModels / _models.length, ); @@ -245,9 +245,8 @@ class ModelManager { // Process based on file type if (model.isCompressed) { // Update progress - _updateDownloadProgress( - context: scaffoldContext, - message: 'Extracting ${model.name}...', + _toastService.showToast( + 'Extracting ${model.name}...', progress: completedModels / _models.length, ); @@ -264,10 +263,7 @@ class ModelManager { } completedModels++; - _updateDownloadProgress( - context: scaffoldContext, - progress: completedModels / _models.length, - ); + _toastService.updateProgress(completedModels / _models.length); } // Create marker file to indicate successful installation @@ -275,65 +271,18 @@ class ModelManager { await markerFile.writeAsString('Models installed on ${DateTime.now()}'); // Hide toast and show success toast - ToastManager.hideToast(); - _showDownloadCompleteToast(scaffoldContext); + _toastService.hideToast(); + _toastService.showSuccess('All models have been downloaded successfully.'); - return true; } catch (e) { debugPrint('Error downloading models: $e'); // Hide progress toast - ToastManager.hideToast(); - - // Show error dialog - await showDialog( - context: scaffoldContext, - builder: (dialogContext) => AlertDialog( - title: const Text('Download Failed'), - content: Text('Failed to download models: $e'), - actions: [ - TextButton( - child: const Text('OK'), - onPressed: () => Navigator.of(dialogContext).pop(), - ), - ], - ), - ); + _toastService.hideToast(); - return false; - } - } - - // Update download progress toast - void _updateDownloadProgress({ - required BuildContext context, - String? message, - required double progress, - }) { - // Update progress values - if (message != null) { - _progressMessage = message; + // Show error toast + _toastService.showError('Failed to download models: $e'); } - _progressValue = progress; - - // Show or update toast - ToastManager.showPersistentToast( - context: context, - message: _progressMessage, - progress: _progressValue, - ); - } - - // Show temporary download complete toast - void _showDownloadCompleteToast(BuildContext context) { - // Show a success toast - ScaffoldMessenger.of(context).showSnackBar( - const SnackBar( - content: Text('All models have been downloaded successfully.'), - backgroundColor: Colors.green, - duration: Duration(seconds: 3), - ), - ); } // Process compressed .bz2 file @@ -414,12 +363,9 @@ class ModelManager { // Trigger model download from settings page Future downloadModelsFromSettings(BuildContext context) async { - // Store context for later use - final scaffoldContext = context; - - final shouldDownload = await showDownloadDialog(scaffoldContext); + final shouldDownload = await showDownloadDialog(context); if (shouldDownload) { - return await downloadModels(scaffoldContext); + return await downloadModels(context); } return false; } diff --git a/team_b/yappy/lib/services/toast_manager.dart b/team_b/yappy/lib/services/toast_manager.dart deleted file mode 100644 index 9f207343..00000000 --- a/team_b/yappy/lib/services/toast_manager.dart +++ /dev/null @@ -1,98 +0,0 @@ -import 'package:flutter/material.dart'; - -/// A service for showing persistent toast notifications -class ToastManager { - static OverlayEntry? _currentToast; - static bool _isVisible = false; - - /// Shows a persistent toast with progress at the bottom of the screen - static void showPersistentToast({ - required BuildContext context, - required String message, - double progress = 0.0, - }) { - // Hide any existing toast first - if (_isVisible) { - hideToast(); - } - - // Create overlay entry - final overlay = Overlay.of(context); - _currentToast = OverlayEntry( - builder: (context) => Positioned( - bottom: 32.0, - left: 16.0, - right: 16.0, - child: Material( - elevation: 4.0, - borderRadius: BorderRadius.circular(8.0), - color: Colors.black87, - child: Padding( - padding: const EdgeInsets.symmetric( - horizontal: 16.0, - vertical: 12.0, - ), - child: Column( - mainAxisSize: MainAxisSize.min, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Row( - children: [ - const SizedBox( - height: 24, - width: 24, - child: CircularProgressIndicator( - strokeWidth: 2.0, - valueColor: AlwaysStoppedAnimation(Colors.white), - ), - ), - const SizedBox(width: 12.0), - Expanded( - child: Text( - message, - style: const TextStyle( - color: Colors.white, - fontSize: 14.0, - ), - ), - ), - ], - ), - const SizedBox(height: 8.0), - LinearProgressIndicator( - value: progress, - backgroundColor: Colors.white24, - valueColor: const AlwaysStoppedAnimation(Colors.white), - ), - ], - ), - ), - ), - ), - ); - - // Show the toast - overlay.insert(_currentToast!); - _isVisible = true; - } - - /// Updates the existing toast message and progress - static void updateToast({ - String? message, - double? progress, - }) { - if (!_isVisible || _currentToast == null) return; - - // Force rebuild of the overlay - _currentToast!.markNeedsBuild(); - } - - /// Hides the toast - static void hideToast() { - if (_currentToast != null) { - _currentToast!.remove(); - _currentToast = null; - _isVisible = false; - } - } -} \ No newline at end of file diff --git a/team_b/yappy/lib/services/toast_service.dart b/team_b/yappy/lib/services/toast_service.dart new file mode 100644 index 00000000..a2df22ea --- /dev/null +++ b/team_b/yappy/lib/services/toast_service.dart @@ -0,0 +1,59 @@ +import 'dart:async'; + +/// A service that manages toast notifications through stream controllers +class ToastService { + // Singleton pattern + static final ToastService _instance = ToastService._internal(); + factory ToastService() => _instance; + ToastService._internal(); + + // Stream controllers + final _messageController = StreamController.broadcast(); + final _progressController = StreamController.broadcast(); + final _visibilityController = StreamController.broadcast(); + final _successController = StreamController.broadcast(); + final _errorController = StreamController.broadcast(); + + // Stream getters + Stream get messageStream => _messageController.stream; + Stream get progressStream => _progressController.stream; + Stream get visibilityStream => _visibilityController.stream; + Stream get successStream => _successController.stream; + Stream get errorStream => _errorController.stream; + + // Methods to control the toast + void showToast(String message, {double progress = 0.0}) { + _messageController.add(message); + _progressController.add(progress); + _visibilityController.add(true); + } + + void updateProgress(double progress) { + _progressController.add(progress); + } + + void updateMessage(String message) { + _messageController.add(message); + } + + void hideToast() { + _visibilityController.add(false); + } + + void showSuccess(String message) { + _successController.add(message); + } + + void showError(String message) { + _errorController.add(message); + } + + // Clean up resources + void dispose() { + _messageController.close(); + _progressController.close(); + _visibilityController.close(); + _successController.close(); + _errorController.close(); + } +} \ No newline at end of file diff --git a/team_b/yappy/lib/toast_widget.dart b/team_b/yappy/lib/toast_widget.dart new file mode 100644 index 00000000..574b07ae --- /dev/null +++ b/team_b/yappy/lib/toast_widget.dart @@ -0,0 +1,146 @@ +import 'package:flutter/material.dart'; +import 'services/toast_service.dart'; + +/// Widget that displays persistent toast notifications across all screens +class ToastWidget extends StatefulWidget { + final Widget child; + + const ToastWidget({super.key, required this.child}); + + @override + State createState() => _ToastWidgetState(); +} + +class _ToastWidgetState extends State { + final _toastService = ToastService(); + bool _visible = false; + String _message = ''; + double _progress = 0.0; + + @override + void initState() { + super.initState(); + + // Listen to service streams + _toastService.messageStream.listen((message) { + if (mounted) { + setState(() => _message = message); + } + }); + + _toastService.progressStream.listen((progress) { + if (mounted) { + setState(() => _progress = progress); + } + }); + + _toastService.visibilityStream.listen((visible) { + if (mounted) { + setState(() => _visible = visible); + } + }); + + _toastService.successStream.listen(_showSuccessSnackBar); + _toastService.errorStream.listen(_showErrorSnackBar); + } + + void _showSuccessSnackBar(String message) { + // Use ScaffoldMessenger to show SnackBar globally + final messenger = ScaffoldMessenger.of(context); + + messenger.clearSnackBars(); + messenger.showSnackBar( + SnackBar( + content: Text(message), + backgroundColor: Colors.green, + duration: const Duration(seconds: 3), + behavior: SnackBarBehavior.floating, + ), + ); + } + + void _showErrorSnackBar(String message) { + // Use ScaffoldMessenger to show SnackBar globally + final messenger = ScaffoldMessenger.of(context); + + messenger.clearSnackBars(); + messenger.showSnackBar( + SnackBar( + content: Text(message), + backgroundColor: Colors.red, + duration: const Duration(seconds: 5), + behavior: SnackBarBehavior.floating, + ), + ); + } + + @override + Widget build(BuildContext context) { + return Material( + // Ensure Material is available for elevation and visual elements + type: MaterialType.transparency, + child: Stack( + children: [ + // Main content + widget.child, + + // Toast notification + AnimatedPositioned( + duration: const Duration(milliseconds: 300), + curve: Curves.easeInOut, + bottom: _visible ? 32.0 : -100.0, + left: 16.0, + right: 16.0, + child: SafeArea( + child: Material( + elevation: 4.0, + borderRadius: BorderRadius.circular(8.0), + color: Colors.black87, + child: Padding( + padding: const EdgeInsets.symmetric( + horizontal: 16.0, + vertical: 12.0, + ), + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + const SizedBox( + height: 24, + width: 24, + child: CircularProgressIndicator( + strokeWidth: 2.0, + valueColor: AlwaysStoppedAnimation(Colors.white), + ), + ), + const SizedBox(width: 12.0), + Expanded( + child: Text( + _message, + style: const TextStyle( + color: Colors.white, + fontSize: 14.0, + ), + ), + ), + ], + ), + const SizedBox(height: 8.0), + LinearProgressIndicator( + value: _progress, + backgroundColor: Colors.white24, + valueColor: const AlwaysStoppedAnimation(Colors.white), + ), + ], + ), + ), + ), + ), + ), + ], + ), + ); + } +} \ No newline at end of file From a051f809ef5582a384b6cac8b52bb97bd5fdd059 Mon Sep 17 00:00:00 2001 From: FlyingWaffle Date: Wed, 5 Mar 2025 12:36:20 -0600 Subject: [PATCH 14/15] more improvements to toasts --- team_b/yappy/lib/home_page.dart | 5 +++++ team_b/yappy/lib/main.dart | 12 +++++++++++- team_b/yappy/lib/services/model_manager.dart | 5 +++++ team_b/yappy/lib/services/toast_service.dart | 8 ++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/team_b/yappy/lib/home_page.dart b/team_b/yappy/lib/home_page.dart index 4a190b57..64f4b26f 100644 --- a/team_b/yappy/lib/home_page.dart +++ b/team_b/yappy/lib/home_page.dart @@ -30,6 +30,11 @@ class _HomePageState extends State { Future _checkModelsExist() async { try { + // Skip check if downloads are already in progress + if (_modelManager.isDownloadInProgress()) { + return; + } + final modelsExist = await _modelManager.modelsExist(); if (!modelsExist && mounted) { // Models don't exist, prompt for download diff --git a/team_b/yappy/lib/main.dart b/team_b/yappy/lib/main.dart index 4e157d11..69ac584f 100644 --- a/team_b/yappy/lib/main.dart +++ b/team_b/yappy/lib/main.dart @@ -1,6 +1,7 @@ import 'package:flutter/material.dart'; import 'package:yappy/home_page.dart'; import 'package:yappy/services/database_helper.dart'; +import './toast_widget.dart'; // Create a global instance of DatabaseHelper final DatabaseHelper dbHelper = DatabaseHelper(); @@ -18,7 +19,16 @@ class MyApp extends StatelessWidget { Widget build(BuildContext context) { return MaterialApp( debugShowCheckedModeBanner: false, - home: const HomePage(), + title: 'Your App', + theme: ThemeData( + primarySwatch: Colors.lightGreen, + visualDensity: VisualDensity.adaptivePlatformDensity, + ), + home: HomePage(), // Your main app page + builder: (context, child) { + // Wrap every screen with ToastWidget + return ToastWidget(child: child ?? Container()); + }, ); } } \ No newline at end of file diff --git a/team_b/yappy/lib/services/model_manager.dart b/team_b/yappy/lib/services/model_manager.dart index fd410a9a..5a2eb760 100644 --- a/team_b/yappy/lib/services/model_manager.dart +++ b/team_b/yappy/lib/services/model_manager.dart @@ -196,6 +196,11 @@ class ModelManager { }, ) ?? false; // Default to false if dialog is dismissed } + + // Check if downloads are currently in progress + bool isDownloadInProgress() { + return _toastService.isToastVisible; + } // Download all models Future downloadModels(BuildContext context) async { diff --git a/team_b/yappy/lib/services/toast_service.dart b/team_b/yappy/lib/services/toast_service.dart index a2df22ea..489e52e7 100644 --- a/team_b/yappy/lib/services/toast_service.dart +++ b/team_b/yappy/lib/services/toast_service.dart @@ -14,6 +14,9 @@ class ToastService { final _successController = StreamController.broadcast(); final _errorController = StreamController.broadcast(); + // Track toast visibility + bool _isToastVisible = false; + // Stream getters Stream get messageStream => _messageController.stream; Stream get progressStream => _progressController.stream; @@ -21,11 +24,15 @@ class ToastService { Stream get successStream => _successController.stream; Stream get errorStream => _errorController.stream; + // Check if toast is visible (i.e., download in progress) + bool get isToastVisible => _isToastVisible; + // Methods to control the toast void showToast(String message, {double progress = 0.0}) { _messageController.add(message); _progressController.add(progress); _visibilityController.add(true); + _isToastVisible = true; } void updateProgress(double progress) { @@ -38,6 +45,7 @@ class ToastService { void hideToast() { _visibilityController.add(false); + _isToastVisible = false; } void showSuccess(String message) { From bdee09e0fcdedd59718e30303af77f5e2b2b87fe Mon Sep 17 00:00:00 2001 From: FlyingWaffle Date: Wed, 5 Mar 2025 12:57:58 -0600 Subject: [PATCH 15/15] disable recording if models not downloaded --- team_b/yappy/lib/industry_menu.dart | 142 ++++++++++++++++---------- team_b/yappy/lib/mechanic.dart | 3 + team_b/yappy/lib/medical_doctor.dart | 4 +- team_b/yappy/lib/medical_patient.dart | 4 +- team_b/yappy/lib/restaurant.dart | 3 + 5 files changed, 102 insertions(+), 54 deletions(-) diff --git a/team_b/yappy/lib/industry_menu.dart b/team_b/yappy/lib/industry_menu.dart index 86fcfc5d..ddd5cdca 100644 --- a/team_b/yappy/lib/industry_menu.dart +++ b/team_b/yappy/lib/industry_menu.dart @@ -3,57 +3,88 @@ import 'package:record/record.dart'; import 'package:yappy/speech_state.dart'; import 'package:yappy/services/database_helper.dart'; import 'package:share_plus/share_plus.dart'; +import 'services/model_manager.dart'; -class IndustryMenu extends StatelessWidget { +class IndustryMenu extends StatefulWidget { final String title; final IconData icon; final SpeechState speechState; + final ModelManager modelManager; // Add model manager - const IndustryMenu({required this.title, required this.icon, required this.speechState, super.key}); - Widget generateTranscript(BuildContext context, String title, String content) { - return AlertDialog( - title: Text(title), - content: SingleChildScrollView( - child: Text(content), - ), - actions: [ - //add export capes - Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - IconButton( - icon: Icon(Icons.share), - onPressed: () { - // Add your share functionality here - Share.share( - content, - subject: title, - ); - }, - ), - IconButton( - icon: Icon(Icons.download), - onPressed: () { - // Add your download functionality here - }, - ), + const IndustryMenu({ + required this.title, + required this.icon, + required this.speechState, + required this.modelManager, // Add to constructor + super.key + }); + + @override + State createState() => _IndustryMenuState(); +} + +class _IndustryMenuState extends State { + bool modelsExist = false; + + @override + void initState() { + super.initState(); + _checkModels(); + } + + Future _checkModels() async { + final exist = await widget.modelManager.modelsExist(); + if (mounted) { + setState(() { + modelsExist = exist; + }); + } + } + + Widget generateTranscript(BuildContext context, String title, String content) { + return AlertDialog( + title: Text(title), + content: SingleChildScrollView( + child: Text(content), + ), + actions: [ + //add export capes + Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ IconButton( - icon: Icon(Icons.delete), - onPressed: () { - // Add your delete functionality here - }, - ), - ], - ), - TextButton( + icon: Icon(Icons.share), onPressed: () { - Navigator.of(context).pop(); + // Add your share functionality here + Share.share( + content, + subject: title, + ); }, - child: Text('Close'), ), - ], - ); - } + IconButton( + icon: Icon(Icons.download), + onPressed: () { + // Add your download functionality here + }, + ), + IconButton( + icon: Icon(Icons.delete), + onPressed: () { + // Add your delete functionality here + }, + ), + ], + ), + TextButton( + onPressed: () { + Navigator.of(context).pop(); + }, + child: Text('Close'), + ), + ], + ); + } Future>> _fetchTranscripts() async { DatabaseHelper dbHelper = DatabaseHelper(); @@ -84,7 +115,7 @@ class IndustryMenu extends StatelessWidget { padding: EdgeInsets.all(12), child: Center( child: Text( - title, + widget.title, style: TextStyle( fontSize: 24, color: Colors.white @@ -102,16 +133,23 @@ class IndustryMenu extends StatelessWidget { Container( decoration: BoxDecoration( shape: BoxShape.circle, - color: speechState.recordState == RecordState.stop ? Colors.grey : Colors.red + color: !modelsExist + ? Color.fromRGBO(128, 128, 128, 0.5) + : (widget.speechState.recordState == RecordState.stop ? Colors.grey : Colors.red) ), padding: EdgeInsets.all(5), - child: IconButton( - icon: Icon( - speechState.recordState == RecordState.stop ? Icons.mic : Icons.stop, - color: Colors.white, - size: screenHeight * .05, + child: Tooltip( + message: !modelsExist + ? "Download required models to enable recording" + : (widget.speechState.recordState == RecordState.stop ? "Start recording" : "Stop recording"), + child: IconButton( + icon: Icon( + widget.speechState.recordState == RecordState.stop ? Icons.mic : Icons.stop, + color: !modelsExist ? Color.fromRGBO(255, 255, 255, 0.5) : Colors.white, + size: screenHeight * .05, + ), + onPressed: !modelsExist ? null : () => widget.speechState.toggleRecording(), ), - onPressed: () => speechState.toggleRecording(), ), ), SizedBox(width: screenWidth * .06), @@ -143,7 +181,7 @@ class IndustryMenu extends StatelessWidget { padding: EdgeInsets.all(5), child: IconButton( icon: Icon( - icon, + widget.icon, color: Colors.white, size: screenHeight * .05, ), @@ -212,7 +250,7 @@ class IndustryMenu extends StatelessWidget { ), onTap: () { Navigator.pop(context); - if (title == 'Restaurant') { + if (widget.title == 'Restaurant') { // Show Kanban style list for restaurant showModalBottomSheet( context: context, diff --git a/team_b/yappy/lib/mechanic.dart b/team_b/yappy/lib/mechanic.dart index bfbcbc27..a1a7a624 100644 --- a/team_b/yappy/lib/mechanic.dart +++ b/team_b/yappy/lib/mechanic.dart @@ -3,6 +3,7 @@ import 'package:yappy/industry_menu.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/transcription_box.dart'; import 'package:yappy/speech_state.dart'; +import 'services/model_manager.dart'; void main() { runApp(MechanicalAidApp()); @@ -23,6 +24,7 @@ class MechanicalAidApp extends StatelessWidget { class MechanicalAidPage extends StatelessWidget { MechanicalAidPage({super.key}); final speechState = SpeechState(); + final modelManager = ModelManager(); @override Widget build(BuildContext context) { @@ -42,6 +44,7 @@ class MechanicalAidPage extends StatelessWidget { title: "Vehicle Maintenance", icon: Icons.directions_car, speechState: speechState, + modelManager: modelManager, ), Expanded( child: Padding( diff --git a/team_b/yappy/lib/medical_doctor.dart b/team_b/yappy/lib/medical_doctor.dart index 1d02842d..20caccb2 100644 --- a/team_b/yappy/lib/medical_doctor.dart +++ b/team_b/yappy/lib/medical_doctor.dart @@ -3,7 +3,7 @@ import 'package:yappy/industry_menu.dart'; import 'package:yappy/speech_state.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/transcription_box.dart'; - +import 'services/model_manager.dart'; class MedicalDoctorApp extends StatelessWidget { const MedicalDoctorApp({super.key}); @@ -20,6 +20,7 @@ class MedicalDoctorApp extends StatelessWidget { class MedicalDoctorPage extends StatelessWidget { MedicalDoctorPage({super.key}); final speechState = SpeechState(); + final modelManager = ModelManager(); @override Widget build(BuildContext context) { @@ -39,6 +40,7 @@ class MedicalDoctorPage extends StatelessWidget { title: "Medical Doctor", icon: Icons.medical_services, speechState: speechState, + modelManager: modelManager, ), Expanded( child: Padding( diff --git a/team_b/yappy/lib/medical_patient.dart b/team_b/yappy/lib/medical_patient.dart index a521a1e7..62393566 100644 --- a/team_b/yappy/lib/medical_patient.dart +++ b/team_b/yappy/lib/medical_patient.dart @@ -3,7 +3,7 @@ import 'package:yappy/speech_state.dart'; import 'package:yappy/tool_bar.dart'; import 'package:yappy/industry_menu.dart'; import 'package:yappy/transcription_box.dart'; - +import 'services/model_manager.dart'; class MedicalPatientApp extends StatelessWidget { const MedicalPatientApp({super.key}); @@ -20,6 +20,7 @@ class MedicalPatientApp extends StatelessWidget { class MedicalPatientPage extends StatelessWidget { MedicalPatientPage({super.key}); final speechState = SpeechState(); + final modelManager = ModelManager(); @override Widget build(BuildContext context) { @@ -39,6 +40,7 @@ class MedicalPatientPage extends StatelessWidget { title: "Medical Patient", icon: Icons.local_pharmacy, speechState: speechState, + modelManager: modelManager, ), Expanded( child: Padding( diff --git a/team_b/yappy/lib/restaurant.dart b/team_b/yappy/lib/restaurant.dart index a1bd3c2c..4ab65b18 100644 --- a/team_b/yappy/lib/restaurant.dart +++ b/team_b/yappy/lib/restaurant.dart @@ -3,6 +3,7 @@ import 'package:yappy/tool_bar.dart'; import 'package:yappy/industry_menu.dart'; import 'package:yappy/transcription_box.dart'; import 'package:yappy/speech_state.dart'; +import 'services/model_manager.dart'; class RestaurantApp extends StatelessWidget { const RestaurantApp({super.key}); @@ -19,6 +20,7 @@ class RestaurantApp extends StatelessWidget { class RestaurantPage extends StatelessWidget { RestaurantPage({super.key}); final speechState = SpeechState(); + final modelManager = ModelManager(); @override Widget build(BuildContext context) { @@ -38,6 +40,7 @@ class RestaurantPage extends StatelessWidget { title: "Restaurant", icon: Icons.restaurant_menu, speechState: speechState, + modelManager: modelManager, ), Expanded( child: Padding(