diff --git a/Friend/firmware/firmware_v1.0/overlay/xiao_ble_sense_devkitv2-adafruit_module.overlay b/Friend/firmware/firmware_v1.0/overlay/xiao_ble_sense_devkitv2-adafruit.overlay similarity index 100% rename from Friend/firmware/firmware_v1.0/overlay/xiao_ble_sense_devkitv2-adafruit_module.overlay rename to Friend/firmware/firmware_v1.0/overlay/xiao_ble_sense_devkitv2-adafruit.overlay diff --git a/Friend/firmware/firmware_v1.0/src/button.c b/Friend/firmware/firmware_v1.0/src/button.c index 679ff2d06..278916904 100644 --- a/Friend/firmware/firmware_v1.0/src/button.c +++ b/Friend/firmware/firmware_v1.0/src/button.c @@ -214,21 +214,23 @@ void check_button_level(struct k_work *work_item) { //If button is pressed for a long time....... notify_long_tap(); + play_haptic_milli(10); //Fire the long mode notify and enter a grace period //turn off herre - if(!from_wakeup) - { - is_off = !is_off; - } - else - { - from_wakeup = false; - } - if (is_off) - { - bt_off(); - turnoff_all(); - } + // TODO: FIXME + //if(!from_wakeup) + //{ + // is_off = !is_off; + //} + //else + //{ + // from_wakeup = false; + //} + //if (is_off) + //{ + // bt_off(); + // turnoff_all(); + //} current_button_state = GRACE; reset_count(); } @@ -254,6 +256,20 @@ void check_button_level(struct k_work *work_item) else if (inc_count_0 > 10) { notify_tap(); //Fire the notify and enter a grace period + if(!from_wakeup) + { + is_off = !is_off; + } + else + { + from_wakeup = false; + } + //Fire the notify and enter a grace period + if (is_off) + { + bt_off(); + turnoff_all(); + } current_button_state = GRACE; reset_count(); } @@ -272,20 +288,22 @@ void check_button_level(struct k_work *work_item) if (inc_count_1 > threshold) { notify_long_tap(); - if(!from_wakeup) - { - is_off = !is_off; - } - else - { - from_wakeup = false; - } - //Fire the notify and enter a grace period - if (is_off) - { - bt_off(); - turnoff_all(); - } + play_haptic_milli(10); + // TODO: FIXME + //if(!from_wakeup) + //{ + // is_off = !is_off; + //} + //else + //{ + // from_wakeup = false; + //} + ////Fire the notify and enter a grace period + //if (is_off) + //{ + // bt_off(); + // turnoff_all(); + //} current_button_state = GRACE; reset_count(); } @@ -422,4 +440,4 @@ void turnoff_all() void force_button_state(FSM_STATE_T state) { current_button_state = state; -} \ No newline at end of file +} diff --git a/app/lib/backend/http/api/messages.dart b/app/lib/backend/http/api/messages.dart index f34de6827..fbf324036 100644 --- a/app/lib/backend/http/api/messages.dart +++ b/app/lib/backend/http/api/messages.dart @@ -1,11 +1,14 @@ import 'dart:convert'; +import 'dart:io'; import 'package:flutter/material.dart'; +import 'package:http/http.dart' as http; import 'package:friend_private/backend/http/shared.dart'; import 'package:friend_private/backend/schema/message.dart'; import 'package:friend_private/env/env.dart'; import 'package:friend_private/utils/logger.dart'; import 'package:instabug_flutter/instabug_flutter.dart'; +import 'package:path/path.dart'; Future> getMessagesServer() async { // TODO: Add pagination @@ -72,3 +75,29 @@ Future getInitialAppMessage(String? appId) { } }); } + +Future> sendVoiceMessageServer(List files) async { + var request = http.MultipartRequest( + 'POST', + Uri.parse('${Env.apiBaseUrl}v1/voice-messages'), + ); + for (var file in files) { + request.files.add(await http.MultipartFile.fromPath('files', file.path, filename: basename(file.path))); + } + request.headers.addAll({'Authorization': await getAuthHeader()}); + + try { + var streamedResponse = await request.send(); + var response = await http.Response.fromStream(streamedResponse); + if (response.statusCode == 200) { + debugPrint('sendVoiceMessageServer response body: ${jsonDecode(response.body)}'); + return ((jsonDecode(response.body) ?? []) as List).map((m) => ServerMessage.fromJson(m)).toList(); + } else { + debugPrint('Failed to upload sample. Status code: ${response.statusCode} ${response.body}'); + throw Exception('Failed to upload sample. Status code: ${response.statusCode}'); + } + } catch (e) { + debugPrint('An error occurred uploadSample: $e'); + throw Exception('An error occurred uploadSample: $e'); + } +} diff --git a/app/lib/providers/capture_provider.dart b/app/lib/providers/capture_provider.dart index 08077f5c1..38cb95570 100644 --- a/app/lib/providers/capture_provider.dart +++ b/app/lib/providers/capture_provider.dart @@ -1,10 +1,13 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:flutter_foreground_task/flutter_foreground_task.dart'; import 'package:flutter_provider_utilities/flutter_provider_utilities.dart'; import 'package:friend_private/backend/http/api/memories.dart'; +import 'package:friend_private/backend/http/api/messages.dart'; import 'package:friend_private/backend/preferences.dart'; import 'package:friend_private/backend/schema/bt_device/bt_device.dart'; import 'package:friend_private/backend/schema/memory.dart'; @@ -25,6 +28,7 @@ import 'package:friend_private/utils/analytics/mixpanel.dart'; import 'package:friend_private/utils/enums.dart'; import 'package:friend_private/utils/logger.dart'; import 'package:internet_connection_checker_plus/internet_connection_checker_plus.dart'; +import 'package:path_provider/path_provider.dart'; import 'package:permission_handler/permission_handler.dart'; import 'package:uuid/uuid.dart'; @@ -75,6 +79,10 @@ class CaptureProvider extends ChangeNotifier get bleBytesStream => _bleBytesStream; + StreamSubscription? _bleButtonStream; + bool _isDeviceButtonLongPressStart = false; + List> _commandBytes = []; + StreamSubscription? _storageStream; get storageStream => _storageStream; @@ -162,12 +170,72 @@ class CaptureProvider extends ChangeNotifier notifyListeners(); } + Future _flushBytesToTempFile(List> chunk, int timerStart) async { + final directory = await getTemporaryDirectory(); + String filePath = '${directory.path}/audio_${timerStart}.bin'; + List data = []; + for (int i = 0; i < chunk.length; i++) { + var frame = chunk[i]; + + // Format: | ; bytes: 4 | n + final byteFrame = ByteData(frame.length); + for (int i = 0; i < frame.length; i++) { + byteFrame.setUint8(i, frame[i]); + } + data.addAll(Uint32List.fromList([frame.length]).buffer.asUint8List()); + data.addAll(byteFrame.buffer.asUint8List()); + } + final file = File(filePath); + await file.writeAsBytes(data); + + return file; + } + + void _processVoiceCommandBytes() async { + debugPrint("Send ${_commandBytes.length} voice frames to backend"); + var file = await _flushBytesToTempFile(_commandBytes, DateTime.now().millisecondsSinceEpoch ~/ 1000); + try { + var messages = await sendVoiceMessageServer([file]); + debugPrint("Command respond: ${messages.map((m) => m.text).join(" | ")}"); + } catch (e) { + debugPrint(e.toString()); + } + } + + Future streamButton(String id) async { + debugPrint('streamButton in capture_provider'); + _bleButtonStream?.cancel(); + _bleButtonStream = await _getBleButtonListener(id, onButtonReceived: (List value) { + if (value.isEmpty) return; + var buttonState = ByteData.view(Uint8List.fromList(value.sublist(0, 4).reversed.toList()).buffer).getUint32(0); + debugPrint("button ${buttonState}"); + + // start long press + if (buttonState == 3) { + _isDeviceButtonLongPressStart = true; + _commandBytes = []; + } + + // release + if (buttonState == 5 && _isDeviceButtonLongPressStart) { + _isDeviceButtonLongPressStart = false; + _processVoiceCommandBytes(); + } + }); + } + Future streamAudioToWs(String id, BleAudioCodec codec) async { debugPrint('streamAudioToWs in capture_provider'); + await streamButton(id); _bleBytesStream?.cancel(); _bleBytesStream = await _getBleAudioBytesListener(id, onAudioBytesReceived: (List value) { if (value.isEmpty) return; + // command button triggered + if (_isDeviceButtonLongPressStart) { + _commandBytes.add(value.sublist(3)); + } + // support: opus codec, 1m from the first device connectes var deviceFirstConnectedAt = _deviceService.getFirstConnectedAt(); var checkWalSupported = codec == BleAudioCodec.opus && @@ -241,6 +309,17 @@ class CaptureProvider extends ChangeNotifier return connection.getBleAudioBytesListener(onAudioBytesReceived: onAudioBytesReceived); } + Future _getBleButtonListener( + String deviceId, { + required void Function(List) onButtonReceived, + }) async { + var connection = await ServiceManager.instance().device.ensureConnection(deviceId); + if (connection == null) { + return Future.value(null); + } + return connection.getBleButtonListener(onButtonReceived: onButtonReceived); + } + Future _recheckCodecChange() async { if (_recordingDevice != null) { BleAudioCodec newCodec = await _getAudioCodec(_recordingDevice!.id); diff --git a/app/lib/services/devices/device_connection.dart b/app/lib/services/devices/device_connection.dart index e27b692f0..43aeb74e4 100644 --- a/app/lib/services/devices/device_connection.dart +++ b/app/lib/services/devices/device_connection.dart @@ -183,10 +183,24 @@ abstract class DeviceConnection { return null; } + Future getBleButtonListener({ + required void Function(List) onButtonReceived, + }) async { + if (await isConnected()) { + return await performGetBleButtonListener(onButtonReceived: onButtonReceived); + } + //_showDeviceDisconnectedNotification(); + return null; + } + Future performGetBleAudioBytesListener({ required void Function(List) onAudioBytesReceived, }); + Future performGetBleButtonListener({ + required void Function(List) onButtonReceived, + }); + Future getAudioCodec() async { if (await isConnected()) { return await performGetAudioCodec(); diff --git a/app/lib/services/devices/frame_connection.dart b/app/lib/services/devices/frame_connection.dart index 67f87ce0c..515c4d8b3 100644 --- a/app/lib/services/devices/frame_connection.dart +++ b/app/lib/services/devices/frame_connection.dart @@ -257,6 +257,8 @@ class FrameDeviceConnection extends DeviceConnection { return Future.value(BleAudioCodec.pcm8); } + Future performGetBleButtonListener({required void Function(List) onButtonReceived}) async {} + @override Future performGetBleAudioBytesListener( {required void Function(List) onAudioBytesReceived}) async { diff --git a/app/lib/services/devices/friend_connection.dart b/app/lib/services/devices/friend_connection.dart index 6404c2406..51718fa87 100644 --- a/app/lib/services/devices/friend_connection.dart +++ b/app/lib/services/devices/friend_connection.dart @@ -19,6 +19,7 @@ class FriendDeviceConnection extends DeviceConnection { BluetoothService? _friendService; BluetoothService? _storageService; BluetoothService? _accelService; + BluetoothService? _buttonService; FriendDeviceConnection(super.device, super.bleDevice); @@ -49,6 +50,11 @@ class FriendDeviceConnection extends DeviceConnection { if (_accelService == null) { logServiceNotFoundError('Accelerometer', deviceId); } + + _buttonService = await getService(buttonDataStreamServiceUuid); + if (_buttonService == null) { + logServiceNotFoundError('Button', deviceId); + } } // Mimic @app/lib/utils/ble/friend_communication.dart @@ -116,6 +122,62 @@ class FriendDeviceConnection extends DeviceConnection { return listener; } + Future performGetBleButtonListener({ + required void Function(List) onButtonReceived, + }) async { + if (_buttonService == null) { + logServiceNotFoundError('Button', deviceId); + return null; + } + + var buttonDataStreamCharacteristic = getCharacteristic(_buttonService!, buttonTriggerCharacteristicUuid); + if (buttonDataStreamCharacteristic == null) { + logCharacteristicNotFoundError('Button data stream', deviceId); + return null; + } + + try { + // TODO: Unknown GATT error here (code 133) on Android. StackOverflow says that it has to do with smaller MTU size + // The creator of the plugin says not to use autoConnect + // https://github.com/chipweinberger/flutter_blue_plus/issues/612 + final device = bleDevice; + if (device.isConnected) { + if (Platform.isAndroid && device.mtuNow < 512) { + await device.requestMtu(512); // This might fix the code 133 error + } + if (device.isConnected) { + try { + await buttonDataStreamCharacteristic.setNotifyValue(true); // device could be disconnected here. + } on PlatformException catch (e) { + Logger.error('Error setting notify value for audio data stream $e'); + } + } else { + Logger.handle(Exception('Device disconnected before setting notify value'), StackTrace.current, + message: 'Device is disconnected. Please reconnect and try again'); + } + } + } catch (e, stackTrace) { + logSubscribeError('Button data stream', deviceId, e, stackTrace); + return null; + } + + debugPrint('Subscribed to button stream from Friend Device'); + var listener = buttonDataStreamCharacteristic.lastValueStream.listen((value) { + debugPrint("new button value ${value}"); + if (value.isNotEmpty) onButtonReceived(value); + }); + + final device = bleDevice; + device.cancelWhenDisconnected(listener); + + // This will cause a crash in OpenGlass devices + // due to a race with discoverServices() that triggers + // a bug in the device firmware. + if (Platform.isAndroid && device.isConnected) await device.requestMtu(512); + + return listener; + } + @override Future performGetBleAudioBytesListener({ required void Function(List) onAudioBytesReceived, diff --git a/app/lib/services/devices/models.dart b/app/lib/services/devices/models.dart index 4b52c316a..3e68c9f18 100644 --- a/app/lib/services/devices/models.dart +++ b/app/lib/services/devices/models.dart @@ -9,6 +9,8 @@ const String friendServiceUuid = '19b10000-e8f2-537e-4f6c-d104768a1214'; const String audioDataStreamCharacteristicUuid = '19b10001-e8f2-537e-4f6c-d104768a1214'; const String audioCodecCharacteristicUuid = '19b10002-e8f2-537e-4f6c-d104768a1214'; +//static struct bt_uuid_128 button_uuid_x = BT_UUID_INIT_128(BT_UUID_128_ENCODE(23ba7925 ,0x0000,0x1000,0x7450,0x346EAC492E92)); +const String buttonDataStreamServiceUuid = '23ba7924-0000-1000-7450-346eac492e92'; const String buttonDataStreamCharacteristicUuid = '23ba7924-0000-1000-7450-346eac492e92'; const String buttonTriggerCharacteristicUuid = '23ba7925-0000-1000-7450-346eac492e92'; diff --git a/app/lib/services/notifications.dart b/app/lib/services/notifications.dart index 5f1bb468b..43afb4fb4 100644 --- a/app/lib/services/notifications.dart +++ b/app/lib/services/notifications.dart @@ -228,6 +228,7 @@ class NotificationUtil { /// Use this method to detect when the user taps on a notification or action button @pragma("vm:entry-point") static Future onActionReceivedMethod(ReceivedAction receivedAction) async { + debugPrint("onActionReceivedMethod called"); if (receivePort != null) { await onActionReceivedMethodImpl(receivedAction); } else { diff --git a/backend/routers/chat.py b/backend/routers/chat.py index 898392432..cd5758757 100644 --- a/backend/routers/chat.py +++ b/backend/routers/chat.py @@ -1,8 +1,10 @@ import uuid +import threading +import time from datetime import datetime, timezone from typing import List, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, UploadFile, File import database.chat as chat_db from database.plugins import record_plugin_usage @@ -10,10 +12,17 @@ from models.plugin import UsageHistoryType from models.memory import Memory from models.chat import Message, SendMessageRequest, MessageSender, ResponseMessage +from models.transcript_segment import TranscriptSegment from utils.apps import get_available_app_by_id from utils.llm import initial_chat_message from utils.other import endpoints as auth from utils.retrieval.graph import execute_graph_chat +from routers.sync import retrieve_file_paths, decode_files_to_wav, retrieve_vad_segments +from utils.stt.pre_recorded import fal_whisperx, fal_postprocessing +from utils.other.storage import get_syncing_file_temporal_signed_url, delete_syncing_temporal_file +from models.notification_message import NotificationMessage +import database.notifications as notification_db +from utils.notifications import send_notification router = APIRouter() @@ -113,3 +122,104 @@ def get_messages(uid: str = Depends(auth.get_current_user_uid)): if not messages: return [initial_message_util(uid)] return messages + +# VOICE MESSAGE +def process_voice_message_segment(path: str, uid: str): + url = get_syncing_file_temporal_signed_url(path) + + def delete_file(): + time.sleep(480) + delete_syncing_temporal_file(path) + + threading.Thread(target=delete_file).start() + + words, language = fal_whisperx(url, 3, 2, True) + transcript_segments: List[TranscriptSegment] = fal_postprocessing(words, 0) + if not transcript_segments: + print('failed to get fal segments') + return [] + + text = " ".join([segment.text for segment in transcript_segments]) + if len(text) == 0: + print('voice message text is empty') + return [] + + # create message + message = Message( + id=str(uuid.uuid4()), text=text, created_at=datetime.now(timezone.utc), sender='human', type='text' + ) + chat_db.add_message(uid, message.dict()) + + # not support plugin + plugin = None + plugin_id = None + + messages = list(reversed([Message(**msg) for msg in chat_db.get_messages(uid, limit=10)])) + response, ask_for_nps, memories = execute_graph_chat(uid, messages, plugin) # plugin + memories_id = [] + # check if the items in the memories list are dict + if memories: + converted_memories = [] + for m in memories[:5]: + if isinstance(m, dict): + converted_memories.append(Memory(**m)) + else: + converted_memories.append(m) + memories_id = [m.id for m in converted_memories] + ai_message = Message( + id=str(uuid.uuid4()), + text=response, + created_at=datetime.now(timezone.utc), + sender='ai', + plugin_id=plugin_id, + type='text', + memories_id=memories_id, + ) + chat_db.add_message(uid, ai_message.dict()) + ai_message.memories = memories if len(memories) < 5 else memories[:5] + if plugin_id: + record_plugin_usage(uid, plugin_id, UsageHistoryType.chat_message_sent, message_id=ai_message.id) + + ai_message_resp = ai_message.dict() + + ai_message_resp['ask_for_nps'] = ask_for_nps + + # send notification + token = notification_db.get_token_only(uid) + send_chat_message_notification(token, "Omi", None, ai_message.text) + + return [message.dict(), ai_message_resp] + +def send_chat_message_notification(token: str, plugin_name: str, plugin_id: str, message: str): + ai_message = NotificationMessage( + text=message, + plugin_id=plugin_id, + from_integration='true', + type='text', + notification_type='plugin', + ) + send_notification(token, plugin_name + ' says', message, NotificationMessage.get_message_as_dict(ai_message)) + + +@router.post("/v1/voice-messages") +async def create_voice_message(files: List[UploadFile] = File(...), uid: str = Depends(auth.get_current_user_uid)): + paths = retrieve_file_paths(files, uid) + if len(paths) == 0: + raise HTTPException(status_code=400, detail='Paths is invalid') + + # wav + wav_paths = decode_files_to_wav(paths) + if len(wav_paths) == 0: + raise HTTPException(status_code=400, detail='Wav path is invalid') + + # segmented + segmented_paths = set() + retrieve_vad_segments(wav_paths[0], segmented_paths) + if len(segmented_paths) == 0: + raise HTTPException(status_code=400, detail='Segmented paths is invalid') + + resp = process_voice_message_segment(list(segmented_paths)[0], uid) + if not resp: + raise HTTPException(status_code=400, detail='Bad params') + + return resp