From ac8d69747f5e9e938a61f594b9cbcaa14274ceb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 27 Nov 2024 15:25:24 +0700 Subject: [PATCH 1/5] Improve & fixe ble button logic on the device for more accurate voice commands --- .../prj_xiao_ble_sense_devkitv2-adafruit.conf | 18 +- Friend/firmware/firmware_v1.0/src/button.c | 481 +++++++++++------- Friend/firmware/firmware_v1.0/src/button.h | 4 +- app/lib/providers/capture_provider.dart | 62 ++- .../services/devices/device_connection.dart | 11 + .../services/devices/frame_connection.dart | 6 + .../services/devices/friend_connection.dart | 20 +- 7 files changed, 393 insertions(+), 209 deletions(-) diff --git a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf index d2e24ce03..b40a7f595 100644 --- a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf +++ b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf @@ -6,7 +6,6 @@ CONFIG_GPIO=y CONFIG_NRFX_PDM=y CONFIG_ADC=y -# CONFIG_SERIAL=n # To flash via dev board #CONFIG_BUILD_OUTPUT_UF2=n #CONFIG_USE_DT_CODE_PARTITION=n @@ -30,7 +29,7 @@ CONFIG_BT_GATT_DYNAMIC_DB=y CONFIG_BT_CTLR_TX_PWR_ANTENNA=8 CONFIG_BT_PHY_UPDATE=y -CONFIG_PM_DEVICE=y + # # Battery and Device Information services # @@ -42,7 +41,7 @@ CONFIG_BT_DIS_MODEL="Omi DevKit 2" CONFIG_BT_DIS_MANUF="Based Hardware" CONFIG_BT_DIS_FW_REV=y CONFIG_BT_DIS_HW_REV=y -CONFIG_BT_DIS_FW_REV_STR="2.0.3" +CONFIG_BT_DIS_FW_REV_STR="2.0.2" CONFIG_BT_DIS_HW_REV_STR="Seeed Xiao BLE Sense" # @@ -71,10 +70,7 @@ CONFIG_BT_AUTO_DATA_LEN_UPDATE=y CONFIG_BT_USER_PHY_UPDATE=y CONFIG_BT_AUTO_PHY_UPDATE=y -CONFIG_FPU=y -CONFIG_NORDIC_QSPI_NOR=n -# CONFIG_BOARD_ENABLE_DCDC=y -# # +# # Debug # # Enable the lines below to enable debug logs via UART/USB @@ -178,14 +174,6 @@ CONFIG_ACCELEROMETER=y CONFIG_ENABLE_BUTTON=y CONFIG_ENABLE_SPEAKER=y -CONFIG_LOG=n - -CONFIG_RING_BUFFER=y -CONFIG_CONSOLE=n -CONFIG_UART_CONSOLE=n - -CONFIG_SERIAL=n - # CONFIG_PM_DEVICE=y # CONFIG_NRFX_USBD=y diff --git a/Friend/firmware/firmware_v1.0/src/button.c b/Friend/firmware/firmware_v1.0/src/button.c index b5264ac34..09ba3cd46 100644 --- a/Friend/firmware/firmware_v1.0/src/button.c +++ b/Friend/firmware/firmware_v1.0/src/button.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -20,11 +21,11 @@ static ssize_t button_data_read_characteristic(struct bt_conn *conn, const struc static struct gpio_callback button_cb_data; static struct bt_uuid_128 button_uuid = BT_UUID_INIT_128(BT_UUID_128_ENCODE(0x23BA7924,0x0000,0x1000,0x7450,0x346EAC492E92)); -static struct bt_uuid_128 button_uuid_x = BT_UUID_INIT_128(BT_UUID_128_ENCODE(0x23BA7925 ,0x0000,0x1000,0x7450,0x346EAC492E92)); +static struct bt_uuid_128 button_characteristic_data_uuid = BT_UUID_INIT_128(BT_UUID_128_ENCODE(0x23BA7925 ,0x0000,0x1000,0x7450,0x346EAC492E92)); static struct bt_gatt_attr button_service_attr[] = { BT_GATT_PRIMARY_SERVICE(&button_uuid), - BT_GATT_CHARACTERISTIC(&button_uuid_x.uuid, BT_GATT_CHRC_READ | BT_GATT_CHRC_NOTIFY, BT_GATT_PERM_READ, button_data_read_characteristic, NULL, NULL), + BT_GATT_CHARACTERISTIC(&button_characteristic_data_uuid.uuid, BT_GATT_CHRC_READ | BT_GATT_CHRC_NOTIFY, BT_GATT_PERM_READ, button_data_read_characteristic, NULL, NULL), BT_GATT_CCC(button_ccc_config_changed_handler, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE), }; @@ -49,35 +50,24 @@ static void button_ccc_config_changed_handler(const struct bt_gatt_attr *attr, u struct gpio_dt_spec d4_pin = {.port = DEVICE_DT_GET(DT_NODELABEL(gpio0)), .pin=4, .dt_flags = GPIO_OUTPUT_ACTIVE}; //3.3 struct gpio_dt_spec d5_pin_input = {.port = DEVICE_DT_GET(DT_NODELABEL(gpio0)), .pin=5, .dt_flags = GPIO_INT_EDGE_RISING}; -static uint32_t current_button_time = 0; -static uint32_t previous_button_time = 0; - -const int max_debounce_interval = 700; static bool was_pressed = false; // // button // -void button_pressed(const struct device *dev, struct gpio_callback *cb, +void button_pressed_callback(const struct device *dev, struct gpio_callback *cb, uint32_t pins) { - current_button_time = k_cycle_get_32(); - if (current_button_time - previous_button_time < max_debounce_interval) - { //too low! - } - else - { //right... - int temp = gpio_pin_get_raw(dev,d5_pin_input.pin); - if (temp) - { - was_pressed = false; - } - else - { - was_pressed = true; - } - } - previous_button_time = current_button_time; + int temp = gpio_pin_get_raw(dev,d5_pin_input.pin); + printf("button_pressed_callback %d\n", temp); + if (temp) + { + was_pressed = false; + } + else + { + was_pressed = true; + } } #define BUTTON_CHECK_INTERVAL 40 // 0.04 seconds, 25 Hz @@ -122,9 +112,11 @@ static inline void notify_unpress() { final_button_state[0] = BUTTON_RELEASE; LOG_INF("unpressed"); + printf("unpressed\n"); struct bt_conn *conn = get_current_connection(); if (conn != NULL) { + printf("unpressed sent"); bt_gatt_notify(conn, &button_service.attrs[1], &final_button_state, sizeof(final_button_state)); } } @@ -162,186 +154,321 @@ static inline void notify_long_tap() } } -#define LONG_PRESS_INTERVAL 25 -#define SINGLE_PRESS_INTERVAL 2 +#define BUTTON_PRESSED 1 +#define BUTTON_RELEASED 0 + +#define TAP_THRESHOLD 300 // 300 ms for single tap +#define DOUBLE_TAP_WINDOW 600 // 600 ms maximum for double-tap +#define LONG_PRESS_TIME 1000 // 1000 ms for long press + +typedef enum { + BUTTON_EVENT_NONE, + BUTTON_EVENT_SINGLE_TAP, + BUTTON_EVENT_DOUBLE_TAP, + BUTTON_EVENT_LONG_PRESS, + BUTTON_EVENT_RELEASE +} ButtonEvent; + +static uint32_t current_time = 0; +static uint32_t btn_press_start_time; +static uint32_t btn_release_time; +static uint32_t btn_last_tap_time; +static bool btn_is_pressed; + +static u_int8_t btn_last_event = BUTTON_EVENT_NONE; + void check_button_level(struct k_work *work_item) { - //insert the current button state here - int state_ = was_pressed ? 1 : 0; - if (current_button_state == IDLE) - { - if (state_ == 0) - { - //Do nothing! + current_time = current_time + 1; + + ButtonEvent event = BUTTON_EVENT_NONE; + + u_int8_t btn_state = was_pressed ? BUTTON_PRESSED : BUTTON_RELEASED; + + // Debouncing pressed state + if (btn_state == BUTTON_PRESSED && !btn_is_pressed) { + btn_is_pressed = true; + btn_press_start_time = current_time; + } else if (btn_state == BUTTON_RELEASED && btn_is_pressed) { + btn_is_pressed = false; + btn_release_time = current_time; + + // Check for double tap + uint32_t press_duration = (btn_release_time - btn_press_start_time)*BUTTON_CHECK_INTERVAL; + if (press_duration < TAP_THRESHOLD) { + if (btn_last_tap_time > 0 && (current_time - btn_last_tap_time)*BUTTON_CHECK_INTERVAL < DOUBLE_TAP_WINDOW) { + event = BUTTON_EVENT_DOUBLE_TAP; + btn_last_tap_time = 0; // Reset double-tap / single-tap detection + } else { + btn_last_tap_time = current_time; + } } - else if (state_ == 1) - { - //Also do nothing, but transition to the next state - notify_press(); - current_button_state = ONE_PRESS; - if (is_off) - { - is_off = false; - bt_on(); - play_haptic_milli(50); - } + } + + // Check for single tap + if (btn_state == BUTTON_RELEASED && !btn_is_pressed) { + uint32_t press_duration = (btn_release_time - btn_press_start_time)*BUTTON_CHECK_INTERVAL; + if (press_duration < TAP_THRESHOLD && btn_last_tap_time > 0 && (current_time - btn_press_start_time)*BUTTON_CHECK_INTERVAL > TAP_THRESHOLD) { + event = BUTTON_EVENT_SINGLE_TAP; + btn_last_tap_time = 0; + } else if ((current_time - btn_press_start_time)*BUTTON_CHECK_INTERVAL > TAP_THRESHOLD) { + event = BUTTON_EVENT_RELEASE; } } - else if (current_button_state == ONE_PRESS) + // Check for long press + if (btn_is_pressed && (current_time - btn_press_start_time)*BUTTON_CHECK_INTERVAL >= LONG_PRESS_TIME) { + event = BUTTON_EVENT_LONG_PRESS; + } + + // Single tap + if (event == BUTTON_EVENT_SINGLE_TAP) { - if (state_ == 0) + printk("single tap detected\n"); + btn_last_event = event; + notify_tap(); + + //Fire the long mode notify and enter a grace period + //turn off herre + if(!from_wakeup) { - - if(inc_count_0 == 0) - { - notify_unpress(); - } - inc_count_0++; //button is unpressed - if (inc_count_0 > SINGLE_PRESS_INTERVAL) - { - //If button is not pressed for a little while....... - //transition to Two_press. button could be a single or double tap - current_button_state = TWO_PRESS; - reset_count(); - } + is_off = !is_off; } - if (state_ == 1) + else { - inc_count_1++; //button is pressed - - if (inc_count_1 > LONG_PRESS_INTERVAL) - { - //If button is pressed for a long time....... - notify_long_tap(); - //play_haptic_milli(10); - //Fire the long mode notify and enter a grace period - //turn off herre - // TODO: FIXME - //if(!from_wakeup) - //{ - // is_off = !is_off; - //} - //else - //{ - // from_wakeup = false; - //} - //if (is_off) - //{ - // bt_off(); - // turnoff_all(); - //} - current_button_state = GRACE; - reset_count(); - } - + from_wakeup = false; + } + if (is_off) + { + bt_off(); + turnoff_all(); } + } + // Double tap + if (event == BUTTON_EVENT_DOUBLE_TAP) + { + printk("double tap detected\n"); + btn_last_event = event; + notify_double_tap(); } - else if (current_button_state == TWO_PRESS) + // Long press, one time event + if (event == BUTTON_EVENT_LONG_PRESS && btn_last_event != BUTTON_EVENT_LONG_PRESS) { - if (state_ == 0) - { - if (inc_count_1 > 0) - { // if button has been pressed...... - notify_unpress(); - notify_double_tap(); - - //Fire the notify and enter a grace period - current_button_state = GRACE; - reset_count(); - } - //single button press - else if (inc_count_0 > 10) - { - notify_tap(); //Fire the notify and enter a grace period - if(!from_wakeup) - { - is_off = !is_off; - } - else - { - from_wakeup = false; - } - //Fire the notify and enter a grace period - if (is_off) - { - bt_off(); - turnoff_all(); - } - current_button_state = GRACE; - reset_count(); - } - else - { - inc_count_0++; //not pressed - } - } - else if (state_ == 1 ) - { - if (inc_count_1 == 0) - { - notify_press(); - inc_count_1++; - } - if (inc_count_1 > threshold) - { - notify_long_tap(); - //play_haptic_milli(10); - // TODO: FIXME - //if(!from_wakeup) - //{ - // is_off = !is_off; - //} - //else - //{ - // from_wakeup = false; - //} - ////Fire the notify and enter a grace period - //if (is_off) - //{ - // bt_off(); - // turnoff_all(); - //} - current_button_state = GRACE; - reset_count(); - } - } + printk("long press detected\n"); + btn_last_event = event; + notify_long_tap(); } - else if (current_button_state == GRACE) + // Releases, one time event + if (event == BUTTON_EVENT_RELEASE && btn_last_event != BUTTON_EVENT_RELEASE) { - if (state_ == 0) - { - if (inc_count_0 == 0 && (inc_count_1 > 0)) - { - notify_unpress(); - } - inc_count_0++; - if (inc_count_0 > 1) - { - current_button_state = IDLE; - reset_count(); - } - } - else if (state_ == 1) - { - inc_count_1++; - } + printk("release detected\n"); + btn_last_event = event; + notify_unpress(); + + // Reset + current_time = 0; + btn_press_start_time = 0; + btn_release_time = 0; + btn_last_tap_time = 0; + } + if (event == BUTTON_EVENT_RELEASE) + { + current_button_state = GRACE; } + k_work_reschedule(&button_work, K_MSEC(BUTTON_CHECK_INTERVAL)); + return 0; } +#define LONG_PRESS_INTERVAL 25 +#define SINGLE_PRESS_INTERVAL 2 +// void check_button_level(struct k_work *work_item) +// { +// //insert the current button state here +// int state_ = was_pressed ? 1 : 0; +// if (current_button_state == IDLE) +// { +// if (state_ == 0) +// { +// //Do nothing! +// } +// else if (state_ == 1) +// { +// //Also do nothing, but transition to the next state +// notify_press(); +// current_button_state = ONE_PRESS; +// if (is_off) +// { +// is_off = false; +// bt_on(); +// play_haptic_milli(50); +// } +// } +// } +// +// else if (current_button_state == ONE_PRESS) +// { +// if (state_ == 0) +// { +// +// if(inc_count_0 == 0) +// { +// notify_unpress(); +// } +// inc_count_0++; //button is unpressed +// if (inc_count_0 > SINGLE_PRESS_INTERVAL) +// { +// //If button is not pressed for a little while....... +// //transition to Two_press. button could be a single or double tap +// current_button_state = TWO_PRESS; +// reset_count(); +// } +// } +// if (state_ == 1) +// { +// inc_count_1++; //button is pressed +// +// if (inc_count_1 > LONG_PRESS_INTERVAL) +// { +// //If button is pressed for a long time....... +// notify_long_tap(); +// //play_haptic_milli(10); +// //Fire the long mode notify and enter a grace period +// //turn off herre +// // TODO: FIXME +// //if(!from_wakeup) +// //{ +// // is_off = !is_off; +// //} +// //else +// //{ +// // from_wakeup = false; +// //} +// //if (is_off) +// //{ +// // bt_off(); +// // turnoff_all(); +// //} +// current_button_state = GRACE; +// reset_count(); +// } +// +// } +// +// } +// +// else if (current_button_state == TWO_PRESS) +// { +// if (state_ == 0) +// { +// if (inc_count_1 > 0) +// { // if button has been pressed...... +// notify_unpress(); +// notify_double_tap(); +// +// //Fire the notify and enter a grace period +// current_button_state = GRACE; +// reset_count(); +// } +// //single button press +// else if (inc_count_0 > 10) +// { +// notify_tap(); //Fire the notify and enter a grace period +// if(!from_wakeup) +// { +// is_off = !is_off; +// } +// else +// { +// from_wakeup = false; +// } +// //Fire the notify and enter a grace period +// if (is_off) +// { +// bt_off(); +// turnoff_all(); +// } +// current_button_state = GRACE; +// reset_count(); +// } +// else +// { +// inc_count_0++; //not pressed +// } +// } +// else if (state_ == 1 ) +// { +// if (inc_count_1 == 0) +// { +// notify_press(); +// inc_count_1++; +// } +// if (inc_count_1 > threshold) +// { +// notify_long_tap(); +// //play_haptic_milli(10); +// // TODO: FIXME +// //if(!from_wakeup) +// //{ +// // is_off = !is_off; +// //} +// //else +// //{ +// // from_wakeup = false; +// //} +// ////Fire the notify and enter a grace period +// //if (is_off) +// //{ +// // bt_off(); +// // turnoff_all(); +// //} +// current_button_state = GRACE; +// reset_count(); +// } +// } +// } +// +// else if (current_button_state == GRACE) +// { +// if (state_ == 0) +// { +// if (inc_count_0 == 0 && (inc_count_1 > 0)) +// { +// notify_unpress(); +// } +// inc_count_0++; +// if (inc_count_0 > 1) +// { +// current_button_state = IDLE; +// reset_count(); +// } +// } +// else if (state_ == 1) +// { +// inc_count_1++; +// } +// } +// k_work_reschedule(&button_work, K_MSEC(BUTTON_CHECK_INTERVAL)); +// } +// static ssize_t button_data_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset) { LOG_INF("button_data_read_characteristic"); - int lint = 1; LOG_INF("was_pressed: %d", was_pressed); - return bt_gatt_attr_read(conn, attr, buf, len, offset, &lint, sizeof(lint)); - + printf("was_pressed: %d\n", final_button_state[0]); + return bt_gatt_attr_read(conn, attr, buf, len, offset, &final_button_state, sizeof(final_button_state)); } + +//static ssize_t audio_data_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset) +//{ +// LOG_DBG("audio_data_read_characteristic"); +// return bt_gatt_attr_read(conn, attr, buf, len, offset, NULL, 0); +//} + int button_init() { if (gpio_is_ready_dt(&d4_pin)) @@ -397,7 +524,7 @@ int button_init() } - gpio_init_callback(&button_cb_data, button_pressed, BIT(d5_pin_input.pin)); + gpio_init_callback(&button_cb_data, button_pressed_callback, BIT(d5_pin_input.pin)); gpio_add_callback(d5_pin_input.port, &button_cb_data); return 0; diff --git a/Friend/firmware/firmware_v1.0/src/button.h b/Friend/firmware/firmware_v1.0/src/button.h index 040bf4459..3b54fa675 100644 --- a/Friend/firmware/firmware_v1.0/src/button.h +++ b/Friend/firmware/firmware_v1.0/src/button.h @@ -3,8 +3,6 @@ typedef enum { IDLE, - ONE_PRESS, - TWO_PRESS, GRACE } FSM_STATE_T; @@ -16,4 +14,4 @@ FSM_STATE_T get_current_button_state(); void force_button_state(FSM_STATE_T state); -#endif \ No newline at end of file +#endif diff --git a/app/lib/providers/capture_provider.dart b/app/lib/providers/capture_provider.dart index 8008d56d9..bce731d1d 100644 --- a/app/lib/providers/capture_provider.dart +++ b/app/lib/providers/capture_provider.dart @@ -80,7 +80,7 @@ class CaptureProvider extends ChangeNotifier get bleBytesStream => _bleBytesStream; StreamSubscription? _bleButtonStream; - bool _isDeviceButtonLongPressStart = false; + DateTime? _voiceCommandSession; List> _commandBytes = []; StreamSubscription? _storageStream; @@ -191,10 +191,15 @@ class CaptureProvider extends ChangeNotifier return file; } - void _processVoiceCommandBytes() async { - debugPrint("Send ${_commandBytes.length} voice frames to backend"); - var file = await _flushBytesToTempFile( - _commandBytes, DateTime.now().millisecondsSinceEpoch ~/ 1000 - (_commandBytes.length / 100).ceil()); + void _processVoiceCommandBytes(List> data) async { + if (data.isEmpty) { + debugPrint("voice frames is empty"); + return; + } + + debugPrint("Send ${data.length} voice frames to backend"); + var file = + await _flushBytesToTempFile(data, DateTime.now().millisecondsSinceEpoch ~/ 1000 - (data.length / 100).ceil()); try { var messages = await sendVoiceMessageServer([file]); debugPrint("Command respond: ${messages.map((m) => m.text).join(" | ")}"); @@ -206,6 +211,28 @@ class CaptureProvider extends ChangeNotifier } } + // Just incase the ble connection get loss + void _watchVoiceCommands(String deviceId, DateTime session) { + Timer.periodic(const Duration(seconds: 3), (t) async { + debugPrint("voice command watch"); + if (session != _voiceCommandSession) { + t.cancel(); + return; + } + var value = await _getBleButtonState(deviceId); + var buttonState = ByteData.view(Uint8List.fromList(value.sublist(0, 4).reversed.toList()).buffer).getUint32(0); + debugPrint("watch device button ${buttonState}"); + + // Force process + if (buttonState == 5 && session == _voiceCommandSession) { + _voiceCommandSession = null; // end session + var data = List>.from(_commandBytes); + _commandBytes = []; + _processVoiceCommandBytes(data); + } + }); + } + Future streamButton(String deviceId) async { debugPrint('streamButton in capture_provider'); _bleButtonStream?.cancel(); @@ -215,18 +242,19 @@ class CaptureProvider extends ChangeNotifier debugPrint("device button ${buttonState}"); // start long press - if (buttonState == 3) { - _isDeviceButtonLongPressStart = true; + if (buttonState == 3 && _voiceCommandSession == null) { + _voiceCommandSession = DateTime.now(); _commandBytes = []; + _watchVoiceCommands(deviceId, _voiceCommandSession!); _playSpeakerHaptic(deviceId, 1); } // release - if (buttonState == 5 && _isDeviceButtonLongPressStart) { - Future.delayed(const Duration(seconds: 1)).then((_) { - _isDeviceButtonLongPressStart = false; - _processVoiceCommandBytes(); - }); + if (buttonState == 5 && _voiceCommandSession != null) { + _voiceCommandSession = null; // end session + var data = List>.from(_commandBytes); + _commandBytes = []; + _processVoiceCommandBytes(data); } }); } @@ -238,7 +266,7 @@ class CaptureProvider extends ChangeNotifier if (value.isEmpty) return; // command button triggered - if (_isDeviceButtonLongPressStart) { + if (_voiceCommandSession != null) { _commandBytes.add(value.sublist(3)); } @@ -334,6 +362,14 @@ class CaptureProvider extends ChangeNotifier return connection.getBleButtonListener(onButtonReceived: onButtonReceived); } + Future> _getBleButtonState(String deviceId) async { + var connection = await ServiceManager.instance().device.ensureConnection(deviceId); + if (connection == null) { + return Future.value([]); + } + return connection.getBleButtonState(); + } + Future _recheckCodecChange() async { if (_recordingDevice != null) { BleAudioCodec newCodec = await _getAudioCodec(_recordingDevice!.id); diff --git a/app/lib/services/devices/device_connection.dart b/app/lib/services/devices/device_connection.dart index 50383259e..4786fdf3c 100644 --- a/app/lib/services/devices/device_connection.dart +++ b/app/lib/services/devices/device_connection.dart @@ -183,6 +183,17 @@ abstract class DeviceConnection { return null; } + Future> getBleButtonState() async { + if (await isConnected()) { + debugPrint('button state called'); + return await performGetButtonState(); + } + debugPrint('button state error'); + return Future.value([]); + } + + Future> performGetButtonState(); + Future getBleButtonListener({ required void Function(List) onButtonReceived, }) async { diff --git a/app/lib/services/devices/frame_connection.dart b/app/lib/services/devices/frame_connection.dart index b2ebe753b..168038ba4 100644 --- a/app/lib/services/devices/frame_connection.dart +++ b/app/lib/services/devices/frame_connection.dart @@ -257,6 +257,12 @@ class FrameDeviceConnection extends DeviceConnection { return Future.value(BleAudioCodec.pcm8); } + @override + Future> performGetButtonState() async { + return Future.value([]); + } + + @override Future performGetBleButtonListener({required void Function(List) onButtonReceived}) async {} @override diff --git a/app/lib/services/devices/friend_connection.dart b/app/lib/services/devices/friend_connection.dart index 99c0356a2..f57be4533 100644 --- a/app/lib/services/devices/friend_connection.dart +++ b/app/lib/services/devices/friend_connection.dart @@ -128,6 +128,23 @@ class FriendDeviceConnection extends DeviceConnection { return listener; } + @override + Future> performGetButtonState() async { + debugPrint('perform button state called'); + if (_buttonService == null) { + return Future.value([]); + } + + var buttonStateCharacteristic = getCharacteristic(_buttonService!, buttonTriggerCharacteristicUuid); + if (buttonStateCharacteristic == null) { + logCharacteristicNotFoundError('Button state', deviceId); + return Future.value([]); + } + var value = await buttonStateCharacteristic.read(); + return value; + } + + @override Future performGetBleButtonListener({ required void Function(List) onButtonReceived, }) async { @@ -281,6 +298,7 @@ class FriendDeviceConnection extends DeviceConnection { return codec; } + @override Future> getStorageList() async { if (await isConnected()) { debugPrint('storage list called'); @@ -291,7 +309,7 @@ class FriendDeviceConnection extends DeviceConnection { return Future.value([]); } - // @override + @override Future> performGetStorageList() async { debugPrint(' perform storage list called'); if (_storageService == null) { From 1e0191d9c0a4fc61665bf75f81e6dbaf84eb8e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 27 Nov 2024 16:00:12 +0700 Subject: [PATCH 2/5] Clean configs, mark deprecated code --- .../prj_xiao_ble_sense_devkitv2-adafruit.conf | 15 +- Friend/firmware/firmware_v1.0/src/button.c | 353 +++++++++--------- 2 files changed, 187 insertions(+), 181 deletions(-) diff --git a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf index b40a7f595..e03f865c7 100644 --- a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf +++ b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf @@ -70,6 +70,17 @@ CONFIG_BT_AUTO_DATA_LEN_UPDATE=y CONFIG_BT_USER_PHY_UPDATE=y CONFIG_BT_AUTO_PHY_UPDATE=y +CONFIG_FPU=y +CONFIG_NORDIC_QSPI_NOR=n +# CONFIG_BOARD_ENABLE_DCDC=y +# + +# +# Console +# +# Disable the lines to enable console log +CONFIG_CONSOLE=n + # # Debug # @@ -170,11 +181,13 @@ CONFIG_SPI_NRFX=y #EVERYTHING ELSE # +CONFIG_RING_BUFFER=y + CONFIG_ACCELEROMETER=y CONFIG_ENABLE_BUTTON=y CONFIG_ENABLE_SPEAKER=y -# CONFIG_PM_DEVICE=y +CONFIG_PM_DEVICE=y # CONFIG_NRFX_USBD=y # CONFIG_FLASH=y diff --git a/Friend/firmware/firmware_v1.0/src/button.c b/Friend/firmware/firmware_v1.0/src/button.c index 09ba3cd46..0e0e5568c 100644 --- a/Friend/firmware/firmware_v1.0/src/button.c +++ b/Friend/firmware/firmware_v1.0/src/button.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -116,7 +115,7 @@ static inline void notify_unpress() struct bt_conn *conn = get_current_connection(); if (conn != NULL) { - printf("unpressed sent"); + printf("unpressed sent\n"); bt_gatt_notify(conn, &button_service.attrs[1], &final_button_state, sizeof(final_button_state)); } } @@ -181,10 +180,10 @@ void check_button_level(struct k_work *work_item) { current_time = current_time + 1; - ButtonEvent event = BUTTON_EVENT_NONE; - u_int8_t btn_state = was_pressed ? BUTTON_PRESSED : BUTTON_RELEASED; + ButtonEvent event = BUTTON_EVENT_NONE; + // Debouncing pressed state if (btn_state == BUTTON_PRESSED && !btn_is_pressed) { btn_is_pressed = true; @@ -283,192 +282,186 @@ void check_button_level(struct k_work *work_item) return 0; } -#define LONG_PRESS_INTERVAL 25 -#define SINGLE_PRESS_INTERVAL 2 -// void check_button_level(struct k_work *work_item) -// { -// //insert the current button state here -// int state_ = was_pressed ? 1 : 0; -// if (current_button_state == IDLE) -// { -// if (state_ == 0) -// { -// //Do nothing! -// } -// else if (state_ == 1) -// { -// //Also do nothing, but transition to the next state -// notify_press(); -// current_button_state = ONE_PRESS; -// if (is_off) +// @deprecated +//#define LONG_PRESS_INTERVAL 25 +//#define SINGLE_PRESS_INTERVAL 2 +//void check_button_level_2(struct k_work *work_item) +//{ +// //insert the current button state here +// int state_ = was_pressed ? 1 : 0; +// if (current_button_state == IDLE) +// { +// if (state_ == 0) +// { +// //Do nothing! +// } +// else if (state_ == 1) +// { +// //Also do nothing, but transition to the next state +// notify_press(); +// current_button_state = ONE_PRESS; +// if (is_off) +// { +// is_off = false; +// bt_on(); +// play_haptic_milli(50); +// } +// } +// } +// +// else if (current_button_state == ONE_PRESS) +// { +// if (state_ == 0) +// { +// +// if(inc_count_0 == 0) +// { +// notify_unpress(); +// } +// inc_count_0++; //button is unpressed +// if (inc_count_0 > SINGLE_PRESS_INTERVAL) +// { +// //If button is not pressed for a little while....... +// //transition to Two_press. button could be a single or double tap +// current_button_state = TWO_PRESS; +// reset_count(); +// } +// } +// if (state_ == 1) +// { +// inc_count_1++; //button is pressed +// +// if (inc_count_1 > LONG_PRESS_INTERVAL) +// { +// //If button is pressed for a long time....... +// notify_long_tap(); +// //play_haptic_milli(10); +// //Fire the long mode notify and enter a grace period +// //turn off herre +// // TODO: FIXME +// //if(!from_wakeup) +// //{ +// // is_off = !is_off; +// //} +// //else +// //{ +// // from_wakeup = false; +// //} +// //if (is_off) +// //{ +// // bt_off(); +// // turnoff_all(); +// //} +// current_button_state = GRACE; +// reset_count(); +// } +// +// } +// +// } +// +// else if (current_button_state == TWO_PRESS) +// { +// if (state_ == 0) +// { +// if (inc_count_1 > 0) +// { // if button has been pressed...... +// notify_unpress(); +// notify_double_tap(); +// +// //Fire the notify and enter a grace period +// current_button_state = GRACE; +// reset_count(); +// } +// //single button press +// else if (inc_count_0 > 10) // { -// is_off = false; -// bt_on(); -// play_haptic_milli(50); +// notify_tap(); //Fire the notify and enter a grace period +// if(!from_wakeup) +// { +// is_off = !is_off; +// } +// else +// { +// from_wakeup = false; +// } +// //Fire the notify and enter a grace period +// if (is_off) +// { +// bt_off(); +// turnoff_all(); +// } +// current_button_state = GRACE; +// reset_count(); // } -// } -// } -// -// else if (current_button_state == ONE_PRESS) -// { -// if (state_ == 0) -// { -// -// if(inc_count_0 == 0) -// { -// notify_unpress(); -// } -// inc_count_0++; //button is unpressed -// if (inc_count_0 > SINGLE_PRESS_INTERVAL) -// { -// //If button is not pressed for a little while....... -// //transition to Two_press. button could be a single or double tap -// current_button_state = TWO_PRESS; -// reset_count(); -// } -// } -// if (state_ == 1) -// { -// inc_count_1++; //button is pressed -// -// if (inc_count_1 > LONG_PRESS_INTERVAL) -// { -// //If button is pressed for a long time....... -// notify_long_tap(); -// //play_haptic_milli(10); -// //Fire the long mode notify and enter a grace period -// //turn off herre -// // TODO: FIXME -// //if(!from_wakeup) -// //{ -// // is_off = !is_off; -// //} -// //else -// //{ -// // from_wakeup = false; -// //} -// //if (is_off) -// //{ -// // bt_off(); -// // turnoff_all(); -// //} -// current_button_state = GRACE; -// reset_count(); -// } -// -// } -// -// } -// -// else if (current_button_state == TWO_PRESS) -// { -// if (state_ == 0) -// { -// if (inc_count_1 > 0) -// { // if button has been pressed...... -// notify_unpress(); -// notify_double_tap(); -// -// //Fire the notify and enter a grace period -// current_button_state = GRACE; -// reset_count(); -// } -// //single button press -// else if (inc_count_0 > 10) -// { -// notify_tap(); //Fire the notify and enter a grace period -// if(!from_wakeup) -// { -// is_off = !is_off; -// } -// else -// { -// from_wakeup = false; -// } -// //Fire the notify and enter a grace period -// if (is_off) -// { -// bt_off(); -// turnoff_all(); -// } -// current_button_state = GRACE; -// reset_count(); -// } -// else -// { -// inc_count_0++; //not pressed -// } -// } -// else if (state_ == 1 ) -// { -// if (inc_count_1 == 0) -// { -// notify_press(); -// inc_count_1++; -// } -// if (inc_count_1 > threshold) -// { -// notify_long_tap(); -// //play_haptic_milli(10); -// // TODO: FIXME -// //if(!from_wakeup) -// //{ -// // is_off = !is_off; -// //} -// //else -// //{ -// // from_wakeup = false; -// //} -// ////Fire the notify and enter a grace period -// //if (is_off) -// //{ -// // bt_off(); -// // turnoff_all(); -// //} -// current_button_state = GRACE; -// reset_count(); -// } -// } -// } -// -// else if (current_button_state == GRACE) -// { -// if (state_ == 0) -// { -// if (inc_count_0 == 0 && (inc_count_1 > 0)) -// { -// notify_unpress(); -// } -// inc_count_0++; -// if (inc_count_0 > 1) -// { -// current_button_state = IDLE; -// reset_count(); -// } -// } -// else if (state_ == 1) -// { -// inc_count_1++; -// } -// } -// k_work_reschedule(&button_work, K_MSEC(BUTTON_CHECK_INTERVAL)); -// } -// +// else +// { +// inc_count_0++; //not pressed +// } +// } +// else if (state_ == 1 ) +// { +// if (inc_count_1 == 0) +// { +// notify_press(); +// inc_count_1++; +// } +// if (inc_count_1 > threshold) +// { +// notify_long_tap(); +// //play_haptic_milli(10); +// // TODO: FIXME +// //if(!from_wakeup) +// //{ +// // is_off = !is_off; +// //} +// //else +// //{ +// // from_wakeup = false; +// //} +// ////Fire the notify and enter a grace period +// //if (is_off) +// //{ +// // bt_off(); +// // turnoff_all(); +// //} +// current_button_state = GRACE; +// reset_count(); +// } +// } +// } +// +// else if (current_button_state == GRACE) +// { +// if (state_ == 0) +// { +// if (inc_count_0 == 0 && (inc_count_1 > 0)) +// { +// notify_unpress(); +// } +// inc_count_0++; +// if (inc_count_0 > 1) +// { +// current_button_state = IDLE; +// reset_count(); +// } +// } +// else if (state_ == 1) +// { +// inc_count_1++; +// } +// } +// k_work_reschedule(&button_work, K_MSEC(BUTTON_CHECK_INTERVAL)); +//} + static ssize_t button_data_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset) { LOG_INF("button_data_read_characteristic"); - LOG_INF("was_pressed: %d", was_pressed); printf("was_pressed: %d\n", final_button_state[0]); return bt_gatt_attr_read(conn, attr, buf, len, offset, &final_button_state, sizeof(final_button_state)); } -//static ssize_t audio_data_read_characteristic(struct bt_conn *conn, const struct bt_gatt_attr *attr, void *buf, uint16_t len, uint16_t offset) -//{ -// LOG_DBG("audio_data_read_characteristic"); -// return bt_gatt_attr_read(conn, attr, buf, len, offset, NULL, 0); -//} - int button_init() { if (gpio_is_ready_dt(&d4_pin)) From 53b0f43bc5a30ef1d8f56adb5e6bad480ba7d523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 27 Nov 2024 16:03:12 +0700 Subject: [PATCH 3/5] Bump Firmware 2.0.5 --- .../firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf index e03f865c7..9581756ba 100644 --- a/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf +++ b/Friend/firmware/firmware_v1.0/prj_xiao_ble_sense_devkitv2-adafruit.conf @@ -41,7 +41,7 @@ CONFIG_BT_DIS_MODEL="Omi DevKit 2" CONFIG_BT_DIS_MANUF="Based Hardware" CONFIG_BT_DIS_FW_REV=y CONFIG_BT_DIS_HW_REV=y -CONFIG_BT_DIS_FW_REV_STR="2.0.2" +CONFIG_BT_DIS_FW_REV_STR="2.0.5" CONFIG_BT_DIS_HW_REV_STR="Seeed Xiao BLE Sense" # From eee958fcf483bf8c6077ba87b4a88762a034bde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 27 Nov 2024 16:23:02 +0700 Subject: [PATCH 4/5] Clean code for main.c --- Friend/firmware/firmware_v1.0/src/main.c | 213 +++++++++++------------ 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/Friend/firmware/firmware_v1.0/src/main.c b/Friend/firmware/firmware_v1.0/src/main.c index 8ffe0e213..4cc1223b2 100644 --- a/Friend/firmware/firmware_v1.0/src/main.c +++ b/Friend/firmware/firmware_v1.0/src/main.c @@ -21,7 +21,7 @@ LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL); static void codec_handler(uint8_t *data, size_t len) { int err = broadcast_audio_packets(data, len); - if (err) + if (err) { LOG_ERR("Failed to broadcast audio packets: %d", err); } @@ -30,7 +30,7 @@ static void codec_handler(uint8_t *data, size_t len) static void mic_handler(int16_t *buffer) { int err = codec_receive_pcm(buffer, MIC_BUFFER_SAMPLES); - if (err) + if (err) { LOG_ERR("Failed to process PCM data: %d", err); } @@ -138,15 +138,15 @@ bool from_wakeup = false; int main(void) { int err; - //for system power off, we have no choice but to handle usb detect wakeup events. if off, and this was the reason, initialize, skip lightshow, start not recording + //for system power off, we have no choice but to handle usb detect wakeup events. if off, and this was the reason, initialize, skip lightshow, start not recording uint32_t reset_reas = NRF_POWER->RESETREAS; NRF_POWER->DCDCEN=1; NRF_POWER->DCDCEN0=1; - + NRF_POWER->RESETREAS=1; bool from_usb_event = (reset_reas & VBUS_DETECT); bool from_wakeup = (reset_reas & WAKEUP_DETECT); - if (from_usb_event) + if (from_usb_event) { k_msleep(100); printf("from reset \n"); @@ -159,135 +159,134 @@ int main(void) } else if (from_wakeup) { - + is_off = false; usb_charge = false; force_button_state(GRACE); k_msleep(1000); - activate_everything_no_lights(); - bt_on(); + activate_everything_no_lights(); + bt_on(); play_haptic_milli(100); - } else { - - LOG_INF("Friend device firmware starting..."); - err = led_start(); - if (err) - { - LOG_ERR("Failed to initialize LEDs: %d", err); - return err; - } - // Run the boot LED sequence - boot_led_sequence(); - // Indicate transport initialization - set_led_green(true); - set_led_green(false); - err = transport_start(); - if (err) - { - LOG_ERR("Failed to start transport: %d", err); - // Blink green LED to indicate error - for (int i = 0; i < 5; i++) + LOG_INF("Friend device firmware starting..."); + err = led_start(); + if (err) { - set_led_green(!gpio_pin_get_dt(&led_green)); - k_msleep(200); + LOG_ERR("Failed to initialize LEDs: %d", err); + return err; } + // Run the boot LED sequence + boot_led_sequence(); + // Indicate transport initialization + set_led_green(true); set_led_green(false); - // return err; - } - play_boot_sound(); - err = mount_sd_card(); - if (err) - { - LOG_ERR("Failed to mount SD card: %d", err); - } - LOG_INF("result of mount:%d",err); - k_msleep(500); - err = storage_init(); - if (err) - { - LOG_ERR("Failed to initialize storage: %d", err); - } - err = init_haptic_pin(); - if (err) - { - LOG_ERR("Failed to initialize haptic pin: %d", err); - } + err = transport_start(); + if (err) + { + LOG_ERR("Failed to start transport: %d", err); + // Blink green LED to indicate error + for (int i = 0; i < 5; i++) + { + set_led_green(!gpio_pin_get_dt(&led_green)); + k_msleep(200); + } + set_led_green(false); + // return err; + } + play_boot_sound(); + err = mount_sd_card(); + if (err) + { + LOG_ERR("Failed to mount SD card: %d", err); + } + LOG_INF("result of mount:%d",err); - set_led_blue(true); - set_codec_callback(codec_handler); - err = codec_start(); - if (err) - { - LOG_ERR("Failed to start codec: %d", err); - // Blink blue LED to indicate error - for (int i = 0; i < 5; i++) + k_msleep(500); + err = storage_init(); + if (err) + { + LOG_ERR("Failed to initialize storage: %d", err); + } + err = init_haptic_pin(); + if (err) + { + LOG_ERR("Failed to initialize haptic pin: %d", err); + } + + set_led_blue(true); + set_codec_callback(codec_handler); + err = codec_start(); + if (err) { - set_led_blue(!gpio_pin_get_dt(&led_blue)); - k_msleep(200); + LOG_ERR("Failed to start codec: %d", err); + // Blink blue LED to indicate error + for (int i = 0; i < 5; i++) + { + set_led_blue(!gpio_pin_get_dt(&led_blue)); + k_msleep(200); + } + set_led_blue(false); + return err; } + play_haptic_milli(500); set_led_blue(false); - return err; - } - play_haptic_milli(500); - set_led_blue(false); - // Indicate microphone initialization - set_led_red(true); - set_led_green(true); - LOG_INF("Starting microphone initialization"); - set_mic_callback(mic_handler); - err = mic_start(); - if (err) - { - LOG_ERR("Failed to start microphone: %d", err); - // Blink red and green LEDs to indicate error - for (int i = 0; i < 5; i++) + // Indicate microphone initialization + set_led_red(true); + set_led_green(true); + LOG_INF("Starting microphone initialization"); + set_mic_callback(mic_handler); + err = mic_start(); + if (err) { - set_led_red(!gpio_pin_get_dt(&led_red)); - set_led_green(!gpio_pin_get_dt(&led_green)); - k_msleep(200); + LOG_ERR("Failed to start microphone: %d", err); + // Blink red and green LEDs to indicate error + for (int i = 0; i < 5; i++) + { + set_led_red(!gpio_pin_get_dt(&led_red)); + set_led_green(!gpio_pin_get_dt(&led_green)); + k_msleep(200); + } + set_led_red(false); + set_led_green(false); + return err; } set_led_red(false); set_led_green(false); - return err; - } - set_led_red(false); - set_led_green(false); - // save_offset(200); - // // Initialize NFC first - // LOG_INF("Initializing NFC..."); - // err = nfc_init(); - // if (err != 0) { - // LOG_ERR("Failed to initialize NFC: %d", err); - // // Consider whether to continue or return based on the severity of the error - // } else { - // LOG_INF("NFC initialized successfully"); - // } + // save_offset(200); + // // Initialize NFC first + // LOG_INF("Initializing NFC..."); + // err = nfc_init(); + // if (err != 0) { + // LOG_ERR("Failed to initialize NFC: %d", err); + // // Consider whether to continue or return based on the severity of the error + // } else { + // LOG_INF("NFC initialized successfully"); + // } - // Indicate successful initialization - err = init_usb(); - if (err) - { - LOG_ERR("Failed to initialize power supply: %d", err); - } + // Indicate successful initialization + err = init_usb(); + if (err) + { + LOG_ERR("Failed to initialize power supply: %d", err); + } - // button_init(); - // register_button_service(); - // activate_button_work(); + // button_init(); + // register_button_service(); + // activate_button_work(); - LOG_INF("Omi firmware initialized successfully\n"); - set_led_blue(true); - k_msleep(1000); - set_led_blue(false); - printf("reset reas:%d\n",reset_reas); + LOG_INF("Omi firmware initialized successfully\n"); + set_led_blue(true); + k_msleep(1000); + set_led_blue(false); + printf("reset reas:%d\n",reset_reas); } printf("reset reas:%d\n",reset_reas); @@ -296,8 +295,8 @@ int main(void) set_led_state(); k_msleep(500); } + // Unreachable return 0; } - From 42c35f2a987ceb2525a3d1c4d585187c7ab08ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Wed, 27 Nov 2024 18:10:04 +0700 Subject: [PATCH 5/5] Recieve command's response with haptic feedback --- app/lib/providers/capture_provider.dart | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/lib/providers/capture_provider.dart b/app/lib/providers/capture_provider.dart index bce731d1d..7759ccb7e 100644 --- a/app/lib/providers/capture_provider.dart +++ b/app/lib/providers/capture_provider.dart @@ -191,7 +191,7 @@ class CaptureProvider extends ChangeNotifier return file; } - void _processVoiceCommandBytes(List> data) async { + void _processVoiceCommandBytes(String deviceId, List> data) async { if (data.isEmpty) { debugPrint("voice frames is empty"); return; @@ -205,6 +205,7 @@ class CaptureProvider extends ChangeNotifier debugPrint("Command respond: ${messages.map((m) => m.text).join(" | ")}"); if (messages.isNotEmpty) { messageProvider?.refreshMessages(); + _playSpeakerHaptic(deviceId, 2); } } catch (e) { debugPrint(e.toString()); @@ -228,7 +229,7 @@ class CaptureProvider extends ChangeNotifier _voiceCommandSession = null; // end session var data = List>.from(_commandBytes); _commandBytes = []; - _processVoiceCommandBytes(data); + _processVoiceCommandBytes(deviceId, data); } }); } @@ -254,7 +255,7 @@ class CaptureProvider extends ChangeNotifier _voiceCommandSession = null; // end session var data = List>.from(_commandBytes); _commandBytes = []; - _processVoiceCommandBytes(data); + _processVoiceCommandBytes(deviceId, data); } }); }