Mithun Das
Published © GPL3+

Boon - A sensory substitution for ASD, better sleep and more

This device classifies human emotions, snores, sign languages, visual objects using EI and provide haptic feedback through Neosensory Buzz.

IntermediateFull instructions provided20 hours1,297
Participation Prizes
Expand your senses
Boon - A sensory substitution for ASD, better sleep and more

Things used in this project

Hardware components

Raspberry Pi 4 Model B
Raspberry Pi 4 Model B
×1
Camera Module V2
Raspberry Pi Camera Module V2
×1
Raspiaudio MIC
×1
Neosensory Buzz
Neosensory Buzz
×1
ESP32S
Espressif ESP32S
×1

Software apps and online services

Edge Impulse Studio
Edge Impulse Studio
balenaCloud
balenaCloud
Arduino IDE
Arduino IDE
Xcode
Apple Xcode
VS Code
Microsoft VS Code

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)

Story

Read more

Code

NEOESP32

Arduino
Code to interface with Neo sensory buzz. Depends on ESP32 SDK for Neosensory
#include <neosensory_esp32.h>
#include <WiFi.h>
#include <PubSubClient.h>
#include <ArduinoJson.h>

#define RXD2 16
#define TXD2 17
#define SLEEP_TIME 1 //1 second

NeosensoryESP32 NeoESP32;

int motor = 0;
float intensity = 0;
float **rumble_frames;
float **motor_frames;
int battery = 0;
String deviceId = "";
String serialNo = "";
long lastUpdated = 0;

const size_t capacity = JSON_OBJECT_SIZE(2) + 20;
DynamicJsonDocument doc(capacity);
DynamicJsonDocument deviceInfo(JSON_OBJECT_SIZE(21) + 440);
bool jsonStarted_ = false;
String jsonMessage_;

const char* ssid = "xxxx";//put your wifi network name here
const char* password = "xxxx";//put your wifi password here
const char* mqtt_server = "broker.hivemq.com";
const int mqtt_port = 1883;
const char* MQTT_SERIAL_PUBLISH_CH = "/neoosensory/boon/xxx/tx";
const char* MQTT_SERIAL_RECEIVE_CH = "/neoosensory/boon/xxx/rx";
const char* MQTT_SERIAL_RECEIVE_CH_BUZZ = "/neoosensory/boon/xxx/rx/buzz";
const char* MQTT_SERIAL_RECEIVE_CH_INFO = "/neoosensory/boon/xxx/rx/info";

WiFiClient espClient;
PubSubClient client(espClient);

void setup_wifi() {
  delay(10);
  // We start by connecting to a WiFi network
  Serial.println();
  Serial.print("Connecting to ");
  Serial.println(ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }
  randomSeed(micros());
  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}

void callback(char* topic, byte* payload, unsigned int length) {

  Serial.print("Message arrived in topic: ");
  Serial.println(topic);

  Serial.print("Message:");
  String message;

  for (int i = 0; i < length; i++) {
    //Serial.print((char)payload[i]);
    message += (char)payload[i];
  }

  Serial.println(message);
  Serial.println("-----------------------");

  if (String(topic) == String(MQTT_SERIAL_RECEIVE_CH_BUZZ)) {
    Serial.println("BUZZ Requested");


    while (message.indexOf(";") != -1) {
      int index = message.indexOf(";");
      processFrames(message.substring(0, index));
      message = message.substring(index + 1);
    }


  }

  if (String(topic) == String(MQTT_SERIAL_RECEIVE_CH_INFO)) {
    NeoESP32.deviceInfo();
    NeoESP32.deviceBattery();
  }

}


void processFrames(String input) {

  if (NeoESP32.isConnected() && NeoESP32.isAuthorized()) {
    Serial.println("Processing frame : " + input);
    int noOfFrames = 1;

    int index1 = input.indexOf(",");
    int* intensity_frame = new int[4];

    if (index1 != -1) {
      noOfFrames = input.substring(0, index1).toInt();
    }

    if (noOfFrames > NeoESP32.max_frames_per_bt_package()) {
      noOfFrames = NeoESP32.max_frames_per_bt_package();
    }

    Serial.printf("No of frames %d \n", noOfFrames);
    input = input.substring(index1 + 1);


    for (int i = 0; i < NeoESP32.num_motors(); i++) {
      index1 = input.indexOf(",");

      if (index1 != -1) {
        String str  = input.substring(0, index1);
        intensity_frame[i] = str.toInt();

        input = input.substring(index1 + 1);
      } else {
        //for the last one
        intensity_frame[i] = input.toInt();

      }

    }

    Serial.printf("Frame is {%d, %d, %d, %d}\n", intensity_frame[0], intensity_frame[1], intensity_frame[2], intensity_frame[3]);

    motor_frames = new float*[noOfFrames];
    for (int i = 0; i < noOfFrames - 1; i++) {
      motor_frames[i] = new float[NeoESP32.num_motors()];
      for (int j = 0; j < NeoESP32.num_motors(); j++) {
        motor_frames[i][j] = map(intensity_frame[j], 0, 255, 0, 100) / 100.0;
      }

    }

    motor_frames[noOfFrames - 1] = new float[NeoESP32.num_motors()];
    for (int p = 0; p < NeoESP32.num_motors(); p++) {
      motor_frames[noOfFrames - 1][p] = 0.0;
    }

    NeoESP32.vibrateMotors(motor_frames, noOfFrames);
    delay((noOfFrames) * NeoESP32.firmware_frame_duration());

  }


}
void publish(String msg) {

  char message[100];
  msg.toCharArray(message, 100);
  client.publish(MQTT_SERIAL_PUBLISH_CH, message);
}

void publish(String msg, char* topic) {
  char message[100];
  msg.toCharArray(message, 100);
  client.publish(topic, message);
}


void reconnect() {
  // Loop until we're reconnected
  while (!client.connected())
  {
    Serial.print("Attempting MQTT connection...");
    // Create a random client ID
    String clientId = "ESP8266Client-";
    clientId += String(random(0xffff), HEX);
    // Attempt to connect
    //if you MQTT broker has clientID,username and password
    //please change following line to    if (client.connect(clientId,userName,passWord))
    if (client.connect(clientId.c_str()))
    {
      Serial.println("connected");
      //once connected to MQTT broker, subscribe command if any
      client.subscribe(MQTT_SERIAL_RECEIVE_CH);
      client.subscribe(MQTT_SERIAL_RECEIVE_CH_BUZZ);
      client.subscribe(MQTT_SERIAL_RECEIVE_CH_INFO);
      publish("C");
      publish(String(battery), "/neoosensory/boon/xxx/tx/bat");
      publish(deviceId + "," + serialNo, "/neoosensory/boon/xxx/tx/info");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" reboot esp32 ...");
      // reboot when MQTT connection fails. 
      esp_deep_sleep(1000000LL * SLEEP_TIME);
    }
  }
} //end reconnect()

void setup() {
  Serial.begin(115200);
  Serial2.begin(9600, SERIAL_8N1, RXD2, TXD2);

  setup_wifi();
  client.setServer(mqtt_server, mqtt_port);
  client.setCallback(callback);


  NeoESP32.setConnectedCallback(onConnected);
  NeoESP32.setReadNotifyCallback(onReadNotify);
  NeoESP32.setDisconnectedCallback(onDisconnected);
  NeoESP32.begin();
  while (!NeoESP32.isConnected() || !NeoESP32.isAuthorized()) {}
  NeoESP32.deviceInfo();
  NeoESP32.deviceBattery();

}

void loop() {
  if (!client.connected()) {
    reconnect();
  }
  client.loop();
  
  if(millis() - lastUpdated > 60*1000){
    lastUpdated = millis();
    NeoESP32.deviceBattery();
    
    
  }
}






void onConnected(bool success) {
  if (!success) {
    Serial.println("Attempted connection but failed.");
    return;
  }
  Serial.println("Successfully connected to Neosensory Buzz...");
  NeoESP32.authorizeDeveloper();
  NeoESP32.acceptTermsAndConditions();
  NeoESP32.stopAlgorithm();


}

void onDisconnected() {
  Serial.println("Neosensory Buzz disconnected");
}

void onReadNotify(BLERemoteCharacteristic *pBLERemoteCharacteristic, uint8_t *pData, size_t length, bool isNotify) {
  for (int i = 0; i < length; i++) {
    Serial.write(pData[i]);
  }
  for (int i = 0; i < length; i++)
  {
    if (pData[i] == '{')
    {
      jsonStarted_ = true;
      jsonMessage_ = "";
    }
    if (jsonStarted_)
    {
      jsonMessage_ += (char)pData[i];
    }
    if (pData[i] == '}')
    {
      jsonStarted_ = false;
      handleJSON(jsonMessage_);

    }
  }
}

void handleJSON(String jsonMessage) {
  Serial.println(jsonMessage);

  if (jsonMessage.indexOf("battery_soc") != -1)
  {
    deserializeJson(doc, jsonMessage);


    battery = doc["battery_soc"];
    publish(String(battery), "/neoosensory/boon/xxx/tx/bat");


  }

  if (jsonMessage.indexOf("device_id") != -1)
  {
    deserializeJson(deviceInfo, jsonMessage);
    serializeJson(deviceInfo, Serial);

    deviceId = deviceInfo["device_id"].as<String>();
    serialNo = deviceInfo["serial_number"].as<String>();
    publish(deviceId + "," + serialNo, "/neoosensory/boon/xxx/tx/info");
  }


}

Neosensory Buzz library for ESP32

Credits

Mithun Das

Mithun Das

33 projects • 170 followers
Hacker and Maker driven by passion. Ambassador at Edge Impulse and Balena. Follow me on Twitter @_mithundas

Comments

Awards
Participation Prizes
Expand your senses
Related channels and tags
  • Apple
  • Edge Impulse
  • Espressif
  • Machine Learning & AI
  • Microsoft
  • Neosensory
  • Raspberry Pi