Salma MayorquinTerry Rodriguez
Published © GPL3+

Person Detection with TensorFlow and Arduino

Using the TensorFlow Lite library, we can flash tiny machine learning models on an Arduino to detect humans from a camera.

AdvancedWork in progress3 hours56,368

Things used in this project

Hardware components

Nano 33 BLE Sense
Arduino Nano 33 BLE Sense
×1
Arducam Mini 2MP
×1
Adafruit Micro-Lipo Charger
×1
Adafruit Lithium Ion Polymer Battery
×1

Software apps and online services

TensorFlow
TensorFlow
Arduino IDE
Arduino IDE

Hand tools and fabrication machines

Breadboard, 170 Pin
Breadboard, 170 Pin
Premium Female/Male Extension Jumper Wires, 40 x 6" (150mm)
Premium Female/Male Extension Jumper Wires, 40 x 6" (150mm)

Story

Read more

Code

Person Detection with BLE

C/C++
This script uses the Arduino BLE library to send a high alert to a peripheral bluetooth device when a person has been detected.
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <ArduinoBLE.h>
#include <TensorFlowLite.h>

#include "main_functions.h"

#include "detection_responder.h"
#include "image_provider.h"
#include "model_settings.h"
#include "person_detect_model_data.h"
#include "tensorflow/lite/experimental/micro/kernels/micro_ops.h"
#include "tensorflow/lite/experimental/micro/micro_error_reporter.h"
#include "tensorflow/lite/experimental/micro/micro_interpreter.h"
#include "tensorflow/lite/experimental/micro/micro_mutable_op_resolver.h"
#include "tensorflow/lite/schema/schema_generated.h"
#include "tensorflow/lite/version.h"

// Globals, used for compatibility with Arduino-style sketches.
namespace {
tflite::ErrorReporter* error_reporter = nullptr;
const tflite::Model* model = nullptr;
tflite::MicroInterpreter* interpreter = nullptr;
TfLiteTensor* input = nullptr;

// An area of memory to use for input, output, and intermediate arrays.
constexpr int kTensorArenaSize = 70 * 1024;
static uint8_t tensor_arena[kTensorArenaSize];
}  // namespace

// The name of this function is important for Arduino compatibility.
void setup() {
  // Set up logging. Google style is to avoid globals or statics because of
  // lifetime uncertainty, but since this has a trivial destructor it's okay.
  // NOLINTNEXTLINE(runtime-global-variables)
  static tflite::MicroErrorReporter micro_error_reporter;
  error_reporter = &micro_error_reporter;

  // Map the model into a usable data structure. This doesn't involve any
  // copying or parsing, it's a very lightweight operation.
  model = tflite::GetModel(g_person_detect_model_data);
  if (model->version() != TFLITE_SCHEMA_VERSION) {
    error_reporter->Report(
        "Model provided is schema version %d not equal "
        "to supported version %d.",
        model->version(), TFLITE_SCHEMA_VERSION);
    return;
  }

  // Pull in only the operation implementations we need.
  // This relies on a complete list of all the ops needed by this graph.
  // An easier approach is to just use the AllOpsResolver, but this will
  // incur some penalty in code space for op implementations that are not
  // needed by this graph.
  //
  // tflite::ops::micro::AllOpsResolver resolver;
  // NOLINTNEXTLINE(runtime-global-variables)
  static tflite::MicroMutableOpResolver micro_mutable_op_resolver;
  micro_mutable_op_resolver.AddBuiltin(
      tflite::BuiltinOperator_DEPTHWISE_CONV_2D,
      tflite::ops::micro::Register_DEPTHWISE_CONV_2D());
  micro_mutable_op_resolver.AddBuiltin(tflite::BuiltinOperator_CONV_2D,
                                       tflite::ops::micro::Register_CONV_2D());
  micro_mutable_op_resolver.AddBuiltin(
      tflite::BuiltinOperator_AVERAGE_POOL_2D,
      tflite::ops::micro::Register_AVERAGE_POOL_2D());

  // Build an interpreter to run the model with.
  static tflite::MicroInterpreter static_interpreter(
      model, micro_mutable_op_resolver, tensor_arena, kTensorArenaSize,
      error_reporter);
  interpreter = &static_interpreter;

  // Allocate memory from the tensor_arena for the model's tensors.
  TfLiteStatus allocate_status = interpreter->AllocateTensors();
  if (allocate_status != kTfLiteOk) {
    error_reporter->Report("AllocateTensors() failed");
    return;
  }

  // Get information about the memory area to use for the model's input.
  input = interpreter->input(0);
  /* BLE initialize */
  BLE.begin();
  //BLE.scanForUuid("1801");
}

// The name of this function is important for Arduino compatibility.
void loop() {
  // Get image from provider.
  if (kTfLiteOk != GetImage(error_reporter, kNumCols, kNumRows, kNumChannels,
                            input->data.uint8)) {
    error_reporter->Report("Image capture failed.");
  }

  // Run the model on this input and make sure it succeeds.
  if (kTfLiteOk != interpreter->Invoke()) {
    error_reporter->Report("Invoke failed.");
  }

  TfLiteTensor* output = interpreter->output(0);

  // Process the inference results.
  uint8_t person_score = output->data.uint8[kPersonIndex];
  uint8_t no_person_score = output->data.uint8[kNotAPersonIndex];
  RespondToDetection(error_reporter, person_score, no_person_score);
  /* Send inference via bluetooth */
  BLE.scanForUuid("1801");
  delay(500); // delay to find uuid
  BLEDevice peripheral = BLE.available();
  Serial.println(peripheral);
  if (peripheral){
    if (peripheral.localName() != "PersonDetectionMonitor") {
      return;
    }
    BLE.stopScan();
    if (person_score > no_person_score){
      sendAlert(peripheral,2);
    }
    else{
      sendAlert(peripheral,0);
    }
    //peripheral.disconnect();
  }
  else{
    Serial.println("Peripheral not available");
  }
}

void sendAlert(BLEDevice peripheral, int sendsignal) {
  // connect to the peripheral
  Serial.println("Connecting ...");

  if (peripheral.connect()) {
    Serial.println("Connected");
  } else {
    Serial.println("Failed to connect!");
    return;
  }

  // discover peripheral attributes
  Serial.println("Discovering attributes ...");
  if (peripheral.discoverAttributes()) {
    Serial.println("Attributes discovered");
  } else {
    Serial.println("Attribute discovery failed!");
    peripheral.disconnect();
    return;
  }
  // retrieve the alert level characteristic
  BLECharacteristic alertLevelChar = peripheral.characteristic("2A06");

   if (!alertLevelChar) {
    Serial.println("Peripheral does not have alert level characteristic!");
    peripheral.disconnect();
    return;
  } else if (!alertLevelChar.canWrite()) {
    Serial.println("Peripheral does not have a writable alert level characteristic!");
    peripheral.disconnect();
    return;
  }

  if (peripheral.connected()) {
    if (sendsignal > 0){
      alertLevelChar.writeValue((byte)0x02);
      Serial.println("Wrote high alert");
    }
    else{
      alertLevelChar.writeValue((byte)0x00);
      Serial.println("Wrote low alert");
    }
  }
  peripheral.disconnect();
  Serial.println("Peripheral disconnected");
}

Peripheral Device

C/C++
This script uses the Arduino BLE library to connect and listen for an alert triggered by the person detection device to switch a relay on/off
#include <ArduinoBLE.h>
#include "Arduino.h"

// Relay on D8
int relay = 8;

 // BLE PersonDetection Service
BLEService PersonDetectionService("1801");

// BLE Alert Level Characteristic
BLEByteCharacteristic alertLevelChar("2A06",  // standard 16-bit characteristic UUID
    BLERead | BLEWrite); // remote clients will be able to get notifications if this characteristic changes


void setup() {
  Serial.begin(9600);    // initialize serial communication
  while (!Serial);

  pinMode(LED_BUILTIN, OUTPUT); // initialize the built-in LED pin to indicate when a central is connected
  //pinMode(LEDR, OUTPUT);
  //pinMode(LEDG, OUTPUT);
  //pinMode(LEDB, OUTPUT);
  //pinMode(relay, OUTPUT);

  //digitalWrite(LEDR, LOW);
  //digitalWrite(LEDG, LOW);
  //digitalWrite(LEDB, LOW);
  digitalWrite(relay, LOW);
  
  // begin initialization
  if (!BLE.begin()) {
    Serial.println("starting BLE failed!");

    while (1);
  }

  /* Set a local name for the BLE device
     This name will appear in advertising packets
     and can be used by remote devices to identify this BLE device
     The name can be changed but maybe be truncated based on space left in advertisement packet
  */
  BLE.setLocalName("PersonDetectionMonitor");
  BLE.setAdvertisedService(PersonDetectionService); // add the service UUID
  PersonDetectionService.addCharacteristic(alertLevelChar); // add the alert level characteristic
  BLE.addService(PersonDetectionService); // Add the battery service
  alertLevelChar.writeValue((byte)0x00);
  
  /* Start advertising BLE.  It will start continuously transmitting BLE
     advertising packets and will be visible to remote BLE central devices
     until it receives a new connection */

  // start advertising
  BLE.advertise();

  Serial.println("Bluetooth device active, waiting for connections...");
}

void loop() {
  // wait for a BLE central
  BLEDevice central = BLE.central();

  // if a central is connected to the peripheral:
  if (central) {
    Serial.print("Connected to central: ");
    // print the central's BT address:
    Serial.println(central.address());
    // turn on the LED to indicate the connection:
    digitalWrite(LED_BUILTIN, HIGH);

    // while the central is connected:
    while (central.connected()) {
      //Serial.println("Getting Alert Level:");
      if (alertLevelChar.written()){
        if (alertLevelChar.value()){
          Serial.println("Got high alert");
          digitalWrite(relay, HIGH);
          Serial.println("Set relay to HIGH");
        } else{
          Serial.println("Got low alert");
          digitalWrite(relay, LOW);
          Serial.println("Set relay to LOW");
        }
      }
    }
    // when the central disconnects, turn off the LED:
    digitalWrite(LED_BUILTIN, LOW);
    Serial.print("Disconnected from central: ");
    Serial.println(central.address());
  }
}

Credits

Salma Mayorquin

Salma Mayorquin

21 projects • 381 followers
Software engineer AI/ML & hardware tinkerer interested in embedded AI. Lets hack the change we want to see!
Terry Rodriguez

Terry Rodriguez

21 projects • 194 followers
hack the change you want to see

Comments