BW21-CBV-Kit——Dynamic Recognition of Gestures and Faces

Project Evaluation

This section details the official Demo evaluation process and effect demonstration for Hand Gesture Detection and RTSP Face Recognition.

Gesture Recognition

Gesture tracking, extraction, simulation, and recognition are achieved through loading a Neural Network (NN) training model.

Code

Note: Modify the WiFi name and password.

#include "WiFi.h"
#include "StreamIO.h"
#include "VideoStream.h"
#include "RTSP.h"
#include "NNGestureDetection.h"
#include "VideoStreamOverlay.h"
#define CHANNEL 0
#define CHANNELNN 3
#define NNWIDTH 192
#define NNHEIGHT 192
VideoSetting config(VIDEO_FHD, 30, VIDEO_H264, 0);
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
NNGestureDetection gesturedetect;
RTSP rtsp;
StreamIO videoStreamer(1, 1);
StreamIO videoStreamerRGBGD(1, 1);
char ssid[] = "xxx";
char pass[] = "xxx";
int status = WL_IDLE_STATUS;
IPAddress ip;
int rtsp_portnum;
void setup() {
Serial.begin(115200);
while (status != WL_CONNECTED) {
Serial.print("Attempting to connect to WPA SSID: ");
Serial.println(ssid);
status = WiFi.begin(ssid, pass);
delay(2000);
}
ip = WiFi.localIP();
config.setBitrate(2 * 1024 * 1024);
Camera.configVideoChannel(CHANNEL, config);
Camera.configVideoChannel(CHANNELNN, configNN);
Camera.videoInit();
rtsp.configVideo(config);
rtsp.begin();
rtsp_portnum = rtsp.getPort();
gesturedetect.configVideo(configNN);
gesturedetect.modelSelect(GESTURE_DETECTION, NA_MODEL, NA_MODEL, NA_MODEL, NA_MODEL, NA_MODEL, DEFAULT_PALMDETECT, DEFAULT_HANDLANDMARK);
gesturedetect.begin();
videoStreamer.registerInput(Camera.getStream(CHANNEL));
videoStreamer.registerOutput(rtsp);
if (videoStreamer.begin() != 0) {
Serial.println("StreamIO link start failed");
}
Camera.channelBegin(CHANNEL);
videoStreamerRGBGD.registerInput(Camera.getStream(CHANNELNN));
videoStreamerRGBGD.setStackSize();
videoStreamerRGBGD.setTaskPriority();
videoStreamerRGBGD.registerOutput(gesturedetect);
if (videoStreamerRGBGD.begin() != 0) {
Serial.println("StreamIO link start failed");
}
Camera.channelBegin(CHANNELNN);
OSD.configVideo(CHANNEL, config);
OSD.configTextSize(CHANNEL, 16, 32);
OSD.begin();
gesturedetect.drawHandRegion();
}
void loop() {
// Do nothing
}

Upload

1. Hold down the BOOT button and press the EN button to enter download mode.

2. Select the corresponding serial port and click Download. Wait for a successful upload prompt.

3. Reset by pressing EN again to run the program.

Configuration

After uploading, open the serial monitor. Once WiFi connection is detected, the camera’s IP address will be displayed.

Enter rtsp://192.168.1.104:554 in VLC software to achieve RTSP streaming.

Face Recognition

Use the BW21-CBV-Kit development board to recognize detected faces.

Code

For details, refer to: Neural Nework – Face Recognition – Realtek IoT/Wi-Fi MCU Solutions

#include "WiFi.h"
#include "StreamIO.h"
#include "VideoStream.h"
#include "RTSP.h"
#include "NNFaceDetectionRecognition.h"
#include "VideoStreamOverlay.h"
#define CHANNEL 0
#define CHANNELNN 3
#define NNWIDTH 576
#define NNHEIGHT 320
VideoSetting config(VIDEO_FHD, 30, VIDEO_H264, 0);
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
NNFaceDetectionRecognition facerecog;
RTSP rtsp;
StreamIO videoStreamer(1, 1);
StreamIO videoStreamerFDFR(1, 1);
StreamIO videoStreamerRGBFD(1, 1);
char ssid[] = "xxx";
char pass[] = "xxx";
int status = WL_IDLE_STATUS;
IPAddress ip;
int rtsp_portnum;
void setup() {
Serial.begin(115200);
while (status != WL_CONNECTED) {
Serial.print("Attempting to connect to WPA SSID: ");
Serial.println(ssid);
status = WiFi.begin(ssid, pass);
delay(2000);
}
ip = WiFi.localIP();
config.setBitrate(2 * 1024 * 1024);
Camera.configVideoChannel(CHANNEL, config);
Camera.configVideoChannel(CHANNELNN, configNN);
Camera.videoInit();
rtsp.configVideo(config);
rtsp.begin();
rtsp_portnum = rtsp.getPort();
facerecog.configVideo(configNN);
facerecog.modelSelect(FACE_RECOGNITION, NA_MODEL, DEFAULT_SCRFD, DEFAULT_MOBILEFACENET);
facerecog.begin();
facerecog.setResultCallback(FRPostProcess);
videoStreamer.registerInput(Camera.getStream(CHANNEL));
videoStreamer.registerOutput(rtsp);
if (videoStreamer.begin() != 0) {
Serial.println("StreamIO link start failed");
}
Camera.channelBegin(CHANNEL);
videoStreamerRGBFD.registerInput(Camera.getStream(CHANNELNN));
videoStreamerRGBFD.setStackSize();
videoStreamerRGBFD.setTaskPriority();
videoStreamerRGBFD.registerOutput(facerecog);
if (videoStreamerRGBFD.begin() != 0) {
Serial.println("StreamIO link start failed");
}
Camera.channelBegin(CHANNELNN);
OSD.configVideo(CHANNEL, config);
OSD.begin();
}
void loop() {
if (Serial.available() > 0) {
String input = Serial.readString();
input.trim();
if (input.startsWith(String("REG="))) {
String name = input.substring(4);
facerecog.registerFace(name);
} else if (input.startsWith(String("DEL="))) {
String name = input.substring(4);
facerecog.removeFace(name);
} else if (input.startsWith(String("RESET"))) {
facerecog.resetRegisteredFace();
} else if (input.startsWith(String("BACKUP"))) {
facerecog.backupRegisteredFace();
} else if (input.startsWith(String("RESTORE"))) {
facerecog.restoreRegisteredFace();
}
}
delay(2000);
OSD.createBitmap(CHANNEL);
OSD.update(CHANNEL);
}
void FRPostProcess(std::vector<FaceRecognitionResult> results) {
uint16_t im_h = config.height();
uint16_t im_w = config.width();
Serial.print("Network URL for RTSP Streaming: ");
Serial.print("rtsp://");
Serial.print(ip);
Serial.print(":");
Serial.println(rtsp_portnum);
Serial.println(" ");
printf("Total number of faces detected = %d\r\n", facerecog.getResultCount());
OSD.createBitmap(CHANNEL);
if (facerecog.getResultCount() > 0) {
for (int i = 0; i < facerecog.getResultCount(); i++) {
FaceRecognitionResult item = results[i];
int xmin = (int)(item.xMin() * im_w);
int xmax = (int)(item.xMax() * im_w);
int ymin = (int)(item.yMin() * im_h);
int ymax = (int)(item.yMax() * im_h);
uint32_t osd_color;
if (String(item.name()) == String("unknown")) {
osd_color = OSD_COLOR_RED;
} else {
osd_color = OSD_COLOR_GREEN;
}
OSD.drawRect(CHANNEL, xmin, ymin, xmax, ymax, 3, osd_color);
char text_str[40];
snprintf(text_str, sizeof(text_str), "Face:%s", item.name());
OSD.drawText(CHANNEL, xmin, ymin - OSD.getTextHeight(CHANNEL), text_str, osd_color);
}
}
OSD.update(CHANNEL);
}

Upload

After uploading, open the serial monitor. Once WiFi connection is detected, the camera’s IP address will be displayed. Enter rtsp://192.168.253.1:554 in VLC software to achieve RTSP streaming.

Aim the target face at the camera. When a face is detected and enclosed by a red box, enter REG=Bean in the serial port to register the person. The box color will change to green, indicating the corresponding person's nam

Effect

Dynamic recognition

It has been observed that face recognition accuracy improves at a greater distance, while accuracy decreases when the face occupies a larger portion of the frame.

Additionally, during operation, the module’s temperature increases significantly, so proper heat dissipation should be considered.

Possible factors include:

the face information collection resolution in the code is set to 192x192, which may be insufficient for dynamic scenes, limiting the model’s ability to handle varying face sizes and expressions; camera resolution limitations; and processing power constraints.

Improvement solutions: enhance hardware computational power, implement cooling solutions, expand sample size, increase training iterations, and refine the model.

Note: If the serial port outputs the error message error: vipnn not applied, reinstall the development board package.