Alistair MacDonald
Published © LGPL

Drive by Smile

Control interactive fun and games by using your facial expressions.

IntermediateFull instructions provided2 hours420

Things used in this project

Hardware components

Google Coral Dev Board Mini
×1
Arduino Micro
Arduino Micro
×1
EyonMe W6 Webcam
×1
USB-C OTG Cable
×1
USB-A to Micro-USB Cable
USB-A to Micro-USB Cable
×1

Software apps and online services

Mendel Linux
This should be preinstalled on the Google Coral Dev Board Mini, but you may want to update it.

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)
The custom design case an be 3D printed, or a generic enclosure can be used.

Story

Read more

Custom parts and enclosures

Drive By Smile Case Top

The main enclosure top

Drive By Smile Case Base

The main enclosure

Schematics

Drive by Smile - Schematic

The overview schematic with pin assignment.

Code

Drive by Smile - Coral head tracking code

Python
This code runs on the Google Coral board to track someone's head and identify head tilting gestures. It uses the PyCoral API ( https://github.com/google-coral/pycoral/ ) and pretrained PoseNet MobileNet V1 model ( https://coral.ai/models/pose-estimation/ ).
import cv2

from pose_engine import PoseEngine, KeypointType
from PIL import Image
from PIL import ImageDraw

import numpy as np
import sys

from periphery import Serial

uart1 = Serial("/dev/ttyS1", 115200)

cap = cv2.VideoCapture()

cap.open(1, apiPreference=cv2.CAP_V4L2)

cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('Y', 'U', 'Y', '2'))
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1024)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 768)
cap.set(cv2.CAP_PROP_FPS, 20.0)

if not cap.isOpened():
	sys.exit('Could not open video device')

engine = PoseEngine('models/mobilenet/posenet_mobilenet_v1_075_481_641_quant_decoder_edgetpu.tflite')


while True:

	cap.grab()
	ret, frame = cap.read()

	if ret:

		pil_image = Image.fromarray(frame)
		poses, inference_time = engine.DetectPosesInImage(pil_image)

		for pose in poses:
			if pose.score < 0.4:
				uart1.write(b"?")
				continue

			left_eye = pose.keypoints[KeypointType.LEFT_EYE]
			right_eye = pose.keypoints[KeypointType.RIGHT_EYE]

			eye_gap = ( left_eye.point[0] - right_eye.point[0] ) / 10

			if left_eye.point[1] > right_eye.point[1] + eye_gap:
				uart1.write(b"l")
			elif right_eye.point[1] > left_eye.point[1] + eye_gap:
				uart1.write(b"r")
			else:
				uart1.write(b".")

Drive by Smile - Keyboard emulation bridge

Arduino
This code is emulates a USB keyboard when instructed over a serial connection. This code will run on most ATmega32U4 based Arduinos and Arduino clones.
#include <Keyboard.h>
#include <SoftwareSerial.h>

SoftwareSerial mySerial(9, 5); // RX, TX

void setup() {
  Keyboard.begin();
  mySerial.begin(115200);
}

int currentKey = 0;

void loop() {

  int newKey;

  if ( mySerial.available() > 0 ) {
    newKey = mySerial.read();

    if ( ( newKey >= ' ' ) && ( newKey != currentKey ) ) {
      Keyboard.releaseAll();
      switch (newKey) {
        case '?':
          break;
        case 'r':
          Keyboard.press(KEY_UP_ARROW);
          Keyboard.press(KEY_RIGHT_ARROW);
          break;
        case 'l':
          Keyboard.press(KEY_UP_ARROW);
          Keyboard.press(KEY_LEFT_ARROW);
          break;
        case '.':
          Keyboard.press(KEY_UP_ARROW);
          break;
      }
      currentKey = newKey;
      delay(100);
    }

  }  
  
}

Credits

Alistair MacDonald

Alistair MacDonald

6 projects • 2 followers

Comments