Shahariar
Published © CC BY

Gaming with Disabilities: Face Tracking Game Controller

An attempt to develop a Gaming Controller for Persons with Disabilities, that works with movement of face/head to control movements in games

IntermediateWork in progress6 hours739

Things used in this project

Hardware components

Person Sensor
Useful Sensors Person Sensor
×1
SparkFun I2C 4 wire JST connector for Person Sensor
×1
Beetle - The Smallest Arduino
DFRobot Beetle - The Smallest Arduino
×1
SG90 Micro-servo motor
SG90 Micro-servo motor
×1
OLED Display 128x64 0.96 inch, I2C Interface
DIYables OLED Display 128x64 0.96 inch, I2C Interface
×1
LilyPad LED White (5pcs)
SparkFun LilyPad LED White (5pcs)
×2
5mW Laser Module emitter - Red Point
Seeed Studio 5mW Laser Module emitter - Red Point
×2
Photodiode, 45 °
Photodiode, 45 °
×2
Grove - Vibration Motor
Seeed Studio Grove - Vibration Motor
×2
2n2222 NPN Transistor
×3
Resistor 1k ohm
Resistor 1k ohm
×2
Resistor 100 ohm
Resistor 100 ohm
×2
Capacitor 100 µF
Capacitor 100 µF
×3

Software apps and online services

Arduino IDE
Arduino IDE

Hand tools and fabrication machines

Soldering iron (generic)
Soldering iron (generic)

Story

Read more

Schematics

sch board

sch v1

Code

person_sensor_game_controller

C/C++
Arduino Leonardo or Beetle
// Include libraries for person sensor, OLED display, HID keyboard etc

#include <Wire.h>
#include <Arduino.h>
#include <Servo.h>
#include <U8g2lib.h>
#include <Keyboard.h>
#include "person_sensor.h"

// keyboard key short defination

#define up       KEY_UP_ARROW
#define down     KEY_DOWN_ARROW
#define left     KEY_LEFT_ARROW
#define right    KEY_RIGHT_ARROW


const int32_t SAMPLE_DELAY_MS = 50;


int x = 0;
int y = 0;
int x_offset = 15;
int y_offset = 47;
int pos = 0;
uint8_t lamp_drive_pwm =50;

// I2C OLED display object
U8G2_SSD1306_128X64_NONAME_F_HW_I2C u8g2(U8G2_R0, /* reset=*/ U8X8_PIN_NONE, /* clock=*/ 16, /* data=*/ 17);   // ESP32 Thing, HW I2C with pin remapping
// create servo object to control a servo for cam movement
Servo camSweepServo;  

void setup() 
{
  // safety delay
  delay(5000);
  camSweepServo.attach(9);  // attaches the servo on pin 9 to the servo object
  
  for (pos = 75; pos <= 105; pos += 1)
  { 
    camSweepServo.write(pos);   // tell servo to go to position in variable 'pos'
    delay(15);                  // waits 15 ms for the servo to reach the position
  }
  
  for (pos = 105; pos >= 75; pos -= 1) 
  { 
    camSweepServo.write(pos);        // tell servo to go to position in variable 'pos'
    delay(15);                       // waits 15 ms for the servo to reach the position
  }
  pos = 90;
 camSweepServo.write(pos);
 
  // You need to make sure you call Wire.begin() in setup, or the I2C access
  // below will fail.
 
  Wire.begin();
  u8g2.begin();

  // these pins drive 2 vibration motors for haptic feedback (tba)
  pinMode(10,OUTPUT);
  pinMode(A0,OUTPUT);
  digitalWrite(10,0);
  digitalWrite(A0,0);

  // this pin outputs PWM to adjust the brightness of the LED lamps
  pinMode(11,OUTPUT);

  // 5 step brightness increase
  analogWrite(11,20);
  delay(200);
  analogWrite(11,40);
  delay(200);
  analogWrite(11,60);
  delay(200);
  analogWrite(11,100);
  delay(200);
  analogWrite(11,150);
  delay(300);

}

void loop() 
{
  // readjust the brightness according to user perference if needed
  
  analogWrite(11,lamp_drive_pwm);

  // poll person sensor and put in an array

  person_sensor_results_t results = {}; 
  if (!person_sensor_read(&results)) 
  {
    delay(SAMPLE_DELAY_MS);
    return;
  }
   const person_sensor_face_t* face = &results.faces[0];

  // clear OLED display buffer 
    u8g2.clearBuffer();

  // if there is a face infront of person sensor
    
  if (results.num_faces>0)
     {

      /* 
       *  the following code will transform 256x256 coordinate into
       *  128x64 by scaling down, this is done to keep the face 
       *  into 128x64 OLED display, using circle and triangle func
       *  an emoji face will be drawn which will represent the user
       *  face, it will move as the person's head moves on the FOV
       *  plane of the person sensor's camera 
       */

        x = (face->box_left)/2; // 63
        y = (face->box_top)/4 ; //31
        x = x + x_offset;
        y = y_offset - y ;
        u8g2.drawCircle(x,y,10, U8G2_DRAW_ALL);
        u8g2.drawDisc(x-3,y-2,2, U8G2_DRAW_ALL);
        u8g2.drawDisc(x+3,y-2,2, U8G2_DRAW_ALL);
        u8g2.drawTriangle(x-4,y+4,x,y+7,x+4,y+5); 
        u8g2.setFont(u8g2_font_ncenB10_tr);
        u8g2.setCursor(0,64);
        u8g2.print("! Face Tracking !");
        u8g2.sendBuffer();
    }
    else 
    {  
        u8g2.drawCircle(63,31,10, U8G2_DRAW_ALL);
        u8g2.drawDisc(60,29,1, U8G2_DRAW_ALL);
        u8g2.drawDisc(66,29,1, U8G2_DRAW_ALL);
        u8g2.drawTriangle(59,35,63,36,67,35);
        u8g2.setFont(u8g2_font_ncenB10_tr);
        u8g2.setCursor(0,64);
        u8g2.print("!No Face Found!");
        u8g2.sendBuffer();
    }

  /*
   * now based of the face position data
   * keyboard press will be emulated and
   * arrow key press will be sent to 
   * host pc over USB
   */
   
  // let go all previouly hold keys
    Keyboard.releaseAll();
    
  // arrow key control based on face position 
   if(results.num_faces>0)
   {   
    if  (y<22)  
    {
      Keyboard.press(up);
    }
    if  (y>36)  
    {
      Keyboard.press(down);
    }
   if  (x<48)  
    {
       Keyboard.press(up);
       Keyboard.press(left);
    }

    if  (x>72)  
    {
      Keyboard.press(up);
      Keyboard.press(right);
    }
 }
 
/*
 * adjust lamp brightness using these two optical beam
 * interrupt switches
 * these can be reprogrammed to use for gaming too
 */

  if (analogRead(A1)<150)
  {
   digitalWrite(A0,1);  
   lamp_drive_pwm--;
  }
  else
  {
   digitalWrite(A0,0);  
  }
  

  if (analogRead(A2)<150)
  {
   digitalWrite(10,1);  
    lamp_drive_pwm++;
   }
  else  
  {
   digitalWrite(10,0);  
  }

  // a bit delay as person sensor can update at 7HZ only
  
  delay(SAMPLE_DELAY_MS);

} 

///////////////////////////////////////////////
////////////// end of void loop////////////////
///////////////////////////////////////////////

full code

C/C++
No preview (download only).

Credits

Shahariar

Shahariar

71 projects • 263 followers
"What Kills a 'Great life' is a 'Good Life', which is Living a Life Inside While Loop"

Comments