Akshayan Sinha
Published © CC BY-SA

Visual Gesture Controlled IoT Car

Remember the opening scene of movie 'Project Almanac'? Controlling a drone with hand? Make it yourself, and to simplify, let's control a CAR

IntermediateFull instructions provided4 hours323
Visual Gesture Controlled IoT Car

Things used in this project

Hardware components

Espressif ESP32 Development Board - Developer Edition
Espressif ESP32 Development Board - Developer Edition
×1
DC Motor, 12 V
DC Motor, 12 V
×1
Dual H-Bridge motor drivers L298
SparkFun Dual H-Bridge motor drivers L298
×1

Software apps and online services

OpenCV
OpenCV

Story

Read more

Schematics

3_562ZhyyDhT.png

Code

CarFirmware

Arduino
/*
 * Akshayan Sinha
 * Complete Project Details https://www.hackster.io/akshayansinha
*/
#include <WiFi.h>
#include <ESPmDNS.h>
#include <WebServer.h>

int LeftA= 33;  //IN1
int LeftB= 25;  //IN2
int RightA= 26; //IN3
int RightB= 27; //IN4

const char* ssid = " ";  // Enter SSID here
const char* password = " ";  //Enter Password here

WebServer server(80);             
 
void setup() {

// ----- MOVEMENT ------
  pinMode(LeftA,OUTPUT);
  pinMode(LeftB,OUTPUT);
  pinMode(RightA,OUTPUT);
  pinMode(RightB,OUTPUT);
  pinMode(2,OUTPUT);
  

 //------ WiFi Connection ----------
  
  Serial.begin(115200);
  delay(100);
  Serial.println("Connecting to ");
  Serial.println(ssid);

  //connect to your local wi-fi network
  WiFi.begin(ssid, password);

  //check wi-fi is connected to wi-fi network
  while (WiFi.status() != WL_CONNECTED) {
  delay(1000);
  Serial.print(".");
  }
  Serial.println("");
  Serial.println("WiFi connected..!");
  Serial.print("Got IP: ");  
  Serial.println(WiFi.localIP());
  digitalWrite(2,HIGH); //Blue LED to display connected WiFi

  
  if (!MDNS.begin("projectalmanac")) {
        Serial.println("Error setting up MDNS responder!");
        while(1) {
            delay(1000);
        }
    }
   Serial.println("mDNS responder started");

    

//---------- Page Endpoints ( Main Control Room ) ----------------

  server.on("/", handle_OnConnect);

  server.on("/left", left);

  server.on("/right", right);

  server.on("/forward", forward);

  server.on("/backward", backward);

  server.on("/stop", halt);
 
  server.onNotFound(handle_NotFound);

//---------- SERVER EXECUTION --------------

  server.begin();
  Serial.println("HTTP server started");
  MDNS.addService("http", "tcp", 80);
  
}
void loop() {
  server.handleClient();
}

void handle_OnConnect() {
  server.send(200, "text/html", SendHTML()); 
}

void forward() {
  digitalWrite(LeftA,LOW);
  digitalWrite(LeftB,HIGH);
  digitalWrite(RightA,HIGH);
  digitalWrite(RightB,LOW);
  
  server.send(200, "text/html", SendHTML()); 
}

void backward() {

  digitalWrite(LeftA,HIGH);
  digitalWrite(LeftB,LOW);
  digitalWrite(RightA,LOW);
  digitalWrite(RightB,HIGH);
  
  server.send(200, "text/html", SendHTML()); 
}

void left() {

  digitalWrite(LeftA,HIGH);
  digitalWrite(LeftB,LOW);
  digitalWrite(RightA,HIGH);
  digitalWrite(RightB,LOW);
  
  server.send(200, "text/html", SendHTML()); 
}

void right() {

  digitalWrite(LeftA,LOW);
  digitalWrite(LeftB,HIGH);
  digitalWrite(RightA,LOW);
  digitalWrite(RightB,HIGH);
  
  server.send(200, "text/html", SendHTML()); 
}

void halt() {

  digitalWrite(LeftA,LOW);
  digitalWrite(LeftB,LOW);
  digitalWrite(RightA,LOW);
  digitalWrite(RightB,LOW);
  
  server.send(200, "text/html", SendHTML()); 
}

void handle_NotFound(){
  server.send(404, "text/plain", "Not found");
}

String SendHTML(){
  String ptr = "<!DOCTYPE html> <html>\n";
  ptr +="<head><meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0, user-scalable=yes\">\n";
  ptr +="<title>IoT Gesture Bot</title>\n";
  ptr +="<style>html { font-family: Courier New; display: inline-block; margin: 0px auto; text-align: center;}\n";
  ptr +="body{margin-top: 50px;} h1 {color: #444444;margin: 50px auto 30px;}\n";
  ptr +="p {font-size: 24px;color: #444444;margin-bottom: 10px;}\n";
  ptr +="</style>\n";
  ptr +="</head>\n";
  ptr +="<body>\n";
  ptr +="<div id=\"webpage\">\n";
  ptr +="<h1>Enter /command on the URL, or contact Admin for WebRemote</h1>\n";
  ptr +="<p>Available Commands - left,  right,  forward,  stop,  backward, open </p>";
  ptr +="</div>\n";
  ptr +="</body>\n";
  ptr +="</html>\n";
  return ptr;
}

CameraController on Python

Python
import cv2, requests, threading
import mediapipe as mp
import numpy as np

# Initialize MediaPipe Hands
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()
mp_drawing = mp.solutions.drawing_utils

# Initialize variables for hand tracking
tracking = False
hand_y = 0
hand_x = 0
prev_dir = ""

URL = "http://projectalmanac.local/"

def send(link):
    try:
        response = requests.get(link)
        print("Response ->", response)
    except Exception as e:
        print(f"Error sending HTTP request: {e}")

# Function to find the direction based on the hand's xy-coordinate
def find_direction(frame, hand_y, hand_x, frame_center):
    width, height, _ = frame.shape[:3]

    if hand_y < frame_center[1] - height * 0.1:
        return "forward"
    elif hand_y > frame_center[1] + height * 0.1:
        return "backward"
    elif hand_x < frame_center[0] - width * 0.2:
        return "left"
    elif hand_x > frame_center[0] + width * 0.2:
        return "right"
    else:
        return "stop"


# Capture video from the default camera (usually 0)
cap = cv2.VideoCapture(0)

with hands:
    hand_landmarks = None  # Initialize hand_landmarks outside the loop
    while True:
        ret, frame = cap.read()
        frame = cv2.flip(frame,1)

        if not ret:
            break

        # Get the frame dimensions
        height, width, _ = frame.shape

        # Create a black background image
        black_background = np.zeros_like(frame)

        # Process the frame with MediaPipe Hands
        rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        # rgb_frame = cv2.flip(rgb_frame, 1)
        results = hands.process(rgb_frame)

        if results.multi_hand_landmarks:
            # Assuming only one hand is in the frame
            hand_landmarks = results.multi_hand_landmarks[0]

            # Extract the y-coordinate of the index finger tip
            index_finger_tip = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
            hand_y = int(index_finger_tip.y * height)
            hand_x = int(index_finger_tip.x * width)

            tracking = True

        # Update the direction based on the finger position
        frame_center = (width // 2, height // 2)
        if tracking:
            direction = find_direction(frame, hand_y, hand_x, frame_center)

            if(direction != prev_dir):
                try:
                    link = URL+direction
                    http_thread = threading.Thread(target=send, args=(link,))
                    http_thread.start()
                except Exception as e:
                    print(e)
                prev_dir = direction
                print(direction)
    
            cv2.putText(black_background, direction, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)

        # Draw a filled circle for the index finger on the black background
        if tracking:
            circle_radius = 20
            circle_color = (0, 0, 255)  # Red circle color
            cv2.circle(black_background, (hand_x, hand_y), circle_radius, circle_color, -1)

        # Add a colored border to the center area on the black background
        # border_color = (0, 255, 0)  # Green border color
        # border_thickness = 5
        # center_x, center_y = frame_center
        # center_width, center_height = int(width * 0.2), int(height * 0.2)
        # cv2.rectangle(black_background, (center_x - center_width, center_y - center_height), 
        #               (center_x + center_width, center_y + center_height), border_color, border_thickness)

        text_x = 10
        text_y = 60
        cv2.putText(black_background, "Forward", (width*2 // 5, text_y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv2.putText(black_background, "Backward", (width*2 // 5, height - text_y - text_x), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv2.putText(black_background, "Left", (text_x, height // 2), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv2.putText(black_background, "Right", (width - text_x - 90, height // 2), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
        cv2.putText(black_background, "Stop", (width // 2 - 20, height // 2), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)


        # Decrease the opacity of the black background
        opacity = 0.8  # Adjust this value for the desired opacity (0.0 to 1.0)
        cv2.addWeighted(black_background, opacity, frame, 1 - opacity, 0, frame)

        # Show the frame with all the elements
        cv2.imshow("Hand Tracking", frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

cap.release()
cv2.destroyAllWindows()

Credits

Akshayan Sinha

Akshayan Sinha

28 projects • 26 followers
Robotics Software Engineer with a makermind.

Comments

Related channels and tags
  • Espressif
  • Internet of Things
  • OpenCV
  • Photos & Video
  • Robotics
  • SparkFun