Rayan Kiwan
Published

Hand gesture controlling rgb led using arduino and python

What if you could control an RGB LED using nothing but your hand? In this project, I built a hand gesture controlled RGB LED

AdvancedFull instructions provided1 hour23
Hand gesture controlling rgb led using arduino and python

Things used in this project

Hardware components

Arduino UNO
Arduino UNO
×1
RGB LED
Arduino RGB LED
×1
Resistor 220 ohm
Resistor 220 ohm
×3
Breadboard (generic)
Breadboard (generic)
×1

Software apps and online services

Arduino IDE
Arduino IDE
PictoBlox
STEMpedia PictoBlox

Story

Read more

Schematics

rgb led connection

Code

arduino ide code

Arduino
#include <Arduino.h>

const int RED_PIN = 9;
const int GREEN_PIN = 10;
const int BLUE_PIN = 11;

void setup() {

  Serial.begin(9600);

  pinMode(RED_PIN, OUTPUT);
  pinMode(GREEN_PIN, OUTPUT);
  pinMode(BLUE_PIN, OUTPUT);

  setColor(0, 0, 0);

  Serial.println("RGB LED READY");
}

void loop() {

  if (Serial.available()) {

    String data = Serial.readStringUntil('\n');

    data.trim();

    int fingers[5];

    int index = 0;

    char buffer[50];

    data.toCharArray(buffer, 50);

    char *token = strtok(buffer, ",");

    while (token != NULL && index < 5) {

      fingers[index] = atoi(token);

      index++;

      token = strtok(NULL, ",");
    }

    // Make sure we received 5 values
    if (index == 5) {

      int red = 0;
      int green = 0;
      int blue = 0;

      // ==================================
      // THUMB  RED
      // ==================================

      if (fingers[0] == 1) {

        red = 255;
      }

      // ==================================
      // INDEX  GREEN
      // ==================================

      if (fingers[1] == 1) {

        green = 255;
      }

      // ==================================
      // MIDDLE  BLUE
      // ==================================

      if (fingers[2] == 1) {

        blue = 255;
      }

      // ==================================
      // RING  YELLOW
      // ==================================

      if (fingers[3] == 1) {

        red = 255;
        green = 255;
      }

      // ==================================
      // PINKY  PURPLE
      // ==================================

      if (fingers[4] == 1) {

        red = 255;
        blue = 255;
      }

      // ==================================
      // OUTPUT RGB
      // ==================================

      setColor(
        red,
        green,
        blue
      );

      // ==================================
      // DEBUG
      // ==================================

      Serial.print("Fingers: ");

      for (int i = 0; i < 5; i++) {

        Serial.print(fingers[i]);

        if (i < 4) {
          Serial.print(",");
        }
      }

      Serial.print(" | RGB: ");

      Serial.print(red);
      Serial.print(",");
      Serial.print(green);
      Serial.print(",");
      Serial.println(blue);
    }
  }
}

// ==========================================
// RGB LED FUNCTION
// ==========================================

void setColor(
  int red,
  int green,
  int blue
) {

  analogWrite(
    RED_PIN,
    red
  );

  analogWrite(
    GREEN_PIN,
    green
  );

  analogWrite(
    BLUE_PIN,
    blue
  );
}

python code

Python
sprite = Sprite('Tobi')
import cv2
import mediapipe as mp
import serial
import time
import math

# ==========================================
# SETTINGS
# ==========================================

SERIAL_PORT = "COM4"
BAUD_RATE = 9600
CAMERA_INDEX = 0

# ==========================================
# CONNECT TO ARDUINO
# ==========================================

print("Opening Arduino...")

arduino = serial.Serial(
    SERIAL_PORT,
    BAUD_RATE,
    timeout=1
)

time.sleep(2)

print("Arduino connected successfully!")

# ==========================================
# MEDIAPIPE
# ==========================================

mp_hands = mp.solutions.hands
mp_draw = mp.solutions.drawing_utils

hands = mp_hands.Hands(
    static_image_mode=False,
    max_num_hands=1,
    min_detection_confidence=0.7,
    min_tracking_confidence=0.7
)

# ==========================================
# CAMERA
# ==========================================

cap = cv2.VideoCapture(CAMERA_INDEX)

if not cap.isOpened():
    print("ERROR: Camera could not be opened.")
    arduino.close()
    exit()

# ==========================================
# VARIABLES
# ==========================================

last_command = ""

# ==========================================
# FINGER DETECTION FUNCTIONS
# ==========================================

def distance(p1, p2):

    return math.sqrt(
        (p1.x - p2.x) ** 2 +
        (p1.y - p2.y) ** 2 +
        (p1.z - p2.z) ** 2
    )


def finger_is_up(landmarks, tip, pip):

    tip_distance = distance(
        landmarks[tip],
        landmarks[0]
    )

    pip_distance = distance(
        landmarks[pip],
        landmarks[0]
    )

    return tip_distance > pip_distance


def thumb_is_up(landmarks):

    thumb_tip_distance = distance(
        landmarks[4],
        landmarks[0]
    )

    thumb_ip_distance = distance(
        landmarks[3],
        landmarks[0]
    )

    return thumb_tip_distance > thumb_ip_distance


# ==========================================
# MAIN LOOP
# ==========================================

while True:

    success, frame = cap.read()

    if not success:
        print("Could not read camera.")
        break

    # --------------------------------------
    # Mirror image
    # --------------------------------------

    frame = cv2.flip(frame, 1)

    # --------------------------------------
    # Convert BGR → RGB
    # --------------------------------------

    rgb_frame = cv2.cvtColor(
        frame,
        cv2.COLOR_BGR2RGB
    )

    # --------------------------------------
    # MediaPipe processing
    # --------------------------------------

    results = hands.process(rgb_frame)

    # ======================================
    # DEFAULT VALUES
    # ======================================

    thumb = 0
    index = 0
    middle = 0
    ring = 0
    pinky = 0

    # ======================================
    # HAND DETECTED
    # ======================================

    if results.multi_hand_landmarks:

        for hand_landmarks, handedness in zip(
            results.multi_hand_landmarks,
            results.multi_handedness
        ):

            # --------------------------------------
            # RIGHT HAND ONLY
            # --------------------------------------

            hand_label = handedness.classification[0].label

            if hand_label != "Right":
                continue

            # --------------------------------------
            # DRAW HAND LANDMARKS
            # --------------------------------------

            mp_draw.draw_landmarks(
                frame,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS
            )

            landmarks = hand_landmarks.landmark

            # ======================================
            # DETECT FIVE FINGERS
            # ======================================

            # Thumb
            thumb = int(
                thumb_is_up(landmarks)
            )

            # Index
            index = int(
                finger_is_up(
                    landmarks,
                    8,
                    6
                )
            )

            # Middle
            middle = int(
                finger_is_up(
                    landmarks,
                    12,
                    10
                )
            )

            # Ring
            ring = int(
                finger_is_up(
                    landmarks,
                    16,
                    14
                )
            )

            # Pinky
            pinky = int(
                finger_is_up(
                    landmarks,
                    20,
                    18
                )
            )

            # ======================================
            # CREATE COMMAND
            # ======================================

            command = (
                str(thumb) + "," +
                str(index) + "," +
                str(middle) + "," +
                str(ring) + "," +
                str(pinky) + "\n"
            )

            # ======================================
            # SEND TO ARDUINO
            # ======================================

            if command != last_command:

                arduino.write(
                    command.encode()
                )

                print(
                    "Finger states:",
                    command.strip()
                )

                last_command = command

            # ======================================
            # DISPLAY HAND
            # ======================================

            cv2.putText(
                frame,
                "Right Hand",
                (20, 40),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.8,
                (0, 255, 0),
                2
            )

            # ======================================
            # DISPLAY FINGER STATUS
            # ======================================

            cv2.putText(
                frame,
                "Thumb: " +
                ("UP" if thumb else "DOWN"),
                (20, 80),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.7,
                (0, 255, 0) if thumb else (0, 0, 255),
                2
            )

            cv2.putText(
                frame,
                "Index: " +
                ("UP" if index else "DOWN"),
                (20, 115),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.7,
                (0, 255, 0) if index else (0, 0, 255),
                2
            )

            cv2.putText(
                frame,
                "Middle: " +
                ("UP" if middle else "DOWN"),
                (20, 150),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.7,
                (0, 255, 0) if middle else (0, 0, 255),
                2
            )

            cv2.putText(
                frame,
                "Ring: " +
                ("UP" if ring else "DOWN"),
                (20, 185),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.7,
                (0, 255, 0) if ring else (0, 0, 255),
                2
            )

            cv2.putText(
                frame,
                "Pinky: " +
                ("UP" if pinky else "DOWN"),
                (20, 220),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.7,
                (0, 255, 0) if pinky else (0, 0, 255),
                2
            )

            # ======================================
            # SHOW COMMAND
            # ======================================

            cv2.putText(
                frame,
                "Command: " +
                command.strip(),
                (20, 260),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.65,
                (255, 255, 255),
                2
            )

            break

    # ==========================================
    # NO HAND DETECTED
    # ==========================================

    else:

        thumb = 0
        index = 0
        middle = 0
        ring = 0
        pinky = 0

        command = "0,0,0,0,0\n"

        if command != last_command:

            arduino.write(
                command.encode()
            )

            print(
                "No Right Hand Detected"
            )

            last_command = command

        cv2.putText(
            frame,
            "No Right Hand Detected",
            (20, 40),
            cv2.FONT_HERSHEY_SIMPLEX,
            0.8,
            (0, 0, 255),
            2
        )

    # ==========================================
    # SHOW CAMERA
    # ==========================================

    cv2.imshow(
        "AI Hand Gesture - RGB LED Control",
        frame
    )

    # Press Q to quit
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break


# ==========================================
# CLEANUP
# ==========================================

print("Stopping...")

arduino.write(
    b"0,0,0,0,0\n"
)

time.sleep(0.2)

cap.release()

cv2.destroyAllWindows()

hands.close()

arduino.close()

print("Finished!")

Credits

Rayan Kiwan
2 projects • 0 followers

Comments

Related channels and tags
  • AI & Machine Learning
  • AI & Machine Learning
  • Arduino
  • Python on Hardware
  • Python on Hardware
  • STEMpedia