Multi-model Hand Gesture Pipeline on Dragonwing IQ-9075 EVK

import os
import cv2
import numpy as np
import time
import ai_edge_litert.interpreter as litert

# ==============================================================================
# 1. VERIFIED HARDWARE AND LOCAL MODEL PATH CONFIGURATION
# ==============================================================================
MODEL_DIR = os.path.expanduser("~/models/mediapipe_hand_gesture-tflite-w8a8")
PALM_DETECTOR_PATH = os.path.join(MODEL_DIR, "palm_detector.tflite")
LANDMARK_DETECTOR_PATH = os.path.join(MODEL_DIR, "hand_landmark_detector.tflite")
CLASSIFIER_PATH = os.path.join(MODEL_DIR, "canned_gesture_classifier.tflite")
QNN_DELEGATE_PATH = "/usr/lib/libQnnTFLiteDelegate.so"

# Updated to match the exact labels from the reference source
GESTURE_LABELS = [
    "None", "Closed_Fist", "Open_Palm", "Pointing_Up",
    "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou",
]

HAND_CONNECTIONS = [
    (0, 1), (1, 2), (2, 3), (3, 4),        
    (0, 5), (5, 6), (6, 7), (7, 8),        
    (5, 9), (9, 10), (10, 11), (11, 12),   
    (9, 13), (13, 14), (14, 15), (15, 16), 
    (13, 17), (0, 17), (17, 18), (18, 19), (19, 20) 
]

# Wrist plus the four finger bases for stable normalisation
CENTER_IDX = np.array([0, 1, 5, 9, 13, 17])

# ==============================================================================
# 2. UTILITIES (SMOOTHING, QUANTIZATION & NORMALIZATION)
# ==============================================================================
class BoxSmoother:
    def __init__(self, alpha=0.35):
        self.alpha = alpha
        self.prev_box = None

    def smooth(self, current_box):
        if self.prev_box is None:
            self.prev_box = np.array(current_box, dtype=np.float32)
            return current_box
        
        self.prev_box = self.alpha * np.array(current_box, dtype=np.float32) + (1.0 - self.alpha) * self.prev_box
        return [int(v) for v in self.prev_box]

def set_quantized_input(interpreter, input_index, float_data):
    input_details = interpreter.get_input_details()[input_index]
    dtype = input_details['dtype']
    
    if dtype == np.float32:
        interpreter.set_tensor(input_details['index'], float_data.astype(np.float32))
        return

    quantization = input_details.get('quantization_parameters', None)
    if quantization is not None and len(quantization['scales']) > 0:
        scale = quantization['scales'][0]
        zero_point = quantization['zero_points'][0]
        quant_data = np.round(float_data / scale) + zero_point
        
        if dtype == np.uint8:
            quant_data = np.clip(quant_data, 0, 255).astype(np.uint8)
        else:
            quant_data = np.clip(quant_data, -128, 127).astype(np.int8)
            
        interpreter.set_tensor(input_details['index'], quant_data)
    else:
        interpreter.set_tensor(input_details['index'], float_data.astype(dtype))

def get_dequantized_output(interpreter, output_index):
    output_details = interpreter.get_output_details()[output_index]
    raw_data = interpreter.get_tensor(output_details['index'])
    
    if output_details['dtype'] == np.float32:
        return raw_data

    quantization = output_details.get('quantization_parameters', None)
    if quantization is not None and len(quantization['scales']) > 0:
        scale = quantization['scales'][0]
        zero_point = quantization['zero_points'][0]
        return (raw_data.astype(np.float32) - zero_point) * scale
    return raw_data.astype(np.float32)

def preprocess_hand_x64(pts, handedness, mirror=False):
    """Normalise 21 landmarks into the classifier's 64-value input vector."""
    pts = pts.astype(np.float32).copy()
    if mirror:
        pts[:, 0] *= -1.0
        handedness = 1.0 - handedness
    center = pts[CENTER_IDX].mean(axis=0)
    normed = pts - center
    range_x = normed[:, 0].max() - normed[:, 0].min()
    range_y = normed[:, 1].max() - normed[:, 1].min()
    scale = max(range_x, range_y) + 1e-5
    flat = (normed / scale).reshape(-1)                            # 63
    return np.concatenate([flat, [handedness]]).astype(np.float32) # 64

# ==============================================================================
# 3. INITIALIZE HARDWARE ACCELERATION (QNN DELEGATE)
# ==============================================================================
print("Loading QNN Hexagon Delegate...")
qnn_delegate = litert.load_delegate(QNN_DELEGATE_PATH, options={"backend_type": "htp"})

print("Initializing Interpreters on NPU...")
palm_interp = litert.Interpreter(model_path=PALM_DETECTOR_PATH, experimental_delegates=[qnn_delegate])
palm_interp.allocate_tensors()

land_interp = litert.Interpreter(model_path=LANDMARK_DETECTOR_PATH, experimental_delegates=[qnn_delegate])
land_interp.allocate_tensors()

class_interp = litert.Interpreter(model_path=CLASSIFIER_PATH, experimental_delegates=[qnn_delegate])
class_interp.allocate_tensors()

print(" Pipeline successfully built inside the Hexagon memory cache!")

# ==============================================================================
# 4. OPENCV LIVE CAMERA CAPTURE LOOP
# ==============================================================================
cap = cv2.VideoCapture(0)
box_filter = BoxSmoother(alpha=0.35)

if not cap.isOpened():
    print(" Error: Could not access the video device file stream.")
    exit(1)

print("Starting live stream tracking loop. Press 'q' inside the window to exit.")

while True:
    ret, frame = cap.read()
    if not ret: continue

    curr_time = time.time()

    frame = cv2.flip(frame, 1)
    orig_h, orig_w, _ = frame.shape

    # --- STAGE 1: PALM DETECTION ---
    palm_input_frame = cv2.resize(frame, (256, 256))
    palm_input_frame = cv2.cvtColor(palm_input_frame, cv2.COLOR_BGR2RGB)
    palm_normalized = palm_input_frame.astype(np.float32) / 255.0
    palm_input_tensor = np.expand_dims(palm_normalized, axis=0)

    set_quantized_input(palm_interp, 0, palm_input_tensor)
    palm_interp.invoke()

    raw_boxes = get_dequantized_output(palm_interp, 0)[0]
    raw_scores = get_dequantized_output(palm_interp, 1)[0]

    max_idx = np.argmax(raw_scores)
    confidence = raw_scores[max_idx]

    if confidence > 0.65:
        if len(raw_boxes[max_idx]) >= 4:
            xmin, ymin, xmax, ymax = raw_boxes[max_idx][:4]
            
            if np.max(raw_boxes[max_idx][:4]) > 1.0:
                xmin, ymin, xmax, ymax = xmin/256.0, ymin/256.0, xmax/256.0, ymax/256.0
                
            px_xmin = int(xmin * orig_w)
            px_ymin = int(ymin * orig_h)
            px_xmax = int(xmax * orig_w)
            px_ymax = int(ymax * orig_h)

            palm_cx = (px_xmin + px_xmax) // 2
            palm_cy = (px_ymin + px_ymax) // 2
            palm_w = px_xmax - px_xmin
            palm_h = px_ymax - px_ymin

            hand_cy = int(palm_cy - (palm_h * 0.4))
            side_px = int(max(palm_w, palm_h) * 2.1)

            raw_x1 = int(palm_cx - side_px // 2)
            raw_y1 = int(hand_cy - side_px // 2)
            raw_x2 = int(palm_cx + side_px // 2)
            raw_y2 = int(hand_cy + side_px // 2)

            x1, y1, x2, y2 = box_filter.smooth([raw_x1, raw_y1, raw_x2, raw_y2])
            x1, y1 = max(0, x1), max(0, y1)
            x2, y2 = min(orig_w, x2), min(orig_h, y2)

            if (x2 - x1) > 20 and (y2 - y1) > 20:
                cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)

                # --- STAGE 2: LANDMARK EXTRACTION ---
                hand_roi = frame[y1:y2, x1:x2]
                
                if hand_roi.size > 0:
                    hand_input = cv2.resize(hand_roi, (224, 224))
                    hand_input = cv2.cvtColor(hand_input, cv2.COLOR_BGR2RGB)
                    hand_normalized = hand_input.astype(np.float32) / 255.0
                    hand_input_tensor = np.expand_dims(hand_normalized, axis=0)

                    set_quantized_input(land_interp, 0, hand_input_tensor)
                    land_interp.invoke()

                    # Extract raw model crop landmarks for the classifier
                    landmarks = get_dequantized_output(land_interp, 0)[0]
                    landmarks_reshaped = landmarks.reshape(-1, 3)[:21]

                    # Dynamically extract handedness (lr) from the landmark model outputs
                    handedness = 0.5
                    for i, d in enumerate(land_interp.get_output_details()):
                        if "lr" in d["name"].lower() or "handedness" in d["name"].lower():
                            handedness = float(get_dequantized_output(land_interp, i).ravel()[0])
                            break

                    is_pixel_scale = np.max(landmarks_reshaped[:, :2]) > 2.0

                    landmark_points = []
                    for lm in landmarks_reshaped:
                        norm_x = lm[0] / 224.0 if is_pixel_scale else lm[0]
                        norm_y = lm[1] / 224.0 if is_pixel_scale else lm[1]
                        
                        lm_x = int(x1 + (norm_x * (x2 - x1)))
                        lm_y = int(y1 + (norm_y * (y2 - y1)))
                        landmark_points.append((lm_x, lm_y))
                        
                    for connection in HAND_CONNECTIONS:
                        start_idx, end_idx = connection
                        if start_idx < len(landmark_points) and end_idx < len(landmark_points):
                            cv2.line(frame, landmark_points[start_idx], landmark_points[end_idx], (0, 255, 255), 2)

                    for pt in landmark_points:
                        cv2.circle(frame, pt, 5, (0, 0, 255), -1)

                    # --- STAGE 3: OFFICIAL MEDIA-PIPE GESTURE CLASSIFIER ---
                    # The classifier expects BOTH a standard and mirrored 64-value hand array
                    feed = {
                        "hand": preprocess_hand_x64(landmarks_reshaped, handedness, mirror=False),
                        "mirrored_hand": preprocess_hand_x64(landmarks_reshaped, handedness, mirror=True),
                    }
                    
                    # Feed the correct normalized array into the corresponding input tensor
                    for i, d in enumerate(class_interp.get_input_details()):
                        key = "mirrored_hand" if "mirror" in d["name"] else "hand"
                        x = np.expand_dims(feed[key], axis=0)
                        set_quantized_input(class_interp, i, x)

                    class_interp.invoke()

                    gesture_probs = get_dequantized_output(class_interp, 0)[0]
                    gesture_id = np.argmax(gesture_probs)
                    gesture_confidence = gesture_probs[gesture_id]
                    
                    gesture_name = GESTURE_LABELS[gesture_id] if gesture_id < len(GESTURE_LABELS) else "Unknown"

                    # Only show if not "None" and confidence is reasonable
                    if gesture_name != "None" and gesture_confidence > 0.4:
                        cv2.putText(frame, f"Gesture: {gesture_name} ({gesture_confidence*100:.1f}%)", 
                                    (x1, max(30, y1 - 10)), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (150, 0, 0), 2)
    else:
        box_filter.prev_box = None

    inference_time = (time.time() - curr_time) * 1000

    cv2.putText(frame, f"NPU: {inference_time:.1f} ms", (15, 40), 
                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)

    cv2.imshow("Qualcomm IQ-9075 NPU Hand Gesture Tracking", frame)
    
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()
print("Cleaned up camera pipeline assets. Terminating successfully.")