Jonathan Xu
Published © GPL3+

Walabot Touchpad

Turn any tabletop surface into a hidden touchpad for your computer

IntermediateFull instructions provided8 hours2,949
Walabot Touchpad

Things used in this project

Hardware components

Walabot Developer Pack
Walabot Developer Pack
×1
Raspberry Pi 3 Model B
Raspberry Pi 3 Model B
Any raspberry pi works but newer models are faster and can give more frequent sensor pings
×1
Power Bank (Generic)
This serves as a remote power supply to allow for the walabot touchpad to be powered wherever
×1
USB-A to Micro-USB Cable
USB-A to Micro-USB Cable
×2

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)
Used to create a custom compartment for the walabot touchpad. It's optional if you want to build your own.

Story

Read more

Code

client.py

Python
The client's role is to detect targets with the walabot and to send the data to the server via a TCP socket. To run this properly, replace the ip address on line 40 with your pc's ip. This can be found by typing ipconfig in cmd on windows. The ip for the server may change periodically.
# Dependencies
from __future__ import print_function # WalabotAPI works on both Python 2 an 3.
from sys import platform
from os import system
from imp import load_source
from os.path import join
import socket
import pickle


def PrintSensorTargets(targets):
    system('cls' if platform == 'win32' else 'clear')
    targetPos = [0, 0]
    detected = 0;
    if targets:
        for i, target in enumerate(targets):
            info = 'Target #{}:\nx: {}\ny: {}\nz: {}\namplitude: {}\n'.format(
                i + 1, target.xPosCm, target.yPosCm, target.zPosCm,
                target.amplitude)
            print(info + "\r")
        if len(targets) == 1:  # If only one target detected
            detected = 1
            targetPos = [target.xPosCm, target.yPosCm]
            print("One target detected")
        elif len(targets) == 2:  # Left click
            detected = 2
            print("Two targets detected")
        elif len(targets) == 3:  # Right click
            detected = 3
            print("The targets detected")
    else:
        print("Target not detected" + "\r")
    message = [detected, targetPos]
    client.send("target:".encode())
    client.recv(1024).decode()
    client.send(pickle.dumps(message))


# Connecting to the server
ip = "192.168.0.20"
print("Connecting to " + ip)
client = socket.socket()
client.connect((ip, 1234))
print("Pinging server:")
client.send("ping".encode())
print("Server says: " + client.recv(1024).decode())

# Finding walabot api file and initializing walabot
if platform == 'win32':
    modulePath = join('C:/', 'Program Files', 'Walabot', 'WalabotSDK',
                      'python', 'WalabotAPI.py')
elif platform.startswith('linux'):
    modulePath = join('/usr', 'share', 'walabot', 'python', 'WalabotAPI.py')
wlbt = load_source('WalabotAPI', modulePath)
print("Initializing walabot")
wlbt.Init()

#Setting arena parameters
minInCm, maxInCm, resInCm = 2, 10, 3  # r parameters
minIndegrees, maxIndegrees, resIndegrees = -15, 15, 5  # theta parameters
minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees = -60, 60, 5  # phi parameters
# Set MTI mode
mtiMode = False
# Configure Walabot database install location (for windows)
wlbt.SetSettingsFolder()
# 1) Connect : Establish communication with walabot.
wlbt.ConnectAny()
# 2) Configure: Set scan profile and arena
# Set Profile - to Sensor.
wlbt.SetProfile(wlbt.PROF_SENSOR)
# Setup arena - specify it by Cartesian coordinates.
wlbt.SetArenaR(minInCm, maxInCm, resInCm)
# Sets polar range and resolution of arena (parameters in degrees).
wlbt.SetArenaTheta(minIndegrees, maxIndegrees, resIndegrees)
# Sets azimuth range and resolution of arena.(parameters in degrees).
wlbt.SetArenaPhi(minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees)
# Moving Target Identification: standard dynamic-imaging filter
filterType = wlbt.FILTER_TYPE_MTI if mtiMode else wlbt.FILTER_TYPE_NONE
wlbt.SetDynamicImageFilter(filterType)
# 3) Start: Start the system in preparation for scanning.
wlbt.Start()
# calibrates scanning to ignore or reduce the signals
if not mtiMode:  # if MTI mode is not set - start calibrartion
    # calibrates scanning to ignore or reduce the signals
    wlbt.StartCalibration()
    while wlbt.GetStatus()[0] == wlbt.STATUS_CALIBRATING:
        wlbt.Trigger()
print("Walabot started and calibrated")
client.send("Walabot started and calibrated".encode())
print("Sending target data:")
while True:
    appStatus, calibrationProcess = wlbt.GetStatus()
    # Triggers sensor for target detection
    wlbt.Trigger()
    # Get targets
    targets = wlbt.GetSensorTargets()
    rasterImage, _, _, sliceDepth, power = wlbt.GetRawImageSlice()
    PrintSensorTargets(targets)
# 7) Stop and Disconnect.
wlbt.Stop()
wlbt.Disconnect()
client.send("Terminating".encode)
client.stop()
print('Terminate successfully')

server.py

Python
The server's role is to listen for data coming in from the rpi and convert the target info into mouse actions. It also does an automatic calibration procedure at the start.
import pyautogui
import socket
import pickle
import time

def moveMouse(sensorArena, sensorTarget):
    sensorWidth = sensorArena[0][1] - sensorArena[0][0] #maxX - minX
    sensorHeight = sensorArena[1][1] - sensorArena[1][0] #maxY - minY
    xRatio = screenWidth / sensorWidth
    yRatio = screenHeight / sensorHeight

    xPos = (sensorTarget[1][1] - sensorArena[0][0]) * xRatio #add minx/miny to set edge point as 0
    yPos = screenHeight -(sensorTarget[1][0] - sensorArena[1][0]) * yRatio
    pyautogui.moveTo(xPos, yPos)
    print("Moving mouse to: " + str(xPos) + " " + str(yPos))


def calibrate(mode):
    min = float("inf")
    max = float("-inf")
    success = False;
    # Setting visible arena in cm
    while True:
        data = client.recv(1024).decode()
        client.send("listening for target".encode())
        target = pickle.loads(client.recv(1024))
        if target[0] == 1:  # If finger detected
            while True:
                data = client.recv(1024).decode()
                client.send("listening for target".encode())
                target = pickle.loads(client.recv(1024))
                print("min: " + str(min) + " max: " + str(max))
                if target[1][1-mode] < min:
                    min = target[1][1-mode]
                if target[1][1-mode] > max:
                    max = target[1][1-mode]
                if target[0] == 0:  # When finger leaves area
                    break
            break
    print("range:" + str((max-min)))
    if mode == 0 and max-min > 8 and max-min < 10:
        success = True
    if mode == 1 and max-min > 4 and max-min < 6:
        success = True
    return [success, min, max]

# Logging the screen size
screenWidth, screenHeight = pyautogui.size()
pyautogui.FAILSAFE = False

# Walabot related variable declarations
arena = []
target = []

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ip = socket.gethostbyname(socket.gethostname())
port = 1234
server.bind((ip, port))
server.listen(1)
print("Started listening on "+ip+":"+str(port))
client, addr = server.accept()
print("Got a connection from " + str(addr[0]) + ":" + str(addr[1]))

xCalibrate = yCalibrate = False

while True:
    data = client.recv(1024).decode()
    if data == "disconnect":
        print("Disconnecting")
        client.send("Closing client".encode())
        client.close()
        break
    elif data == "ping":
        print("Client ping")
        client.send("ping".encode())
    elif data == "target:":
        client.send("listening for target".encode())
        target = pickle.loads(client.recv(1024))
        if not xCalibrate or not yCalibrate:
            print("Calibrating visible arena dimensions")
            while not xCalibrate:
                print("Slowly slide one finger from the far left to the far right.")
                calibration = calibrate(0)
                if calibration[0]:  # If calibration success
                    minX = calibration[1]
                    maxX = calibration[2]
                    xCalibrate = True
                    print("Success. Lift your finger.")
                else:
                    print("Calibration failure. Retrying..")
            time.sleep(3)
            while not yCalibrate:
                print("Slowly slide one finger form the far bottom to the far top.")
                calibration = calibrate(1)
                if calibration[0]:  # If calibration success
                    minY = calibration[1]
                    maxY = calibration[2]
                    yCalibrate = True
                    print("Success.")
                else:
                    print("Calibration failure. Retrying..")
        arena = [
            [minX, maxX, 3],
            [minY, maxY, 3],
        ]
        '''
        arena = [
            [-7, 3.5, 3],
            [-2.5, 2.5, 3],
        ]
        # minx -7, 3.5
        # y -2.5, 2.5
        '''
        print("Arena calibrated:")
        print(arena)
    else:
        print("Target info:")
        if target[0] == 1:  # If one finger detected
            print("x: " + str(target[1][0]) + " y: " + str(target[1][1]))
            moveMouse(arena, target)
        elif target[0] == 2:  # If two fingers detected
            print("Left click")
            pyautogui.click(button='left')
        elif target[0] == 3:  # If three fingers detected
            print("Right click")
            pyautogui.click(button='right')
        else:
            print("Target not detected")

btClient.py

Python
This is the client code, similar to client.py but serial communication is done on Bluetooth. To use this properly, replace the host variable on line 39 with your computer's Bluetooth address. This can be found by typing ipconfig/all in cmd on Windows.
# Dependencies
from __future__ import print_function # WalabotAPI works on both Python 2 an 3.
from sys import platform
from os import system
from imp import load_source
from os.path import join
from bluetooth import *


def PrintSensorTargets(targets):
    system('cls' if platform == 'win32' else 'clear')
    targetPos = [0, 0]
    detected = 0;
    if targets:
        for i, target in enumerate(targets):
            info = 'Target #{}:\nx: {}\ny: {}\nz: {}\namplitude: {}\n'.format(
                i + 1, target.xPosCm, target.yPosCm, target.zPosCm,
                target.amplitude)
            print(info + "\r")
        if len(targets) == 1:  # If only one target detected
            detected = 1
            targetPos = [target.xPosCm, target.yPosCm]
            print("One target detected")
        elif len(targets) == 2:  # Left click
            detected = 2
            print("Two targets detected")
        elif len(targets) == 3:  # Right click
            detected = 3
            print("The targets detected")
    else:
        print("Target not detected" + "\r")
    message = [detected, targetPos]
    client.send("target:".encode())
    client.recv(1024).decode()
    client.send(pickle.dumps(message))


# Connecting to the server
host = "48:5A:B6:AE:A4:7D"
print("Connecting to " + host)
client = BluetoothSocket(RFCOMM)
client.connect((host, 3))
print("Pinging server:")
client.send("ping".encode())
print("Server says: " + client.recv(1024).decode())

# Finding walabot api file and initializing walabot
if platform == 'win32':
    modulePath = join('C:/', 'Program Files', 'Walabot', 'WalabotSDK',
                      'python', 'WalabotAPI.py')
elif platform.startswith('linux'):
    modulePath = join('/usr', 'share', 'walabot', 'python', 'WalabotAPI.py')
wlbt = load_source('WalabotAPI', modulePath)
print("Initializing walabot")
wlbt.Init()

#Setting arena parameters
minInCm, maxInCm, resInCm = 2, 10, 3  # r parameters
minIndegrees, maxIndegrees, resIndegrees = -15, 15, 5  # theta parameters
minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees = -60, 60, 5  # phi parameters
# Set MTI mode
mtiMode = False
# Configure Walabot database install location (for windows)
wlbt.SetSettingsFolder()
# 1) Connect : Establish communication with walabot.
wlbt.ConnectAny()
# 2) Configure: Set scan profile and arena
# Set Profile - to Sensor.
wlbt.SetProfile(wlbt.PROF_SENSOR)
# Setup arena - specify it by Cartesian coordinates.
wlbt.SetArenaR(minInCm, maxInCm, resInCm)
# Sets polar range and resolution of arena (parameters in degrees).
wlbt.SetArenaTheta(minIndegrees, maxIndegrees, resIndegrees)
# Sets azimuth range and resolution of arena.(parameters in degrees).
wlbt.SetArenaPhi(minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees)
# Moving Target Identification: standard dynamic-imaging filter
filterType = wlbt.FILTER_TYPE_MTI if mtiMode else wlbt.FILTER_TYPE_NONE
wlbt.SetDynamicImageFilter(filterType)
# 3) Start: Start the system in preparation for scanning.
wlbt.Start()
# calibrates scanning to ignore or reduce the signals
if not mtiMode:  # if MTI mode is not set - start calibrartion
    # calibrates scanning to ignore or reduce the signals
    wlbt.StartCalibration()
    while wlbt.GetStatus()[0] == wlbt.STATUS_CALIBRATING:
        wlbt.Trigger()
print("Walabot started and calibrated")
client.send("Walabot started and calibrated".encode())
print("Sending target data:")
while True:
    appStatus, calibrationProcess = wlbt.GetStatus()
    # Triggers sensor for target detection
    wlbt.Trigger()
    # Get targets
    targets = wlbt.GetSensorTargets()
    rasterImage, _, _, sliceDepth, power = wlbt.GetRawImageSlice()
    PrintSensorTargets(targets)
# 7) Stop and Disconnect.
wlbt.Stop()
wlbt.Disconnect()
client.send("Terminating".encode)
client.stop()
print('Terminate successfully')

btHost.py

Python
This is the host code for Bluetooth serial communication. It does the same thing as server.py but through Bluetooth.
import pyautogui
from bluetooth import *
import time


def moveMouse(sensorArena, sensorTarget):
    sensorWidth = sensorArena[0][1] - sensorArena[0][0] #maxX - minX
    sensorHeight = sensorArena[1][1] - sensorArena[1][0] #maxY - minY
    xRatio = screenWidth / sensorWidth
    yRatio = screenHeight / sensorHeight

    xPos = (sensorTarget[1][1] - sensorArena[0][0]) * xRatio #add minx/miny to set edge point as 0
    yPos = screenHeight -(sensorTarget[1][0] - sensorArena[1][0]) * yRatio
    pyautogui.moveTo(xPos, yPos)
    print("Moving mouse to: " + str(xPos) + " " + str(yPos))


def calibrate(mode):
    min = float("inf")
    max = float("-inf")
    success = False;
    # Setting visible arena in cm
    while True:
        data = client.recv(1024).decode()
        client.send("listening for target".encode())
        target = pickle.loads(client.recv(1024))
        if target[0] == 1:  # If finger detected
            while True:
                data = client.recv(1024).decode()
                client.send("listening for target".encode())
                target = pickle.loads(client.recv(1024))
                print("min: " + str(min) + " max: " + str(max))
                if target[1][1-mode] < min:
                    min = target[1][1-mode]
                if target[1][1-mode] > max:
                    max = target[1][1-mode]
                if target[0] == 0:  # When finger leaves area
                    break
            break
    print("range:" + str((max-min)))
    if mode == 0 and max-min > 8 and max-min < 10:
        success = True
    if mode == 1 and max-min > 4 and max-min < 6:
        success = True
    return [success, min, max]

# Logging the screen size
screenWidth, screenHeight = pyautogui.size()
pyautogui.FAILSAFE = False

# Walabot related variable declarations
arena = []
target = []

server = BluetoothSocket(RFCOMM)
host = ""
#48-5A-B6-AE-A4-7D
port = 3
server.bind((host, port))
server.listen(1)
print("Started listening on "+host+":"+str(port))
client, addr = server.accept()
print("Got a connection from " + str(addr[0]) + ":" + str(addr[1]))

xCalibrate = yCalibrate = False

while True:
    data = client.recv(1024).decode()
    if data == "disconnect":
        print("Disconnecting")
        client.send("Closing client".encode())
        client.close()
        break
    elif data == "ping":
        print("Client ping")
        client.send("ping".encode())
    elif data == "target:":
        client.send("listening for target".encode())
        target = pickle.loads(client.recv(1024))
        if not xCalibrate or not yCalibrate:
            print("Calibrating visible arena dimensions")
            while not xCalibrate:
                print("Slowly slide one finger from the far left to the far right.")
                calibration = calibrate(0)
                if calibration[0]:  # If calibration success
                    minX = calibration[1]
                    maxX = calibration[2]
                    xCalibrate = True
                    print("Success. Lift your finger.")
                else:
                    print("Calibration failure. Retrying..")
            time.sleep(3)
            while not yCalibrate:
                print("Slowly slide one finger form the far bottom to the far top.")
                calibration = calibrate(1)
                if calibration[0]:  # If calibration success
                    minY = calibration[1]
                    maxY = calibration[2]
                    yCalibrate = True
                    print("Success.")
                else:
                    print("Calibration failure. Retrying..")
        arena = [
            [minX, maxX, 3],
            [minY, maxY, 3],
        ]
        print("Arena calibrated:")
        print(arena)
    else:
        print("Target info:")
        if target[0] == 1:  # If one finger detected
            print("x: " + str(target[1][0]) + " y: " + str(target[1][1]))
            moveMouse(arena, target)
        elif target[0] == 2:  # If two fingers detected
            print("Left click")
            pyautogui.click(button='left')
        elif target[0] == 3:  # If three fingers detected
            print("Right click")
            pyautogui.click(button='right')
        else:
            print("Target not detected")

Credits

Jonathan Xu

Jonathan Xu

7 projects • 16 followers
Hardware enthusiast who's good at shorting out Arduinos. @ University of Waterloo Software Engineering 2024

Comments