Leon Chu
Published © CC BY

Interactive Visual Motion Synth

Create visual sounds canvas with your body motion. Use voice command to change sound, patterns and set position markers.

IntermediateFull instructions provided4 hours770

Things used in this project

Hardware components

Walabot
Walabot
×1
Echo Dot
Amazon Alexa Echo Dot
×1

Software apps and online services

Unity
Unity
ngrok
Alexa Skills Kit
Amazon Alexa Alexa Skills Kit

Story

Read more

Schematics

Data flow

Dataflow 01 orxnyrdows

Code

SynthHub

Python
#!/usr/bin/env python
""" 
Set up message hub to pass Walabot coordinate data to
connected client.

Process Alexa Skill requests and relay commands to connected client

Based on socket IO chat room example 
"""
from threading import Lock
from flask import Flask, render_template, session, request
from flask_ask import Ask, statement, question, session
from flask_socketio import SocketIO, emit, join_room, leave_room, \
    close_room, rooms, disconnect
from WalabotService import *
import random
  
POLL_INTERVAL_SEC = 0.2
DEBUG =False

# Set this variable to "threading", "eventlet" or "gevent" to test the
# different async modes, or leave it set to None for the application to choose
# the best option based on installed packages.
async_mode = None

app = Flask(__name__)

ask = Ask(app, "/")
#logging.getLogger("flask_ask").setLevel(logging.DEBUG)

# Initialize Walabot libraries
global walabotStarted
global walabot
walabot = WalabotService()
walabotStarted = False

app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
thread_lock = Lock()


def background_thread():
    """Runs in the background and relay Walabot detected movement coordinates to connected client """
    count = 0
    while True:
        if DEBUG:
            count +=1
            walabotStarted = False
        if count == 0:  # start up walabot on first socket connection
              walabotStarted = walabot.StartService()
              count += 1
        socketio.sleep(POLL_INTERVAL_SEC)
        if walabotStarted :
            x,y,z,a = walabot.GetTargetPosition()
            socketio.emit('coord',
                          {'x': x, 'y': y, 'z':z, 'a': a} )
        elif DEBUG == True:
            """ Send Random Sample coordinates for testing """
            socketio.emit('coord',  
              {'x': random.uniform(-40,40), 'y': random.uniform(-40,40), 'z':0, 'a': random.uniform(1,5)} )
       

def get_walabot_status():
	global walabotStarted
	state = "offline"
	if not walabotStarted:
		state = "connected"
	return "Walabot is " + state


@app.route('/')
def index():
    return render_template('index.html', async_mode=socketio.async_mode)

@ask.launch
def new_ask():
    #return question(welcome) \
    #    .reprompt(reprompt)
	return statement("Welcome to Visual Synth." + get_walabot_status())

@ask.intent("TriggerZoneIntent", convert={'zone_size': int})
def trigger_zone(zone_size):
    if zone_size == None:
        zone_size = ""
    socketio.emit('set_trigger_zone', {'data': zone_size} )
    return statement("zone set")


@ask.intent("MusicScaleIntent", convert={'music_mode': int})
def music_scale(music_mode):
    if music_mode == None:
        music_mode = ""
    socketio.emit('music_mode', {'data': music_mode} )
    return statement("mode set")

 
@ask.intent("GraphicStyleIntent")
def graphic_style( visual_style):
    if visual_style == None:
        visual_style = ""
    #text = render_template('hit',  style= visual_style)
    socketio.emit('graphic_style', {'data':  visual_style} )
    return statement("style is " + visual_style)
	# change_color(visual_style)

@ask.intent("InstrumentIntent")
def change_instrument( instrument_name, instrument_number= None):
	if instrument_number <> None:
		socketio.emit('program_change', {'data': instrument_number} )
		return statement("program changed " + instrument_number)

	if instrument_name == None:
		instrument_name = ""
	print('Change Instrument Intent' + instrument_name)
	socketio.emit('program_change', {'data': instrument_name} )
	return statement("changed to " + instrument_name)


@ask.intent("PlayIntent", convert={'song': int})
def play_song(name):
    socketio.emit('play', {'data': name} )
    return statement("playing" )


@ask.intent("StopIntent")
def stop_song():
    socketio.emit('stop', {'data': 'none'}  )
    return statement("stopped" )


@ask.intent("CalibrationIntent")
def calibrate_area( ):
    socketio.emit('calibrate', {'data': 'none'} )
   
    return statement("OK status " + str(walabot.GetStatusCode()))

@ask.intent("AMAZON.HelpIntent")
def help_message():
	help_msg = render_template('help')
	return statement(help_msg)


""" Handle socket connection """
@socketio.on('change_color' )
def change_color( color ):
    print("Color change to " + str(color))
    #change_color(color)


@socketio.on('connect'  )
def connect():
    global thread
    with thread_lock:
        if thread is None:
            thread = socketio.start_background_task(target=background_thread)
    emit('my_response', {'data': 'Connected', 'count': 0})


@socketio.on('disconnect' )
def disconnect():
    print('Client disconnected', request.sid)


if __name__ == '__main__':
    socketio.run(app, host='0.0.0.0', debug=True)
  

""""
#Raspberry pi GPIO LED light control

import sys, time
import RPi.GPIO as GPIO
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)
redPin   = 11
greenPin = 13
bluePin  = 15
GPIO.setup(redPin,GPIO.OUT)   
GPIO.setup(greenPin,GPIO.OUT)   
GPIO.setup(bluePin,GPIO.OUT)   


def change_color(color):
	print("Change color "+ color)
	if color in ["red", "yellow","magenta"]:
		GPIO.output(redPin,GPIO.HIGH)
	else:
		GPIO.output(redPin,GPIO.LOW)

	if color in ["green", "yellow","cyan"]:
		GPIO.output(greenPin,GPIO.HIGH)
	else:
		GPIO.output(greenPin,GPIO.LOW)
	
	if color in ["blue", "cyan","magenta"]:
		GPIO.output(bluePin,GPIO.HIGH)
	else:
		GPIO.output(bluePin,GPIO.LOW)
	
"""

WalabotService

Python
#!/usr/bin/env python
""" 
Start up Walabot and wait until calibration is complete
"""

from __future__ import print_function
from sys import platform
from os import system
from imp import load_source
from os.path import join
import time
#import WalabotAPI as wlbt 
 
MAX_RECONNECT = 10 
SMALL_ROOM = True

class WalabotService:
    def __init__(self):
        system('cls' if platform == 'win32' else 'clear')  # clear the terminal

        modulePath = join('C:/', 'Program Files', 'Walabot', 'WalabotSDK',
                'python', 'WalabotAPI.py')
        # self.wlbt = wlbt
        self.wlbt = load_source('WalabotAPI', modulePath) #disable for linux
        self.retry = MAX_RECONNECT
        self.status = 0
        self.wlbt.Init()
        self.status += 1

    def PrintSensorTargets(self,targets):
        system('cls' if platform == 'win32' else 'clear')
        if targets:
            for i, target in enumerate(targets):
                print('Target #{}:\nx: {}\ny: {}\nz: {}\namplitude: {}\n'.format(
                    i + 1, target.xPosCm, target.yPosCm, target.zPosCm,
                    target.amplitude))
        else:
            print('No Target Detected')

   
    def Configure(self):
            # skip if already configured
            if self.status > 2:
                return

            if SMALL_ROOM:
                # wlbt.SetArenaR - input parameters
                minInCm, maxInCm, resInCm = 50, 300, 3
                # wlbt.SetArenaTheta - input parameters
                minIndegrees, maxIndegrees, resIndegrees = -15, 15, 5
                # wlbt.SetArenaPhi - input parameters
                minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees = -60, 60, 5
            else:
                minInCm, maxInCm, resInCm = 100, 500, 3
                minIndegrees, maxIndegrees, resIndegrees = -25, 25, 5
                minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees = -60, 60, 5



            # Set MTI mode
            mtiMode = True
            threashHold = 4
            # Configure Walabot database install location (for windows)
            self.wlbt.SetSettingsFolder()
            # 1) Connect : Establish communication with walabot.
            print ("Connecting Walabot ...")
            while self.retry  > 0 and self.status < 2:
                try:
                    time.sleep(1)
                    self.wlbt.ConnectAny()
                    self.status += 1
                    print ("Found Walbot")
                except Exception, e:
                    self.retry-=1
            self.retry = MAX_RECONNECT
            # 2) Configure: Set scan profile and arena 
  
            # Set Profile - to Sensor.
            self.wlbt.SetProfile(self.wlbt.PROF_SENSOR)
            # Setup arena - specify it by Cartesian coordinates.
            self.wlbt.SetArenaR(minInCm, maxInCm, resInCm)
            # Sets polar range and resolution of arena (parameters in degrees).
            self.wlbt.SetArenaTheta(minIndegrees, maxIndegrees, resIndegrees)
            # Sets azimuth range and resolution of arena.(parameters in degrees).
            self.wlbt.SetArenaPhi(minPhiInDegrees, maxPhiInDegrees, resPhiInDegrees)
            # Moving Target Identification: standard dynamic-imaging filter

            self.wlbt.SetThreshold(threashHold)

            filterType = self.wlbt.FILTER_TYPE_MTI if mtiMode else self.wlbt.FILTER_TYPE_NONE
            self.wlbt.SetDynamicImageFilter(filterType)
            # 3) Start: Start the system in preparation for scanning.
            print ("Starting Walabot")
            self.wlbt.Start()
            self.status += 1


    def GetStatusCode(self):
        try:
           msg = self.wlbt.GetStatus()
        except Exception, e:
           msg = e.message
        return msg

    def Calibrate(self):
            print ('Calibrating ',end="")
            # calibrates scanning to ignore or reduce the signals
            self.wlbt.StartCalibration()
            while self.wlbt.GetStatus()[0] == self.wlbt.STATUS_CALIBRATING:
                self.wlbt.Trigger()
                time.sleep(1)
                print (".", end="")

            self.status += 1


    def ReadSensor(self):
            self.wlbt.Trigger()  # initiates a scan and records signals 
            targets = self.wlbt.GetSensorTargets()  # provides a list of identified targets
            self.PrintSensorTargets(targets)

    def GetTargetPosition(self):
         self.wlbt.Trigger() 
         targets = self.wlbt.GetSensorTargets() 
         if targets:
            target = targets[0]
            return ( target.xPosCm, target.yPosCm, target.zPosCm, target.amplitude)
         else:
            return (0,0,0,0)

    def Close(self):                     
        self.wlbt.Stop()  # stops Walabot when finished scanning
        self.wlbt.Disconnect() # stops communication with Walabot
        print('Walabot disconnected')

    def StartService(self): 
        if self.status >= 104 :
            print ('Already started')
            return True
        while self.status <> 104 and self.retry > 0:
            try:
                print ('status is %d' % self.status)
                self.Configure()
                print('Walabot configured')
                self.Calibrate()
                print('Walabot calibrated')
                self.status = 104
                return True
            except Exception, e:
                 print ("Error starting Walabot - %s :" %e )
                 self.retry-=1 
                 time.sleep (1)
        return False

retry = MAX_RECONNECT
if __name__ == '__main__':
    wala = WalabotService()
    connected = wala.StartService()
    while connected:
        wala.ReadSensor()
        #wala.GetTargetPosition()
    wala.Close()
 

Alexa Skill Utterance

JSON
{
  "languageModel": {
    "types": [
      {
        "name": "string",
        "values": [
          {
            "id": null,
            "name": {
              "value": "Piano",
              "synonyms": []
            }
          },
          {
            "id": null,
            "name": {
              "value": "guitar",
              "synonyms": []
            }
          },
          {
            "id": null,
            "name": {
              "value": "flute",
              "synonyms": []
            }
          },
          {
            "id": null,
            "name": {
              "value": "sax",
              "synonyms": []
            }
          }
        ]
      }
    ],
    "intents": [
      {
        "name": "AMAZON.CancelIntent",
        "samples": []
      },
      {
        "name": "AMAZON.HelpIntent",
        "samples": []
      },
      {
        "name": "AMAZON.StopIntent",
        "samples": []
      },
      {
        "name": "CalibrationIntent",
        "samples": [
          "begin calibration",
          "start calibration",
          "initialize",
          "calibrate"
        ],
        "slots": []
      },
      {
        "name": "ConfigurationIntent",
        "samples": [
          "set range to {meter}"
        ],
        "slots": [
          {
            "name": "meter",
            "type": "AMAZON.NUMBER"
          }
        ]
      },
      {
        "name": "GraphicStyleIntent",
        "samples": [
          "set visual style to {visual_style}",
          "change graphics to {visual_style}",
          "switch visual to {visual_style}",
          "change visual {visual_style}",
          "change style",
          "change graphics"
        ],
        "slots": [
          {
            "name": "visual_style",
            "type": "AMAZON.Color"
          }
        ]
      },
      {
        "name": "InstrumentIntent",
        "samples": [
          "change instrument to {instrument_number}",
          "switch sound to {instrument_name}",
          "instrument number {instrument_number}",
          "change instrument",
          "change sound",
          "switch sound"
        ],
        "slots": [
          {
            "name": "instrument_number",
            "type": "AMAZON.NUMBER"
          },
          {
            "name": "instrument_name",
            "type": "string"
          }
        ]
      },
      {
        "name": "MusicScaleIntent",
        "samples": [
          "switch music scale to {music_mode}",
          "set music mode to {music_mode}",
          "change mode to {music_mode}",
          "change scale to {music_mode}",
          "change scale"
        ],
        "slots": [
          {
            "name": "music_mode",
            "type": "string"
          }
        ]
      },
      {
        "name": "PlayIntent",
        "samples": [
          "play ",
          "resume",
          "play {track_number}",
          "play track {track_number}",
          "play next track"
        ],
        "slots": [
          {
            "name": "track_number",
            "type": "AMAZON.NUMBER"
          }
        ]
      },
      {
        "name": "StopIntent",
        "samples": [
          "stop",
          "pause",
          "stop song",
          "stop play",
          "stop music"
        ],
        "slots": []
      },
      {
        "name": "TriggerZoneIntent",
        "samples": [
          "add zone {zone_size}",
          "set position ",
          "set zone",
          "mark zone",
          "set trigger",
          "mark position size {zone_size}",
          "set trigger size {zone_size}"
        ],
        "slots": [
          {
            "name": "zone_size",
            "type": "AMAZON.NUMBER"
          }
        ]
      }
    ],
    "invocationName": "v synth"
  }
}

templates.yaml

YAML
welcome: Welcome to visual synth.
help: Here are list of Visual Synth's commands.  change instrument, change color, set trigger zone, chamge music style, play song, or stop

Visual Synth Unity 3D GUI

Unity 3D Scripts, Scene file and resources

Credits

Leon Chu

Leon Chu

8 projects • 8 followers
Indie Developer / Artist / Lifelong learner

Comments