Scott WilliamsJim Walkoski
Published © MIT

Baby Checkr

An Alexa skill for determining if your baby is sleeping. "Check the little stink before having your next drink".

IntermediateShowcase (no instructions)3 hours1,579

Things used in this project

Story

Read more

Schematics

Baby Checkr Architecture

Architecture, what we did & how we did it

Code

index.js

JavaScript
Lambda function to analyze movement in photos from a S3 bucket
'use strict';

const jpeg = require('jpeg-js');
const fs = require('fs');

const aws = require('aws-sdk');
const s3 = new aws.S3({ apiVersion: '2006-03-01' });
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();

exports.handler = (event, context, callback) => {
    //console.log('Received event:', JSON.stringify(event, null, 2));

    //Find the S3 key of the last uploaded image
    var lastItemKey;
    var lastItemBucket;
    const lastItemParams = {
        TableName: 'last-image',
        Key: {userid : '111'}
    }
    dynamo.getItem(lastItemParams, (err, data) => {
        console.log(data);
        lastItemBucket = data.Item.bucket;
        lastItemKey = data.Item.key;

        //Get the S3 object from the last image
        var lastImageParams = {
            Bucket: lastItemBucket,
            Key: lastItemKey
        };
        var imgData1;
        s3.getObject(lastImageParams, (err, data) => {
            if (err) {
                console.log(err);
                const message = `Error getting object ${lastImageParams.Key} from bucket ${lastImageParams.Bucket}. Make sure they exist and your bucket is in the same region as this function.`;
                console.log(message);
            } else {
                console.log('CONTENT TYPE:', data.ContentType);
                imgData1 = data;

                //Get the S3 object from the event
                var eventImageParams = {
                    Bucket: event.Records[0].s3.bucket.name,
                    Key: event.Records[0].s3.object.key,
                };
                var imgData2;
                s3.getObject(eventImageParams, (err, data) => {
                    if (err) {
                        console.log(err);
                        const message = `Error getting object ${eventImageParams.Key} from bucket ${eventImageParams.Bucket}. Make sure they exist and your bucket is in the same region as this function.`;
                        console.log(message);
                    } else {
                        imgData2 = data;
                        const lastImageParams = {
                            TableName: 'last-image',
                            Item: {
                                userid: '111',
                                bucket: eventImageParams.Bucket,
                                key: eventImageParams.Key
                            }
                        };
                        //Update the last-image table
                        dynamo.putItem(lastImageParams, (err, data) => {
                            if(err) {
                                console.log(err);
                            }

                            // Decode Images
                            var rawImageData = jpeg.decode(imgData1.Body);
                            var rawImageData2 = jpeg.decode(imgData2.Body);

                            var rawDiff = [];
                            rawDiff.length = rawImageData.width * rawImageData.height;
                            for(var i = 0; i < rawImageData.width * rawImageData.height; i++) {
                              //diff each channel separately
                              var pos = i * 4;
                              rawDiff[pos+0] = Math.abs(rawImageData.data[pos+0] - rawImageData2.data[pos+0]); //R
                              rawDiff[pos+1] = Math.abs(rawImageData.data[pos+1] - rawImageData2.data[pos+1]); //G
                              rawDiff[pos+2] = Math.abs(rawImageData.data[pos+2] - rawImageData2.data[pos+2]); //B
                              rawDiff[pos+3] = 255;  //ignore alpha channel
                            }

                            let cumulativeDiff = 0;
                            for(let i = 0; i < rawImageData.width * rawImageData.height; i++) {
                              let pos = i * 4;
                              cumulativeDiff += rawDiff[pos+0] + rawDiff[pos+1] + rawDiff[pos+2]; //ignore alpha channel
                            }
                            console.log('cumulativeDiff', cumulativeDiff);
                            //A ratio allows tuning values to apply to images of different sizes (as long as each set has the same size for each upload)
                            const diffRatio = cumulativeDiff / rawImageData.width / rawImageData.height;
                            console.log('diffRatio', diffRatio);

                            //Update ratio in lookup table
                            const moveRatioParams = {
                                TableName: 'baby-state',
                                Item: {
                                    userid: '111',
                                    moveratio: diffRatio
                                }
                            }
                            dynamo.putItem(moveRatioParams, (err, data) => {
                                if(err) {
                                    console.log(err);
                                }
                            });
                        });
                    }
                });
            }
        });
    });
};

Package.json

JavaScript
Package file for index.js
{
  "name": "baby-awake-skill",
  "version": "1.0.0",
  "description": "Ask Alexa to spy on your baby",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "Baby Checkr",
  "license": "",
  "dependencies": {
    "aws-sdk": "^2.7.9",
    "dynamodb-doc": "^1.0.0",
    "jpeg-js": "^0.2.0"
  }
}

alexa.js

JavaScript
Lambda function to fetch baby state for Alexa
/**
    Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
    Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at
        http://aws.amazon.com/apache2.0/
    or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
*/
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();

/**
 * App ID for the skill
 */
var APP_ID = undefined; //OPTIONAL: replace with "amzn1.echo-sdk-ams.app.[your-unique-value-here]";

/**
 * The AlexaSkill prototype and helper functions
 */
function AlexaSkill(appId) {
    this._appId = appId;
}

AlexaSkill.speechOutputType = {
    PLAIN_TEXT: 'PlainText',
    SSML: 'SSML'
}

AlexaSkill.prototype.requestHandlers = {
    LaunchRequest: function (event, context, response) {
        this.eventHandlers.onLaunch.call(this, event.request, event.session, response);
    },

    IntentRequest: function (event, context, response) {
        this.eventHandlers.onIntent.call(this, event.request, event.session, response);
    },

    SessionEndedRequest: function (event, context) {
        this.eventHandlers.onSessionEnded(event.request, event.session);
        context.succeed();
    }
};

/**
 * Override any of the eventHandlers as needed
 */
AlexaSkill.prototype.eventHandlers = {
    /**
     * Called when the session starts.
     * Subclasses could have overriden this function to open any necessary resources.
     */
    onSessionStarted: function (sessionStartedRequest, session) {
    },

    /**
     * Called when the user invokes the skill without specifying what they want.
     * The subclass must override this function and provide feedback to the user.
     */
    onLaunch: function (launchRequest, session, response) {
        throw "onLaunch should be overriden by subclass";
    },

    /**
     * Called when the user specifies an intent.
     */
    onIntent: function (intentRequest, session, response) {
        var intent = intentRequest.intent,
            intentName = intentRequest.intent.name,
            intentHandler = this.intentHandlers[intentName];
        if (intentHandler) {
            console.log('dispatch intent = ' + intentName);
            intentHandler.call(this, intent, session, response);
        } else {
            throw 'Unsupported intent = ' + intentName;
        }
    },

    /**
     * Called when the user ends the session.
     * Subclasses could have overriden this function to close any open resources.
     */
    onSessionEnded: function (sessionEndedRequest, session) {
    }
};

/**
 * Subclasses should override the intentHandlers with the functions to handle specific intents.
 */
AlexaSkill.prototype.intentHandlers = {};

AlexaSkill.prototype.execute = function (event, context) {
    try {
        console.log("session applicationId: " + event.session.application.applicationId);

        // Validate that this request originated from authorized source.
        if (this._appId && event.session.application.applicationId !== this._appId) {
            console.log("The applicationIds don't match : " + event.session.application.applicationId + " and "
                + this._appId);
            throw "Invalid applicationId";
        }

        if (!event.session.attributes) {
            event.session.attributes = {};
        }

        if (event.session.new) {
            this.eventHandlers.onSessionStarted(event.request, event.session);
        }

        // Route the request to the proper handler which may have been overriden.
        var requestHandler = this.requestHandlers[event.request.type];
        requestHandler.call(this, event, context, new Response(context, event.session));
    } catch (e) {
        console.log("Unexpected exception " + e);
        context.fail(e);
    }
};

var Response = function (context, session) {
    this._context = context;
    this._session = session;
};

function createSpeechObject(optionsParam) {
    if (optionsParam && optionsParam.type === 'SSML') {
        return {
            type: optionsParam.type,
            ssml: optionsParam.speech
        };
    } else {
        return {
            type: optionsParam.type || 'PlainText',
            text: optionsParam.speech || optionsParam
        }
    }
}

Response.prototype = (function () {
    var buildSpeechletResponse = function (options) {
        var alexaResponse = {
            outputSpeech: createSpeechObject(options.output),
            shouldEndSession: options.shouldEndSession
        };
        if (options.reprompt) {
            alexaResponse.reprompt = {
                outputSpeech: createSpeechObject(options.reprompt)
            };
        }
        if (options.cardTitle && options.cardContent) {
            alexaResponse.card = {
                type: "Simple",
                title: options.cardTitle,
                content: options.cardContent
            };
        }
        var returnResult = {
                version: '1.0',
                response: alexaResponse
        };
        if (options.session && options.session.attributes) {
            returnResult.sessionAttributes = options.session.attributes;
        }
        return returnResult;
    };

    return {
        tell: function (speechOutput) {
            this._context.succeed(buildSpeechletResponse({
                session: this._session,
                output: speechOutput,
                shouldEndSession: true
            }));
        },
        tellWithCard: function (speechOutput, cardTitle, cardContent) {
            this._context.succeed(buildSpeechletResponse({
                session: this._session,
                output: speechOutput,
                cardTitle: cardTitle,
                cardContent: cardContent,
                shouldEndSession: true
            }));
        },
        ask: function (speechOutput, repromptSpeech) {
            this._context.succeed(buildSpeechletResponse({
                session: this._session,
                output: speechOutput,
                reprompt: repromptSpeech,
                shouldEndSession: false
            }));
        },
        askWithCard: function (speechOutput, repromptSpeech, cardTitle, cardContent) {
            this._context.succeed(buildSpeechletResponse({
                session: this._session,
                output: speechOutput,
                reprompt: repromptSpeech,
                cardTitle: cardTitle,
                cardContent: cardContent,
                shouldEndSession: false
            }));
        }
    };
})();

/**
 * SpaceGeek is a child of AlexaSkill.
 * To read more about inheritance in JavaScript, see the link below.
 *
 * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Introduction_to_Object-Oriented_JavaScript#Inheritance
 */
var BabyCam = function () {
    AlexaSkill.call(this, APP_ID);
};

// Extend AlexaSkill
BabyCam.prototype = Object.create(AlexaSkill.prototype);
BabyCam.prototype.constructor = BabyCam;

BabyCam.prototype.eventHandlers.onSessionStarted = function (sessionStartedRequest, session) {
    //console.log("onSessionStarted requestId: " + sessionStartedRequest.requestId + ", sessionId: " + session.sessionId);
    // any initialization logic goes here
};

BabyCam.prototype.eventHandlers.onLaunch = function (launchRequest, session, response) {
    //console.log("onLaunch requestId: " + launchRequest.requestId + ", sessionId: " + session.sessionId);
    handleBabyCamRequest(response);
};

/**
 * Overridden to show that a subclass can override this function to teardown session state.
 */
BabyCam.prototype.eventHandlers.onSessionEnded = function (sessionEndedRequest, session) {
    //console.log("onSessionEnded requestId: " + sessionEndedRequest.requestId + ", sessionId: " + session.sessionId);
    // any cleanup logic goes here
};

BabyCam.prototype.intentHandlers = {
    "CheckOnBaby": function (intent, session, response) {
        handleBabyCamRequest(response);
    },

    "AMAZON.HelpIntent": function (intent, session, response) {
        response.ask("You can ask me in the baby is sleeping, or, you can say exit... What can I help you with?", "What can I help you with?");
    },

    "AMAZON.StopIntent": function (intent, session, response) {
        var speechOutput = "Goodbye";
        response.tell(speechOutput);
    },

    "AMAZON.CancelIntent": function (intent, session, response) {
        var speechOutput = "Goodbye";
        response.tell(speechOutput);
    }
};

const AWAKE_THRESHOLD = 25;
const RESTLESS_THRESHOLD = 15;
function handleBabyCamRequest(response) {
    var cardTitle = "Baby Status";
    var speechOutput = "";
    console.log('Checking baby');
    const params = {
        TableName: "baby-state",
        Key: {userid : '111'}
    };

    //Retrieve the movement ratio
    var moveRatio = 1000;
    dynamo.getItem(params, function(err, data) {
        console.log('moveRatio item', data);
        if(err) {
            console.log('moveRatio error: ' + err);
            return false;
        } else {
            console.log('moveRatio item', data);
            moveRatio = data.Item.moveratio;

            //Determine the appropriate response
            if(moveRatio > AWAKE_THRESHOLD) {
                speechOutput = "The baby is awake.";
            } else if(moveRatio > RESTLESS_THRESHOLD) {
                speechOutput = "The baby is restless.";
            } else {
                speechOutput = "The baby is asleep.";
            }
            response.tellWithCard(speechOutput, cardTitle, speechOutput);
        }
    });
}

// Create the handler that responds to the Alexa Request.
exports.handler = function (event, context) {
    var babyCam = new BabyCam();
    babyCam.execute(event, context);
};

algo.js

JavaScript
Algorithm for image comparison used for testing
'use strict';

//Can be run with `node algo.js` to test arbitrary images
const jpeg = require('jpeg-js');
const fs = require('fs');

// First load Image
const jpegData = fs.readFileSync('931.jpg');

// Decode Image
const rawImageData = jpeg.decode(jpegData);
// console.log(rawImageData);
// console.log('rawImageData', rawImageData.data[0], rawImageData.data[1], rawImageData.data[2], rawImageData.data[3]);

const jpegData2 = fs.readFileSync('940.jpg');
const rawImageData2 = jpeg.decode(jpegData2);

const rawDiff = [];
rawDiff.length = rawImageData.width * rawImageData.height;
for(let i = 0; i < rawImageData.width * rawImageData.height; i++) {
  //diff each channel separately
  let pos = i * 4;
  rawDiff[pos+0] = Math.abs(rawImageData.data[pos+0] - rawImageData2.data[pos+0]); //R
  rawDiff[pos+1] = Math.abs(rawImageData.data[pos+1] - rawImageData2.data[pos+1]); //G
  rawDiff[pos+2] = Math.abs(rawImageData.data[pos+2] - rawImageData2.data[pos+2]); //B
  rawDiff[pos+3] = 255;  //ignore alpha channel
}

let cumulativeDiff = 0;
for(let i = 0; i < rawImageData.width * rawImageData.height; i++) {
  let pos = i * 4;
  cumulativeDiff += rawDiff[pos+0] + rawDiff[pos+1] + rawDiff[pos+2]; //ignore alpha channel
}
console.log('cumulativeDiff', cumulativeDiff);
const diffRatio = cumulativeDiff / rawImageData.width / rawImageData.height;
console.log('diffRatio', diffRatio);

intent-schema.json

JSON
Simpe intent schema for Alexa skill
{
  "intents": [
    {
      "intent": "CheckOnBaby"
    },
    {
      "intent": "AMAZON.HelpIntent"
    },
    {
      "intent": "AMAZON.StopIntent"
    },
    {
      "intent": "AMAZON.CancelIntent"
    }
  ]
 }

sample-utterances.txt

Plain text
Sample utterances for Alexa skill
CheckOnBaby Check if my baby sleeping
CheckOnBaby Check if my baby awake
CheckOnBaby Check if my son asleep
CheckOnBaby Check if my daughter up
CheckOnBaby Is my baby sleeping
CheckOnBaby Is my baby awake
CheckOnBaby Is my son asleep
CheckOnBaby Is my daughter up

Credits

Scott Williams

Scott Williams

1 project • 0 followers
Jim Walkoski

Jim Walkoski

1 project • 0 followers

Comments