Rich Huber
Published © MIT

Face Match Doorbell

Use the NXP FRDM-K82F FlexIO interface to capture video and match against saved images to ring the doorbell with a custom ring.

AdvancedFull instructions provided8 hours995
Face Match Doorbell

Things used in this project

Hardware components

Kinetis Freedom Board with FlexIO
NXP Kinetis Freedom Board with FlexIO
NXP freedom board (FRDM-K82F) with FlexIO
×1
OV7670 VGA Camera Module for Arduino
×1
5 mm LED: Yellow
5 mm LED: Yellow
×1
5 mm LED: Green
5 mm LED: Green
×1
47 ohm resitor
×2

Story

Read more

Schematics

Camera LEDs Schematic

Code

imageproc.c

C/C++
The Image processing functions
/*
 * imageproc.c
 *
 *  Created on: May 30, 2016
 *      Author: rhuber
 */

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <stdint.h>

// Lots of dependencies, so we need to include all of these files
#include "flexio_ov7670.h"
#include "board.h"
#include "usb_device_config.h"
#include "usb.h"
#include "usb_device_stack_interface.h"
#include "usb_descriptor.h"

#include "camera.h"


#define ROI_W	72
#define ROI_H	80
#define ROI_X	((160 - ROI_W) / 2)
#define ROI_Y	((120 - ROI_H) / 2)
#define ROI_X2 (ROI_X + ROI_W)
#define ROI_Y2 (ROI_Y + ROI_H)


uint16_t image1[ROI_H][ROI_W];
uint8_t edge_image[ROI_H][ROI_W];
uint8_t edge_ar[ROI_H][ROI_W];

void capture_image(int imagenum, volatile uint16_t ar[120][160] )
{
	uint32_t x, y;
		for(y = 0 ; y < ROI_H ; y++) {
			for(x = 0 ; x < ROI_W ; x++) {
				image1[y][x] = ar[ROI_Y + y][ROI_X + x];
			}
		}
}



#define RED(a)      ((((a) & 0xf800) >> 11) << 3)
#define GREEN(a)    ((((a) & 0x07e0) >> 5) << 2)
#define BLUE(a)     (((a) & 0x001f) << 3)
uint16_t conv_bw_rgb565(uint16_t src) {
	int r = RED(src);	// 5 bits
	int g = GREEN(src);		// 6 bits
	int b = BLUE(src);			// 5 bits
	int gray = (r + g + b) / 3;
	uint16_t output = ((gray >> 3) & 0x1F) << 11 | ((gray >> 2) & 0x3f) << 5 | ((gray >> 3) & 0x1F);
	return output;
}

///  ( ((a&0xf800)>>11) + ((a&0x03e0)>>5) + (a&0x1f) )		// RGB565
uint16_t diff_rgb565(uint16_t src, uint16_t dest) {
	int rSrc = (src & 0xf800) >> 11;	// 5 bits
	int gSrc = (src & 0x07e0) >> 5;		// 6 bits
	int bSrc = (src & 0x001f);			// 5 bits
	int rDst = (dest & 0xf800) >> 11;
	int gDst = (dest & 0x07e0) >> 5;
	int bDst = (dest & 0x001f);

	rDst -= rSrc;
	if(rDst > 31)
		rDst = 31;
	else if(rDst < 0)
		rDst = 0;

	gDst -= gSrc;
	if(gDst > 63)
		gDst = 63;
	else if(gDst < 0)
		gDst = 0;

	bDst -= bSrc;
	if(bDst > 31)
		bDst = 31;
	else if(bDst < 0)
		bDst = 0;

	uint16_t output = (rDst & 0x1F) << 11 | (gDst & 0x3f) << 5 | (bDst & 0x1F);
	return output;
}

void diff_image(int imagenum, volatile uint16_t ar[120][160] )
{
	uint32_t x, y;
	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			ar[ROI_Y + y][ROI_X + x] = diff_rgb565(image1[y][x], ar[ROI_Y + y][ROI_X + x]);
		}
	}
}

int totalDiff = 0;


//#define RED(a)      ((((a) & 0xf800) >> 11) << 3)
//#define GREEN(a)    ((((a) & 0x07e0) >> 5) << 2)
//#define BLUE(a)     (((a) & 0x001f) << 3)
uint16_t diffshow_rgb565(uint16_t src, uint16_t dest) {
	int r = RED(src);
	int g = GREEN(src);
	int b = BLUE(src);
	int graySrc = (r + g + b) / 3;

	r = RED(dest);
	g = GREEN(dest);
	b = BLUE(dest);
	int grayDest = (r + g + b) / 3;

	int grayDiff = graySrc - grayDest;
	if(grayDiff < 0)
		grayDiff = -grayDiff;
	totalDiff += grayDiff;
	int redDest = grayDest;
	if(grayDiff > 0x08) {
		redDest = grayDest + (grayDiff );
		if(redDest > 255)
			redDest = 255;
	}

	uint16_t output = ((redDest >> 3) & 0x1F) << 11 | ((grayDest >> 2) & 0x3f) << 5 | ((grayDest >> 3) & 0x1F);

	return output;
}

#define RGB(r, g, b)  ((r >> 3) & 0x1F) << 11 | ((g >> 2) & 0x3f) << 5 | ((b >> 3) & 0x1F)
void diffshow_image(int imagenum, volatile uint16_t ar[120][160] )
{
	totalDiff = 0;	// reset our diff counter
	uint32_t x, y;

	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			ar[ROI_Y + y][ROI_X + x] = diffshow_rgb565(image1[y][x], ar[ROI_Y + y][ROI_X + x]);
		}
	}

	// totalDiff level: 347000 bad match, 15000 good match
	int level = (totalDiff * ROI_W) / 347000;
	if(level > ROI_W)
		level = ROI_W;
	// First color bar is red
	uint16_t color16 = RGB(255, 0, 0);
	for(x = 0 ; x < level ; x++) {
		ar[ROI_Y-2][ROI_X + x] = color16;
		ar[ROI_Y-1][ROI_X + x] = color16;
	}
	// 2nd bar is green
	color16 = RGB(0, 255, 0);
	for( ; x < ROI_W ; x++) {
		ar[ROI_Y-2][ROI_X + x] = color16;
		ar[ROI_Y-1][ROI_X + x] = color16;
	}
}


void conv_bw_image(int imagenum, volatile uint16_t ar[120][160] )
{
	uint32_t x, y;
	for(y = ROI_Y ; y < ROI_Y2 ; y++) {
		for(x = ROI_X ; x < ROI_X2 ; x++) {
			ar[y][x] = conv_bw_rgb565(ar[y][x]);
		}
	}
}


void show_saved_image(int imagenum, volatile uint16_t ar[120][160] )
{
	uint32_t x, y;
	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			ar[ROI_Y + y][ROI_X + x] = conv_bw_rgb565(image1[y][x]);
		}
	}
}

uint32_t hist[256];

void histogram_image(int flags, uint16_t color16, volatile uint16_t ar[120][160], uint8_t edge_input1[ROI_H][ROI_W] )
{
	int x, y;
	totalDiff = 0;
	for(x = 0 ; x < 256 ; x++) {
		hist[x] = 0;
	}

	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			int edge1 = edge_input1[y][x];
			hist[edge1]++;
		}
	}

//	uint16_t color16 = RGB(255, 0, 0);		// Show hist in red


	// Total counts: 60*80 = 4800 (~0x1000)
	for(x = 0 ; x < 128 ; x++) {
		y = (hist[x << 1] + hist[(x << 1) + 1]) >> 2;
		if(y > 119)
			y = 119;
		else if(y < 0)
			y = 0;
		ar[119 - y][x] = color16;
	}

}


void scan_edges_image(int imagenum, volatile uint16_t ar[120][160], uint8_t edge_output[ROI_H][ROI_W])
{
	//edge_image[ROI_H][ROI_W];
	// iterators
	unsigned int x, y;
	// initialize our top/bottom and left/right borders
	for(x = 0; x < ROI_W; x++){
		edge_output[0][x] = 0;
		edge_output[ROI_H - 1][x] = 0;
	}
	for(y = 0; y < ROI_H; y++){
		edge_output[y][0] = 0;
		edge_output[y][ROI_W - 1] = 0;
	}
	for(x = 1; x < ROI_W - 1; x++){
		for(y = 1; y < ROI_H - 1; y++){
			// initialize Gx and Gy to 0
			int Gx = 0;
			int Gy = 0;
			unsigned int intensity = 0;
			uint16_t pixel;

			// Left column
			pixel = ar[ROI_Y + y - 1][ROI_X + x - 1];
			int r = RED(pixel);
			int g = GREEN(pixel);
			int b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += -intensity;
			Gy += -intensity;

			pixel = ar[ROI_Y + y][ROI_X + x - 1];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += -2 * intensity;

			pixel = ar[ROI_Y + y + 1][ROI_X + x - 1];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += -intensity;
			Gy += +intensity;

			// middle column
			pixel = ar[ROI_Y + y - 1][ROI_X + x];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gy += -2 * intensity;

			pixel = ar[ROI_Y + y + 1][ROI_X + x];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gy += +2 * intensity;

			// right column
			pixel = ar[ROI_Y + y - 1][ROI_X + x + 1];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += +intensity;
			Gy += -intensity;

			pixel = ar[ROI_Y + y][ROI_X + x + 1];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += +2 * intensity;

			pixel = ar[ROI_Y + y + 1][ROI_X + x + 1];
			r = RED(pixel);
			g = GREEN(pixel);
			b = BLUE(pixel);
			intensity = (r + g + b);
			Gx += +intensity;
			Gy += +intensity;

			// calculate the gradient length
			unsigned int length = (unsigned int)sqrt( (float)(Gx * Gx) + (float)(Gy * Gy) );

			// normalize the length to 0 to 255
			length = length / 17;

			// draw the pixel on the edge image
			length *= 3;
			if(length > 255)
				length = 255;
			edge_output[y][x] = length;
		}
	}

}


void overlay_edge_image(int flags, volatile uint16_t ar[120][160], uint8_t edge_input[ROI_H][ROI_W] )
{
	uint32_t x, y;
	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			uint16_t pixel = ar[ROI_Y + y][ROI_X + x];
			uint32_t edge = edge_input[y][x];
			int r = RED(pixel);
			int g = GREEN(pixel);
			int b = BLUE(pixel);
			int gray = ((r + g + b) / 3);
			edge += gray;
			if(edge > 255)
				edge = 255;
			uint16_t color16 = RGB(gray, gray, edge);		// Show edges in blue
			ar[ROI_Y + y][ROI_X + x] = color16;
		}
	}
}

void show_edge_image(int flags, volatile uint16_t ar[120][160], uint8_t edge_input1[ROI_H][ROI_W])
{
	uint32_t x, y;
	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			uint8_t edge = edge_input1[y][x];
			uint16_t color16 = RGB(edge, edge, edge);		// Show edges in green
			ar[ROI_Y + y][ROI_X + x] = color16;
		}
	}
}


void show_edge_image2(int flags, volatile uint16_t ar[120][160], uint8_t edge_input1[ROI_H][ROI_W], uint8_t edge_input2[ROI_H][ROI_W]  )
{
	uint32_t x, y;
	totalDiff = 0;
	for(y = 0 ; y < ROI_H ; y++) {
		for(x = 0 ; x < ROI_W ; x++) {
			int edge1 = edge_input1[y][x];
			int edge2 = edge_input2[y][x];
			int grayDiff = edge1 - edge2;
			if(grayDiff < 0)
				grayDiff = -grayDiff;
			totalDiff += grayDiff;
			uint16_t color16 = RGB(edge1, edge2, 0);		// Show edges in green
			ar[ROI_Y + y][ROI_X + x] = color16;
		}
	}
	// totalDiff level: 347000 bad match, 15000 good match
	int level = (totalDiff * ROI_W) / 347000;
	if(level > ROI_W)
		level = ROI_W;
	// First color bar is red
	uint16_t color16 = RGB(255, 0, 0);
	for(x = 0 ; x < level ; x++) {
		ar[ROI_Y-2][ROI_X + x] = color16;
		ar[ROI_Y-1][ROI_X + x] = color16;
	}
	// 2nd bar is green
	color16 = RGB(0, 255, 0);
	for( ; x < ROI_W ; x++) {
		ar[ROI_Y-2][ROI_X + x] = color16;
		ar[ROI_Y-1][ROI_X + x] = color16;
	}
}

#define SEARCH_W		12
#define SEARCH_H		12
#define SEARCH_X		(SEARCH_W / 2)
#define SEARCH_Y		(SEARCH_H / 2)
#define SEARCH_STEP_X	2
#define SEARCH_STEP_Y	2

void search_edge_image2(int flags, volatile uint16_t ar[120][160], uint8_t edge_live1[ROI_H][ROI_W], uint8_t edge_input2[ROI_H][ROI_W]  )
{
	int x, y;
	int offX, offY;
	long minDiff = 80 * 60 * 255;
	int minX;
	int minY;

	for(offY = 0 ; offY < SEARCH_H ; offY+=SEARCH_STEP_Y) {
		for(offX = 0 ; offX < SEARCH_W ; offX+=SEARCH_STEP_X) {
			totalDiff = 0;

			for(y = 0 ; y < ROI_H-SEARCH_H ; y++) {
				for(x = 0 ; x < ROI_W-SEARCH_W ; x++) {
					int edge1 = edge_live1[y + SEARCH_Y][x + SEARCH_X];
					int edge2 = edge_input2[y + offY+ SEARCH_Y][x + offX + SEARCH_X];
					int grayDiff = edge1 - edge2;
					if(grayDiff < 0)
						grayDiff = -grayDiff;
					totalDiff += grayDiff;
				}
			}
			if(totalDiff < minDiff) {
				minDiff = totalDiff;
				minX = offX;
				minY = offY;
			}

		}
	}

	totalDiff = 0;
	for(y = 0 ; y < ROI_H-SEARCH_H ; y++) {
		for(x = 0 ; x < ROI_W-SEARCH_W ; x++) {
			int edge1 = edge_live1[y + SEARCH_Y][x + SEARCH_X];
			int edge2 = edge_input2[y+minY + SEARCH_Y][x+minX + SEARCH_X];
			int grayDiff = edge1 - edge2;
			if(grayDiff < 0)
				grayDiff = -grayDiff;
			totalDiff += grayDiff;
			uint16_t color16 = RGB(edge1, edge2, 0);		// Show edges in green
			ar[ROI_Y + y + SEARCH_Y][ROI_X + x + SEARCH_X] = color16;
		}
	}

	// totalDiff level: 347000 bad match, 15000 good match
	int level = (totalDiff * ROI_W) / 347000;
	if(level > ROI_W)
		level = ROI_W;
	// First color bar is red
	uint16_t color16 = RGB(255, 0, 0);
	for(x = 0 ; x < level ; x++) {
		ar[ROI_Y+ROI_H][ROI_X + x] = color16;
		ar[ROI_Y+ROI_H+1][ROI_X + x] = color16;
	}
	// 2nd bar is green
	color16 = RGB(0, 255, 0);
	for( ; x < ROI_W ; x++) {
		ar[ROI_Y+ROI_H][ROI_X + x] = color16;
		ar[ROI_Y+ROI_H+1][ROI_X + x] = color16;
	}
}

/*
 * Array indexing ar[row][col] also known as ar[y][x]
 *
 */
#define add_color(a) ( ((a&0xf800)>>11) + ((a&0x03e0)>>5) + (a&0x1f) )

enum _gpio_pins_pinNames2{
  kGpioLED_A = GPIO_MAKE_PIN(GPIOA_IDX, 15U),
  kGpioLED_B = GPIO_MAKE_PIN(GPIOA_IDX, 16U),
};

const gpio_output_pin_user_config_t ledPins2[] = {
  {
    .pinName = kGpioLED_A,
    .config.outputLogic = 1,
    .config.slewRate = kPortSlowSlewRate,
    .config.isOpenDrainEnabled = false,
    .config.driveStrength = kPortLowDriveStrength,
  },
  {
    .pinName = kGpioLED_B,
    .config.outputLogic = 1,
    .config.slewRate = kPortSlowSlewRate,
    .config.isOpenDrainEnabled = false,
    .config.driveStrength = kPortLowDriveStrength,
  },
  {
    .pinName = GPIO_PINS_OUT_OF_RANGE,
  }
};

gpio_input_pin_user_config_t inputPinSW_A =
  {
	.pinName = kGpioSW2,
	.config.isPullEnable = true,
	.config.pullSelect = kPortPullUp,
	.config.isPassiveFilterEnabled = false,
	.config.isDigitalFilterEnabled = false,
	.config.interrupt = kPortIntDisabled
  };

// OV7680 GPIO
//  PTA1
//  PTA2

//  PTB0
//  PTB2
//	PTB3

//	PTC3
//	PTC8
//  PTC9

//	PTB10
//	PTB11

//	PTB18
//	PTB19
//	PTB21
//	PTB20
//	PTB22
//	PTB23
//

void setGPIO(uint32_t pinName, int enabled) {
	GPIO_DRV_WritePinOutput(pinName, enabled ? 1 : 0);
}

void config_gpio_pins2() {
    /* Affects PORTA_PCR4 register */
    PORT_HAL_SetMuxMode(PORTA,15UL,kPortMuxAsGpio);
    GPIO_DRV_SetPinDir(kGpioLED_A, kGpioDigitalOutput);
    PORT_HAL_SetMuxMode(PORTA,16UL,kPortMuxAsGpio);
    GPIO_DRV_SetPinDir(kGpioLED_B, kGpioDigitalOutput);

//    PORT_HAL_SetPullMode(PORTA,4UL,kPortPullUp);
//    PORT_HAL_SetPullCmd(PORTA,4UL,true);
}





enum displayState_e {
	DISP_Normal,
	DISP_Saved,
	DISP_BW,
	Disp_Diff,
	Disp_Edge_Overlay,
	Disp_Edge,
	Disp_Edge_Diff,
	Disp_Edge_Search,
	Disp_MAX,
};

void ImageProc_Init()
{

}


void ImageProc_Task(void)
{
	int buf;
//	int i,j;
	int pending_request_flag = 0;
	int pending_request_buf = 0;
//	int last_button = 0;
	int button1 = 0;
	int lastbutton1 = 0;
	int button2 = 0;
	int lastbutton2 = 0;
	int capturePending = 0;
//	int showDiff = 0;
//	int showBW = 0;
	int diffMax = 0;
	int diffMin = 0x10000000;
	int displayState = DISP_Normal;
	int led_on_time = 0;
	int led_off_time = 0;
	int blinks=0;
	unsigned int b2counts = 0;
	unsigned int capCountdown = 0;

	//SW_EN;
	PORT_HAL_SetMuxMode(PORTA, 4, kPortMuxAsGpio);
	PORT_HAL_SetMuxMode(PORTC, 6, kPortMuxAsGpio);

	config_gpio_pins2();

//	GPIO_DRV_InputPinInit(&inputPinSW2);

	button1 = GPIO_DRV_ReadPinInput(kGpioSW1);
	lastbutton1 = button1;
	button2 = GPIO_DRV_ReadPinInput(kGpioSW2);
	lastbutton2 = button2;

	printf("Button1 = %d, Button2 = %d\r\n", button1, button2);
	while(1)
	{

		/*
		 * The 'portb_callback' function of the camera is where we think the camera indicates that an image
		 * capture has completed.  It let's us know this information through the global variable 'buf_indicator'.
		 * '1' indicates buffer 0 is ready, '2' indicates buffer 1 is ready, and '0' means nothing happening.
		 *
		 * When a PC's program is grabbing video (is_sending flag TRUE), the USB initiates image capture.
		 * If not (is_sending flag FALSE), we initiate image capture.
		 */
		if ( buf_indicator  )
		{
			// buf_indicator flips between 1 and 2.  Adjust value to line up with 0 referenced buffer
			buf=buf_indicator-1;

			// USB capture indicated by '.', capture initiated by our routine is ':'
			if ( pending_request_flag ) printf(":"); else printf(".");

			buf_indicator=0;
			pending_request_flag= 0;
			uint16_t color = RGB(255, 0, 0);

			if(capturePending) {
				capture_image(1, (void *)&(u8CameraFrameBuffer[buf][32]) );
				scan_edges_image(1,  (void *)&(u8CameraFrameBuffer[buf][32]), edge_image );
				capturePending = 0;
//				showDiff = 1;
			} else {
				switch(displayState) {
				case DISP_Normal:
					break;
				case DISP_Saved:
					show_saved_image(1, (void *)&(u8CameraFrameBuffer[buf][32]) );
					overlay_edge_image(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_image );
					color = RGB(255, 0, 0);
					histogram_image(0, color, (void *)&(u8CameraFrameBuffer[buf][32]), edge_image);
					break;

				case DISP_BW:
					conv_bw_image(1, (void *)&(u8CameraFrameBuffer[buf][32]) );
					overlay_edge_image(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					break;

				case Disp_Diff:
					diffshow_image(1, (void *)&(u8CameraFrameBuffer[buf][32]) );
					if(totalDiff < diffMin)
						diffMin = totalDiff;
					if(totalDiff > diffMax)
						diffMax = totalDiff;
					break;

				case Disp_Edge_Overlay:
					scan_edges_image(1,  (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					overlay_edge_image(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					break;

				case Disp_Edge:
					scan_edges_image(1,  (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					show_edge_image(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					color = RGB(0, 255, 0);
					histogram_image(0, color, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar);
					break;

				case Disp_Edge_Diff:
					scan_edges_image(1,  (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					show_edge_image2(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar, edge_image );
					color = RGB(255, 0, 0);
					histogram_image(0, color, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar);
					if(totalDiff < diffMin)
						diffMin = totalDiff;
					if(totalDiff > diffMax)
						diffMax = totalDiff;
					break;

				case Disp_Edge_Search:
					scan_edges_image(1,  (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar );
					search_edge_image2(1, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar, edge_image );
					color = RGB(255, 0, 0);
				//	histogram_image(0, color, (void *)&(u8CameraFrameBuffer[buf][32]), edge_ar);
					if(totalDiff < diffMin)
						diffMin = totalDiff;
					if(totalDiff > diffMax)
						diffMax = totalDiff;
					break;

				default:
					displayState = DISP_Normal;
					break;
				}
			}
		}
		else
		{
			if (!virtual_camera.is_sending && !pending_request_flag )
			{
				// since USB is not requesting image capture, let's do it ourselves
//				pending_request_flag = 1;
//				flexio_ov7670_start_capture( pending_request_buf+1 );
//				pending_request_buf = 1-pending_request_buf;

				//LED3_TOGGLE;
			//	OSA_TimeDelay(5); /* 5ms */
			}
			else
			{
				OSA_TimeDelay(5); /* 5ms */

			}
		}

		button1 = GPIO_DRV_ReadPinInput(kGpioSW1);
		button2 = GPIO_DRV_ReadPinInput(kGpioSW2);

		if(button2 == 0) {
			b2counts++;
			if(capCountdown != 0) {
				printf(" - Capture CANCELLED, countDown=%x\r\n", capCountdown);
				capCountdown = 0;	// Cancel capture
			}
		}

		if(capCountdown > 0) {
			--capCountdown;
			if((capCountdown & 0x1FF) == 0x100) {
				LED3_ON;
				setGPIO(kGpioLED_B, 0);	// Light Green LED
				setGPIO(kGpioLED_A, 0);	// Light Orange LED
				printf(" -  On, countDown=%x\r\n", capCountdown);
			} else if((capCountdown & 0x1FF) == 0x000) {
				LED3_OFF;
				setGPIO(kGpioLED_B, 1);	// clear green LED
				setGPIO(kGpioLED_A, 1);	// clear orange LED
				if(capCountdown == 0) {
					capturePending = 1;
					displayState = Disp_Edge_Search;
					printf(" - Capture started, countDown=%x\r\n", capCountdown);
				} else {
					printf(" - Off, countDown=%x\r\n", capCountdown);
				}
			}

		}

		if(lastbutton1 != button1 || lastbutton2 != button2 ) {
			printf("Button1 = %d, Button2 = %d, diffMin=%d, diffMax=%d, state=%d\r\n", button1, button2, diffMin, diffMax, displayState);
//			setGPIO(kGpioLED_A, button1);
//			setGPIO(kGpioLED_B, button2);
			if(button1 == 0 && button2 == 0) {
				capturePending = 1;
//				capture_image(1, (void *)&(u8CameraFrameBuffer[buf][32]) );
			} else if(button2 == 0) {
//				showDiff = !showDiff;
				diffMax = 0;
				diffMin = 0x1000000;
				displayState = Disp_Edge;
			} else if(button1 == 0) {
				if(++displayState >= Disp_MAX)
					displayState = 0;
				blinks = displayState;
				led_on_time = 10;
				LED3_ON;
			} else {
				// All buttons released
				if(lastbutton2 == 0) {
					if(b2counts > 300) {
					    capCountdown = 0x1000;
					    displayState = Disp_Edge;
					} else {
						displayState = Disp_Edge_Search;
					}
					printf("\r\nCapture=%d: b2counts=%u\r\n", capturePending, b2counts);


					b2counts = 0;
				}
			}

			lastbutton1 = button1;
			lastbutton2 = button2;
		}

		if(led_on_time > 0) {
			if(--led_on_time <= 0) {
				LED3_OFF;
				led_on_time = 0;
				if(blinks > 0) {
					led_off_time = 40;
					--blinks;
				}
			}
		} else if(led_off_time > 0) {
			if(--led_off_time <= 0) {
				LED3_ON;
				led_on_time = 10;
			}
		} else if(displayState == Disp_Diff || displayState == Disp_Edge_Diff || displayState == Disp_Edge_Search) {
			if(totalDiff < 50000) {
				setGPIO(kGpioLED_B, 0);	// Light Green LED
				setGPIO(kGpioLED_A, 1);	// clear orange LED
//				led_on_time = 1;
//				led_off_time = 20;
//				LED3_ON;
			} else if(totalDiff < 100000) {
				setGPIO(kGpioLED_B, 0);	// Light Green LED
				setGPIO(kGpioLED_A, 0);	// Light Orange LED
//				led_on_time = 1;
//				led_off_time = 100;
//				LED3_ON;
			} else if(totalDiff < 150000) {
				setGPIO(kGpioLED_A, 0);	// Light Orange LED
				setGPIO(kGpioLED_B, 1);	// clear green LED
//				led_on_time = 1;
//				led_off_time = 200;
//				LED3_ON;
			} else {
				setGPIO(kGpioLED_B, 1);	// clear green LED
				setGPIO(kGpioLED_A, 1);	// clear orange LED
			}

		}

	}

//	OSA_TimeDelay(2000U);
//	LED3_TOGGLE;
}

camera.c

C/C++
My modified camera.c file. Has extra USB debugging printfs too
/**HEADER********************************************************************
* 
* Copyright (c) 2004-2010, 2014-2015 Freescale Semiconductor;
* All Rights Reserved
*
*************************************************************************** 
*
* THIS SOFTWARE IS PROVIDED BY FREESCALE "AS IS" AND ANY EXPRESSED OR 
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  
* IN NO EVENT SHALL FREESCALE OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 
* THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************
*
* $FileName: virtual_camera.c$
* $Version : 
* $Date    : 
*
* Comments:
*        The file contains Macro's and functions required for Virtual Camera
*        Loopback Application
*
*****************************************************************************/

/******************************************************************************
 * Includes
 *****************************************************************************/
#include "usb_device_config.h"
#include "usb.h"
#include "usb_device_stack_interface.h"

#include "usb_descriptor.h"
#include "camera.h"   /* Virtual camera Application Header File */
//#include "imageproc.h"
#include "usb_request.h"

#if (OS_ADAPTER_ACTIVE_OS == OS_ADAPTER_SDK)
#include "fsl_device_registers.h"
#include "fsl_clock_manager.h"
#include "board.h"
#include "fsl_debug_console.h"
#include "fsl_port_hal.h"

#include <stdio.h>
#include <stdlib.h>
#include <math.h>

#include "flexio_ov7670.h"
#endif

/*****************************************************************************
 * Constant and Macro's - None
 *****************************************************************************/
 #define MAIN_TASK       10
/*****************************************************************************
 * Global Functions Prototypes
 *****************************************************************************/
void APP_init(void);
void USB_Prepare_Data(void);
extern void Main_Task(uint32_t param);

/****************************************************************************
 * Global Variables
 ****************************************************************************/
extern usb_desc_request_notify_struct_t  desc_callback;
virtual_camera_struct_t                  virtual_camera;

#if (OS_ADAPTER_ACTIVE_OS == OS_ADAPTER_MQX)
extern void Main_Task(uint32_t param);
#define MAIN_TASK       10
TASK_TEMPLATE_STRUCT  MQX_template_list[] =
{
   { MAIN_TASK, Main_Task, 2*3000L, 7L, "Main", MQX_AUTO_START_TASK, 0, 0},
   { 0L, 0L, 0L, 0L, 0L, 0L, 0, 0}
};
#endif
/*****************************************************************************
 * Local Types - None
 *****************************************************************************/

/*****************************************************************************
 * Local Functions Prototypes
 *****************************************************************************/

/*****************************************************************************
 * Local Variables
 *****************************************************************************/
volatile int buf_indicator=0;
extern void ImageProc_Task();

/*****************************************************************************
 * Local Functions
 *****************************************************************************/

 /******************************************************************************
 *
 *   @name        USB_Prepare_Data
 *
 *   @brief       This function prepares data to send
 *
 *   @param       None
 *
 *   @return      None
 *****************************************************************************
 * This function prepare data before sending
 *****************************************************************************/
void USB_Prepare_Data(void)
{
    uint32_t                             frame_interval = 0;
//    uint32_t                             buffer_count = 0;
    uint32_t                             send_data = 0;
    video_mjpeg_payload_header_struct_t* video_mjpeg_header_ptr = NULL;
//    uint16_t                             k;
    uint8_t                              j;
    uint8_t                              format_index = 0;

    if (virtual_camera.still_image_transmit > 0)
    {
        format_index = virtual_camera.still_commit_struct.bFormatIndex;
        frame_interval = virtual_camera.commit_struct.dwFrameInterval;
    }
    else
    {
        format_index = virtual_camera.commit_struct.bFormatIndex;
        frame_interval = virtual_camera.commit_struct.dwFrameInterval;
    }

    video_mjpeg_header_ptr = (video_mjpeg_payload_header_struct_t*)virtual_camera.image_header;
    virtual_camera.image_buffer = virtual_camera.image_header;

    if (VS_FORMAT_UNCOMPRESSED_DESC_INDEX == format_index)
    {
        if (virtual_camera.image_position[format_index-1] < OV7670_FRAME_BYTES)
        {
            virtual_camera.image_buffer = (uint8_t *)&u8CameraFrameBuffer[virtual_camera.full_buffer_index][virtual_camera.image_position[format_index-1] - HEADER_PACKET_SIZE + 32];
            video_mjpeg_header_ptr = (video_mjpeg_payload_header_struct_t*)virtual_camera.image_buffer;
        }
    }
    
    video_mjpeg_header_ptr->bHeaderLength = HEADER_PACKET_SIZE;
    for (j = 0;j < sizeof(video_mjpeg_header_ptr->bSourceClockReference);j++)
    {
        video_mjpeg_header_ptr->bSourceClockReference[j] = 0;
    }
    video_mjpeg_header_ptr->HeaderInfo.bmHeaderInfo = 0;
    video_mjpeg_header_ptr->dwPresentationTime = 0;
    video_mjpeg_header_ptr->HeaderInfo.bitMap.frame_id = virtual_camera.frame_id;
    video_mjpeg_header_ptr->HeaderInfo.bitMap.end_of_frame = 0;
    video_mjpeg_header_ptr->HeaderInfo.bitMap.still_image = virtual_camera.still_image_transmit;
    
    virtual_camera.frame_progress += 10000;
    if (virtual_camera.frame_sent > 0)
    {
        virtual_camera.frame_send_length = HEADER_PACKET_SIZE;
        if (virtual_camera.frame_progress < frame_interval)
        {
            return;
        }
        else
        {
            video_mjpeg_header_ptr->HeaderInfo.bitMap.end_of_frame = 1;
            virtual_camera.frame_id = 1 - virtual_camera.frame_id;
            virtual_camera.frame_sent = 0;
            virtual_camera.frame_progress = 0;
            
            virtual_camera.image_position[format_index-1] = 0;
            
            if(virtual_camera.still_image_transmit > 0)
            {
                virtual_camera.still_image_transmit = 0;
                virtual_camera.transmit_type = 0;
            }
            
            if ((virtual_camera.transmit_type > 0) && (virtual_camera.transmit_type < 3))
            {
                virtual_camera.still_image_transmit = 1;
            }
            return;
        }
    }
  
    if (VS_FORMAT_UNCOMPRESSED_DESC_INDEX == format_index)
    {
        if (virtual_camera.image_position[format_index-1] < OV7670_FRAME_BYTES)
        {
            send_data = OV7670_FRAME_BYTES - virtual_camera.image_position[format_index-1];
            if (send_data > MAX_PAYLOAD_DATA_SIZE)
            {
               send_data = MAX_PAYLOAD_DATA_SIZE;
            }
//            for (int i = 0;i < send_data;i++)
//            {                
//                virtual_camera.image_buffer[HEADER_PACKET_SIZE + i] = camera_frame_buffer[virtual_camera.image_position[format_index-1] + i];
//            }
            virtual_camera.image_position[format_index-1] += send_data;
            
            if (virtual_camera.image_position[format_index-1] >= OV7670_FRAME_BYTES)
            {
                flexio_ov7670_start_capture(virtual_camera.full_buffer_index+1);
                virtual_camera.full_buffer_index = 1 - virtual_camera.full_buffer_index;
                buf_indicator=virtual_camera.full_buffer_index+1;
                if (virtual_camera.frame_progress < frame_interval)
                {
                    virtual_camera.frame_sent = 1;
                }
                else
                {
                    video_mjpeg_header_ptr->HeaderInfo.bitMap.end_of_frame = 1;
                    virtual_camera.frame_id = 1 - virtual_camera.frame_id;
                    virtual_camera.frame_sent = 0;
                    virtual_camera.frame_progress = 0;

                    virtual_camera.image_position[format_index-1] = 0;

                    if(virtual_camera.still_image_transmit > 0)
                    {
                        virtual_camera.still_image_transmit = 0;
                        virtual_camera.transmit_type = 0;
                    }

                    if ((virtual_camera.transmit_type > 0) && (virtual_camera.transmit_type < 3))
                    {
                        virtual_camera.still_image_transmit = 1;
                    }
                }
            }
        }
    }
    virtual_camera.frame_send_length = HEADER_PACKET_SIZE + send_data;
}

/******************************************************************************
 * 
 *    @name        USB_App_Param_Callback
 *
 *    @brief       This function handles the callback for Get/Set report req  
 *                  
 *    @param       request  :  request type
 *    @param       value    :  give report type and id
 *    @param       data     :  pointer to the data 
 *    @param       size     :  size of the transfer
 *
 *    @return      status
 *                  USB_OK  :  if successful
 *                  else return error
 *
 *****************************************************************************/
 uint8_t USB_App_Param_Callback
 (
    uint8_t request, 
    uint16_t value, 
    uint8_t ** data, 
    uint32_t* size,
    void* arg
) 
{
    uint8_t error = USB_OK;

    if ((request == USB_DEV_EVENT_SEND_COMPLETE) && (value == USB_VIDEO_ISO_REQ_VAL_INVALID))
    {
        virtual_camera.is_sending = (uint8_t)FALSE;
        
        if (NULL != arg)
        {
            if (((uint8_t)TRUE == virtual_camera.attached) && ((uint8_t)TRUE == virtual_camera.start_send))
            {
                if ((uint8_t)FALSE == virtual_camera.is_sending)
                {
//                	LED3_TOGGLE;
                    virtual_camera.is_sending = (uint8_t)TRUE;
                    USB_Prepare_Data();
                    (void)USB_Class_Video_Send_Data(virtual_camera.video_handle,VIDEO_ISO_ENDPOINT,virtual_camera.image_buffer,virtual_camera.frame_send_length);
                }
            }
        }
        return error;
    }


    error = USB_Class_Get_feature(virtual_camera.video_handle, value, data, size);
    if (error == USBERR_INVALID_REQ_TYPE)
    {
        error = USB_Class_Set_feature(virtual_camera.video_handle, value, data, size);
    }
    USB_PRINTF("param cb(req=%d, val=0x%x, err=0x%x\r\n", request, value, error);

    return error; 
}   

/******************************************************************************
 * 
 *    @name        USB_App_Callback
 *    
 *    @brief       This function handles the callback  
 *                  
 *    @param       handle : handle to Identify the controller
 *    @param       event_type : value of the event
 *    @param       val : gives the configuration value 
 * 
 *    @return      None
 *
 *****************************************************************************/
void USB_App_Callback(uint8_t event_type, void* val,void* arg) 
{
    uint16_t interface_setting;
    uint8_t interface_alternate;
    uint8_t interface_num;


    USB_PRINTF("app cb(type=%d\r\n", event_type);
    if ((event_type == USB_DEV_EVENT_BUS_RESET) || (event_type == USB_DEV_EVENT_CONFIG_CHANGED))
    {
        virtual_camera.attached=FALSE;
        if (USB_OK == USB_Class_Video_Get_Speed(virtual_camera.video_handle, &virtual_camera.app_speed))
        {
            USB_Desc_Set_Speed(virtual_camera.video_handle, virtual_camera.app_speed);
        }
    }
    else if (event_type == USB_DEV_EVENT_ENUM_COMPLETE) 
    {
        virtual_camera.attached=TRUE; 
        USB_PRINTF("USB camera is working ... \r\n");
    }
    else if (event_type == USB_DEV_EVENT_ERROR)
    {
        /* add user code for error handling */
    }
    else if (event_type == USB_DEV_EVENT_INTERFACE_CHANGED)
    {
        interface_setting = *((uint16_t*)val);
        interface_alternate = (uint8_t)(interface_setting&0x00FF);
        interface_num = (uint8_t)((interface_setting>>8)&0x00FF);
        USB_PRINTF("USB iface changed, sett=%x, alt=%x, num=%x, vc[att=%x, sta=%x, is_s=%x]\r\n",
        		interface_setting, interface_alternate, interface_num, virtual_camera.attached, virtual_camera.start_send, virtual_camera.is_sending);
        
        if (VIDEO_STREAM_IF_INDEX == interface_num)
        {
            if (0 != interface_alternate)
            {
                virtual_camera.start_send = TRUE;
                if (((uint8_t)TRUE == virtual_camera.attached) && ((uint8_t)TRUE == virtual_camera.start_send))
                {
                    if ((uint8_t)FALSE == virtual_camera.is_sending)
                    {
                        virtual_camera.is_sending = (uint8_t)TRUE;
                        virtual_camera.frame_sent = 0;
                        virtual_camera.frame_progress = 0;
                        USB_Prepare_Data();
                        (void)USB_Class_Video_Send_Data(virtual_camera.video_handle,VIDEO_ISO_ENDPOINT,virtual_camera.image_buffer,virtual_camera.frame_send_length);
                    }
                }
            }
            else
            {
                virtual_camera.transmit_type = 0;
                virtual_camera.start_send = (uint8_t)FALSE;
                if ((uint8_t)TRUE == virtual_camera.is_sending)
                {
                    virtual_camera.is_sending = (uint8_t)FALSE;
                    (void)USB_Class_Video_Cancel(virtual_camera.video_handle,VIDEO_ISO_ENDPOINT,USB_SEND);
                }
            }
        }
    }
  
}



 /******************************************************************************
 *
 *   @name        APP_init
 *
 *   @brief       This function is the entry for the Virtual camera application
 *
 *   @param       None
 *
 *   @return      None
 *****************************************************************************
 * This function starts the virtual camera application
 *****************************************************************************/

void APP_init(void)
{
    video_config_struct_t video_config;
    video_config.video_application_callback.callback = USB_App_Callback;
    video_config.video_application_callback.arg = &virtual_camera.video_handle;
    video_config.class_specific_callback.callback = USB_App_Param_Callback;
    video_config.class_specific_callback.arg = &virtual_camera.video_handle;
    video_config.board_init_callback.callback = NULL;
    video_config.board_init_callback.arg = 0;
    video_config.desc_callback_ptr = &desc_callback;

    OS_Mem_zero((uint8_t*)&virtual_camera.probe_struct, sizeof(virtual_camera.probe_struct));
    OS_Mem_zero((uint8_t*)&virtual_camera.commit_struct, sizeof(virtual_camera.commit_struct));
    OS_Mem_zero((uint8_t*)&virtual_camera.still_probe_struct, sizeof(virtual_camera.still_probe_struct));
    OS_Mem_zero((uint8_t*)&virtual_camera.still_commit_struct, sizeof(virtual_camera.still_commit_struct));
    
    virtual_camera.probe_struct.bFormatIndex = VS_FORMAT_UNCOMPRESSED_DESC_INDEX;
    virtual_camera.probe_struct.bFrameIndex = 1;
    virtual_camera.probe_struct.dwFrameInterval = 0x1E8480;
    virtual_camera.probe_struct.dwMaxPayloadTransferSize = VIDEO_ISO_ENDPOINT_PACKET_SIZE;
    virtual_camera.probe_struct.dwMaxVideoFrameSize = 0x025800;
    
    virtual_camera.commit_struct.bFormatIndex = VS_FORMAT_UNCOMPRESSED_DESC_INDEX;
    virtual_camera.commit_struct.bFrameIndex = 1;
    virtual_camera.commit_struct.dwFrameInterval = 0x1E8480;
    virtual_camera.commit_struct.dwMaxPayloadTransferSize = VIDEO_ISO_ENDPOINT_PACKET_SIZE;
    virtual_camera.commit_struct.dwMaxVideoFrameSize = 0x025800;
    
    virtual_camera.probe_length = 26;
    virtual_camera.commit_length = 26;
    virtual_camera.probe_info = 0x03;
    virtual_camera.commit_info = 0x03;
    virtual_camera.frame_progress = 0;
    
    virtual_camera.still_probe_struct.bFormatIndex = VS_FORMAT_UNCOMPRESSED_DESC_INDEX;
    virtual_camera.still_probe_struct.bFrameIndex = 1;
    virtual_camera.still_probe_struct.bCompressionIndex = 1;
    virtual_camera.still_probe_struct.dwMaxPayloadTransferSize = VIDEO_ISO_ENDPOINT_PACKET_SIZE;
    virtual_camera.still_probe_struct.dwMaxVideoFrameSize = 0x025800;
    
    virtual_camera.still_commit_struct.bFormatIndex = VS_FORMAT_UNCOMPRESSED_DESC_INDEX;
    virtual_camera.still_commit_struct.bFrameIndex = 1;
    virtual_camera.still_commit_struct.bCompressionIndex = 1;
    virtual_camera.still_commit_struct.dwMaxPayloadTransferSize = VIDEO_ISO_ENDPOINT_PACKET_SIZE;
    virtual_camera.still_commit_struct.dwMaxVideoFrameSize = 0x025800;

    virtual_camera.still_probe_length = 26;
    virtual_camera.still_commit_length = 26;
    virtual_camera.still_probe_info = 0x03;
    virtual_camera.still_commit_info = 0x03;
    virtual_camera.transmit_type = 0;
    virtual_camera.full_buffer_index = 1;

    extern void flexio_ov7670_init();
    /* Initialize the USB interface */
    USB_Class_Video_Init(CONTROLLER_ID, &video_config, &virtual_camera.video_handle);
   // OSA_TimeDelay(200);
    flexio_ov7670_init();
   // OSA_TimeDelay(500);
}

/******************************************************************************
 *
 *    @name        APP_task
 *
 *    @brief       This function use to send data
 *
 *    @param       None
 *
 *    @return      None
 *
 *****************************************************************************
 *
 *****************************************************************************/

void APP_task(void)
{
//	ImageProc_Init();
//	while ( 1 )
//	{
		ImageProc_Task();
//		//OSA_TimeDelay(5);
//	}
}


#if (OS_ADAPTER_ACTIVE_OS == OS_ADAPTER_MQX)

/*FUNCTION*----------------------------------------------------------------
* 
* Function Name  : Main_Task
* Returned Value : None
* Comments       :
*     First function called.  Calls Test_App
*     callback functions.
* 
*END*--------------------------------------------------------------------*/
void Main_Task( uint32_t param )
{   
    UNUSED_ARGUMENT (param)
    APP_init();
}
#endif

#if (OS_ADAPTER_ACTIVE_OS == OS_ADAPTER_SDK)

#if defined(FSL_RTOS_MQX)
void Main_Task(uint32_t param);
TASK_TEMPLATE_STRUCT  MQX_template_list[] =
{
   { 1L,     Main_Task,      2500L,  MQX_MAIN_TASK_PRIORITY, "Main",      MQX_AUTO_START_TASK},
   { 0L,     0L,             0L,    0L, 0L,          0L }
};
#endif


#if (USE_RTOS)
static void Task_Start(void *arg)
{
    APP_init();
    for (;; )
    {
        APP_task();
    }
}
#endif

#if defined(FSL_RTOS_MQX)
void Main_Task(uint32_t param)
#else

#if defined(__CC_ARM) || defined(__GNUC__)
int main(void)
#else
void main(void)
#endif

#endif
{
    OSA_Init();
    hardware_init();
    dbg_uart_init();

    PRINTF("Video Startup\r\n");
    LED3_EN;

#if !(USE_RTOS)
    APP_init();
#else
    OS_Task_create(Task_Start, NULL, 4L, 1000L, "task_start", NULL);
#endif

    OSA_Start();
#if (!defined(FSL_RTOS_MQX))&(defined(__CC_ARM) || defined(__GNUC__))
    return 1;
#endif
}
#endif

/* EOF */

Credits

Rich Huber

Rich Huber

1 project • 1 follower

Comments

Related channels and tags
  • NXP
  • Photos & Video