As we completed W5300 control and High-speed TCP transmission test using STM32F4-RP board, let’s run a test to transmit camera images through network.
Using the camera interface DCMI of STM32, we will obtain camera images, send images to the network, and write a program that can receive images from PC host programs and process them.
W5300 allows TCP transmission of more than 60 Mbps, meaning that we will be able to transmit 10 640x480x2 Byte RAW (RGB565) images per second. The reason for using RAW Data is to ensure that there is no loss in image processing as much as possible.
Let’s create a network camera that will send 10 frames per second of RAW image with resolution 640x480 and do simple video processing.
Project related source code is available for download at https://github.com/elabsystem/STM32_RP.
STM32 camera settings can be done in DCMI device.
Let’s configure DCMI using STM32CubeIDE.
Configure interrupt usage for DMA.
The camera module requires a master clock to be entered, but by setting PA8 of STM32 to MCO, a system clock output is set to 16Mhz.
Camera related configuration can be done through I2C interface.
Let's configure using I2C2.
After the configuration is completed and saved, basic code is generated.
FirmwareWe used file ov2640.c (https://github.com/STMicroelectronics/stm32-ov2640) from ST Micro for camera control.
Camera related functions are defined in_user_code_cam.h
The following functions were redefined; HAL_DCMI_LineEventCallback() when line data is received from camera, and HAL_DCMI_VsyncEventCallback() when one frame is completed, so that the main function is notified.
void HAL_DCMI_LineEventCallback(DCMI_HandleTypeDef *hdcmi)
{
gCameraLineFlag++;
}
void HAL_DCMI_VsyncEventCallback(DCMI_HandleTypeDef *hdcmi)
{
gCameraFrameFlag++;
}
void CamImagTransferLine(unsigned int Cmd, unsigned int Para)
{
int ret ;
unsigned char data_buf[2048*_UDP_TX_BUF_RATE];
unsigned int size = _IMAGE_SIZE_X*2*_UDP_TX_BUF_RATE;
data_buf[0] = Cmd;
data_buf[1] = 0;
data_buf[2] = Para>>8;
data_buf[3] = Para;
memcpy((uint8_t *)&data_buf[4], (uint8_t *)&FrameBuffer, size);
if(gImgTransferFlag)
{
ret = send(_CAM_SOCK_NUM, (uint8_t *)data_buf, size+4);
}
}
void CamImagTransferFrame(unsigned int Cmd, unsigned int Para)
{
int ret ;
unsigned char data_buf[2048*_UDP_TX_BUF_RATE];
unsigned int size = _IMAGE_SIZE_X*2*_UDP_TX_BUF_RATE;
data_buf[0] = Cmd;
data_buf[1] = 0;
data_buf[2] = Para>>8;
data_buf[3] = Para;
if(gImgTransferFlag)
{
HAL_Delay(1);
ret = send(_CAM_SOCK_NUM, data_buf, size+4);
HAL_Delay(1);
}
}
When a line interrupt signal is recognized in the main loop, the video data stored in the camera buffer is sent to TCP, and when a frame interrupt signal is recognized, the PC host program is sent a frame completion signal to update the screen.
while (1)
{
/* USER CODE END WHILE */
ProcessCamTcps(_CAM_SOCK_NUM, ethBuf0, dDestport);
if(gCameraFrameFlag>0)
{
gCameraFrameFlag = 0;
CamImagTransferFrame(_CMD_CAM_FRAME, gLineCnt);
printf(">%d, %d\r\n", gFrameCnt, gLineCnt);
gFrameCnt++;
gLineCnt = 0;
}
if(gCameraLineFlag>0)
{
gCameraLineFlag = 0;
CamImagTransferLine(_CMD_CAM_LINE, gLineCnt);
gLineCnt++;
}
/* USER CODE BEGIN 3 */
}
The ProcessCamTcps() function handles commands sent from a PC.
Commands such as video transmission start and end or camera control can be sent through TCP communication.
int32_t ProcessCamTCPS(uint8_t sn, uint8_t* buf, uint16_t port)
{
int32_t ret;
uint16_t size = 0, sentsize=0;
unsigned int cmd;
unsigned int para;
switch(getSn_SR(sn))
{
case SOCK_ESTABLISHED :
if(getSn_IR(sn) & Sn_IR_CON)
{
setSn_IR(sn, Sn_IR_CON);
}
// Don't need to check SOCKERR_BUSY because it doesn't not occur.
if((size = getSn_RX_RSR(sn)) > 0)
{
if(size > ETH_MAX_BUF_SIZE) size = ETH_MAX_BUF_SIZE;
ret = recv(sn, buf, size);
cmd = buf[0];
para = buf[2]<<8 | buf[3];
if(cmd == _CMD_LED)
{
if(para == 0)
{
Led1Off();
}
else if(para == 1)
{
Led1On();
}
}
else if(cmd == _CMD_CAM_START)
{
gImgTransferFlag = 1;
printf("Cam Stream Start\r\n");
}
else if(cmd == _CMD_CAM_STOP)
{
gImgTransferFlag = 0;
printf("Cam Stream Stop\r\n");
}
//sendto(sock, data_buf, len, gDestip, dDestport);
printf("rx size=%d \r\n", size);
}
break;
case SOCK_CLOSE_WAIT :
if((ret = disconnect(sn)) != SOCK_OK) return ret;
break;
case SOCK_INIT :
if( (ret = listen(sn)) != SOCK_OK) return ret;
break;
case SOCK_CLOSED:
if((ret = socket(sn, Sn_MR_TCP, port, 0x00)) != sn) return ret;
break;
default:
break;
}
return 1;
}
PC SoftwareA Client program that will be able to receive data sent from W5300 board is needed.
Let's create a NetImagPlay program for printing images on the PC using Visual Studio C++& OpenCV library,
Download OpenCV from https://opencv.org/releases/ and add its directory to Visual Studio environment settings.
Configure the OpenCV library file too.
PC Program to enter network information
Executable file can be downloaded from https://github.com/elabsystem/STM32_RP/tree/master/HostSW_NetPlay/x64/Release
If OpenCV is not installed, file opencv_world430.dll is needed.
No, create function SendCommand() to transmit control commands via TCP.
int CNetImagePlayDlg::SendCommand(unsigned int Cmd, unsigned int Para)
{
CString str;
unsigned char buf[4];
buf[0] = Cmd;
buf[1] = 0;
buf[2] = Para >> 8;
buf[3] = Para;
m_ClientSocket.Send(buf, sizeof(buf));
return 0;
}
When the image data is transmitted from W5300 to TCP, OnReceive() funciton will be called to distinguish whether it is line or frame, and print it on screen.
int CNetImagePlayDlg::OnReceive(unsigned char* pDataBuffer)
{
// TODO: add the code here
CString str;
int cmd;
int line = 0;
unsigned int size = _IMAGE_SIZE_X * 2 * _UDP_TX_BUF_RATE;
cmd = pDataBuffer[0];
line = pDataBuffer[2] << 8 | pDataBuffer[3];
if (cmd == _CMD_CAM_LINE)
{
str.Format(L"%d, %d", m_gCamBufferIndex, line);
SetDlgItemText(IDC_STATIC_MSG, str);
if (m_gCamBufferIndex < _IMAGE_SIZE_Y)
{
memcpy((unsigned char*)&gImageBuffer[m_gCamBufferIndex * _IMAGE_SIZE_X * 2 * _UDP_TX_BUF_RATE], (unsigned char*)&pDataBuffer[4], _IMAGE_SIZE_X * 2 + _UDP_TX_BUF_RATE);
m_gCamBufferIndex++;
}
}
else if (cmd == _CMD_CAM_FRAME)
{
m_gCamBufferIndex = 0;
DisplayCamImag(gImageBuffer, _IMAGE_SIZE_Y, _IMAGE_SIZE_X);
}
return 0;
}
A function that prints images on the screen when images come in, and prints them out with a simple image processing (Canny Edge)
int CNetImagePlayDlg::OnThreadProc()
{
Mat scale_img;
int i;
Mat gray_image;
//convert image from grey to color cvtColor(m_DisplayImg, gray_image, COLOR_BGR2GRAY);
cv::waitKey(10);
imshow("Sensor", m_DisplayImg);
cvtColor(m_DisplayImg, gray_image, COLOR_BGR2GRAY);
Mat img_edge;
Canny(gray_image, img_edge, 50, 150);
imshow("img_edge", img_edge);
return 0;
}
Test video
Testing a network camera based on W5300 that is able to transmit 10 frames per second (6Mbyte/s - 640x480x2x10), each frame is a 640x480 image
Jong Ho Park
Jake Lee
Comments