Jallson SuryoSamuel AlexanderNicholas Patrick
Published © Apache-2.0

Counting for Inspection and Quality Control with TensorRT

An Nvidia Jetson Nano computer vision project with TensorRT acceleration, to perform counting for manufacturing quality control.

AdvancedFull instructions providedOver 1 day827
AI at the Edge Applications (Open Category): 3rd Place
AI Innovation Challenge
Counting for Inspection and Quality Control with TensorRT

Things used in this project

Hardware components

NVIDIA Jetson Nano Developer Kit
NVIDIA Jetson Nano Developer Kit
×1
USB webcam (eg. Logitech C270 or C920)
×1
SparkFun Qwiic Pro Micro - USB-C (ATmega32U4)
SparkFun Qwiic Pro Micro - USB-C (ATmega32U4)
Could be any microcontroller, really.
×1
Mini conveyor belt system
Check Bill of Materials for details
×1
Objects: Mini pizza with toppings
Dough or printed paper scaled model
×1
Ethernet Cable, 1 m
Ethernet Cable, 1 m
×1

Software apps and online services

NVIDIA Jetpack SDK (inc Tensor RT, OpenCV, PyTorch, CUDA C++ GPU accelerated dev toolkit)
Edge Impulse Studio
Edge Impulse Studio
EI Linux, Python, C++ SDK
Arduino IDE
Arduino IDE

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)

Story

Read more

Custom parts and enclosures

cam_mount_krT5IiB4b1.stl

cam_stem_mount_yUVlU8b1Zo.stl

cylinder_iGWK5PRVyd.stl

dc_motor_bracket_vsKrOioAtg.stl

holder_mirror_9yeLLK0YYI.stl

holder_X5QxrVATJb.stl

indicator_base_6pzPNdEuaU.stl

indicator_diffuser_8KZDyDrFPL.stl

indicator_mid1_a5TmT9EbX9.stl

indicator_mid2_SVC3A3dRIB.stl

indicator_top_4NHqIM7wbT.stl

pizza_mold_base_oKUepEwHVB.stl

pizza_mold_lid_Pg1Nv2Eqvy.stl

pro_micro_holder_kNf9cDsIiT.stl

pusher_long_xJt7YCq9Xl.stl

pusher_short_541M4hXruE.stl

reject_slide_CIHTXzLZST.stl

servo_mount_x8GWpc6lex.stl

Schematics

pizzatopping_schematic_eQSXug8ArN.png

Code

pizzaToppingProMicro.ino

Arduino
Arduino code for the microcontroller (see step 8)
#define red 5
#define green 6
#define on 100      //brightness of LED when it is turned on
#define off 0

#define open 122    //gate open and close values
#define close 74

#include <Servo.h>
Servo gate;
int pos = open;     //position of gate
bool isClose = false;

void setup()
{
  pinMode(A2, INPUT_PULLUP);
  pinMode(A1, INPUT_PULLUP);
  pinMode(red, OUTPUT);
  pinMode(green, OUTPUT);
  gate.attach(3);
  gate.write(open);
  Serial.begin(9600);
  Serial.println("Initialize Serial Monitor");
}

void loop()
//LOW < 115 , HIGH > 690 , PULLUP > 1015
{
  //Serial.println(analogRead(A1));
  //Serial.println(analogRead(A2));
  //when GPIO 11 on jetson set to LOW (OK pizza)
  if (analogRead(A1) < 600){
    analogWrite(green, on);
    if (isClose) {
      // goes from close to open
      for (pos = close; pos <= open; pos += 1) {
        gate.write(pos);
        delay(15);
      }
      isClose = false;
    }
    delay(2000);
    analogWrite(green, off);
  }
  //when GPIO 7 on jetson set to LOW (Bad pizza)
  if (analogRead(A2) < 600){
    analogWrite(red, on);
    if (!isClose) {
      for (pos = open; pos >= close; pos -= 1) {
        gate.write(pos);
        delay(15);
      }
      isClose = true;
    }
    delay(2000);
    analogWrite(red, off);
  }
}

topping2.py

Python
topping.py with actuators; servo and LED program added. (see step 5).
#!/usr/bin/env python

import RPi.GPIO as GPIO
GPIO.cleanup()
GPIO.setmode(GPIO.BOARD)
GPIO.setup(7,GPIO.OUT)
GPIO.setup(11,GPIO.OUT)
GPIO.output(7,1)
GPIO.output(11,1)

import device_patches       # Device specific patches for Jetson Nano (needs to be before importing cv2)

import cv2
import os
import sys, getopt
import signal
import time
from edge_impulse_linux.image import ImageImpulseRunner

runner = None
# if you don't want to see a camera preview, set this to False
show_camera = True
if (sys.platform == 'linux' and not os.environ.get('DISPLAY')):
    show_camera = False

def now():
    return round(time.time() * 1000)

def get_webcams():
    port_ids = []
    for port in range(5):
        print("Looking for a camera in port %s:" %port)
        camera = cv2.VideoCapture(port)
        if camera.isOpened():
            ret = camera.read()[0]
            if ret:
                backendName =camera.getBackendName()
                w = camera.get(3)
                h = camera.get(4)
                print("Camera %s (%s x %s) found in port %s " %(backendName,h,w, port))
                port_ids.append(port)
            camera.release()
    return port_ids

def sigint_handler(sig, frame):
    print('Interrupted')
    if (runner):
        runner.stop()
    sys.exit(0)

signal.signal(signal.SIGINT, sigint_handler)

def help():
    print('python classify.py <path_to_model.eim> <Camera port ID, only required when more than 1 camera is present>')

def main(argv):
    try:
        opts, args = getopt.getopt(argv, "h", ["--help"])
    except getopt.GetoptError:
        help()
        sys.exit(2)

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            help()
            sys.exit()

    if len(args) == 0:
        help()
        sys.exit(2)

    model = args[0]

    dir_path = os.path.dirname(os.path.realpath(__file__))
    modelfile = os.path.join(dir_path, model)

    print('MODEL: ' + modelfile)

    with ImageImpulseRunner(modelfile) as runner:
        try:
            model_info = runner.init()
            print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
            labels = model_info['model_parameters']['labels']
            if len(args)>= 2:
                videoCaptureDeviceId = int(args[1])
            else:
                port_ids = get_webcams()
                if len(port_ids) == 0:
                    raise Exception('Cannot find any webcams')
                if len(args)<= 1 and len(port_ids)> 1:
                    raise Exception("Multiple cameras found. Add the camera port ID as a second argument to use to this script")
                videoCaptureDeviceId = int(port_ids[0])

            camera = cv2.VideoCapture(videoCaptureDeviceId)
            ret = camera.read()[0]
            if ret:
                backendName = camera.getBackendName()
                w = camera.get(3)
                h = camera.get(4)
                print("Camera %s (%s x %s) in port %s selected." %(backendName,h,w, videoCaptureDeviceId))
                camera.release()
            else:
                raise Exception("Couldn't initialize selected camera.")

            next_frame = 0 # limit to ~10 fps here

            # topping counting helper variables
            topping_names = ["mush", "papri", "roni"]
            topping_good_count = [3, 3, 3]
            topping_prev_count = [0] * len(topping_names)
            movingIn = True

            for res, img in runner.classifier(videoCaptureDeviceId):
                if (next_frame > now()):
                    time.sleep((next_frame - now()) / 1000)

                # print('classification runner response', res)

                if "classification" in res["result"].keys():
                    print('Result (%d ms.) ' % (res['timing']['dsp'] + res['timing']['classification']), end='')
                    for label in labels:
                        score = res['result']['classification'][label]
                        print('%s: %.2f\t' % (label, score), end='')
                    print('', flush=True)

                elif "bounding_boxes" in res["result"].keys():
                    # count the occurrence of each topping in this frame
                    topping_real_count = [0] * len(topping_names)
                    for bb in res["result"]["bounding_boxes"]:
                        topping_real_count[topping_names.index(bb['label'])] += 1

                    # determine if the pizza is currently moving out or moving in
                    switch = False
                    for i in range(len(topping_names)):
                        if movingIn:
                            # if any topping disappeared, it's moving out
                            if topping_prev_count[i] > topping_real_count[i]:
                                switch = True
                                break
                        else:
                            # if there are no toppings, it's moving in
                            if sum(topping_real_count) == 0:
                                switch = True
                                GPIO.output(7,1)
                                GPIO.output(11,1)
                                break
                    report = movingIn and switch
                    movingIn = movingIn ^ switch

                    # report the results if the pizza was moving in and now moving out
                    if report:
                        toPrint = ""
                        if topping_prev_count == topping_good_count:
                            toPrint += "Ok:"
                            GPIO.output(7,0)
                        else:
                            toPrint += "Bad:"
                            GPIO.output(11,0)
                        for i in range(len(topping_names)):
                            toPrint += " %s: %d" % (topping_names[i], topping_prev_count[i])
                            if i + 1 < len(topping_names): toPrint += ","
                        print(toPrint)

                    # set the previous count to be the current count
                    topping_prev_count = topping_real_count

                if (show_camera):
                    cv2.imshow('edgeimpulse', cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
                    if cv2.waitKey(1) == ord('q'):
                        break

                next_frame = now() + 100
        finally:
            if (runner):
                runner.stop()

if __name__ == "__main__":
   main(sys.argv[1:])

topping.py

Python
Program to detect whether the pizza toppings on the moving conveyer belt are the correct amount. Will give OK or Bad output for each pizza that passes. (see step 5).
'''
	Author: Jallson Suryo, Samuel Alexander, & Nicholas Patrick
	Date: 2023-12-15
	License: CC0
	Description: Program to detect whether the pizza toppings on the moving conveyer belt are the correct amount. Will give OK or Bad output for each pizza that passes.
'''
#!/usr/bin/env python

import device_patches       # Device specific patches for Jetson Nano (needs to be before importing cv2)

import cv2
import os
import sys, getopt
import signal
import time
from edge_impulse_linux.image import ImageImpulseRunner

runner = None
# if you don't want to see a camera preview, set this to False
show_camera = True
if (sys.platform == 'linux' and not os.environ.get('DISPLAY')):
    show_camera = False

def now():
    return round(time.time() * 1000)

def get_webcams():
    port_ids = []
    for port in range(5):
        print("Looking for a camera in port %s:" %port)
        camera = cv2.VideoCapture(port)
        if camera.isOpened():
            ret = camera.read()[0]
            if ret:
                backendName =camera.getBackendName()
                w = camera.get(3)
                h = camera.get(4)
                print("Camera %s (%s x %s) found in port %s " %(backendName,h,w, port))
                port_ids.append(port)
            camera.release()
    return port_ids

def sigint_handler(sig, frame):
    print('Interrupted')
    if (runner):
        runner.stop()
    sys.exit(0)

signal.signal(signal.SIGINT, sigint_handler)

def help():
    print('python classify.py <path_to_model.eim> <Camera port ID, only required when more than 1 camera is present>')

def main(argv):
    try:
        opts, args = getopt.getopt(argv, "h", ["--help"])
    except getopt.GetoptError:
        help()
        sys.exit(2)

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            help()
            sys.exit()

    if len(args) == 0:
        help()
        sys.exit(2)

    model = args[0]

    dir_path = os.path.dirname(os.path.realpath(__file__))
    modelfile = os.path.join(dir_path, model)

    print('MODEL: ' + modelfile)

    with ImageImpulseRunner(modelfile) as runner:
        try:
            model_info = runner.init()
            print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
            labels = model_info['model_parameters']['labels']
            if len(args)>= 2:
                videoCaptureDeviceId = int(args[1])
            else:
                port_ids = get_webcams()
                if len(port_ids) == 0:
                    raise Exception('Cannot find any webcams')
                if len(args)<= 1 and len(port_ids)> 1:
                    raise Exception("Multiple cameras found. Add the camera port ID as a second argument to use to this script")
                videoCaptureDeviceId = int(port_ids[0])

            camera = cv2.VideoCapture(videoCaptureDeviceId)
            ret = camera.read()[0]
            if ret:
                backendName = camera.getBackendName()
                w = camera.get(3)
                h = camera.get(4)
                print("Camera %s (%s x %s) in port %s selected." %(backendName,h,w, videoCaptureDeviceId))
                camera.release()
            else:
                raise Exception("Couldn't initialize selected camera.")

            next_frame = 0 # limit to ~10 fps here

            # topping counting helper variables
            topping_names = ["mush", "papri", "roni"]
            topping_good_count = [3, 3, 3]
            topping_prev_count = [0] * len(topping_names)
            movingIn = True

            for res, img in runner.classifier(videoCaptureDeviceId):
                if (next_frame > now()):
                    time.sleep((next_frame - now()) / 1000)

                # print('classification runner response', res)

                if "classification" in res["result"].keys():
                    print('Result (%d ms.) ' % (res['timing']['dsp'] + res['timing']['classification']), end='')
                    for label in labels:
                        score = res['result']['classification'][label]
                        print('%s: %.2f\t' % (label, score), end='')
                    print('', flush=True)

                elif "bounding_boxes" in res["result"].keys():
                    # count the occurrence of each topping in this frame
                    topping_real_count = [0] * len(topping_names)
                    for bb in res["result"]["bounding_boxes"]:
                        topping_real_count[topping_names.index(bb['label'])] += 1

                    # determine if the pizza is currently moving out or moving in
                    switch = False
                    for i in range(len(topping_names)):
                        if movingIn:
                            # if any topping disappeared, it's moving out
                            if topping_prev_count[i] > topping_real_count[i]:
                                switch = True
                                break
                        else:
                            # if there are no toppings, it's moving in
                            if sum(topping_real_count) == 0:
                                switch = True
                                break
                    report = movingIn and switch
                    movingIn = movingIn ^ switch

                    # report the results if the pizza was moving in and now moving out
                    if report:
                        toPrint = ""
                        if topping_prev_count == topping_good_count: toPrint += "Ok:"
                        else: toPrint += "Bad:"
                        for i in range(len(topping_names)):
                            toPrint += " %s: %d" % (topping_names[i], topping_prev_count[i])
                            if i + 1 < len(topping_names): toPrint += ","
                        print(toPrint)

                    # set the previous count to be the current count
                    topping_prev_count = topping_real_count

                if (show_camera):
                    cv2.imshow('edgeimpulse', cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
                    if cv2.waitKey(1) == ord('q'):
                        break

                next_frame = now() + 100
        finally:
            if (runner):
                runner.stop()

if __name__ == "__main__":
   main(sys.argv[1:])

Credits

Jallson Suryo

Jallson Suryo

4 projects • 18 followers
Tech integrator for schools. Also works as a maker and his activities include disassembling, fixing, and making things.
Samuel Alexander

Samuel Alexander

4 projects • 18 followers
Nicholas Patrick

Nicholas Patrick

1 project • 1 follower

Comments

Awards
AI at the Edge Applications (Open Category): 3rd Place
AI Innovation Challenge
Related channels and tags
  • Edge Impulse
  • Machine Learning & AI
  • NVIDIA
  • Robotics
  • SparkFun