TeleAI Claw Machine

import cv2
import numpy as np
from openvino.runtime import Core
import datetime
from shapely.geometry import Polygon
import paho.mqtt.publish as publish




def draw_bbox(bgr_image, resized_image, boxes,confidence):
    colors = {"blue": (255, 0, 0), "green": (0, 255, 0), "red":(0, 0, 255)}

    (real_y, real_x), (resized_y, resized_x) = bgr_image.shape[:2], resized_image.shape[:2]
    ratio_x, ratio_y = real_x / resized_x, real_y / resized_y
    #rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
    rgb_image = bgr_image

    #roi
    video_xmax, video_ymax,video_size = 1920-250, 1080-50,500
    roi_point=[[video_xmax, video_ymax], [video_xmax, video_ymax-video_size], [video_xmax-video_size, video_ymax-video_size], [video_xmax-video_size, video_ymax]]
    roi = Polygon(roi_point)
    points = np.array(roi_point, np.int32)
    rgb_image = cv2.polylines(rgb_image, pts=[points], isClosed=True, color=colors["blue"], thickness=3)
    rgb_image = cv2.putText(rgb_image, "winner-zone", (video_xmax-video_size, video_ymax-video_size-10), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["blue"] , 2, cv2.LINE_AA)


    for box in boxes:
        conf = box[-1]
        
        if conf > confidence:
            (x_min, y_min, x_max, y_max) = [
                int(max(corner_position * ratio_y, 10)) if idx % 2 
                else int(corner_position * ratio_x)
                for idx, corner_position in enumerate(box[:-1])
            ]
            if roi.intersects(Polygon([(x_min,y_max),(x_max, y_max),(x_max,y_min),(x_min, y_min)])):
                rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["blue"], 3)
                rgb_image = cv2.putText(rgb_image, str(int(conf*100))+"%", (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["blue"] , 2, cv2.LINE_AA)
                rgb_image = cv2.putText(rgb_image, "amd yes!!", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["blue"] , 2, cv2.LINE_AA)
                payload = str([x_min, y_min, x_max, y_max])
                publish.single(topic, payload, qos=1, hostname=host)
            else:
                rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["green"], 3)
                rgb_image = cv2.putText(rgb_image, str(int(conf*100))+"%", (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["green"] , 2, cv2.LINE_AA)                
            
                #if x_max>1500 and y_max <800:
                    #rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["blue"], 3)
                    #rgb_image = cv2.putText(rgb_image, str(int(conf*100))+"%", (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["blue"] , 2, cv2.LINE_AA)                
                #else:
                    #rgb_image = cv2.rectangle(rgb_image, (x_min, y_min), (x_max, y_max), colors["green"], 3)
                    #rgb_image = cv2.putText(rgb_image, str(int(conf*100))+"%", (x_min, y_min-5), cv2.FONT_HERSHEY_SIMPLEX, 1,colors["green"] , 2, cv2.LINE_AA)                
                                    
            

    return rgb_image



host = "localhost"
topic = "test"
payload = "amdyes"

ie = Core()
model_xml_path = "model/openvino/openvino.xml"
model = ie.read_model(model=model_xml_path)
model.reshape([1,3,736,992])
compiled_model = ie.compile_model(model=model, device_name="GPU")
input_layer_ir = compiled_model.input(0)
N, C, H, W  = (1,3,736,992)

#fourcc = cv2.VideoWriter_fourcc(*'MP4V')
#out = cv2.VideoWriter('result/output_WIN_20240716_16_52_14_Pro.mp4', fourcc, 30.0, (1920,1080))

cap = cv2.VideoCapture('image/WIN_20240716_16_52_14_Pro.mp4')
while cap.isOpened():
    ret, image = cap.read()
    if not ret:
        print("Can't receive frame (stream end?). Exiting ...")
        break

    resized_image = cv2.resize(image, (W, H))
    resized_image = cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB)
    input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0)


    result = compiled_model([input_image])

    boxes = result[0][0]
    image_detection = draw_bbox(image, resized_image, boxes,0.3)

    #out.write(image_detection)
    image_detection = cv2.resize(image_detection, (W, H))
    cv2.imshow('amd yes', image_detection)
    if cv2.waitKey(1) == ord('q'):
        break


#out.release()
cap.release()
cv2.destroyAllWindows()

import torch
import torch.nn as nn
import onnxruntime
import numpy as np
import argparse
from utils import (
    LoadImages,
    non_max_suppression,
    plot_images,
    output_to_target,
)
import sys
import pathlib
CURRENT_DIR = pathlib.Path(__file__).parent
sys.path.append(str(CURRENT_DIR))

def preprocess(img):
    img = torch.from_numpy(img)
    img = img.float()  # uint8 to fp16/32
    img /= 255  # 0 - 255 to 0.0 - 1.0
    return img


class DFL(nn.Module):
    # Integral module of Distribution Focal Loss (DFL) proposed in Generalized Focal Loss https://ieeexplore.ieee.org/document/9792391
    def __init__(self, c1=16):
        super().__init__()
        self.conv = nn.Conv2d(c1, 1, 1, bias=False).requires_grad_(False)
        x = torch.arange(c1, dtype=torch.float)
        self.conv.weight.data[:] = nn.Parameter(x.view(1, c1, 1, 1))
        self.c1 = c1

    def forward(self, x):
        b, c, a = x.shape  # batch, channels, anchors
        return self.conv(x.view(b, 4, self.c1, a).transpose(2, 1).softmax(1)).view(
            b, 4, a
        )


def dist2bbox(distance, anchor_points, xywh=True, dim=-1):
    """Transform distance(ltrb) to box(xywh or xyxy)."""
    lt, rb = torch.split(distance, 2, dim)
    x1y1 = anchor_points - lt
    x2y2 = anchor_points + rb
    if xywh:
        c_xy = (x1y1 + x2y2) / 2
        wh = x2y2 - x1y1
        return torch.cat((c_xy, wh), dim)  # xywh bbox
    return torch.cat((x1y1, x2y2), dim)  # xyxy bbox


def post_process(x):
    dfl = DFL(16)
    anchors = torch.tensor(
        np.load(
            "./anchors.npy",
            allow_pickle=True,
        )
    )
    strides = torch.tensor(
        np.load(
            "./strides.npy",
            allow_pickle=True,
        )
    )
    box, cls = torch.cat([xi.view(x[0].shape[0], 144, -1) for xi in x], 2).split(
        (16 * 4, 80), 1
    )
    dbox = dist2bbox(dfl(box), anchors.unsqueeze(0), xywh=True, dim=1) * strides
    y = torch.cat((dbox, cls.sigmoid()), 1)
    return y, x


def make_parser():
    parser = argparse.ArgumentParser("onnxruntime inference sample")
    parser.add_argument(
        "-m",
        "--onnx_model",
        type=str,
        default="./yolov8m.onnx",
        help="input your onnx model.",
    )
    parser.add_argument(
        "-i",
        "--image_path",
        type=str,
        default='./demo.jpg',
        help="path to your input image.",
    )
    parser.add_argument(
        "-o",
        "--output_path",
        type=str,
        default='./demo_infer.jpg',
        help="path to your output directory.",
    )
    parser.add_argument(
        "--ipu", action='store_true', help='flag for ryzen ai'
    )
    parser.add_argument(
        "--provider_config", default='', type=str, help='provider config for ryzen ai'
    )
    return parser

classnames = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',
        'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
        'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
        'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
        'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
        'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
        'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',
        'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
        'hair drier', 'toothbrush'] 
names = {k: classnames[k] for k in range(80)}
imgsz = [640, 640]


if __name__ == '__main__':
    args = make_parser().parse_args()
    print(args)
    source = args.image_path 
    dataset = LoadImages(
        source, imgsz=imgsz, stride=32, auto=False, transforms=None, vid_stride=1
    )
    onnx_weight = args.onnx_model
    if args.ipu:
        providers = ["VitisAIExecutionProvider"]
        provider_options = [{"config_file": args.provider_config}]
        onnx_model = onnxruntime.InferenceSession(onnx_weight, providers=providers, provider_options=provider_options)
    else:
        onnx_model = onnxruntime.InferenceSession(onnx_weight)
    for batch in dataset:
        path, im, im0s, vid_cap, s = batch
        im = preprocess(im)
        if len(im.shape) == 3:
            im = im[None]
        # outputs = onnx_model.run(None, {onnx_model.get_inputs()[0].name: im.cpu().numpy()})
        # outputs = [torch.tensor(item) for item in outputs]
        outputs = onnx_model.run(None, {onnx_model.get_inputs()[0].name: im.permute(0, 2, 3, 1).cpu().numpy()})
        outputs = [torch.tensor(item).permute(0, 3, 1, 2) for item in outputs]
        preds = post_process(outputs)
        preds = non_max_suppression(
            preds, 0.25, 0.7, agnostic=False, max_det=300, classes=None
        )
        plot_images(
            im,
            *output_to_target(preds, max_det=15),
            source,
            fname=args.output_path,
            names=names,
        )

import uuid
import requests
from flask import request, jsonify
from flask.views import MethodView
from flask_smorest import Blueprint, abort
from sqlalchemy.exc import SQLAlchemyError,IntegrityError
from db import db
#from modules.inference import Inference
import json
from schemas import EventSchema,InferenceJobEventSchema
from models import EventModel
import datetime
from shapely.geometry import Polygon
#from multiprocessing import Process
import threading
from threading import Event
import time
from sqlalchemy.exc import SQLAlchemyError, IntegrityError
from db import db
from models import EventModel,InferenceJobModel
import cv2
import grpc
import numpy as np
from tensorflow import make_tensor_proto, make_ndarray
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
from openvino.runtime import Core
from socket_io import socket_io

blp = Blueprint("SIS","sis", description="Stream inference server")

processes = []
check = True
new_process = None
new_socket = None
temp_data = None
exit_event = Event()
last_time = None

class SocketMessage:

    def socket_send(self):
        global temp_data, exit_event, last_time
        while True:
            if exit_event.is_set():
                exit_event.clear()
                break

            if last_time == None:
                data =[{
                    "event_type": {
                    "description": "Alert when detected trespassing",
                    "id": 1,
                    "name": "Human detection Alert"
                    },
                    "image_shot": {"image":"image"},
                    "inference_result": [],
                    "source_media_id_offset": {"media": "media"},
                    "time": datetime.datetime.now().isoformat()
                }]
                return data
            data =[{
                    "event_type": {
                    "description": "Alert when detected trespassing",
                    "id": 1,
                    "name": "Human detection Alert"
                    },
                    "image_shot": {"image":"image"},
                    "inference_result": temp_data["inference_result"],
                    "source_media_id_offset": {"media": "media"},
                    "time": temp_data["time"]
                }]
            
            this_time = data[0]["time"]
            print(this_time)
            print(last_time)
            if last_time == this_time:
                data[0]["time"] = datetime.datetime.now().isoformat()
                data[0]["inference_result"] = []
            else:
                last_time = this_time
                
            socket_io.emit('event', data)
            socket_io.sleep(1)

    def start_inference(self):
        socket_io.start_background_task(target=self.socket_send)
    def stop_inference(self):
        global exit_event
        exit_event.set()




@blp.route("/api/inference/start/<int:inference_job_id>")
class Inference_start(MethodView):
    def post(self, inference_job_id):
        global check, new_process, new_socket
        if check == True:
            new_process = Inference(inference_job_id)
            new_process.start_inference()
            processes.append(new_process)
            new_socket = SocketMessage()
            new_socket.start_inference()
            processes.append(new_socket)
            print("Start Success!!")
            check = False
            return ("Success")
        else:
            return("Error, can only start 1 job")


@blp.route("/api/inference/stop")
class Inference_stop(MethodView):
    def post(self):
        global check, new_process, new_socket
        if not processes:
            return("nothing is running")
        else:
            for p in processes:
                p.stop_inference()
            processes.clear()
            new_process = None
            new_socket = None
            check = True
            print("Stop Success!!")
            return ("All process killed")


@blp.route("/api/event/get_by_inference_job/<string:target_inference_job_id>")
class Event(MethodView):
    #@blp.response(200, InferenceJobEventSchema(many=True))
    @blp.response(200)
    def get(self, target_inference_job_id):
        global last_time, new_process, temp_data
        '''
        event = EventModel.query.order_by(EventModel.time.desc()).limit(1).all()
        this_time = event[0].time
        '''
        if last_time == None:
            data =[{
                "event_type": {
                "description": "Alert when detected trespassing",
                "id": 1,
                "name": "Human detection Alert"
                },
                "image_shot": {"image":"image"},
                "inference_result": [],
                "source_media_id_offset": {"media": "media"},
                "time": datetime.datetime.now().isoformat()
            }]
            return data

        data =[{
                "event_type": {
                "description": "Alert when detected trespassing",
                "id": 1,
                "name": "Human detection Alert"
                },
                "image_shot": {"image":"image"},
                "inference_result": temp_data["inference_result"],
                "source_media_id_offset": {"media": "media"},
                "time": temp_data["time"]
            }]
        
        #event = EventModel(**new_process.event_data)
        this_time = data[0]["time"]
        print(this_time)
        print(last_time)
        if last_time == this_time:
            data[0]["time"] = datetime.datetime.now().isoformat()
            data[0]["inference_result"] = []
        else:
            last_time = this_time
        '''
        if last_time == this_time:
            event[0].time = datetime.datetime.now()
            event[0].inference_result = []
        else:
            last_time = this_time
        '''
        return data
    
@blp.route("/api/event/get_by_inference_job_all/<string:target_inference_job_id>")
class Event2(MethodView):
    @blp.response(200, InferenceJobEventSchema(many=True))
    def get(self, target_inference_job_id):
        global last_time, check, new_process,new_process, temp_data
        if check == True:
            new_process = Inference(target_inference_job_id)
            new_process.start_inference()
            processes.append(new_process)
            new_socket = SocketMessage()
            new_socket.start_inference()
            processes.append(new_socket)
            print("Start Success!!")
            check = False

        last_time = None
        event = EventModel.query.order_by(EventModel.time.desc()).limit(20).all()
        print(event)
        if event == []:
            event = EventModel.query.order_by(EventModel.time.desc()).limit(20)
            last_time = None
        else:
            last_time = event[0].time
            data ={
                    "time": last_time,
                    "image_shot": {"img": "img"},
                    "source_media_id_offset": {"media": "media"},
                    "inference_result": event[0].inference_result,
                    "inference_job_id": new_process.job_id,
                    "event_type_id" : 1
                }
            temp_data = data
            print(temp_data)
            #event = EventModel.query.order_by(EventModel.time.desc()).limit(20)
        return event
    

class Inference:
    def __init__(self,job_id):
        self.interest = None
        self.inference = None
        self.model = "person-detection-0303"
        self.job = InferenceJobModel.query.get_or_404(job_id)
        region = self.job.region.first()
        camera = region.camera
        fence = region.region_sets['fence']
        self.fence = Polygon(fence).convex_hull
        self.media_url = camera.RTSP
        #self.media_url = "./input/videos/test1.avi"
        self.job_id = job_id
        self.count = 0
        self.event_data = None
        self.lock = False

    def set_interest(self, interest):
        self.interest = Polygon(interest).convex_hull

    def set_fence(self, fence):
        self.fence = Polygon(fence).convex_hull

    def convert_to_corners(self,coords):
        xmin, ymin, xmax, ymax, confidence = coords
        if confidence>=0.4:
            return [[xmin, ymin], [xmax, ymin], [xmax, ymax], [xmin, ymax]]
        else:
            return None
    
    def find_label_index(self, result):
        #labels = result_dict["inference_result"]["labels"]
        boxes = result["boxes"]
        temp = []
        #for index, label in enumerate(labels):
        for rows in boxes:
            '''
            if label == 1:
                temp.append(self.convert_to_corners(boxes[index]))
            '''
            corners = self.convert_to_corners(rows)
            if corners != None:
                temp.append(self.convert_to_corners(rows))
        return temp
    
    def send_event(self, result):
        global temp_data
        self.event_data = {
	        "time": datetime.datetime.now().isoformat(),
	        "image_shot": {"img": "img"},
            "source_media_id_offset": {"media": "media"},
            "inference_result": result,
            "inference_job_id": self.job_id,
            "event_type_id" : 1
        }
        
        temp_data = self.event_data
        print("Alert!!")
        #print(event_data)
        '''
        event = EventModel(**self.event_data)
        
        try:
            db.session.add(event)
            db.session.commit()
        except SQLAlchemyError:
            print("error when sending event")

        self.count = self.count + 1
        '''
        '''
        if self.count == 1500:
            subquery = db.session.query(EventModel.time).order_by(EventModel.time.desc()).limit(60).subquery()
            delete_query = EventModel.query.filter(~EventModel.time.in_(subquery))
            deleted_count = delete_query.delete(synchronize_session=False)
            db.session.commit()
            self.count = 0
        '''

    def read_now(self):
        return self.event_data

    def if_intersect(self,human):
        #print("here")
        human_poly = Polygon(human).convex_hull
        if self.fence.intersects(human_poly):
            #print("ALERT!!!!")
            self.send_event(human)

    def sis(self):
        address = "127.0.0.1:9000"
        vidcap = cv2.VideoCapture(self.media_url)

        ie = Core()
        model_xml_path = "./ai-model/openvino.xml"
        model = ie.read_model(model=model_xml_path)
        model.reshape([1,3,736,992])
        compiled_model = ie.compile_model(model=model, device_name="GPU")
        input_layer_ir = compiled_model.input(0)
        N, C, H, W  = (1,3,736,992)

        frame_index = 0
        while vidcap.isOpened():
            success,img = vidcap.read()
            if frame_index ==2:
                frame_index = 0
                continue
            frame_index = frame_index +  1
            if success:
                resized_image = cv2.resize(img, (W, H))
                resized_image = cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB)
                input_image = np.expand_dims(resized_image.transpose(2, 0, 1), 0)

                result = compiled_model([input_image])
                #boxes = result['boxes']
                boxes = result[0][0]
                #labels = result['labels']
                labels = [0]*len(boxes)

                #grpc_req = predict_pb2.PredictRequest()
                #grpc_req.model_spec.name = self.model
                #grpc_req.inputs["image"].CopyFrom(make_tensor_proto(img, shape=(img.shape)))
        
                #result = stub.Predict(grpc_req, 10.0) 
                #boxes = make_ndarray(result.outputs["boxes"])
                #labels = make_ndarray(result.outputs["labels"])
                
                output = {
                    "boxes":boxes.tolist(),
                    #"labels":labels.tolist()
                    "labels":labels
                }
                        
                #pub = {"video_stream":self.media_url,"frame_index":frame_index,"offset_ms":vidcap.get(cv2.CAP_PROP_POS_MSEC),"inference_result": output}
                #pub = {"inference_result": output}
                #print(pub)
                #pub = json.dumps(pub)+';'          
                #frame_index = frame_index +1
                human = self.find_label_index(output)
                for rows in human:
                    self.if_intersect(rows)
                #if human == []:
                    #print("No Human")
                if self.lock == True:
                    break
                

                
        '''
        pub = { 
            "mediaurl":self.media_url,
            "model":self.model
        }
        r = requests.post('http://10.10.80.46:5000/api/inference',stream=True, data = pub)
        for line in r.iter_content(None,True):
            line = line[:-1]
            line = ast.literal_eval(line)
            human = self.find_label_index(line)
            
            for rows in human:
                self.if_intersect(rows)
            if human == []:
                print("No Human")
        '''

    def start_inference(self):
        #self.inference = Process(target = self.sis)
        self.inference = threading.Thread(target= self.sis)
        self.inference.start()
    
    def stop_inference(self):
        #self.inference.terminate()
        self.lock = True
        print("test killed")

'''
@blp.route("/api/test/<int:inference_job_id>")
class testingpurpose(MethodView):
    def post(self, inference_job_id):
        subquery = db.session.query(EventModel.time).order_by(EventModel.time.desc()).limit(20).subquery()
        delete_query = EventModel.query.filter(~EventModel.time.in_(subquery))
        deleted_count = delete_query.delete(synchronize_session=False)
        db.session.commit()
        print(deleted_count)
        return("safe")

        new_process = Inference(inference_job_id)
        print(new_process.fence)
        print(new_process.media_url)
        print(new_process.model)
        return("safe")

        inference_job = InferenceJobModel.query.get_or_404(inference_job_id)
        region = inference_job.region.first()
        region_sets = region.region_sets['fence']
        camera = region.camera
        camera_url = camera.RTSP
        print(region_sets)
        print(camera_url)
        return("test")
        region_ids = [region.id for region in inference_job.region.all()]
        print(region_ids)
        return jsonify({'region_ids': region_ids})



@blp.route("/sis/<string:sis_key_name>")
class Metric(MethodView):
    @blp.response(200, SisSchema)
    def get(self, sis_key_name):
        sis_key_name = SisModel.query.filter_by(name=sis_key_name).all()
        return sis_key_name

@blp.route("/sis")
class Sis(MethodView):
    @blp.response(200, SisSchema(many=True))
    def get(self):
        #return SisModel.query.all()
        return None
    def post(self):
        pub = { 
            "mediaurl":"./input/images/people1.jpg",
            "model":"face-detection"
        }
        r = requests.post('http://10.10.80.228:5000/api/inference',stream=True, data = pub)
        sis_data = {}
        for line in r.iter_content(None,True): #line is string
            line_json = json.loads(line) #convert line from string into dict
            sis_data = {
                "data" : line_json
            }
            print(line)
            sis = SisModel(**sis_data)
            try:
                db.session.add(sis)
                db.session.commit()
            except SQLAlchemyError:
                abort(500, message="An error occurred while inserting the sis.")
            sis_data = {}
            
            break
        return jsonify({'Success': 'successfully added'}), 200
'''