Kutluhan Aktar
Published © CC BY

Stardew Valley Villager Recognition and Gift Preferences Bot

Via OpenCV, recognize villagers to display their birthdays, loves, likes, and hates while playing Stardew Valley on LattePanda Alpha.

ExpertFull instructions provided6 hours1,247

Things used in this project

Hardware components

LattePanda Alpha 864s
×1
Arduino Leonardo
Arduino Leonardo
Built-in on LattePanda Alpha
×1
DFRobot 8.9″ 1920x1200 IPS Touch Display
×1
3.5″ 320x480 TFT LCD Touch Screen (ILI9488)
×1
SparkFun Logic Level Converter - Bi-Directional
SparkFun Logic Level Converter - Bi-Directional
×2
Buzzer
Buzzer
×1
5 mm LED: Green
5 mm LED: Green
×1
Resistor 220 ohm
Resistor 220 ohm
×1
Solderless Breadboard Full Size
Solderless Breadboard Full Size
×1
Jumper wires (generic)
Jumper wires (generic)
×1

Software apps and online services

OpenCV
OpenCV
Arduino IDE
Arduino IDE
Thonny
Windows 10
Microsoft Windows 10

Story

Read more

Custom parts and enclosures

Fritzing

Stardew_Valley_Vision.zip

Schematics

Connections

LattePanda Alpha

Code

main.py

Python
# Stardew Valley Villager Recognition and Gift Preferences Bot w/ OpenCV
#
# Via OpenCV, recognize villagers to display their birthdays, loves, likes, and hates
# while playing Stardew Valley on LattePanda Alpha.
#
# LattePanda Alpha 864s
#
# By Kutluhan Aktar

import cv2 as cv
import numpy as np
from windowframe import captureWindowFrame
from vision import computerVision
from arduino_serial_comm import arduino_serial_comm
from pics import character_pics

# Define a character (villager) name in Stardew Valley:
_character = 'Penny'
# Create a list of cropped images:
cropped_images_to_detect = [character_pics[_character][0], character_pics[_character][1], character_pics[_character][2]]

# Define the class objects:
winframe = captureWindowFrame('Stardew Valley')
game_vision = computerVision(cropped_images_to_detect)

# Define the detection status to avoid repetitive messages to the Arduino Leonardo:
detection_status = True

def detect_character(character, result_type, port):
    global detection_status
    # Get a new frame (screenshot):
    new_frame = winframe.get_frames()
    # Detect the cropped images in the given list:
    game_vision.detect_cropped_image(new_frame, 0.65)
    # Get the output image:
    output = game_vision.get_and_draw_points(result_type)
    # Get the detected cropped image points:
    points = len(game_vision.get_center_points())
    print(points)
    # Send message (data) to the Arduino Leonardo via serial communication:
    if(points == 0):
        detection_status = True
    elif (points > 0 and detection_status):
        detection_status = False
        arduino_serial_comm(port, character)
        # Save the output image:
        cv.imwrite('result_{}.jpg'.format(character), output)
    # Show the output image:
    cv.imshow('OpenCV Computer Vision on LattePanda', output)

while True:
    # Initialize villager (character) recognition:
    detect_character(_character, 'boxes', 'COM6')
    #detect_character(_character, 'markers', 'COM6')
    
    # Press 'q' to exit:
    if(cv.waitKey(1) == ord('q')):
        cv.destroyAllWindows()
        break
    
print('Window Closed!')

windowframe.py

Python
import numpy as np
import win32gui, win32ui, win32con

class captureWindowFrame:
    def __init__(self, window_name):
        # Define the handle for the selected window.
        # Full Screen:
        # self.hwnd = win32gui.GetDesktopWindow()
        self.hwnd = win32gui.FindWindow(None, window_name)
        if not self.hwnd:
            raise Exception('Window Not Found: {}'.format(window_name))
        # Window Sizes:
        window_rect = win32gui.GetWindowRect(self.hwnd)
        self.w = window_rect[2] - window_rect[0]
        self.h = window_rect[3] - window_rect[1]
        # Adjust the window to remove unnecessary pixels:
        self.border_px = 8
        self.titlebar_px = 30
        self.w = self.w - (self.border_px * 2)
        self.h = self.h - self.titlebar_px - self.border_px
    def get_frames(self):
        # Get the frame (screenshot) data:
        wDC = win32gui.GetWindowDC(self.hwnd)
        dcObj = win32ui.CreateDCFromHandle(wDC)
        cDC = dcObj.CreateCompatibleDC()
        dataBitMap = win32ui.CreateBitmap()
        dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
        cDC.SelectObject(dataBitMap)
        cDC.BitBlt((0, 0), (self.w, self.h), dcObj, (self.border_px, self.titlebar_px), win32con.SRCCOPY)
        # Convert the raw frame data for OpenCV:
        signedIntsArray = dataBitMap.GetBitmapBits(True)
        img = np.fromstring(signedIntsArray, dtype='uint8')
        img.shape = (self.h, self.w, 4)
        # Clear the frame data:
        dcObj.DeleteDC()
        cDC.DeleteDC()
        win32gui.ReleaseDC(self.hwnd, wDC)
        win32gui.DeleteObject(dataBitMap.GetHandle())
        # Drop the alpha channel and make the image C_CONTIGUOUS to avoid errors while running OpenCV:
        img = img[...,:3]
        img = np.ascontiguousarray(img)
        # Return:
        return img
    # List window names:
    def get_window_names(self):
        def winEnumHandler(hwnd, ctx):
            if win32gui.IsWindowVisible(hwnd):
                print(hex(hwnd), win32gui.GetWindowText(hwnd))
        win32gui.EnumWindows(winEnumHandler, None)
    

vision.py

Python
import cv2 as cv
import numpy as np

class computerVision:
    def __init__(self, cropped_images_to_detect):
        # Define parameters:
        self.templates = []
        self.template_sizes = []
        # Get and save the required parameters (template and sizes) for all cropped images in the given list:
        for img in cropped_images_to_detect:
            # Template:
            cropped_img = cv.imread(img, cv.IMREAD_UNCHANGED)
            self.templates.append(cropped_img)
            # Sizes:
            cropped_w = cropped_img.shape[1]
            cropped_h = cropped_img.shape[0]
            self.template_sizes.append((cropped_w, cropped_h))
    # Detect cropped images on the main image:
    def detect_cropped_image(self, main_img, threshold, method=cv.TM_CCOEFF_NORMED):
        self.main_img = main_img
        # Generate results for each template:
        self.rectangles = []
        for i in range(len(self.templates)):
            # Get detected cropped image locations on the main image:
            result = cv.matchTemplate(self.main_img, self.templates[i], method)
            locations = np.where(result >= threshold)
            # Zip the detected cropped image locations into (X, Y) position tuples:
            locations = list(zip(*locations[::-1]))
            # Create the rectangles list [x, y, w, h] to avert overlapping positions:
            for loc in locations:
                w, h = self.template_sizes[i]
                rect = [int(loc[0]), int(loc[1]), int(w), int(h)]
                self.rectangles.append(rect)
                # For single points:
                self.rectangles.append(rect)
        # Group the rectangles list:
        self.rectangles, weights = cv.groupRectangles(self.rectangles, 1, 0.7)
    # Draw rectangles or markers on the detected croppped images:
    def get_and_draw_points(self, result_type, rect_color=(255,0,255), rect_type=cv.LINE_4, rect_thickness=3, marker_color=(255,0,255), marker_type=cv.MARKER_STAR, marker_size=25, marker_thickness=3):
        self.detected_center_points = []
        if(len(self.rectangles)):
            # Loop all detected image locations in the rectangles list:
            for (x, y, w, h) in self.rectangles:
                # Define the rectangle box points:
                top_left = (x, y)
                bottom_right = (x + w, y + h)
                # Define the rectangle center points for drawing markers:
                center_x = x + int(w/2)
                center_y = y + int(h/2)
                # Save the center points:
                self.detected_center_points.append((center_x, center_y))
                # Draw results on the main image:
                if(result_type == 'boxes'):
                    # Draw rectangles around the detected cropped images:
                    cv.rectangle(self.main_img, top_left, bottom_right, color=rect_color, thickness=rect_thickness, lineType=rect_type)
                elif(result_type == 'markers'):
                    # Draw markers on the detected cropped images:
                    cv.drawMarker(self.main_img, (center_x, center_y), marker_color, marker_type, marker_size, marker_thickness)
        return self.main_img
        # Clear the rectangles list to avoid errors:
        self.rectangles = []
    # Get the center points of the detected cropped images:
    def get_center_points(self):
        return self.detected_center_points

arduino_serial_comm.py

Python
import json
import serial
from time import sleep

def arduino_serial_comm(port, character):
    # Define the data file (json):
    json_file = 'assets/villagers.json'
    # Define the build-in Arduino Leonardo port on the LattePanda:
    arduino = serial.Serial(port, 9600, timeout=1)
    # Elicit information:
    with open (json_file) as data:
        villagers = json.load(data)
        c = villagers[character]
        msg = "Character: {}\n\nBirthday: {}\n\nLoves: {}\n\nLikes: {}\n\nHates: {}".format(character, c['birthday'], c['loves'], c['likes'], c['hates'])
        print(msg)
        arduino.write(msg.encode())
        sleep(1)
        

pics.py

Python
character_pics = {
        'Abigail' : ['assets/abigail_1.jpg', 'assets/abigail_2.jpg', 'assets/abigail_3.jpg'],
        'Emily' : ['assets/emily_1.jpg', 'assets/emily_2.jpg'],
        'Haley' : ['assets/haley_1.jpg', 'assets/haley_2.jpg', 'assets/haley_3.jpg'],
        'Maru' : ['assets/maru_1.jpg', 'assets/maru_2.jpg', 'assets/maru_3.jpg'],
        'Penny' : ['assets/penny_1.jpg', 'assets/penny_2.jpg', 'assets/penny_3.jpg'],
        'Alex' : ['assets/alex_1.jpg', 'assets/alex_2.jpg'],
        'Harvey' : ['assets/harvey_1.jpg', 'assets/harvey_2.jpg'],
        'Sam' : ['assets/sam_1.jpg', 'assets/sam_2.jpg', 'assets/sam_3.jpg']
    }

lattepanda_detection_results_display.ino

Arduino
         /////////////////////////////////////////////  
        // Stardew Valley Villager Recognition and //
       //      Gift Preferences Bot w/ OpenCV     //
      //             ---------------             //
     //         (LattePanda Alpha 864s)         //           
    //             by Kutluhan Aktar           // 
   //                                         //
  /////////////////////////////////////////////

//
// Via OpenCV, recognize villagers to display their birthdays, loves, likes, and hates while playing Stardew Valley on LattePanda Alpha.
//
// For more information:
// https://www.theamplituhedron.com/projects/Stardew_Valley_Villager_Recognition_and_Gift_Preferences_Bot_w_OpenCV
//
//
// Connections
// LattePanda Alpha 864s (Arduino Leonardo) :  
//                                3.5'' 320x480 TFT LCD Touch Screen (ILI9488)
// D10 --------------------------- CS 
// D8  --------------------------- RESET 
// D9  --------------------------- D/C
// MOSI -------------------------- SDI 
// SCK --------------------------- SCK 
// MISO -------------------------- SDO(MISO) 
//                                Buzzer
// D11 --------------------------- S     
//                                5mm Green LED
// D12 --------------------------- S


// Include the required libraries:
#include "SPI.h"
#include <Adafruit_GFX.h>
#include <ILI9488.h>

// Define the required pins for the 320x480 TFT LCD Touch Screen (ILI9488)
#define TFT_DC 9
#define TFT_CS 10
#define TFT_RST 8

// Use hardware SPI (on Leonardo, SCK, MISO, MOSI) and the above for DC/CS/RST
ILI9488 tft = ILI9488(TFT_CS, TFT_DC, TFT_RST);

// Define the buzzer pin:
#define buzzer 11
// Define the green LED pin:
#define green_LED 12

// Define the data holders:
String gameData = "";

void setup() {
  pinMode(green_LED, OUTPUT);
  // Initialize the serial communication:
  Serial.begin(9600);
  // Initialize the TFT LCD Touch Screen (ILI9488):
  tft.begin();
  tft.setRotation(1);
  tft.fillScreen(ILI9488_RED);
  tft.setCursor(0, 0);
  tft.setTextColor(ILI9488_BLACK); tft.setTextSize(4);
  tft.println("\nStardew Valley\n");
  tft.println("Villager Recognition\n");
  tft.println("and Gift Preferences\n");
  tft.println("Bot w/ OpenCV\n");

}

void loop() {
  // Via the serial communication, elicit the detected character's information:
  while (Serial.available()) {
    char c = (char)Serial.read();
    gameData += c;
  }
  // Display the transferred character information:
  if(gameData != ""){
    // Notify the user:
    tft.fillScreen(ILI9488_BLACK);
    tft.setCursor(10, 5);
    tft.setTextColor(ILI9488_RED); tft.setTextSize(4);
    tft.println("Stardew Valley\n");
    tft.setTextColor(ILI9488_GREEN); tft.setTextSize(2);
    tft.println(gameData);
    tone(buzzer, 500);
    digitalWrite(green_LED, HIGH);
    delay(500);
    noTone(buzzer);
    digitalWrite(green_LED, LOW);
  }
  // Clear:
  delay(250);
  gameData = "";
}

villagers.json

JSON
{
	"Abigail": {
		"birthday" : "Fall 13",
		"loves" : "Amethyst, Banana Pudding, Blackberry Cobbler, Chocolate Cake, Pufferfish, Pumpkin, Spicy Eel",
		"likes" : "Quartz",
		"hates" : "Clay, Holly",
		"introduction" : "Abigail is a villager who lives at Pierre's General Store in Pelican Town. She is one of the twelve characters available to marry. "
	},
	
	"Emily": {
		"birthday" : "Spring 27",
		"loves" : "Amethyst, Aquamarine, Cloth, Emerald, Jade, Ruby, Survival Burger, Topaz, Wool",
		"likes" : "Daffodil, Quartz",
		"hates" : "Fish Taco, Holly, Maki Roll, Salmon Dinner, Sashimi",
		"introduction" : "Emily is a villager who lives in Pelican Town. She is one of the twelve characters available to marry. Her home is south of the town square, right next to Jodi's house, at the address 2 Willow Lane. She works most evenings at The Stardrop Saloon starting at about 4:00 PM."
	},
	
	"Haley": {
		"birthday" : "Spring 14",
		"loves" : "Coconut, Fruit Salad, Pink Cake, Sunflower",
		"likes" : "Daffodil",
		"hates" : "All Fish, Clay, Prismatic Shard, Wild Horseradish",
		"introduction" : "Haley is a villager who lives in Pelican Town. She's one of the twelve characters available to marry."
	},
	
	"Maru": {
		"birthday" : "Summer 10",
		"loves" : "Battery Pack, Cauliflower, Cheese Cauliflower, Diamond, Gold Bar, Iridium Bar, Miner's Treat, Pepper Poppers, Radioactive Bar, Rhubarb Pie, Strawberry",
		"likes" : "Copper Bar, Iron Bar, Oak Resin, Pine Tar, Quartz, Radioactive Ore",
		"hates" : "Holly, Honey, Pickles, Snow Yam, Truffle",
		"introduction" : "Maru is a villager who lives in The Mountains north of Pelican Town. She's one of the twelve characters available to marry. She lives north of town with her family in a house attached to Robin's carpenter shop. In the Social Status menu, Maru's outfit will change to a nursing uniform when she is at her job at the clinic."
	},
	
	"Penny": {
		"birthday" : "Fall 02",
		"loves" : "Diamond, Emerald, Melon, Poppy, Poppyseed Muffin, Red Plate, Roots Platter, Sandfish, Tom Kha Soup",
		"likes" : "All Artifacts, All Milk, Dandelion, Leek",
		"hates" : "Beer, Grape, Holly, Hops, Mead, Pale Ale, Pina Colada, Rabbits Foot, Wine",
		"introduction" : "Penny is a villager who lives in Pelican Town. She's one of the twelve characters available to marry. Her trailer is just east of the center of town, west of the river. She teaches Vincent and Jas."
	},

	"Alex": {
		"birthday" : "Summer 13",
		"loves" : "Complete Breakfast, Salmon Dinner",
		"likes" : "All Eggs (except Void Egg)",
		"hates" : "Holly, Quartz",
		"introduction" : "Alex is a villager who lives in the house southeast of Pierre's General Store. Alex is one of the twelve characters available to marry."
	},
	
	"Harvey": {
		"birthday" : "Winter 14",
		"loves" : "Coffee, Pickles, Super Meal, Truffle Oil, Wine",
		"likes" : "Daffodil, Dandelion, Duck Egg, Duck Feather, Ginger, Goat Milk, Hazelnut, Holly, Large Goat Milk, Leek, Quartz, Snow Yam, Spring Onion, Wild Horseradish, Winter Root",
		"hates" : "Coral, Nautilus Shell, Rainbow Shell, Salmonberry, Spice Berry",
		"introduction" : "Harvey is a villager who lives in Pelican Town. He runs the town's medical clinic and is passionate about the health of the townsfolk. He's one of the twelve characters available to marry."
	},
	
	"Sam": {
		"birthday" : "Summer 17",
		"loves" : "Cactus Fruit, Maple Bar, Pizza, Tigerseye",
		"likes" : "All Eggs (except Void Egg), Joja Cola",
		"hates" : "Coal, Copper Bar, Duck Mayonnaise, Gold Bar, Gold Ore, Iridium Bar, Iridium Ore, Iron Bar, Mayonnaise, Pickles, Refined Quartz",
		"introduction" : "Sam is a villager who lives in Pelican Town. He's one of the twelve characters available to marry. He lives in the southern part of town, just north of the river at 1 Willow Lane. Sam is an outgoing and energetic young man with a passion for music. He works part-time at JojaMart."
	}
}

Credits

Kutluhan Aktar

Kutluhan Aktar

53 projects • 139 followers
Self-Taught Full Stack Developer | Programmer | Maker | Physics Enthusiast

Comments