Gerrit Niezen
Published © CC BY-SA

Using AI and IoT to help grow food in remote greenhouses

We can use embedded ML on low-power microcontrollers to analyse crops in greenhouses without power and send the results over LPWAN networks

IntermediateFull instructions provided3 hours1,555

Things used in this project

Hardware components

Spresense boards (main & extension)
Sony Spresense boards (main & extension)
Only the main board, as we'll be using the LTE extension board
×1
Spresense camera board
Sony Spresense camera board
×1
Spresense LTE extension board
Sony Spresense LTE extension board
×1
Li-Ion Battery 1000mAh
Li-Ion Battery 1000mAh
or whatever size battery you need
×1
Flash Memory Card, MicroSD Card
Flash Memory Card, MicroSD Card
×1
JST-PH 2-Pin Through Hole Right Angle Connector
×1
LTE-M/NB-IoT SIM card
×1
M5 bolt (25 mm / 1 inch) and nut
To attach the clamp to the enclosure
×1

Software apps and online services

Edge Impulse Studio
Edge Impulse Studio
Arduino IDE
Arduino IDE

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)

Story

Read more

Custom parts and enclosures

Enclosure

STL file that can be 3D-printed.

Clamp

STL file for 1-inch clamp that can be 3D-printed.

Enclosure source

OpenSCAD file that can be edited to generate new STL file

Clamp source

OpenSCAD file that can be edited to generate a new STL file

Library for enclosure source

Should be placed in a "lib" subfolder.

Code

plant-sketch.ino

Arduino
Main Arduino sketch
#include <SDHCI.h>
#include <RTC.h>
#include <LowPower.h>
#include <GNSS.h>

#define DEBUG true
static SDClass  theSD;
static int batteryVoltage;
static SpNavData data;

void setup() {
  if (DEBUG) {
    Serial.begin(115200); 
    Serial.println("INFO: camera initializing on wakeup...");
  }

  LowPower.begin();

  /* Initialize SD */
  while (!theSD.begin()) 
  {
    /* wait until SD card is mounted. */
    if (DEBUG) Serial.println("Insert SD card.");
  }

  if (DEBUG) Serial.println("SD card mounted.");

  // Initialize RTC at first
  RTC.begin();

  // Set the temporary RTC time
  RtcTime compiledDateTime(__DATE__, __TIME__);
  RTC.setTime(compiledDateTime);

  setupGnss();
  if (DEBUG) Serial.println("INFO: gnss started");

  // device attempts to connect to cellular
  ei_camera_connect_cellular(DEBUG);

  //camera starts continuously classifying video feed at 5fps
  ei_camera_start_continuous(DEBUG);
}

void loop() {
  // this routine is used to validate when we have valid GNSS data
  // camera video feed events have generally been paused if this is running
  if (!loopGnss()) {

    if (DEBUG) {
      Serial.println("gnss update failed, data:");
      Serial.println(sprintNavData());
    }
  } else if (DEBUG) {
    Serial.println("gnss update:");
    Serial.println(sprintNavData());
  }

  sleep(60);
  
  // retry cellular connection
  Serial.println("Reconnect cellular..");
  ei_camera_connect_cellular(DEBUG);
  // restart continuously classifying video feed at 5fps
  Serial.println("Start streaming again..");
  ei_camera_start_continuous(DEBUG);
}

4g_camera.ino

Arduino
Arduino code for LTE comms, camera and ML algorithm
/*
   Image processing and 4G transmit functionality. Derived from:
   https://developer.sony.com/develop/spresense/docs/arduino_developer_guide_en.html#_camera_library
*/

#include <plant-disease_inferencing.h>
#include <LTE.h>
#include <Camera.h>

#define APP_LTE_APN "lpwa.pelion"   // replace with your APN
#define APP_LTE_USER_NAME "streamip"   // replace with your username
#define APP_LTE_PASSWORD  "streamip"   // replace with your password
// replace with authentication method based on APN
#define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_CHAP) // Authentication : CHAP
// #define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_PAP) // Authentication : PAP
// #define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_NONE) // Authentication : NONE

// APN IP type
#define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4V6) // IP : IPv4v6
// #define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4) // IP : IPv4
// #define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V6) // IP : IPv6

/* RAT to use
 * Refer to the cellular carriers information
 * to find out which RAT your SIM supports.
 * The RAT set on the modem can be checked with LTEModemVerification::getRAT().
 */

// #define APP_LTE_RAT (LTE_NET_RAT_CATM) // RAT : LTE-M (LTE Cat-M1)
#define APP_LTE_RAT (LTE_NET_RAT_NBIOT) // RAT : NB-IoT


/* Defines to center crop and resize a image to the Impulse image size the speresense hardware accelerator

   NOTE: EI_CLASSIFIER_INPUT width and height must be less than RAW_HEIGHT * SCALE_FACTOR, and must
   simultaneously meet the requirements of the spresense api:
   https://developer.sony.com/develop/spresense/developer-tools/api-reference/api-references-arduino/group__camera.html#ga3df31ea63c3abe387ddd1e1fd2724e97
*/
#define SCALE_FACTOR 1
#define RAW_WIDTH CAM_IMGSIZE_QVGA_H
#define RAW_HEIGHT CAM_IMGSIZE_QVGA_V
#define CLIP_WIDTH (EI_CLASSIFIER_INPUT_WIDTH * SCALE_FACTOR)
#define CLIP_HEIGHT (EI_CLASSIFIER_INPUT_HEIGHT * SCALE_FACTOR)
#define OFFSET_X  ((RAW_WIDTH - CLIP_WIDTH) / 2)
#define OFFSET_Y  ((RAW_HEIGHT - CLIP_HEIGHT) / 2)

// enable for very verbose logging from edge impulse sdk
#define DEBUG_NN false

#define CLASSIFIER_THRESHOLD 0.7

/* static variables */
static CamImage sized_img;
static ei_impulse_result_t ei_result = { 0 };
static char names[190];

static LTE lteAccess;
static LTEClient client;

char server[] = "ai.gerritniezen.com";
char api_key[] = "1234";
char path[] = "/observations";
int port = 80; // port 80 is the default for HTTP

/* prototypes */
void printError(enum CamErr err);
void CamCB(CamImage img);

/**
   @brief      Convert monochrome data to rgb values

   @param[in]  mono_data  The mono data
   @param      r          red pixel value
   @param      g          green pixel value
   @param      b          blue pixel value
*/
static inline void mono_to_rgb(uint8_t mono_data, uint8_t *r, uint8_t *g, uint8_t *b) {
  uint8_t v = mono_data;
  *r = *g = *b = v;
}


int ei_camera_cutout_get_data(size_t offset, size_t length, float *out_ptr) {
  size_t bytes_left = length;
  size_t out_ptr_ix = 0;

  uint8_t *buffer = sized_img.getImgBuff();

  // read byte for byte
  while (bytes_left != 0) {

    // grab the value and convert to r/g/b
    uint8_t pixel = buffer[offset];

    uint8_t r, g, b;
    mono_to_rgb(pixel, &r, &g, &b);

    // then convert to out_ptr format
    float pixel_f = (r << 16) + (g << 8) + b;
    out_ptr[out_ptr_ix] = pixel_f;

    // and go to the next pixel
    out_ptr_ix++;
    offset++;
    bytes_left--;
  }

  // and done!
  return 0;
}

/**
   Print error message
*/
void printError(enum CamErr err)
{
  Serial.print("Error: ");
  switch (err)
  {
    case CAM_ERR_NO_DEVICE:
      Serial.println("No Device");
      break;
    case CAM_ERR_ILLEGAL_DEVERR:
      Serial.println("Illegal device error");
      break;
    case CAM_ERR_ALREADY_INITIALIZED:
      Serial.println("Already initialized");
      break;
    case CAM_ERR_NOT_INITIALIZED:
      Serial.println("Not initialized");
      break;
    case CAM_ERR_NOT_STILL_INITIALIZED:
      Serial.println("Still picture not initialized");
      break;
    case CAM_ERR_CANT_CREATE_THREAD:
      Serial.println("Failed to create thread");
      break;
    case CAM_ERR_INVALID_PARAM:
      Serial.println("Invalid parameter");
      break;
    case CAM_ERR_NO_MEMORY:
      Serial.println("No memory");
      break;
    case CAM_ERR_USR_INUSED:
      Serial.println("Buffer already in use");
      break;
    case CAM_ERR_NOT_PERMITTED:
      Serial.println("Operation not permitted");
      break;
    default:
      break;
  }
}

/**
   @brief run inference on the static sized_image buffer using the provided impulse
*/
static void ei_camera_classify(bool debug) {
  ei::signal_t signal;
  signal.total_length = EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT;
  signal.get_data = &ei_camera_cutout_get_data;

  EI_IMPULSE_ERROR err = run_classifier(&signal, &ei_result, DEBUG_NN);
  if (err != EI_IMPULSE_OK) {
    ei_printf("ERROR: Failed to run classifier (%d)\n", err);
    return;
  }
  // print the predictions
  if (debug) {
    ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
              ei_result.timing.dsp, ei_result.timing.classification, ei_result.timing.anomaly);
#if EI_CLASSIFIER_OBJECT_DETECTION == 1
    bool bb_found = ei_result.bounding_boxes[0].value > 0;
    for (size_t ix = 0; ix < EI_CLASSIFIER_OBJECT_DETECTION_COUNT; ix++) {
      auto bb = ei_result.bounding_boxes[ix];
      if (bb.value == 0) {
        continue;
      }

      ei_printf("    %s (", bb.label);
      ei_printf_float(bb.value);
      ei_printf(") [ x: %u, y: %u, width: %u, height: %u ]\n", bb.x, bb.y, bb.width, bb.height);
    }

    if (!bb_found) {
      ei_printf("    No objects found\n");
    }
#else
    for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
      ei_printf("    %s: ", ei_result.classification[ix].label);
      ei_printf_float(ei_result.classification[ix].value);
      ei_printf("\n");
    }
#if EI_CLASSIFIER_HAS_ANOMALY == 1
    ei_printf("    anomaly score: ");
    ei_printf_float(ei_result.anomaly);
    ei_printf("\n");
#endif
#endif
  }

  return;
}

/**
 * @brief callback that checks for the presence of a plant in the camera preview window, and then 
 *   executes ei_camera_snapshot() if found
 */
void CamCB(CamImage img)
{
  if (!img.isAvailable()) return; // fast path if image is no longer ready
  CamErr err;
  Serial.println("INFO: new frame processing...");
  
  // resize and convert image to grayscale to prepare for inferencing
  err = img.clipAndResizeImageByHW(sized_img
                                   , OFFSET_X, OFFSET_Y
                                   , OFFSET_X + CLIP_WIDTH - 1
                                   , OFFSET_Y + CLIP_HEIGHT - 1
                                   , EI_CLASSIFIER_INPUT_WIDTH, EI_CLASSIFIER_INPUT_HEIGHT);
  if (err) printError(err);

  err = sized_img.convertPixFormat(CAM_IMAGE_PIX_FMT_GRAY);
  if (err) printError(err);

  // get inference results on resized grayscale image
  ei_camera_classify(true);

  names[0] = '\0';
  for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) {
    if (ei_result.classification[ix].value >= CLASSIFIER_THRESHOLD) {
      sprintf(names + strlen(names), "%s_", ei_result.classification[ix].label); 
    }
  }

  if (strlen(names) > 0) {
    Serial.println("INFO: Plant detected!");
    ei_camera_snapshot(true);
    // if a plant snapshot is captured, pause to check for connectivity prior to taking followup photos
    err = theCamera.startStreaming(false, CamCB);
  }
}

/**
   @brief initialize the camera for continuous monitoring of video feed
*/
void ei_camera_start_continuous(bool debug) {
  CamErr err;

  if (debug) Serial.println("Starting camera..");

  err = theCamera.begin(1, CAM_VIDEO_FPS_5, RAW_WIDTH, RAW_HEIGHT);
  if (err && debug) printError(err);

  // start streaming the preview images to the classifier
  err = theCamera.startStreaming(true, CamCB);
  if (err && debug) printError(err);
    
  // still image format must be jpeg to allow for compressed storage/transmit
  err = theCamera.setStillPictureImageFormat(
    RAW_WIDTH,
    RAW_HEIGHT,
    CAM_IMAGE_PIX_FMT_JPG);
  if (err && debug) printError(err);

  if (debug) Serial.println("INFO: started camera recording");
}

/**
   @brief connect to cellular network for communication
*/
void ei_camera_connect_cellular(bool debug) {
  if ((lteAccess.begin() != LTE_SEARCHING) && debug) {
      Serial.println("ERROR: Could not start LTE modem to LTE_SEARCHING.");
      Serial.println("Please check the status of the LTE board.");
  }
  if (!(lteAccess.attach(APP_LTE_RAT,
                         APP_LTE_APN,
                         APP_LTE_USER_NAME,
                         APP_LTE_PASSWORD,
                         APP_LTE_AUTH_TYPE,
                         APP_LTE_IP_TYPE) == LTE_READY) && debug) {
      Serial.println("ERROR: Failed to attach to LTE network");
      Serial.println("Check SIM card, APN settings, and coverage in current area");
  } else {
    if (debug) Serial.println("LTE attach succeeded.");
  }
}

/**
   @brief take a jpeg picture, save to SD card, and transmit over 4G LTE if available
*/
void ei_camera_snapshot(bool debug)
{
  char filename[200];
  char json[400];

  if (data.type == 0) {
    Serial.println("Time from GPS not yet valid.");
    theCamera.end();
    return;
  }

  Serial.println("Getting time and voltage");
  RtcTime rtc = RTC.getTime();
  batteryVoltage = LowPower.getVoltage();
  
  // snapshot and save a jpeg
  CamImage img = theCamera.takePicture();
  if (theSD.begin() && img.isAvailable()) {
    sprintf(filename, "PLANT.%s.%lu.jpg", names, rtc.unixtime());
    if (debug) ei_printf("INFO: saving %s to SD card...\n", filename);
    theSD.remove(filename);
    File myFile = theSD.open(filename, FILE_WRITE);
    myFile.write(img.getImgBuff(), img.getImgSize());
    myFile.close();
  } else if (debug) {
    Serial.println("failed to compress and save image, check that camera and SD card are connected properly");
  }

  Serial.println("Building JSON");

  sprintf(json, "{ \"date\": \"%04d/%02d/%02d\", \"time\": \"%02d:%02d:%02d\", \"observation\": \"%s\", \"battery\": %d }\0",
    rtc.year(), rtc.month(), rtc.day(), rtc.hour(), rtc.minute(), rtc.second(), names, batteryVoltage);

  Serial.println(json);

  // if connected, transmit using 4G LTE
  Serial.println("Connecting to network..");
  Serial.println(server);
  
  if (client.connect(server, port)) {
    if (debug) Serial.println("INFO: connected to network, attempting to send observation....");
    // Make a HTTP request:
    client.print("POST ");
    client.print(path);
    client.print(" HTTP/1.1\r\n");
    client.print("Host: ");
    client.print(server);
    client.print("\r\n");
    client.print("Content-Type: application/json\r\n");
    client.print("User-Agent: curl/7.79.1\r\n");
    client.print("Accept: */*\r\n");
    client.print("x-api-key: ");
    client.print(api_key);
    client.print("\r\n");
    client.print("Content-Length: ");
    client.print(strlen(json), DEC);
    client.print("\r\n");
    client.println();
    client.print(json);
    client.println();
  } else {
    Serial.println("connection failed");
  }

  Serial.println("Sent data");

  sleep(5);

  // if there are incoming bytes available
  // from the server, read them and print them:
  while (int len = client.available() && debug) {
    char buff[len + 1];
    buff[len] = '\0';
    client.read((uint8_t*)buff, len);
    Serial.print(buff);
  }
  Serial.print("\n");

  Serial.println("Shutting down LTE");
  lteAccess.shutdown();
  Serial.println("Shutting down GPS");
  shutdownGnss();

  Serial.println("Going to deep sleep..");
  LowPower.deepSleep(20800); // sleep for 3 hours
}

low_power_gnss.ino

Arduino
Arduino code for retrieving accurate date/time with GPS
/* Low power GNSS utility functions, derived from:
 *  https://developer.sony.com/file/download/spresense-low-power-demo
 */

#include <LowPower.h>
#include <stdlib.h>

#define GNSS_UPDATE_MAX_TRIES 5

#define STRING_BUFFER_SIZE   200

/* static variables */
static SpGnss  Gnss;
static RtcTime ref_time(2019, 1, 1, 0, 0, 0, 0);
static char buffer[STRING_BUFFER_SIZE];

/* prototypes */
bool updateRtc(SpGnssTime *satTime);


/* Print navigation data contents on serial output*/
char * sprintNavData() {
  Gnss.getNavData(&data);
  if (data.posDataExist) {
    SpGnssTime time = data.time;

    snprintf(buffer, STRING_BUFFER_SIZE,
            "%d-%02d-%02d_%02d-%02d-%02d_lat%.6f_long%.6f",
            time.year, time.month, time.day,
            time.hour, time.minute, time.sec,
            data.latitude, data.longitude);
  } else {
    snprintf(buffer, STRING_BUFFER_SIZE, "[no data available]   Satellites:%d\n",
             data.numSatellites);
  }

  return buffer;
}

/* Update RTC with GNSS time if it is valid and they are not in sync */
bool updateRtc(SpGnssTime *satTime) {
  RtcTime rtc = RTC.getTime();
  RtcTime gps(satTime->year, satTime->month, satTime->day,
              satTime->hour, satTime->minute, satTime->sec,
              satTime->usec * 1000);

  /* Sanity check */
  if (gps.unixtime() < ref_time.unixtime()) {
    return false;
  }

  Serial.println("Syncing time.");

  /* Sync RTC with GPS time */
  if (abs((int)gps - (int)rtc) >= 1) {
    RTC.setTime(gps);
  }

  /* Allow sleep if we have a valid time */
  if (rtc.unixtime() > 100) {
    return true;
  }
  return false;
}

bool getGnssUpdate() {
  Serial.println("Waiting for GNSS update.");
  if (Gnss.waitUpdate()) {
    Gnss.getNavData(&data);

    /* Check that we got data that is not cached etc. but from GNSS (type 1) */
    if (data.posDataExist && data.posFixMode && data.type == 1) {
      return updateRtc(&data.time);
    }
  }
  return false;
}

/* Check if RTC has been set by comparing to a reference time/date */
bool isRtcValid() {
  RtcTime rtc = RTC.getTime();
  return rtc.unixtime() > ref_time.unixtime();
}

/* Set GNSS time from RTC if RTC is valid */
void setGnssTime() {
  if (isRtcValid()) {
    SpGnssTime gnssTime;
    RtcTime rtc = RTC.getTime();
    gnssTime.year = rtc.year();
    gnssTime.month = rtc.month();
    gnssTime.day = rtc.day();
    gnssTime.hour = rtc.hour();
    gnssTime.minute = rtc.minute();
    gnssTime.sec = rtc.second();
    gnssTime.usec = rtc.nsec() / 1000;

    Gnss.setTime(&gnssTime);
  }
}

/*
 * Initialize GNSS 
 */
void setupGnss() {
  Gnss.begin();
  Gnss.select(GPS);
  Gnss.select(GLONASS);
  setGnssTime();
  Gnss.start(HOT_START);
}

void shutdownGnss() {
  Gnss.stop();
  Gnss.end();
}

/*
 * Fetch updates until we reach GNSS_MIN_UPDATE_CNT valid ones
 */
bool loopGnss() {
  if (getGnssUpdate()) {
    Gnss.saveEphemeris();
    return true;
  } else {
    return false;
  }
}

Code for web server

Credits

Gerrit Niezen

Gerrit Niezen

5 projects • 10 followers
Maker of open-source hardware and software

Comments