Save the Japan Deer.

/* 
 *  2022/08/05 
 *  Spresense Developer Challenge 2022 with Sony
 *  one project "Save tha japan deer"
 *  
 */

// libraries
#include <ArduinoHttpClient.h>
#include <RTC.h>
#include <SDHCI.h>
#include <LTE.h>
#include <string.h>
#include <SDHCI.h> // SDカード用
#include <stdio.h>  /* for sprintf */
#include <Camera.h> // カメラ用
#include <DNNRT.h> // 推論用
#include <Audio.h>

#define PLAYBACK_FILE_NAME "Sound.wav"

const int cam_width = 320; // キャプチャした画像の幅・高さ
const int cam_height = 240; 
const int dscale_x = 8; // キャプチャ画像を内部バッファに収める時の縮小率(1/4)
const int dscale_y = 6; 
const int img_size = (cam_width / dscale_x)* (cam_height / dscale_y) * 3; // 内部バッファのサイズ
const int  img_width = ((cam_width) / (dscale_x)); // 内部バッファの幅・高さ
const int  img_height = ((cam_height) / (dscale_y));
uint8_t img_buf[img_size]; // 内部バッファ

unsigned int busy = 0;
int index_out = 0;
char filename[16] = {0};
String gStrResult;


// APN name
#define APP_LTE_APN "iijmio.jp" // replace your APN
#define APP_LTE_USER_NAME "mio@iij" // replace with your username
#define APP_LTE_PASSWORD  "iij" // replace with your password

// APN IP type
#define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4V6) // IP : IPv4v6
// #define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4) // IP : IPv4
// #define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V6) // IP : IPv6

// APN authentication type
#define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_CHAP) // Authentication : CHAP
// #define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_PAP) // Authentication : PAP
// #define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_NONE) // Authentication : NONE



#define APP_LTE_RAT (LTE_NET_RAT_CATM) // RAT : LTE-M (LTE Cat-M1)
// #define APP_LTE_RAT (LTE_NET_RAT_NBIOT) // RAT : NB-IoT

// URL, path & port (for example: httpbin.org)
char server[] = "notify-api.line.me";
char getPath[] = "/get";
char postPath[] = "/api/notify";
int port = 443; // port 443 is the default for HTTPS

#define ROOTCA_FILE "CERTS/line-right.pem"   // Define the path to a file containing CA
                                      // certificates that are trusted.
#define CERT_FILE   "path/to/certfile" // Define the path to a file containing certificate
                                       // for this client, if required by the server.
#define KEY_FILE    "path/to/keyfile"  // Define the path to a file containing private key
                                       // for this client, if required by the server.
char apn[LTE_NET_APN_MAXLEN] = APP_LTE_APN;
LTENetworkAuthType authtype = APP_LTE_AUTH_TYPE;
char user_name[LTE_NET_USER_MAXLEN] = APP_LTE_USER_NAME;
char password[LTE_NET_PASSWORD_MAXLEN] = APP_LTE_PASSWORD;

char notify_token[] = "(input your token)";
String message="Detected wild deer";

// initialize the library instance
LTE lteAccess;
LTETLSClient tlsClient;
HttpClient client = HttpClient(tlsClient, server, port);
SDClass theSD;
DNNRT dnnrt;
AudioClass *theAudio;

File myFile;
WavContainerFormatParser theParser;

const int32_t sc_buffer_size = 6144;
uint8_t s_buffer[sc_buffer_size];
static const uint32_t sc_prestore_frames = 10;
static const uint32_t sc_store_frames = 10;
uint32_t s_remain_size = 0;
bool ErrEnd = false;

void CamCB(CamImage img)
{

  /* Check the img instance is available or not. */

  if (img.isAvailable())
    {
      img.convertPixFormat(CAM_IMAGE_PIX_FMT_RGB565);

      if(busy == 0) { // バッファを処理中の時は書き込まない（ロック機構としては弱いがとりあえずで…）
        uint8_t* buf = img_buf; 
        uint16_t* bf = (uint16_t*)img.getImgBuff(); ; 

        // QVGA のままだと学習済みパラメータが大きくなってしまい、
        // メモリに乗らなくなってしまうため、
        // キャプチャした時点で画像サイズを小さくする。
        for(int y=0;y<cam_height;y += dscale_y) {
          for(int x=0;x<cam_width;x += dscale_x) {
            uint8_t r = ((*bf) >> 11 & 0x1f) << 3; 
            uint8_t g = ((*bf) >> 5 & 0x3f) << 2; 
            uint8_t b = ((*bf) >> 0 & 0x1f) << 3; 
            *buf = g; 
            buf++; 
            *buf = b; 
            buf++; 
            *buf = r; 
            buf++; 
            bf += dscale_x; 
          }
          bf += cam_width * (dscale_y - 1); 
        }
      }
      sprintf(filename, "aiueo.jpg");
      writePPM(filename);
      Serial.print("save ");
      Serial.println(filename);
      dnnrt_set();
    }
  else
    {
      Serial.print("Failed to get video stream image\n");
    }
    // 推論結果のディスプレイ表示
    img.convertPixFormat(CAM_IMAGE_PIX_FMT_RGB565);
    uint16_t* imgBuf = (uint16_t*)img.getImgBuff();
//    tft.drawRGBBitmap(0, 0, (uint16_t*)imgBuf, 320, 224);
//    putStringOnLcd(gStrResult, ILI9341_YELLOW);
    
    delay(100); //1000
}

static void audio_attention_cb(const ErrorAttentionParam *atprm)
{
  puts("Attention!");
  
  if (atprm->error_code >= AS_ATTENTION_CODE_WARNING)
    {
      ErrEnd = true;
    }
}

void LTE_set(){   //LTE   
  Serial.println("Starting secure HTTP client.");

  /* Set if Access Point Name is empty */
  if (strlen(APP_LTE_APN) == 0) {
    Serial.println("This sketch doesn't have a APN information.");
    readApnInformation(apn, &authtype, user_name, password);
  }
  Serial.println("=========== APN information ===========");
  Serial.print("Access Point Name  : ");
  Serial.println(apn);
  Serial.print("Authentication Type: ");
  Serial.println(authtype == LTE_NET_AUTHTYPE_CHAP ? "CHAP" :
                 authtype == LTE_NET_AUTHTYPE_NONE ? "NONE" : "PAP");
  if (authtype != LTE_NET_AUTHTYPE_NONE) {
    Serial.print("User Name          : ");
    Serial.println(user_name);
    Serial.print("Password           : ");
    Serial.println(password);
  }                   
  while (true) {

    /* Power on the modem and Enable the radio function. */

    if (lteAccess.begin() != LTE_SEARCHING) {
      Serial.println("Could not transition to LTE_SEARCHING.");
      Serial.println("Please check the status of the LTE board.");
      for (;;) {
        sleep(1);
      }
    }

    /* The connection process to the APN will start.
     * If the synchronous parameter is false,
     * the return value will be returned when the connection process is started.
     */
    if (lteAccess.attach(APP_LTE_RAT,
                         apn,
                         user_name,
                         password,
                         authtype,
                         APP_LTE_IP_TYPE) == LTE_READY) {
      Serial.println("attach succeeded.");
      break;
    }

    /* If the following logs occur frequently, one of the following might be a cause:
     * - APN settings are incorrect
     * - SIM is not inserted correctly
     * - If you have specified LTE_NET_RAT_NBIOT for APP_LTE_RAT,
     *   your LTE board may not support it.
     * - Rejected from LTE network
     */
    Serial.println("An error has occurred. Shutdown and retry the network attach process after 1 second.");
    lteAccess.shutdown();
    sleep(1);
  }   
}

void writePPM(char* filename)
{
// 追記させないように、あらかじめファイルを消しておく。
  if(theSD.exists(filename)) {
   theSD.remove(filename); 
  }

  File f = theSD.open(filename, FILE_WRITE); 
  f.println("P6"); 
  f.println(String(img_width) + String(" ") + String(img_height)); 
  f.println("255"); 

  busy = 1; // バッファをロックする
  f.write(img_buf, img_size); 
  busy = 0; 

  f.close(); 
}

void setup_nnb()  //推論開始
{
//  theCamera.begin(); // 初期化
//  theCamera.startStreaming(true, CamCB); // CamCB については後述

  // 学習・推論共に同じ場所で、常に撮影環境を同じ場所にできるならば、
  // ホワイトバランスのモードをその場所に最も合うように選択するのが良い。
  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_INCANDESCENT ); 

  theCamera.setStillPictureImageFormat(
   CAM_IMGSIZE_QVGA_H,
   CAM_IMGSIZE_QVGA_V,
   CAM_IMAGE_PIX_FMT_JPG);
   
  File nnbfile = theSD.open("model.nnb");
  if (!nnbfile) {
    Serial.print("nnb not found");
    return;
  }
  int ret = dnnrt.begin(nnbfile);
  if (ret < 0) {
    Serial.print("Runtime initialization failure. ");
    if (ret == -16) {
      Serial.println("Please update bootloader!");
    } else {
      Serial.println(ret);
    }
    nnbfile.close();
    return;
  }
  
  nnbfile.close(); 
}


void dnnrt_set(){
  DNNVariable input(img_size);
  float *fbuf = input.data();
  char buf[255]; 

  busy = 1; // バッファをロック
  for (int y = 0; y < img_height; y++) {
    for (int x = 0; x < img_width; x++) {
      // img_buf: ＳＤに保存される時のデータ順
      // fbuf: DNNVariable として格納される時のデータ順
      int p, p2; 
      p = (y * img_width + x) * 3 + 2; 
      p2 = (y * img_width + x) + (img_width * img_height) * 0; 
      fbuf[p2] = float(img_buf[p]) / 255.0;

      p = (y * img_width + x) * 3 + 0; 
      p2 = (y * img_width + x) + (img_width * img_height) * 1; 
      fbuf[p2] = float(img_buf[p]) / 255.0;

      p = (y * img_width + x) * 3 + 1; 
      p2 = (y * img_width + x) + (img_width * img_height) * 2; 
      fbuf[p2] = float(img_buf[p]) / 255.0;
    }
  }
  busy = 0;
  
  dnnrt.inputVariable(input, 0);
  dnnrt.forward();
  DNNVariable output = dnnrt.outputVariable(0);
  int index = output.maxIndex();
  index_out = index;
  if (index < 2) {
    gStrResult = String(index) + String(":") + String(output[index]);
    Serial.println(gStrResult);
  } else {
    gStrResult = String("Error");    
  }
  for(int i=0;i<output.size();i++) {
    sprintf(buf, "$%d:%.5f\r\n", i, output[i]); 
    Serial.print(buf); 
  }
}


String readFromSerial() {
  /* Read String from serial monitor */
  String str;
  int  read_byte = 0;
  while (true) {
    if (Serial.available() > 0) {
      read_byte = Serial.read();
      if (read_byte == '\n' || read_byte == '\r') {
        Serial.println("");
        break;
      }
      Serial.print((char)read_byte);
      str += (char)read_byte;
    }
  }
  return str;
}

void readApnInformation(char apn[], LTENetworkAuthType *authtype,
                       char user_name[], char password[]) {
  /* Set APN parameter to arguments from readFromSerial() */

  String read_buf;

  while (strlen(apn) == 0) {
    Serial.print("Enter Access Point Name:");
    readFromSerial().toCharArray(apn, LTE_NET_APN_MAXLEN);
  }

  while (true) {
    Serial.print("Enter APN authentication type(CHAP, PAP, NONE):");
    read_buf = readFromSerial();
    if (read_buf.equals("NONE") == true) {
      *authtype = LTE_NET_AUTHTYPE_NONE;
    } else if (read_buf.equals("PAP") == true) {
      *authtype = LTE_NET_AUTHTYPE_PAP;
    } else if (read_buf.equals("CHAP") == true) {
      *authtype = LTE_NET_AUTHTYPE_CHAP;
    } else {
      /* No match authtype */
      Serial.println("No match authtype. type at CHAP, PAP, NONE.");
      continue;
    }
    break;
  }

  if (*authtype != LTE_NET_AUTHTYPE_NONE) {
    while (strlen(user_name)== 0) {
      Serial.print("Enter username:");
      readFromSerial().toCharArray(user_name, LTE_NET_USER_MAXLEN);
    }
    while (strlen(password) == 0) {
      Serial.print("Enter password:");
      readFromSerial().toCharArray(password, LTE_NET_PASSWORD_MAXLEN);
    }
  }

  return;
}
void Audio_setup(){
  // Get wav file info

  fmt_chunk_t fmt;

  handel_wav_parser_t *handle
    = (handel_wav_parser_t *)theParser.parseChunk("/mnt/sd0/" PLAYBACK_FILE_NAME, &fmt);
  if (handle == NULL)
    {
      printf("Wav parser error.\n");
      exit(1);
    }

  // Get data chunk info from wav format
  uint32_t data_offset = handle->data_offset;
  s_remain_size = handle->data_size;

  theParser.resetParser((handel_wav_parser *)handle);

  // start audio system
  theAudio = AudioClass::getInstance();

  theAudio->begin(audio_attention_cb);

  puts("initialization Audio Library");

  /* Set clock mode to normal */

  theAudio->setRenderingClockMode((fmt.rate <= 48000) ? AS_CLKMODE_NORMAL : AS_CLKMODE_HIRES);

  theAudio->setPlayerMode(AS_SETPLAYER_OUTPUTDEVICE_SPHP, AS_SP_DRV_MODE_LINEOUT);

  err_t err = theAudio->initPlayer(AudioClass::Player0, AS_CODECTYPE_WAV, "/mnt/sd0/BIN", fmt.rate, fmt.bit, fmt.channel);

  /* Verify player initialize */
  if (err != AUDIOLIB_ECODE_OK)
    {
      printf("Player0 initialize error\n");
      exit(1);
    }

  /* Open file placed on SD card */
  myFile = theSD.open(PLAYBACK_FILE_NAME);

  /* Verify file open */
  if (!myFile)
    {
      printf("File open error\n");
      exit(1);
    }
  printf("Open! %s\n", myFile.name());

  /* Set file position to beginning of data */
  myFile.seek(data_offset);

  for (uint32_t i = 0; i < sc_prestore_frames; i++)
    {
      size_t supply_size = myFile.read(s_buffer, sizeof(s_buffer));
      s_remain_size -= supply_size;
      
      err = theAudio->writeFrames(AudioClass::Player0, s_buffer, supply_size);
      if (err != AUDIOLIB_ECODE_OK)
        {
          break;
        }
        
      if (s_remain_size == 0)
        {
          break;
        }
    }
    
  /* Main volume set to -16.0 dB */
    
  theAudio->setVolume(-100);
  theAudio->startPlayer(AudioClass::Player0);
  printf("Play!");
  int stop_player = 0;
  while(1){
  static bool is_carry_over = false;
  static size_t supply_size = 0;

  /* Send new frames to decode in a loop until file ends */
  for (uint32_t i = 0; i < sc_store_frames; i++)
    {
      if (!is_carry_over)
        {
          supply_size = myFile.read(s_buffer, (s_remain_size < sizeof(s_buffer)) ? s_remain_size : sizeof(s_buffer));
          s_remain_size -= supply_size;
        }
      is_carry_over = false;

      int err = theAudio->writeFrames(AudioClass::Player0, s_buffer, supply_size);

      if (err == AUDIOLIB_ECODE_SIMPLEFIFO_ERROR)
        {
          is_carry_over = true;
          break;
        }

      if (s_remain_size == 0)
        {
          stop_player = 1;
        }
    }

  if (ErrEnd)
    {
      printf("Error End\n");
      stop_player = 1;
    }

    usleep(1000);

    if(stop_player){
      stop_player = 0;  
      theAudio->stopPlayer(AudioClass::Player0);
      myFile.close();
      theAudio->setReadyMode();
      theAudio->end();
      printf("Exit player\n");
      break;
    }
  }
}
void LINE(){
  Serial.print("LINE_START");
// Set certifications via a file on the SD card before connecting to the server
  File rootCertsFile = theSD.open(ROOTCA_FILE, FILE_READ);
  tlsClient.setCACert(rootCertsFile, rootCertsFile.available());
  rootCertsFile.close();

  String contentType = "application/x-www-form-urlencoded\nAuthorization: Bearer " + String(notify_token);
  String postData = "message="+String(message);
  Serial.print("Post code");
  client.post(postPath, contentType, postData);

  // read the status code and body of the response
  int statusCode = client.responseStatusCode();
  String response = client.responseBody();

  Serial.print("Status code: ");
  Serial.println(statusCode);
  Serial.print("Response: ");
  Serial.println(response);
}

void setup(){
  Serial.begin(115200);
  /* Initialize SD */
  while (!theSD.begin())
    {
      /* wait until SD card is mounted. */
      Serial.println("Insert SD card.");
    }
  theCamera.begin();
  theCamera.startStreaming(true, CamCB);
    theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_AUTO); //自動
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_INCANDESCENT); //白熱電球
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_FLUORESCENT); //蛍光灯
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_DAYLIGHT); //晴天
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_FLASH); //フラッシュ
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_CLOUDY);  //曇り空
//  theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_SHADE); //影
//  theCamera.setHDR(CAM_HDR_MODE_ON); // HDR ON
//  theCamera.setHDR(CAM_HDR_MODE_OFF); // HDR OFF
  theCamera.setHDR(CAM_HDR_MODE_AUTO); // HDR AUTO
  // 露出の設定
  Serial.println("Set Auto exposure");
  theCamera.setAutoExposure(false);
  setup_nnb();  

}
void loop()
{
  if(index_out == 1){
    index_out = 0;
    theCamera.startStreaming(false, CamCB);
    theCamera.end();
    dnnrt.end();
    Audio_setup();
    LTE_set();
    LINE();
    setup_nnb();
    theCamera.begin();
    theCamera.startStreaming(true, CamCB);
  }
  
}