Published March 7, 2019

Kernel Density Distribution Analytics from Data Sensor

IoT project and data analytics to get insight of soil (soil moisture and soil temperature).

IntermediateWork in progress5 hours616

Kernel Density Distribution Analytics from Data Sensor

Things used in this project

Hardware components

Espressif Wemos D1 Mini

SparkFun Soil Moisture Sensor (with Screw Terminals)

Raspberry Pi 3 Model B

Adafruit Waterproof DS18B20 Digital temperature sensor

Software apps and online services

Node-RED

Microsoft Visual Studio Code Extension for Arduino

Story

this project explains how to analyze data from influx database. there are 2 sensor to measure soil parameter (soil humidity and soil temperature).

System Schematic

data sensors are sent to MQTT gateway (raspberry) using Node-RED, then we can see the visualization of data from our laptop from Node-RED dashboard.

dashboard

Flow diagram in Node-RED

we can set all the data in database over time.

Soil humidity and soil temperature can plotted in python.

Raw Data of Soil Temperature

Raw Data of Soil Humidity

1 / 2 • Histogram of soil temperature and humidty

Kernel Density Distribution of Soil Temperature and Humidity

I am still working to finish this project.

#include <PubSubClient.h>
#include <ESP8266WiFi.h>
#include <Wire.h>
#include<OneWire.h>
#include<DallasTemperature.h>

#define ONE_WIRE_BUS D3
OneWire oneWire(ONE_WIRE_BUS);
DallasTemperature sensors(&oneWire);

const char* ssid = "";
const char* password = "";
const char* mqtt_server = "";

int adcSoil = 0;

WiFiClient espClient;
PubSubClient client(espClient);
long lastData = 0;

void setup_wifi() {

  delay(10);
  Serial.println();
  Serial.print("Connecting to ");
  Serial.println(ssid);
  //WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  randomSeed(micros());

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}


void reconnect() {

  while (!client.connected()) {

    Serial.print("Attempting MQTT connection...");
    String clientId = "ESP8266Client-";
    clientId += String(random(0xffff), HEX); 

    if (client.connect(clientId.c_str())) { 

      Serial.println("connected");
      digitalWrite(LED_BUILTIN, LOW);
      //client.publish("outputsensor/soildata", "soil_sensor value");
      

    } else {
      
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      digitalWrite(LED_BUILTIN, HIGH);

      delay(5000);
      
    }
  }
}

void setup() {

  Serial.begin(9600);
  pinMode(LED_BUILTIN, OUTPUT);
  Wire.begin();
  sensors.begin();
  setup_wifi();
  client.setServer(mqtt_server, 1883);
}

void loop() {

  if (!client.connected()) {
    
    reconnect();
  }

  client.loop();

  
  float tS = sensors.getTempCByIndex(0);
  adcSoil = analogRead(A0);
  sensors.requestTemperatures();
  
  String soilTemp = String(tS);
  String soilMoisture = String(adcSoil);
  
  
  String payload = "{\"soilTemperature\":";
  payload += soilTemp;
  payload += ",\"soilMoisture\":";
  payload += soilMoisture;
  payload += "}";
  
  char dataSensorSoil[150];
  
  long now = millis();

  if (now - lastData > 5000) {

    lastData = now;
    payload.toCharArray(dataSensorSoil, 150);
    client.publish("outputsensor/soildata", dataSensorSoil);
    Serial.println(dataSensorSoil);

   
  }
}

from influxdb import InfluxDBClient
import matplotlib.pyplot as plt 
import argparse
import numpy as np 

values_temp  = []
values_hum = []
timeseries_temp = []
timeseries_hum = []

client = InfluxDBClient(host='localhost', port=8086)
print(client.get_list_database())

client.switch_database('datasoil')
client.get_list_measurements()

result = client.query('select *from soil')
points_temp = result.get_points()


for point in points_temp:
    timeseries_temp.append(point['time'])
    values_temp.append(point['soil_temperature'])

plt.plot(timeseries_temp, values_temp, 'green')
plt.ylabel("Soil Temperature")
plt.xlabel("time")
plt.show()


points_hum = result.get_points()
for point in points_hum:
    timeseries_hum.append(point['time'])
    values_hum.append(point['soil_humidity'])

plt.plot(timeseries_hum, values_hum, 'red')
plt.ylabel("Soil Humidity")
plt.xlabel("time")
plt.show()

plt.scatter(values_temp, values_hum)
plt.xlabel('soil humidity')
plt.ylabel('soil temperature')
plt.show()

from numpy import mean
from numpy import std
from scipy.stats import spearmanr
from scipy.stats import pearsonr

print('temp: mean=%.3f stdv=%.3f' % (mean(values_temp), std(values_temp)))
print('hum: mean=%.3f stdv=%.3f' % (mean(values_hum), std(values_hum)))


from scipy.stats.kde import gaussian_kde
from scipy.stats import norm

my_pdf_temperature = gaussian_kde(values_temp)
my_pdf_humidty = gaussian_kde(values_hum)

figure = plt.subplot(2, 1, 1)
plt.title("Soil Temperature Histogram", fontsize=14)
plt.hist(values_temp, label='Temperature', density=False, color='C1', alpha=0.5, bins=30)
plt.xlabel("Temperature")
plt.ylabel("Frequency")
plt.legend(fontsize=8)

figure = plt.subplot(2, 1, 2)
plt.title("Soil Humidity Histogram", fontsize=14)
plt.hist(values_hum, label='Humidity', density=False, color='C0', alpha=0.5, bins=30)
plt.xlabel("Humidity")
plt.ylabel("Frequency")

plt.legend(fontsize=8)
plt.tight_layout()
plt.show()


figure = plt.subplot(2, 1, 1)
ax = figure.axes
plt.title("Kernel Density Estimation Temperature", fontsize=14)
x = np.linspace(0,50)
plt.plot(x, my_pdf_temperature(x), label='Temperature', color = 'C1')
plt.xlabel("Temperature")
plt.ylabel("Probability Distribution")
plt.legend(fontsize=8)

figure = plt.subplot(2,1,2)
ax = figure.axes
plt.title("Kernel Density Estimation Humidity", fontsize=14)
y = np.linspace(0,1023)
plt.plot(y, my_pdf_humidty(y), label='Humidity', color = 'C0')
plt.xlabel("Humidity")
plt.ylabel("Probability Distribution")
plt.legend(fontsize=8)

plt.tight_layout()
plt.show()