Using qsort to average readings

great,

I added a timeout in the case where the sensor is not producing any PWM signal (not tested obviously): you could even extend it to report a bad sensor…

double analyze(pin_t pulsePin)
{
  double temperature = 0.0;// Local variable
  const int num_pulses = 160;
  const int timeout = 25;  // we should see nearly 100 pulses at 4kHz in 25milliseconds
  int periodDutyCycle[num_pulses] = {0};
  RGB.control(true); // control RGB LED
  RGB.color(255, 0, 0);// RGB LED Red
  for (int i = 0; i < num_pulses; i++)  // get num_pulses samples of HIGH and LOW pulses
  {
    bool goodRead = true;
    int startReadTime = millis();
    int highPulseTime = 0;
    while(highPulseTime == 0)
    {
      highPulseTime = pulseIn(pulsePin, HIGH);
      if(millis() - startReadTime > timeout)
      {
        goodRead = false;
        break;
      }
    }
    int lowPulseTime = 0;
    startReadTime = millis();
    while(lowPulseTime == 0)
    {
      lowPulseTime = pulseIn(pulsePin, LOW);
      if(millis() - startReadTime > timeout)
      {
        goodRead = false;
        break;
      }
    }
    if(goodRead)  // will only do the math if you have received values, otherwise it will stay zero...
    {
      periodDutyCycle[i] = highPulseTime / (highPulseTime + lowPulseTime);  // save individual period duty cycle
    }
  }
  int dutyCycle = 0;
  int num_goodReads = 0;
  for(int i = 0; i < num_pulses; i++)
  {
    if(periodDutyCycle[i] != 0)
    {
      dutyCycle += periodDutyCycle[i];
      num_goodReads++;
    }
  }
  dutyCycle /= num_goodReads;
  temperature = (212.77 * dutyCycle) - 68.085;
  RGB.control(false); // end RGB LED control
  return temperature;
}

compiles as part of this:

/**************************************************************
 * Duty Cycle analyzer FUNCTION for PWM temperature sensors
 Any pin can be sensor input pin: Specify in function call in the loop()
 For debugging, the serial monitor shows all temperatures on one line
 **************************************************************/

//double Temp[16] = {0}; // All the temperatures into an array!!
double Tav; // Only variables needed in the loop() function.
int pulsePins[] = { D0, D2, D3, D5, D7, A1, A3, A5, A7, TX };
double Temp[10] = {0}; // All the temperatures into an array!!  // 10 in this example
char resultstr[256] = "";

void setup()
{
 Serial.begin(9600);
 for (int i = 0; i < sizeof(pulsePins)/sizeof(pulsePins[0]); i++)
 {
   pinMode(pulsePins[i], INPUT_PULLUP);
 }
 Particle.variable("result", resultstr, STRING);
}

void loop()
{
// 1) Data collection (All 16 Photon pins!)
strcpy(resultstr, "");  // start with an empty buffer
Tav = 0;  // get ready to average what we recieve
for (int i = 0; i < sizeof(pulsePins)/sizeof(pulsePins[0]); i++)  // count our way through all of the pins
{
  Temp[i] = analyze(pulsePins[i]);
  Tav += Temp[i];  // add all of the readings together
  char myText[10] = "";  // create a littl buffer for your reading.
  sprintf(myText, "T%d:%3.2f%d ", i, Temp[i], i < sizeof(pulsePins)/sizeof(pulsePins[0])? "," : "");  // Build your buffer reading by reading
  strcat(resultstr, myText); // bolt on each segment to the buffer
}
Tav /= sizeof(pulsePins)/sizeof(pulsePins[0]); // compute the average by dividing by number of readings
Serial.print("temperatures:");
Serial.println(resultstr);
Serial.print("Average = ");
Serial.println(Tav,1);

 // 4) Test conditions and send status & alert messages, take actions: Pumps, valves...
 if (Temp[0] > 30.0) // Example...
 {
  Serial.println("T1 > 30 deg");
  Particle.publish("Status","T1 > 30 deg",60,PRIVATE);
  RGB.control(true); // start control RGB LED
  RGB.color(255, 255, 255);// RGB LED White
  delay(1000);
  RGB.control(false); // stop control RGB LED
 }
 delay(500); // Slow down!
}

// In this function I'd like to integrate a "median filter":

double analyze(pin_t pulsePin)
{
  double temperature = 0.0;// Local variable
  const int num_pulses = 160;
  const int timeout = 25;  // we should see nearlyu 100 pulses at 4kHz in 25milliseconds
  int periodDutyCycle[num_pulses] = {0};
  RGB.control(true); // control RGB LED
  RGB.color(255, 0, 0);// RGB LED Red
  for (int i = 0; i < num_pulses; i++)  // get num_pulses samples of HIGH and LOW pulses
  {
    bool goodRead = true;
    int startReadTime = millis();
    int highPulseTime = 0;
    while(highPulseTime == 0)
    {
      highPulseTime = pulseIn(pulsePin, HIGH);
      if(millis() - startReadTime > timeout)
      {
        goodRead = false;
        break;
      }
    }
    int lowPulseTime = 0;
    startReadTime = millis();
    while(lowPulseTime == 0)
    {
      lowPulseTime = pulseIn(pulsePin, LOW);
      if(millis() - startReadTime > timeout)
      {
        goodRead = false;
        break;
      }
    }
    if(goodRead)
    {
      periodDutyCycle[i] = highPulseTime / (highPulseTime + lowPulseTime);  // save individual period duty cycle
    }
  }
  int dutyCycle = 0;
  int num_goodReads = 0;
  for(int i = 0; i < num_pulses; i++)
  {
    if(periodDutyCycle[i] != 0)
    {
      dutyCycle += periodDutyCycle[i];
      num_goodReads++;
    }
  }
  dutyCycle /= num_goodReads;
  temperature = (212.77 * dutyCycle) - 68.085;
  RGB.control(false); // end RGB LED control
  return temperature;
}
1 Like

That sketch looks great, @BulldogLowell but when I try it out on a Photon, it locks up with or without sensors…
When I try with my original sketch, all works fine…

This is what I get on the serial monitor, when starting with 4 sensors and then one by one removing them:

22.6=D0 22.5=D1 22.5=D2 22.1=D3 T-av = 22.4
22.9=D0 22.5=D1 22.4=D2 22.3=D3 T-av = 22.5
22.9=D0 22.5=D1 22.8=D2 22.0=D3 T-av = 22.5
22.9=D0 22.5=D1 23.0=D2 22.3=D3 T-av = 22.7
22.9=D0 22.5=D1 22.5=D2 22.7=D3 T-av = 22.6
22.8=D0 22.3=D1 22.5=D2 T-av = 22.6
23.0=D0 22.8=D1 22.7=D2 T-av = 22.8
23.1=D0 22.5=D1 22.7=D2 T-av = 22.8
22.9=D0 22.3=D1 22.1=D2 T-av = 22.4
22.9=D0 22.5=D1 T-av = 22.7
23.1=D0 22.4=D1 T-av = 22.7
22.9=D0 22.6=D1 T-av = 22.8
23.0=D0 22.7=D1 T-av = 22.9
22.8=D0 T-av = 22.8
23.0=D0 T-av = 23.0
23.6=D0 T-av = 23.6
23.0=D0 T-av = 23.0

So, if you have no time, please don’t bother!
I think it’s an interesting exercise, but for me this is not essential.
I’m more interested how I can use “qsort” (or another command) to create a “median” filter for my sampled data.

In case you could find some time, you could flash your sketch to a Photon and see what happens without sensors:
The RGB led should stay Red continuously.
With another Photon and Tinker, you could connect a PWM pin to one of the inputs, simulating a sensor.
Then you should see the RGB led flash red and cyan each time a line of results is written to the serial monitor.

I cannot debug that sketch as I have to focus on work at hand for the time being.

Anyway, thanks for your help! :wave: :older_man:

Hello @Ric, I am trying to apply your example of a “median filter” to another smaller project:
I will install an ultrasonic sensor on top of a 20.000 liter rain water tank. I want to monitor the height of the water level in order to be warned when it’s near to empty.
Below is the “Portable Range finder” sketch I made today, to test on site tomorrow.
It works perfectly but it is quite “nervous”. So, it’s probably a perfect example to try out your filtering method.

I have integrated your above code already as far as I could, between comment lines (/* … */) because it is not ready for testing yet.

Could you have a look and show how to complete it?

/**************************************************************
 * Portable Distance sensor.ino (Ultrasonic HC-SR04 sensor)
 -------------------------------------------------------------
 This sketch reads the time between sent and received pulse and converts it to a distance.
 Connections:

  Module => Particle
  VCC => Vin (+5V)
  GND => GND
  TRIG => D1
  ECHO => D2

 We use the SSD1306 Mini OLED Display.
 Connections:

  OLED=>Particle
  GND=>GND
  Vcc=>3V3
  SCL=>D1
  SDA=>D0
  No need for pull-up resistors in this case!

*/

// Libraries for use in the "Particle DEV" program:
 #include "Adafruit_GFX.h"
 #include "Adafruit_SSD1306.h"

// #include "application.h" (Is this needed? Some include this with the PING program)

 #define OLED_RESET D4 // Is this needed for I2C version? (Seems not to work without it... Test!)
Adafruit_SSD1306 display1(OLED_RESET);

// I would prefer to create it in the function, but probably I have to create it in all functions
float duration, cm;

void setup()
{
 // For the wifi ON/OFF switch
 pinMode(D7, OUTPUT);// To use the blue LED as indicator that wifi is ON/OFF
 pinMode(D3, INPUT_PULLUP);// To switch wifi ON/OFF: Connected to GND: Wifi = OFF
 // OLED Display initialization
 display1.begin(SSD1306_SWITCHCAPVCC, 0x3C);
 display1.clearDisplay();
}

void loop()
{
  // We sometimes use this portable range finder in open air: Switch wifi ON/OFF.
  if (digitalRead(D3) == HIGH)
  {
   digitalWrite(D7, HIGH); // LED ON to show wifi = ON
   WiFi.on();
   WiFi.connect();
  }
  else
  {
   digitalWrite(D7, LOW); // LED OFF, wifi OFF
   WiFi.disconnect();
   WiFi.off();
  }

 ping(D6, D5, 0);  // Trigger pin, Echo pin, delay (ms) => Now 0 to sample as fast as possible
}


void ping(pin_t trig_pin, pin_t echo_pin, uint32_t wait)
{
 static bool init = false;
 int storedData[10]; // Array to store the 10 results

 if (!init) // In order to initialize pins only once...
 {
  pinMode(trig_pin, OUTPUT);
  digitalWriteFast(trig_pin, LOW);
  pinMode(echo_pin, INPUT);
  delay(50);
  init = true;
 }

/* Comment: We want to use a "median filter" (to normalize and smoothen the output):

  // Store 10 results in an array:

  for(int i=0; i<10; i=i+1) // Record 10 measurements
  {
    digitalWriteFast(trig_pin, HIGH);
    delayMicroseconds(10);
    digitalWriteFast(trig_pin, LOW);
    duration = pulseIn(echo_pin, HIGH);
    cm = duration / 58; // Convert the time into a distance
    storedData[i]=cm;
  }

  // Sort the array of 10 distances and take the average of the center 4 values:
  qsort(storedData, 10, sizeof(float), compare); // sort to put any outliers on either end of the array
  int center = 5;
  float avg = (storedData[center -2] + storedData[center-1] + storedData[center] + storedData[center+1])/ 4.0;  // take the average of the center 4 values

  int compare (const void * a, const void * b)
  {
    float fa = *(const float*) a;
    float fb = *(const float*) b;
    return (fa > fb) - (fa < fb);
  }

End of comment */

 display();
 delay(wait);
}



void display(void) // Display the distance on the Mini OLED display: Value + bargraph.
{
  display1.clearDisplay();
  display1.setTextColor(WHITE);
  display1.setTextSize(2);
  display1.setCursor(0,0);
  display1.println("Distance:");
  display1.setTextSize(3);
  display1.setCursor(0,30);
  display1.println(cm,0);
  display1.setCursor(60,30);
  display1.println(" cm");

  display1.drawLine(0,60, cm/2,60, WHITE); // Line1
  display1.drawLine(0,61, cm/2,61, WHITE); // Line2
  display1.drawLine(0,62, cm/2,62, WHITE); // Line3
  display1.drawLine(0,63, cm/2,63, WHITE); // Line4
  display1.drawLine(0,64, cm/2,64, WHITE); // Line5

  if(cm < 150)
  {
    display1.invertDisplay(false);
  }
  else if(cm >= 150)
  {
   display1.invertDisplay(true);
  }
  display1.display();
}

Thanks!
:older_man:

It looks like it is pretty much complete. You just need to do whatever it is you want to do with the variable, “avg”. You should make “avg” a global variable though, so you can use it outside of the ping function.

1 Like

OK, thanks for the advice @Ric.
I’ll try that tomorrow evening.

Oops!
There must be something wrong in my sketch. The data is not shown on the display…
Can you see anything wrong @Ric ?

/**************************************************************
 * Portable Range Finder.ino (Ultrasonic HC-SR04 sensor)
 -------------------------------------------------------------
 UPDATE from FiDel: This sketch has been updated a few times after it's original posting and works perfectly now.

 It reads the time between sent and received pulse and converts it to a distance.
 Connections:

  Module => Particle
  VCC => Vin (+5V)
  GND => GND
  TRIG => D1
  ECHO => D2

 We use the SSD1306 Mini OLED Display.
 Connections:

  OLED=>Particle
  GND=>GND
  Vcc=>3V3
  SCL=>D1
  SDA=>D0
  No need for pull-up resistors in this case!

*/

// Libraries for use in the "Particle DEV" program:
 #include "Adafruit_GFX.h"
 #include "Adafruit_SSD1306.h"

//  #include "application.h" (Is this needed? Some include this with the PING program)

 #define OLED_RESET D4 // Is this needed for I2C version? (Seems not to work without it... Test!)
Adafruit_SSD1306 display1(OLED_RESET);

// I would prefer to create it in the function, but probably I have to create it in all functions
float duration, cm, avg;

void setup()
{
 // For the wifi ON/OFF switch
 pinMode(D7, OUTPUT);// To use the blue LED as indicator that wifi is ON/OFF
 pinMode(D3, INPUT_PULLUP);// To switch wifi ON/OFF: Connected to GND: Wifi = OFF
 // OLED Display initialization
 display1.begin(SSD1306_SWITCHCAPVCC, 0x3C);
 display1.clearDisplay();
}

void loop()
{
  // We sometimes use this portable range finder in open air: Switch wifi ON/OFF.
  if (digitalRead(D3) == HIGH)
  {
   digitalWrite(D7, HIGH); // LED ON to show wifi = ON
   WiFi.on();
   WiFi.connect();
  }
  else
  {
   digitalWrite(D7, LOW); // LED OFF, wifi OFF
   WiFi.disconnect();
   WiFi.off();
  }

 ping(D6, D5, 0);  // Trigger pin, Echo pin, delay (ms) => Now 0 to sample as fast as possible
}


void ping(pin_t trig_pin, pin_t echo_pin, uint32_t wait)
{
 static bool init = false;
 int storedData[10]; // Array to store the 10 results

 if (!init) // In order to initialize pins only once...
 {
  pinMode(trig_pin, OUTPUT);
  digitalWriteFast(trig_pin, LOW);
  pinMode(echo_pin, INPUT);
  delay(50);
  init = true;
 }

// Comment: We want to use a "median filter" (to normalize and smoothen the output):

  for(int i=0; i<10; i=i+1) // Store 10 results in an array
  {
    delay(50); // Use a delay between 50 and 200 us for a stable reading
    digitalWriteFast(trig_pin, HIGH);
    delayMicroseconds(10);
    digitalWriteFast(trig_pin, LOW);
    duration = pulseIn(echo_pin, HIGH);
    cm = duration / 58; // Convert the time into a distance
    storedData[i]=cm;
  }

  // Sort the array of 10 distances and take the average of the center 4 values:
  qsort(storedData, 10, sizeof(float), compare); // sort to put any outliers on either end of the array
  int center = 5;
  avg = (storedData[center -2] + storedData[center-1] + storedData[center] + storedData[center+1])/ 4.0;  // take the average of the center 4 values

 display();
 delay(wait);
}

int compare (const void * a, const void * b)
{
  float fa = *(const float*) a;
  float fb = *(const float*) b;
  return (fa > fb) - (fa < fb);
}


void display(void) // Display the distance on the Mini OLED display: Value + bargraph.
{
  display1.clearDisplay();
  display1.setTextColor(WHITE);
  display1.setTextSize(2);
  display1.setCursor(0,0);
  display1.println("Distance:");
  display1.setTextSize(3);
  display1.setCursor(0,30);
  display1.println(avg,0);
  display1.setCursor(60,30);
  display1.println(" cm");

  display1.drawLine(0,60, avg/2,60, WHITE); // Line1
  display1.drawLine(0,61, avg/2,61, WHITE); // Line2
  display1.drawLine(0,62, avg/2,62, WHITE); // Line3
  display1.drawLine(0,63, avg/2,63, WHITE); // Line4
  display1.drawLine(0,64, avg/2,64, WHITE); // Line5

  if(avg < 150)
  {
    display1.invertDisplay(false);
  }
  else if(avg >= 150)
  {
   display1.invertDisplay(true);
  }
  display1.display();
}

You said it was working perfectly before; I assume that means your were seeing the value of “cm” displayed before. One problem I see is that you typed storedData as “int[]”, when what you’re storing (the value of cm) is a float, so change storedData’s type to float, and see if that fixes the problem.

Spotted one more problem. When you’re computing the average, you have “float avg = …”. You don’t want the “float” there since you’ve already declared avg as a global float at the top of the file. This is creating a new local variable called float, so it’s value can’t be seen outside the ping function.

Great, thanks @Ric!
Indeed, removing the second “float” fixed it!
I was not aware of that, but it’s logical of course. Lesson learned!

After that, I tested again after changing the array storedData from “int[]” to “float[]” .
That seems not to change anything. (256 cm is sufficient a range :wink: )
But I left it as “float storedData[10];”


The result of this all is rather noticeable:
Overall, the distances appearing on the display change more gradually, with less erratic jumps.
But the reliability of the measurements is not (yet) convincing…

The strange effect is now that it seems to have difficulties displaying values higher than 90 cm. When I point into open air it will show higher values, but not stable…
That behaviour is not yet suitable for what we want to achieve: Monitor the water surface in a tank between 10 and 200 cm.

It will need more thought on the correct processing of the data before we can install it.

If you like, I can send you a short video tomorrow morning…

:hand::older_man:

I’m doing almost the same thing you are; measuring the height of water in a 2600 gallon water tank that’s filled from my well. I haven’t tested my sensor at 200 cm distances yet (that’s about my max too), so hopefully we can make this work. That distance is certainly within the specs of the sensor.

1 Like

Oh that’s good!
Here’s my demo video of the behaviour today: VIDEO

And these are the water tanks, installed:

The location to install the sensor:

Today and tomorrow I cannot continue as I will be on the road…

:hand: :older_man:

ping(D6, D5, 0);

It could be that transient sound waves are reflecting/scattering around and affecting the readings. Are you adequately spacing out your pings to eliminate the return of spurious reflections from other hard surfaces?

You may find at least 50milliseconds minimum between pings may help. If your fluid level is not so dynamic, perhaps waiting longer. I’m not sure how long your display() function takes, but that certainly will eat into that 50milliseconds.

something to try… if you have not already.

1 Like

Good tip @bulldoglowell I will try that tomorrow when back home!
:+1::nerd_face:

I agree with @BulldogLowell about the delay. I am currently using a 200 ms delay between triggerings, and when I put my sensor on the floor pointed at the ceiling, I get very consistent readings of from 239.6 to 240.6 cm. I don’t think the readings in “open air” are very reliable. That might be due to reflections off objects, but it also might reflect the fact that the timeout period for the sensor is not reliable. The specs say that it should timeout at 38 ms, but my tests of the 4 sensors I have, show timeouts between 100 and 240 ms.

2 Likes

or presence of bats :wink:

1 Like

Thanks for the tips @BulldogLowell and @Ric
The delay certainly makes a difference.
Also logical when you think about it of course!

I played with different delays between 50 and 200 us and found the stability of readings increases with the delay.
Also indeed Ric, there’s a clear difference between use inside and outside. I suppose the bats are now sleeping, but outside there’s all kinds of sounds, obviously influencing the readings.
Even inside, when I turn the volume of my stereo up, the readings become erratic… :stuck_out_tongue_winking_eye:

Also, I increased the number of samples for averaging to 20 and that also increases the stability of readings, of course decreasing the refresh rate. But for the application we will use it for it’s OK.

So, now the next challenge comes to install it inside the water tank and protect it from moisture…
:hand: :older_man:

1 Like

With some of these sensors, there is a nuisance error that returns a zero read (on a timeout). You may find that excluding any zero readings while averaging your data will also help. Something like this where you have collected your distance into an array called distance[]:

int num_goodReads = 0
int averageDistance = 0;
for(int i = 0; i < sizeof(distance)/sizeof(distance[0]); i++)
{
  if(distance[i] != 0)
  {
    averageDistance += distance[i];
    num_goodReads++;
  }
} 
averageDistance /= num_goodReads;
1 Like

This reminded me of finding the most frequent value in an array. By counting repetition of all the different numbers, and using the highest count. If it’s sorted, that makes it easier because each time it changes, you have the max count and can just hold that value and count if higher than the one that’s already stored as highest.

1 Like