Electron freezes with solid green or cyan LED

The code at post #17 crashed, so now I am trying the following:

#include "application.h"

SYSTEM_MODE(MANUAL);
SYSTEM_THREAD(ENABLED);
STARTUP(System.enableFeature(FEATURE_RETAINED_MEMORY));

#define SENSOR_FAIL 2
#define SENSOR_SUCCESS 3

int dataPin = D3;
int powerPin = C0;
int status;
uint32_t timerStart;
uint32_t timerStop;
uint16_t rpm;
uint32_t delayTimer = 3000;

unsigned long rpmCountFrozen = 0;
volatile unsigned long rpmCount;

retained uint8_t location;
uint32_t getRpmDelay = 0;

void getRpm();
void interrupt();

void setup() {
  Serial.begin(9600);
  while(!Serial.available()) {}
  Serial.print("Last location: ");
  Serial.println(location);
  pinMode(dataPin, INPUT);
  pinMode(powerPin, OUTPUT);
}

void loop() {
  if(millis() - getRpmDelay > 1000) {
    location = 0;
    getRpm();
    Serial.print("RPM: ");
    Serial.print(rpm);
    Serial.print(", location: ");
    Serial.print(location);
    Serial.print(", millis: ");
    Serial.println(millis());
    getRpmDelay = millis();
  }
}

void getRpm() {
  digitalWrite(powerPin, LOW);
  delay(50);

  location = 1;
  rpm = 0;
  rpmCount = 0;
  rpmCountFrozen = 0;
  location = 2;
  attachInterrupt(dataPin, interrupt, RISING);

  location = 3;
  timerStart = millis();

  location = 4;
  while(millis() - timerStart < delayTimer) {
    Particle.process();
  }

  location = 5;
  timerStop = millis();
  rpmCountFrozen = rpmCount;
  
  //Disable interrupt when calculating
  location = 6;
  noInterrupts();
  location = 7;
  detachInterrupt(dataPin);
  location = 8;
  interrupts();
  
  location = 9;
  rpm = (uint16_t)(rpmCountFrozen / ((timerStop - timerStart) / 1000));
  
  location = 10;
  rpmCount = 0;
  if(rpm >= 0 && rpm < 5000) {
    status = SENSOR_SUCCESS;
  } else {
    status = SENSOR_FAIL;
    rpm = 0;
  }

  location = 11;
  digitalWrite(powerPin, HIGH);
  location = 12;
  return;
}

void interrupt() {
  rpmCount++;
}

Both with and without the Particle.process call in the while loop.

This shouldn't be an issue with the current code, but may become a time bomb if you ever decide to reduce the delayTimer to less than 1000 as this might cause a DIV/0 exeption.

I'm currently running this code as a test with 0.6.4

//SYSTEM_MODE(MANUAL);
SYSTEM_THREAD(ENABLED);
STARTUP(System.enableFeature(FEATURE_RETAINED_MEMORY));

enum SENSOR_STATUS 
{ SENSOR_FAIL                   = 2
, SENSOR_SUCCESS                = 3
};

const uint32_t    msMeasuring   =  3000;
const uint32_t    msRefresh     =     0; // as rapid as possible 10000;
const int         pinData       = D3;
const int         pinPower      = C0;
const int         pinPWM        = D0;           // for isolated test bridge pinPWM to pinData

volatile uint32_t rpmCount;
uint32_t          rpm;
uint32_t          timerStart;
uint32_t          timerStop;
SENSOR_STATUS     status;

void getRpm();
void interrupt();

void setup() {
  Serial.begin();
  pinMode(pinData , INPUT);
  pinMode(pinPower, OUTPUT);
  pinMode(pinPWM  , OUTPUT);                    // for isolated test
}

void loop() {
  static uint32_t msLastRefresh = 0;
  static uint32_t pwmFrequency  = 0;
  if (millis() - msLastRefresh >= msRefresh) {
    getRpm();
    Serial.printlnf("RPM: %4lu, PWM: %4lu, millis: %10lu --------|---------|---", rpm, pwmFrequency, millis());
    Serial.flush();
    pwmFrequency = random(10, 5000);
    analogWrite(pinPWM, 1, pwmFrequency);       // set the test frequency for next turn
    msLastRefresh = millis();
  }
}

void getRpm() {
  pinResetFast(pinPower);
  delay(50);

  rpmCount   = 0;
  timerStart = millis();
  attachInterrupt(pinData, interrupt, RISING);
  while(millis() - timerStart < msMeasuring) Particle.process();  // in MANUAL mode no need
  timerStop = millis();
  noInterrupts();
  detachInterrupt(pinData);
  interrupts();

  rpm    = (1000 * rpmCount) / (timerStop - timerStart);
  status = (0 <= rpm && rpm < 5000) ? SENSOR_SUCCESS : SENSOR_FAIL;
  
  pinSetFast(pinPower);
}

void interrupt() {
  rpmCount++;
}

@ScruffR, yes, I agree that is a time bomb – not only for the reason you pointed out, but also because it limits my measurement intervals to whole seconds. Thanks for pointing that out.

@peekay123, the code in post #20 made it to location 8 and then crashed. Which I think means it crashed when interrupts() was called, re-enabling interrupts. Is this the right way to read this? Is there any other possible answer?

I can now see the same issue (but only at high trigger frequencies 30000+) and am now trying this

  attachInterrupt(pinData, interrupt, RISING);
  while(millis() - timerStart < msMeasuring) Particle.process();  // in MANUAL mode no need
  timerStop = millis();
  //noInterrupts();
ATOMIC_BLOCK() {
  detachInterrupt(pinData);
}
  //interrupts();

let’s see if that helps


Update:
That didn’t help.
Now testing 0.7.0


Update:
0.7.0 is no better in this regard.
How about not detaching the interrupt?
You depower the sensor, so no interrupts should happen anyway, right?

My code is failing when I attempt to start interrupts back up

location = 6;
noInterrupts();
location = 7;
detachInterrupt(dataPin);
location = 8;
interrupts();

location prints out to 8 when I start the system back up

With the interrupts() in place like in your code the same happens to me.
With ATOMIC_BLOCK() the freeze happens in there.
Without noInterrupts()/interrupts() or ATOMIC_BLOCK() it happens with detachInterrupt().

This is definetly a bug that needs reporting, but for the time being you may have to find a workaround, right?

That is a good idea. One caveat would be that my 1-Wire sensor runs in the same application, and it calls noInterrupts and interrupts quite a bit. Code: GitHub - semiosBIO/altrac-one-wire-particle: Provides support for the Dallas One Wire protocol on the Spark core..

But if I never try to detach the interrupt, maybe I can get around this. I don't utilize interrupts anywhere else.

Would it be possible to add a divide-by-10 counter in between your sensor and the Electron, or is your hardware set in stone? One of those would get you back to about the same rate going into your device as you had before when things were working ok.

I have filed a GitHub issue

@Ric, I don’t think that 5kHz signals should be too excessive for a 120MHz controller to handle interrupts with (not even 50kHz for very short ISRs).
While this might be a viable workaround, I’d still think it’s a bug that needs to be addressed.

I originally tested all this on a atmega328p, so I made the assumption the more powerful processor in the Electron would be able to handle this pulse rate.

Also, I’m not sure why the device would run for hours at that rate, and then crash randomly, unless it is a software limitation.

I’m currently running this test code

// https://community.particle.io/t/electron-freezes-with-solid-green-or-cyan-led/41022/22

//SYSTEM_MODE(MANUAL);
SYSTEM_THREAD(ENABLED);
STARTUP(System.enableFeature(FEATURE_RETAINED_MEMORY));

ApplicationWatchdog wd(10000, System.reset, 1536);

enum SENSOR_STATUS 
{ SENSOR_FAIL                   = 2
, SENSOR_SUCCESS                = 3
};

const uint32_t    msMeasuring   =  3000;
const uint32_t    msRefresh     =     0; // as rapid as possible 10000;
const int         pinData       = D3;
const int         pinPower      = C0;
const int         pinPWM        = D0;           // for isolated test bridge pinPWM to pinData

volatile bool     doCount       = false;
volatile uint32_t rpmCount;
uint32_t          rpm;
uint32_t          timerStart;
uint32_t          timerStop;
SENSOR_STATUS     status;

retained int      lastLocation  = 0;
retained uint32_t lastTime      = 0;
int               lastCrash     = 0;
uint32_t          lastCrashTime = 0;
void getRpm();
void interrupt();

void setup() {
  lastCrash     = lastLocation;
  lastCrashTime = lastTime;
  
  Serial.begin();
  pinMode(pinData , INPUT);
  pinMode(pinPower, OUTPUT);
  pinMode(pinPWM  , OUTPUT);                    // for isolated test

  attachInterrupt(pinData, interrupt, RISING);
}

void loop() {
  static uint32_t msLastRefresh = 0;
  static uint32_t pwmFrequency  = 0;
  if (millis() - msLastRefresh >= msRefresh) {
    getRpm();
    Serial.printlnf("RPM: %5lu, PWM: %5lu, status: %s, millis: %10lu, lastCrash: %d (%10lu)", 
                    rpm, pwmFrequency, status == SENSOR_SUCCESS ? "OK" : "XX", millis(), lastCrash, lastCrashTime);
    pwmFrequency = random(1, 65535);
    analogWrite(pinPWM, 1, pwmFrequency);       // set the test frequency for next turn
    Serial.printf("next PWM: %5lu\t-> ", pwmFrequency);
    msLastRefresh = millis();
  }
}

void getRpm() {                                                 //  checkpoins to locate the offending line 
                                                                    lastLocation = 0;
  pinResetFast(pinPower);
  delay(50);
  rpmCount   = 0;
  timerStart = millis();
                                                                    lastLocation++; lastTime = millis();
  //attachInterrupt(pinData, interrupt, RISING);
  doCount = true;
                                                                    lastLocation++; lastTime = millis();
  while(millis() - timerStart < msMeasuring) Particle.process();
  doCount=false;                                                    
                                                                    lastLocation++; lastTime = millis();    
  timerStop = millis();
                                                                    lastLocation++; lastTime = millis();
  //noInterrupts();
                                                                    lastLocation++; lastTime = millis();
  //detachInterrupt(pinData);
                                                                    lastLocation++; lastTime = millis();
  //interrupts();                                                 // offending line    
                                                                    lastLocation++; lastTime = millis();
  rpm    = (1000 * rpmCount) / (timerStop - timerStart);
  status = (0 <= rpm && rpm < 5000) ? SENSOR_SUCCESS : SENSOR_FAIL;
  pinSetFast(pinPower);
}

void interrupt() {
  if (doCount) rpmCount++;
}

Maybe you can use this with your test rig too.
The PWM hack is just as replacement for your sensor.

I agree. I was thinking of it more as a workaround, if the fix takes time in coming (or even figuring out what the fix needs to be). Btw, do you ever sleep? :wink:

Yes, 10 seconds power-naps between posts :wink:

This is the new test code I am running:

#include "application.h"

SYSTEM_MODE(MANUAL);
SYSTEM_THREAD(ENABLED);
STARTUP(System.enableFeature(FEATURE_RETAINED_MEMORY));

#define SENSOR_FAIL 2
#define SENSOR_SUCCESS 3

int dataPin = D3;
int powerPin = C0;
int status;
uint32_t timerStart;
uint32_t timerStop;
uint16_t rpm;
uint32_t delayTimer = 3000;

unsigned long rpmCountFrozen = 0;
volatile unsigned long rpmCount;

retained uint8_t location;
uint32_t getRpmDelay = 0;

void getRpm();
void interrupt();

void setup() {
  Serial.begin(9600);
  while(!Serial.available()) {}
  Serial.print("Last location: ");
  Serial.println(location);
  pinMode(dataPin, INPUT);
  pinMode(powerPin, OUTPUT);

  attachInterrupt(dataPin, interrupt, RISING);
}

void loop() {
  if(millis() - getRpmDelay > 1000) {
    location = 0;
    getRpm();
    Serial.print("RPM: ");
    Serial.print(rpm);
    Serial.print(", location: ");
    Serial.print(location);
    Serial.print(", millis: ");
    Serial.println(millis());
    Serial.flush();
    getRpmDelay = millis();
  }
}

void getRpm() {
  digitalWrite(powerPin, LOW);
  delay(50);

  location = 1;
  rpm = 0;
  rpmCount = 0;
  rpmCountFrozen = 0;
  location = 2;

  location = 3;
  timerStart = millis();

  location = 4;
  while(millis() - timerStart < delayTimer) {}

  location = 5;
  timerStop = millis();
  rpmCountFrozen = rpmCount;
  
  //Disable interrupt when calculating
  // ATOMIC_BLOCK() {
  //   location = 6;
  //   noInterrupts();
  //   location = 7;
  //   detachInterrupt(dataPin);
  //   location = 8;
  //   interrupts();
  // }
  
  location = 9;
  rpm = (uint16_t)(rpmCountFrozen / ((timerStop - timerStart) / 1000));
  
  location = 10;
  rpmCount = 0;
  if(rpm >= 0 && rpm < 5000) {
    status = SENSOR_SUCCESS;
  } else {
    status = SENSOR_FAIL;
    rpm = 0;
  }

  location = 11;
  digitalWrite(powerPin, HIGH);
  location = 12;
  return;
}

void interrupt() {
  rpmCount++;
}

I am also running something similar in my full application code

Thanks for helping me reproduce the issue and filing the bug report!

BTW, you are aware that you don’t really measure RPM there but RPS, right?

And if you slightly reorder your calculation, you’ll increase the accuracy by reducing rounding errors.

rpm = (uint16_t)(rpmCountFrozen / ((timerStop - timerStart) / 1000));

should better be

 rpm = (uint16_t)( (1000 * rpmCountFrozen) / (timerStop - timerStart) );

The workaround has been working for 15 hours or so, both in my normal application and in the smaller application above. I would say this is a good workaround for now, and I am looking forward to seeing this get fixed for real.

One question I had after thinking about this some more is what would a maximum pulse rate be for the Electron? 10kHz? 100kHz?

Yes, I've realized this for some time... this function originally was used to measure pulses per second from a variable reluctance sensor pointed at a ring gear on an engine. The ring gear had 60 teeth, so RPS == actual engine RPM. I should change the name now that we are doing other things with this function...

As my test with the PWM shows 65kHz is no problem by itself.
The theoretic maximum speed with attachInterrupt() having an interrupt latency of 5~8µS would be in the reagon of just above 100kHz, but with the new 0.8.0 feature of low level interrupts the latency could hopefully be reduced to under 2µs which would allow for 500kHz interrupts, but only hypothetically, since with anything above 100kHz your application code will spend more time being interrupted than actually running :wink:
Additionally higher priority interrupts will start interfering considerably with the timing of these high frequency low priority interrupts.