Boron sporadically unresponsive

I have a Boron (1.5.2) that in the last few days has been sporadically going unresponsive to any variable calls or vitals update commands. When this happens it continues to blink steady cyan but I get a failed to call error. After a few minutes it works but a few hours later it happens again. The device does not go to sleep but continuously checks a pin status. Any idea what could it be, what I should look for? Memory utilization is at 75%. Can this is a carrier issue? Signal strength hovers around 35 - 40%. Any help great;y appropriated.

Also, I am using #include <PublishQueueAsyncRK.h> but besides that there is no “advance code” just two nested if statements to check a pin state.

@oraclerouter, can you post your code so we can have a look? Is this an LTE-M1 or 2G/3G Boron?

This is for the LTE version

#include <PublishQueueAsyncRK.h>

#define DELAY_BEFORE_REBOOT (3 * 1000)

SYSTEM_THREAD(ENABLED);

retained uint8_t publishQueueRetainedBuffer[2048];
PublishQueueAsync publishQueue(publishQueueRetainedBuffer, sizeof(publishQueueRetainedBuffer));

int pin = D4;
int led1 = D7;
int sensorStatus;
int powerStatus =1;

int setPhoneEEPROMLocation = 1;
int setEmailEEPROMLocation = 100;
int setAlertEEPROMLocation = 200;

int setAlertEmailPowerEEPROMLocation = 500;
int setAlertEmailSensorEEPROMLocation = 600;

int setAlertPhonePowerEEPROMLocation = 700;
int setAlertPhoneSensorEEPROMLocation = 800;

char email[40];
char phoneNumber[40];

char alert[10];

String setEmailString = "Not Set";
String setPhoneString = "Not Set";

int A = 0;
int PS = 0;
int ES = 0;
int PP = 0;
int EP= 0;

String alertSensorSms = "0";
String alertSensorEmail= "0";
String alertPowerSms = "0";
String alertPowerEmail= "0";

char dev_name[32];
String deviceName;

double voltage;
double power;
double charging;

unsigned int rebootDelayMillis = DELAY_BEFORE_REBOOT;
unsigned long rebootSync = millis();

bool resetFlag = false;

int SG;
double batterySoc = System.batteryCharge();
int powerSource = System.powerSource();


void setup()
{

    Particle.variable("S", sensorStatus);
    Particle.variable("SPS", setPhoneString);
    Particle.variable("SES", setEmailString);

    Particle.function("SES", setEmail);
    Particle.function("SPS", setPhone);
    Particle.function("EA", enableAlerts);
    Particle.function("EP", enableEmailPowerAlerts);
    Particle.function("ES", enableEmailSensorAlerts);
    Particle.function("PP", enablePhonePowerAlerts);
    Particle.function("PS", enablePhoneSensorAlerts);
    Particle.variable("A", A);
    Particle.variable("EP", EP);
    Particle.variable("ES", ES);
    Particle.variable("PP", PP);
    Particle.variable("PS", PS);
    Particle.variable("SG", SG);
    Particle.variable("BAT", batterySoc);
    Particle.variable("PWR", powerSource);
    Particle.function("RT", cloudResetFunction);
    
    getStoredInfo();

    pinMode(pin, INPUT_PULLUP);
    pinMode(led1, OUTPUT);

    
    if (digitalRead(pin) == LOW)
    {

        digitalWrite(led1, LOW);
    }
    else {
        
        digitalWrite(led1, HIGH);
    }

    Particle.subscribe("particle/device/name", handler);
    waitFor(Particle.connected, 600000);
    Particle.publish("particle/device/name");
    
}

void loop()
{   
     batterySoc = System.batteryCharge();
     powerSource = System.powerSource();

    system_display_rssi();

    for (uint32_t ms = millis(); millis() - ms < 200; Particle.process());


    if (powerSource == 5)
    {

        if (powerStatus != 0)
        {


            if ((A == 1) && (PP == 1))
            {

               alertPowerSms = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setPhoneString.c_str());
               publishQueue.publish("test5", alertPowerSms, 60, PRIVATE);
            }

            if ((A == 1) && (EP == 1))
            {
                alertPowerEmail = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setEmailString.c_str());
                publishQueue.publish("test6", alertPowerEmail, 60, PRIVATE);
            }

            powerStatus = 0;
        }
    }
    else if (powerSource < 5)
    {

        if (powerStatus != 1)
        {
        if ((A == 1) && (PP ==1)){
        alertPowerSms = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(),  setPhoneString.c_str());
        publishQueue.publish("test3", alertPowerSms, 60, PRIVATE);

       
        
        
      }
      
      if ((A == 1) && (EP ==1)) {
        alertPowerEmail = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setEmailString.c_str());

        publishQueue.publish("test4", alertPowerEmail, 60, PRIVATE);
      }
            powerStatus = 1;
        }
    }

    if (digitalRead(pin) == HIGH)
    {

        if (sensorStatus != 0)
        {

            digitalWrite(led1, HIGH);


            if ((A == 1) && (PS == 1))
            {
                alertSensorSms = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setPhoneString.c_str());
                publishQueue.publish("test1", alertSensorSms, 60, PRIVATE);
            }

            if ((A == 1) && (ES == 1))
            {
                alertSensorEmail = String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setEmailString.c_str());
                publishQueue.publish("test2", alertSensorEmail, 60, PRIVATE);
            }

            sensorStatus = 0;
        }
    }
    else if (digitalRead(pin) == LOW)
    {

        if (sensorStatus != 1)
        {
            digitalWrite(led1, LOW);
            sensorStatus = 1;
        }
    }

    if ((resetFlag) && (millis() - rebootSync >= rebootDelayMillis))
    {
        delay(1000);
        System.reset();
    }

    if ((PP == 0) && (EP == 0) && (PS == 0) && (ES == 0))

    {
        enableAlerts("0");
    }
    
       if ((PP == 1) | (EP == 1) | (PS== 1) | (ES == 1))
    
    {
        enableAlerts("1");
    }
}

int enableAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        A = 1;
         EEPROM.put(setAlertEEPROMLocation, A);
        return 1;
    }
    else
    {
        A = 0;
        EEPROM.put(setAlertEEPROMLocation, A);
        return 0;
    }
}

int enableEmailPowerAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        EP = 1;
        EEPROM.put(setAlertEmailPowerEEPROMLocation, EP);
        return 1;
    }
    else
    {

        EP = 0;
        EEPROM.put(setAlertEmailPowerEEPROMLocation, EP);
        return 0;
    }
}

int enableEmailSensorAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        ES = 1;
        EEPROM.put(setAlertEmailSensorEEPROMLocation, ES);
        return 1;
    }
    else
    {
        ES = 0;
        EEPROM.put(setAlertEmailSensorEEPROMLocation, ES);
        return 0;
    }
}

int enablePhonePowerAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        PP = 1;
        EEPROM.put(setAlertPhonePowerEEPROMLocation, PP);
        return 1;
    }
    else
    {
        PP = 0;
        EEPROM.put(setAlertPhonePowerEEPROMLocation, PP);
        return 0;
    }
}

int enablePhoneSensorAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        PS = 1;
        EEPROM.put(setAlertPhoneSensorEEPROMLocation, PS);
        return 1;
    }
    else
    {
        PS = 0;
        EEPROM.put(setAlertPhoneSensorEEPROMLocation, PS);
        return 0;
    }
}

int setPhone(const char *data)
{
    setPhoneString = data;
    strncpy(phoneNumber, data, sizeof(phoneNumber) - 1);
    phoneNumber[sizeof(phoneNumber) - 1] = '\0';
    EEPROM.put(setPhoneEEPROMLocation, phoneNumber);
}

int setEmail(const char *data)
{
    setEmailString = data;
    strncpy(email, data, sizeof(email) - 1);
    email[sizeof(email) - 1] = '\0';
    EEPROM.put(setEmailEEPROMLocation, email);
}

void getStoredInfo()
{
    EEPROM.get(setPhoneEEPROMLocation, phoneNumber);
    setPhoneString = phoneNumber;

    EEPROM.get(setEmailEEPROMLocation, email);
    setEmailString = email;

    EEPROM.get(setAlertEEPROMLocation, A);
    A = A;

    EEPROM.get(setAlertEmailSensorEEPROMLocation, ES);
    ES = ES;

    EEPROM.get(setAlertEmailPowerEEPROMLocation, EP);
    EP = EP;

    EEPROM.get(setAlertPhoneSensorEEPROMLocation, PS);
    PS = PS;

    EEPROM.get(setAlertPhonePowerEEPROMLocation, PP);
    PP = PP;
}

void handler(const char *topic, const char *data)
{
    strncpy(dev_name, data, sizeof(dev_name) - 1);
    deviceName = dev_name;

}

int cloudResetFunction(String command)
{
    resetFlag = true;
    rebootSync = millis();
    return 0;
}

void system_display_rssi()
{

    int rssi = 0;

#if Wiring_WiFi == 1
    rssi = WiFi.RSSI();
#elif Wiring_Cellular == 1
    CellularSignal sig = Cellular.RSSI();
    rssi = sig.getStrength();
#endif
    if (rssi < 0)
    {
        if (rssi >= -57)
        {
            SG = 5;
        }
        else if (rssi > -68)
        {
            SG = 4;
        }
        else if (rssi > -80)
        {
            SG = 3;
        }
        else if (rssi > -92)
        {
            SG = 2;
        }
        else if (rssi > -104)
        {
            SG = 1;
        }
    }

    if (rssi >= 0)
    {
        if (rssi >= 60)
        {
            SG = 5;
        }
        else if (rssi > 40)
        {
            SG = 4;
        }
        else if (rssi > 20)
        {
            SG = 3;
        }
        else if (rssi > 1)
        {
            SG = 2;
        }
        else if (rssi >= 0)
        {
            SG = 1;
        }
    }
}

As always ( :wink: ) my first take would be to replace as many String instances as possible with C strings (aka character arrays) and use snprintf instead of String::format().

The reasoning behind that was explained loads of times all over this forum - heap fragmentation being the key word.

2 Likes

@ScruffR can you provide an example on how to convert for instance this one? If you don’t mind ofcourse

String::format("{\"N\": \"%s\", \"T\": \"%s\"}", deviceName.c_str(), setPhoneString.c_str())

@ScruffR never mind i think I got it.
BTW Do you think the heap fragmentation is causing unresponsiveness?

We have seen many instances where users did experience cases where their devices did run fine for hours or days and suddenly started to get flaky reception and sometimes finally failed to reconnect at all but started to work again after a simple reset, just to have the same problem creep up again after a while.
But after replacing dynamic memory with static or automatic variables the issue went aways.

So yes, I do believe this could be at least a contributing factor.

2 Likes

Also, I one more question. Is there any reason to have a
for (uint32_t ms = millis(); millis() - ms < 200; Particle.process());
in the loop() as a “good practice”?

It depends whether your code requires that 200ms delay at all.
But having a delay done this way ensures maximum responsiveness to cloud requests.

So I made changes per your suggestion but I still come across the occasional unresponsiveness to command calls.

Here is the updated code. Any thing else I may be going wrong here. Also, not sure if the ongoing outrage has something to do with it.

#include <PublishQueueAsyncRK.h>

#define DELAY_BEFORE_REBOOT (3 * 1000)


SYSTEM_THREAD(ENABLED);

retained uint8_t publishQueueRetainedBuffer[2048];
PublishQueueAsync publishQueue(publishQueueRetainedBuffer, sizeof(publishQueueRetainedBuffer));

int pin = D4;
int led1 = D7;
int sensorStatus;
int powerStatus =1;

int setPhoneEEPROMLocation = 1;
int setEmailEEPROMLocation = 100;
int setAlertEEPROMLocation = 200;

int setAlertEmailPowerEEPROMLocation = 500;
int setAlertEmailSensorEEPROMLocation = 600;

int setAlertPhonePowerEEPROMLocation = 700;
int setAlertPhoneSensorEEPROMLocation = 800;

char dev_name[40];
char phone_str[40] = "Not Set";
char email_str[40] = "Not Set";
char alertPowerSms[256] = "0";
char alertPowerEmail[256] = "0";
char alertSensorEmail[256] = "0";
char alertSensorSms[256] = "0";



char alert[10];

int A = 0;
int PS = 0;
int ES = 0;
int PP = 0;
int EP= 0;

double voltage;
double power;
double charging;

unsigned int rebootDelayMillis = DELAY_BEFORE_REBOOT;
unsigned long rebootSync = millis();

bool resetFlag = false;

int SG;
double batterySoc = System.batteryCharge();
int powerSource = System.powerSource();


void setup()
{

    Particle.variable("S", sensorStatus);
    Particle.variable("SPS", phone_str);
    Particle.variable("SES", email_str);

    Particle.function("SES", setEmail);
    Particle.function("SPS", setPhone);
    Particle.function("EA", enableAlerts);
    Particle.function("EP", enableEmailPowerAlerts);
    Particle.function("ES", enableEmailSensorAlerts);
    Particle.function("PP", enablePhonePowerAlerts);
    Particle.function("PS", enablePhoneSensorAlerts);
    Particle.variable("A", A);
    Particle.variable("EP", EP);
    Particle.variable("ES", ES);
    Particle.variable("PP", PP);
    Particle.variable("PS", PS);
    Particle.variable("SG", SG);
    Particle.variable("BAT", batterySoc);
    Particle.variable("PWR", powerSource);
    Particle.function("RT", cloudResetFunction);
    
    getStoredInfo();

    pinMode(pin, INPUT_PULLUP);
    pinMode(led1, OUTPUT);

    
    if (digitalRead(pin) == LOW)
    {

        digitalWrite(led1, LOW);
    }
    else {
        
        digitalWrite(led1, HIGH);
    }

    Particle.subscribe("particle/device/name", handler);
    waitFor(Particle.connected, 600000);
    Particle.publish("particle/device/name");
    
}

void loop()
{   
     batterySoc = System.batteryCharge();
     powerSource = System.powerSource();
    system_display_rssi();

    if (powerSource == 5)
    {

        if (powerStatus != 0)
        {

            if ((A == 1) && (PP == 1))
            {
               snprintf(alertPowerSms, sizeof(alertPowerSms)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , phone_str  );
               publishQueue.publish("test1", alertPowerSms, 60, PRIVATE);
            }

            if ((A == 1) && (EP == 1))
            {

                snprintf(alertPowerEmail, sizeof(alertPowerEmail)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , email_str  );
                publishQueue.publish("test2", alertPowerEmail, 60, PRIVATE);
            }

            powerStatus = 0;
        }
    }
    else if (powerSource < 5)
    {

        if (powerStatus != 1)
        {

         if ((A == 1) && (PP ==1)){
		snprintf(alertPowerSms, sizeof(alertPowerSms)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , phone_str  );
        publishQueue.publish("test3", alertPowerSms, 60, PRIVATE);
     
      }
      
      if ((A == 1) && (EP ==1)) {
		snprintf(alertPowerEmail, sizeof(alertPowerEmail)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , email_str  );
        publishQueue.publish("test4", alertPowerEmail, 60, PRIVATE);
      }
            powerStatus = 1;
        }
    }

    if (digitalRead(pin) == HIGH)
    {

        if (sensorStatus != 0)
        {

            digitalWrite(led1, HIGH);


            if ((A == 1) && (PS == 1))
            {
                 snprintf(alertSensorSms, sizeof(alertSensorSms)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , phone_str  );
                publishQueue.publish("test5", alertSensorSms, 60, PRIVATE);
            }

            if ((A == 1) && (ES == 1))
            {
                snprintf(alertSensorEmail, sizeof(alertSensorEmail)  , "{\"N\": \"%s\", \"T\": \"%s\"}"   , dev_name , email_str  );
                publishQueue.publish("test6", alertSensorEmail, 60, PRIVATE);
            }

            sensorStatus = 0;
        }
    }
    else if (digitalRead(pin) == LOW)
    {

        if (sensorStatus != 1)
        {
            digitalWrite(led1, LOW);
            sensorStatus = 1;
        }
    }

    if ((resetFlag) && (millis() - rebootSync >= rebootDelayMillis))
    {

        delay(1000);
        System.reset();
    }

    if ((PP == 0) && (EP == 0) && (PS == 0) && (ES == 0))

    {
        enableAlerts("0");
    }
    
       if ((PP == 1) | (EP == 1) | (PS== 1) | (ES == 1))
    
    {
        enableAlerts("1");
    }
}

int enableAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        A = 1;
        EEPROM.put(setAlertEEPROMLocation, A);
        return 1;
    }
    else
    {

        A = 0;
        EEPROM.put(setAlertEEPROMLocation, A);
        return 0;
    }
}

int enableEmailPowerAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        EP = 1;
        EEPROM.put(setAlertEmailPowerEEPROMLocation, EP);
        return 1;
    }
    else
    {

        EP = 0;
        EEPROM.put(setAlertEmailPowerEEPROMLocation, EP);
        return 0;
    }
}

int enableEmailSensorAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        ES = 1;
        EEPROM.put(setAlertEmailSensorEEPROMLocation, ES);
        return 1;
    }
    else
    {
        ES = 0;
        EEPROM.put(setAlertEmailSensorEEPROMLocation, ES);
        return 0;
    }
}

int enablePhonePowerAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        PP = 1;
        EEPROM.put(setAlertPhonePowerEEPROMLocation, PP);
        return 1;
    }
    else
    {

        PP = 0;
        EEPROM.put(setAlertPhonePowerEEPROMLocation, PP);
        return 0;
    }
}

int enablePhoneSensorAlerts(String command)
{

    if (command.equalsIgnoreCase("1"))
    {
        PS = 1;
        EEPROM.put(setAlertPhoneSensorEEPROMLocation, PS);
        return 1;
    }
    else
    {

        PS = 0;
        EEPROM.put(setAlertPhoneSensorEEPROMLocation, PS);
        return 0;
    }
}

int setPhone(const char *data)
{

    strncpy(phone_str, data, sizeof(phone_str) - 1);
    phone_str[sizeof(phone_str) - 1] = '\0';
    EEPROM.put(setPhoneEEPROMLocation, phone_str);
}

int setEmail(const char *data)
{
    strncpy(email_str, data, sizeof(email_str) - 1);
    email_str[sizeof(email_str) - 1] = '\0';
    EEPROM.put(setEmailEEPROMLocation, email_str);
}

void getStoredInfo()
{
    EEPROM.get(setPhoneEEPROMLocation, phone_str);
    EEPROM.get(setEmailEEPROMLocation,   email_str );
    EEPROM.get(setAlertEEPROMLocation, A);
    EEPROM.get(setAlertEmailSensorEEPROMLocation, ES);
    EEPROM.get(setAlertEmailPowerEEPROMLocation, EP);
    EEPROM.get(setAlertPhoneSensorEEPROMLocation, PS);
    EEPROM.get(setAlertPhonePowerEEPROMLocation, PP);

}

void handler(const char *topic, const char *data)
{
    strncpy(dev_name, data, sizeof(dev_name) - 1);
   
}

int cloudResetFunction(String command)
{
    resetFlag = true;
    rebootSync = millis();
    return 0;
}

void system_display_rssi()
{

    int rssi = 0;

#if Wiring_WiFi == 1
    rssi = WiFi.RSSI();
#elif Wiring_Cellular == 1
    CellularSignal sig = Cellular.RSSI();
    rssi = sig.getStrength();
#endif
    if (rssi < 0)
    {
        if (rssi >= -57)
        {
            SG = 5;
        }
        else if (rssi > -68)
        {
            SG = 4;
        }
        else if (rssi > -80)
        {
            SG = 3;
        }
        else if (rssi > -92)
        {
            SG = 2;
        }
        else if (rssi > -104)
        {
            SG = 1;
        }
    }

    if (rssi >= 0)
    {
        if (rssi >= 60)
        {
            SG = 5;
        }
        else if (rssi > 40)
        {
            SG = 4;
        }
        else if (rssi > 20)
        {
            SG = 3;
        }
        else if (rssi > 1)
        {
            SG = 2;
        }
        else if (rssi >= 0)
        {
            SG = 1;
        }
    }
}

The RSSI call will also take some extra time (especially on Electrons).
Hence you may want to take that reading less frequently.

2 Likes

Yes, what ScruffR said. Calling Cellular.RSSI() can block for up to 5 minutes, and you should not do it from the main loop thread.

2 Likes

@rickkas7 @ScruffR
Thanks for the suggestion. I’ll take it out from the loop and call on it when needed.

@rickkas7 @ScruffR Thanks guys again! I removed the Calling Cellular.RSSI() in the loop() and responsiveness improved.

If there are any other obvious red flags in my code please let me know.

No immediate red flags, but I’d probably unify some of the Particle.function() instances into one single one which takes a more elaborate command parameter and uses that to distinguish between the different use cases.
I’d possibly also combine some of your variables into a structured string variable for simple consumption in one go.
And I’d unify the individual EEPROM fields into a struct.

2 Likes