Hard Fault, PublishQueueAsyncRK, Boron v1.5.0

Hi All,

I am trying to utilise Rick’s great library to queue publish statements in an unconnected state for a certain amount of time before connecting and sending the data.

When I flash the following code to my particle I get the SOS and hard fault error, does anyone have any ideas of where my issue might lie. I’m probably not utilising the library correctly.

#include "Particle.h"
// This #include statement was automatically added by the Particle IDE.
#include <SdFat.h>

// This #include statement was automatically added by the Particle IDE.
#include <PublishQueueAsyncRK.h>
SYSTEM_MODE(SEMI_AUTOMATIC);
// SYSTEM_THREAD(ENABLED);
//------------------------------------------------------------------------------
// SD card Var Inits
const uint8_t SD_CHIP_SELECT = A2;
SdFat sdCard;
//------------------------------------------------------------------------------

//------------------------------------------------------------------------------
// Publish Queue Var Inits
PublishQueueAsyncSdFat publishQueue(sdCard, "events.dat");
const size_t MAX_PARAM = 4;
const unsigned long PUBLISH_PERIOD_MS = 30000;
unsigned long lastPublish = 8000 - PUBLISH_PERIOD_MS;
int counter = 0;
// int testNum;
// int intParam[MAX_PARAM];
// String stringParam[MAX_PARAM];
// size_t numParam;

// int testHandler(String cmd);
void publishCounter(bool withAck);
void publishPaddedCounter(int size);
//------------------------------------------------------------------------------


// ********************
// Test Vars Delete
int counter1 = 0;
// String data = "";
bool CLOUD_HAS_CONNECTED = FALSE;
// ********************

void setup() {
    delay(10000);
    Serial.println("Setup Complete");
  
    // Sd Card INIT
	if (sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED)) {
		publishQueue.setup();
	}
	else {
		Serial.println("failed to initialize sd card");
	}
}

void loop() {
  // nothing happens after setup
  if(counter1>=60){
        connectCloudAndCell(5);
        if(counter1>=90){
            disconnectCloudAndCell();
            counter1=0;
    }
  } else {
      
  }
 
  String data = "Time: " + String(Time.timeStr());
  publishQueue.publish("testEvent", data ,PRIVATE, WITH_ACK);
  Serial.println("Queued data" + String(publishQueue.getNumEvents()));
  delay(10000);
  counter++;
    
}

void connectCloudAndCell(int mins){
    Serial.println(" ");
    Serial.println("CONNECT FUNCTION:");
    Serial.println("----->TURNING ON CELL...");
    Cellular.on();
    Serial.println("----->CONNECTING TO CELL...");
    Cellular.connect();
    if (waitFor(Cellular.ready, mins*60000)) {
        Serial.println("----->TURNING ON CLOUD...");
        if (!Particle.connected()) Particle.connect();
        if (waitFor(Particle.connected, mins*60000)) {
            connectionStatus();
        } else {
            Serial.println("------------------------>ERROR COULD NOT CONNECT TO CLOUD");
            connectionStatus();
        }
    } else {
        Serial.println("------------------------>ERROR COULD NOT CONNECT TO CELLULAR");
        connectionStatus();
    }
    if (Particle.connected()) CLOUD_HAS_CONNECTED = TRUE;
    Serial.println(" ");
}

void disconnectCloudAndCell(){
    if (Particle.connected()) {
        Serial.println(" ");
        Serial.println("DISCONNECT FUNCTION:");
        Serial.println("-----> TURNING OFF CLOUD...");
        Serial.println("-----> TURNING OFF CELL...");
        Particle.disconnect();
    }
    Cellular.off();
    connectionStatus();
    Serial.println(" ");
}

void connectionStatus(){
    Serial.println("CELL CONNECTED: " + String(Cellular.ready()));
    Serial.println("CLOUD CONNECTED: " + String(Particle.connected()));
}

I should also add, that I can successfully use the following example from Ricks Repo. When using the above code the SD card failed to initialise prior to the hard fault. If I use the below example from the repo, the SD card inits successfully and I do not see the hard fault error.

// This #include statement was automatically added by the Particle IDE.
#include <SdFat.h>

// This #include statement was automatically added by the Particle IDE.
#include <PublishQueueAsyncRK.h>

SYSTEM_THREAD(ENABLED);

SerialLogHandler logHandler;

const uint8_t SD_CHIP_SELECT = A2;

SdFat sdCard;

PublishQueueAsyncSdFat publishQueue(sdCard, "events.dat");

enum {
	TEST_IDLE = 0, // Don't do anything
	TEST_COUNTER, // 1 publish, period milliseconds is param0
	TEST_PUBLISH_FAST, // 2 publish events as fast as possible, number is param0, optional size in param2
	TEST_PUBLISH_OFFLINE, // 3 go offline, publish some events, then go back online, number is param0, optional size in param2
	TEST_COUNTER_WITH_ACK, // 4 publish, period milliseconds is param0 but use WITH_ACK mode
	TEST_PAUSE_PUBLISING, // 5 pause publishing
	TEST_RESUME_PUBLISING // 6 resume publishing
};

// Example:
// particle call ivanDev01 test "4,30000"
// Replace electron1 with the name of your device
// "4,30000" is test 4, with a period of 30000 milliseconds or 30 seconds

const size_t MAX_PARAM = 4;
const unsigned long PUBLISH_PERIOD_MS = 30000;
unsigned long lastPublish = 8000 - PUBLISH_PERIOD_MS;
int counter = 0;
int testNum;
int intParam[MAX_PARAM];
String stringParam[MAX_PARAM];
size_t numParam;

int testHandler(String cmd);
void publishCounter(bool withAck);
void publishPaddedCounter(int size);

void setup() {
	Serial.begin();

	Particle.function("test", testHandler);

	// For testing purposes, wait 10 seconds before continuing to allow serial to connect
	// before doing publishQueue.setup() so the debug log messages can be read.
	waitFor(Serial.isConnected, 10000);

	if (sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED)) {
		publishQueue.setup();
	}
	else {
		Log.info("failed to initialize sd card");
	}
}

void loop() {

	if (testNum == TEST_COUNTER || testNum == TEST_COUNTER_WITH_ACK) {
		int publishPeriod = intParam[0];
		if (publishPeriod < 1) {
			publishPeriod = 15000;
		}

		if (millis() - lastPublish >= (unsigned long) publishPeriod) {
			lastPublish = millis();

			Log.info("TEST_COUNTER period=%d", publishPeriod);
			publishCounter(testNum == TEST_COUNTER_WITH_ACK);
		}
	}
	else
	if (testNum == TEST_PUBLISH_FAST) {
		testNum = TEST_IDLE;

		int count = intParam[0];
		int size = intParam[1];

		Log.info("TEST_PUBLISH_FAST count=%d", count);

		for(int ii = 0; ii < count; ii++) {
			publishPaddedCounter(size);
		}
	}
	else
	if (testNum == TEST_PUBLISH_OFFLINE) {
		testNum = TEST_IDLE;

		int count = intParam[0];
		int size = intParam[1];

		Log.info("TEST_PUBLISH_OFFLINE count=%d", count);

		Log.info("Going to Particle.disconnect()...");
		Particle.disconnect();
		delay(2000);

		Log.info("before publishing numEvents=%d", publishQueue.getNumEvents());

		for(int ii = 0; ii < count; ii++) {
			publishPaddedCounter(size);
		}

		Log.info("after publishing numEvents=%d", publishQueue.getNumEvents());

		Log.info("Going to Particle.connect()...");
		Particle.connect();
	}
}

void publishCounter(bool withAck) {
	Log.info("publishing counter=%d", counter);

	char buf[32];
	snprintf(buf, sizeof(buf), "%d", counter++);
	publishQueue.publish("testEvent", buf, 50, (withAck ? (PRIVATE | WITH_ACK) : PRIVATE));
}

void publishPaddedCounter(int size) {
	Log.info("publishing padded counter=%d size=%d", counter, size);

	char buf[256];
	snprintf(buf, sizeof(buf), "%05d", counter++);

	if (size > 0) {
		if (size > (int)(sizeof(buf) - 1)) {
			size = (int)(sizeof(buf) - 1);
		}

		char c = 'A';
		for(size_t ii = strlen(buf); ii < (size_t)size; ii++) {
			buf[ii] = c;
			if (++c > 'Z') {
				c = 'A';
			}
		}
		buf[size] = 0;
	}

	publishQueue.publish("testEvent", buf, PRIVATE | WITH_ACK);
}


int testHandler(String cmd) {
	char *mutableCopy = strdup(cmd.c_str());

	char *cp = strtok(mutableCopy, ",");

	int tempTestNum = atoi(cp);

	switch(tempTestNum) {
	case TEST_PAUSE_PUBLISING:
		Log.info("pausing publishing from test handler");
		publishQueue.setPausePublishing(true);
		break;

	case TEST_RESUME_PUBLISING:
		Log.info("resuming publishing from test handler");
		publishQueue.setPausePublishing(false);
		break;

	default:
		testNum = tempTestNum;

		for(numParam = 0; numParam < MAX_PARAM; numParam++) {
			cp = strtok(NULL, ",");
			if (!cp) {
				break;
			}
			intParam[numParam] = atoi(cp);
			stringParam[numParam] = cp;
		}
		for(size_t ii = numParam; ii < MAX_PARAM; ii++) {
			intParam[ii] = 0;
			stringParam[ii] = "";
		}
		break;
	}

	free(mutableCopy);
	return 0;
}

Once your code learns that the SD card is unavailble it would be negligent to use any calls that rely on its presence.

Your code only prints a warning message but then carries on as if everything worked as expected - you need to guard the rest of the code against that situation.

In particular you are using the publishQueue object without it ever being setup().

I’d expect that the sample also panics when the SD card cannot be initialised and hence the publishQueue was not initialised.

1 Like

Thanks Scruff.

Do you see anything obvious in the second sketch that enables the SD to be initialised correctly, or any issue in the first sketch that is preventing the SD card from being initialised?

If my SD card does not initialise, do you have any smarter behaviour then simply reset particle and try again?

I sometimes also have the SD card not initialise properly for reasons not always obvious.
It usually doesn’t fail after a cold boot but after a mere reset if fails more often.

However, if your SD fails on first attempt you could retry a few times, maybe with reduced speed.
When that doesn’t help you can try a reset.
When that doesn’t help either then you could opt for RAM based publishQueue as fallback.

But maybe @rickkas7 has some additional suggestions and can also comment on the SD card issue.

BTW, if the sample consistently works and your version consistently doesn’t then you can maybe incorporate your functionality bit by bit into the sample and see if or when it breaks.

1 Like

Good ideas Scruff. I will do some more testing with the SD card init, and try piecemeal integrate functionality to the second sketch to try and locate what is causing the hard fault. Thanks

Aside from all the valid points that @ScruffR has made - you may want to consider using FRAM instead of the SD - at last reading an I2C FRAM of 1Mbit (128KB) is available and personal experience with PublishQueueAsyncRK is very good whereas SPI and SD cards - it depends a lot on the quality of the connector and the SD card and the circuity (pull-ups) as to what speed of SPI bus works.

3 Likes

Thanks @armor, We may have to go that route if we cannot get reliable results with SD setup.

Currently I am now seeing two issues. I find that the SD card will not initialise unless I remove sd and reinsert into the sd card holder will trying to call

sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED);

I am also now seeing the following error:

0000119040 [app.pubq] INFO: initialized events file
Setup Complete
Connected:  0, Queued data: 1
Connected:  0, Queued data: 2
0000139248 [app.pubq] ERROR: writeBytes seek failed seekTo=-1
0000139249 [app.pubq] ERROR: writeBytes seek failed seekTo=0
Connected:  0, Queued data: 3

Does anyone know what might be causing the writeBytes seek error?

I am using the following sketch, it is a modified version of Ricks SD card example from his PublishQueueAsync lib:

// This #include statement was automatically added by the Particle IDE.
#include <SdFat.h>

// This #include statement was automatically added by the Particle IDE.
#include <PublishQueueAsyncRK.h>
SYSTEM_MODE(SEMI_AUTOMATIC);
SYSTEM_THREAD(ENABLED);

SerialLogHandler logHandler;

const uint8_t SD_CHIP_SELECT = A2;

SdFat sdCard;

PublishQueueAsyncSdFat publishQueue(sdCard, "events.dat");

// enum {
// 	TEST_IDLE = 0, // Don't do anything
// 	TEST_COUNTER, // 1 publish, period milliseconds is param0
// 	TEST_PUBLISH_FAST, // 2 publish events as fast as possible, number is param0, optional size in param2
// 	TEST_PUBLISH_OFFLINE, // 3 go offline, publish some events, then go back online, number is param0, optional size in param2
// 	TEST_COUNTER_WITH_ACK, // 4 publish, period milliseconds is param0 but use WITH_ACK mode
// 	TEST_PAUSE_PUBLISING, // 5 pause publishing
// 	TEST_RESUME_PUBLISING // 6 resume publishing
// };

// Example:
// particle call ivanDev01 test "4,30000"
// Replace electron1 with the name of your device
// "4,30000" is test 4, with a period of 30000 milliseconds or 30 seconds

// const size_t MAX_PARAM = 4;
// const unsigned long PUBLISH_PERIOD_MS = 30000;
// unsigned long lastPublish = 8000 - PUBLISH_PERIOD_MS;
// int counter = 0;
// int testNum;
// int intParam[MAX_PARAM];
// String stringParam[MAX_PARAM];
// size_t numParam;

// int testHandler(String cmd);
// void publishCounter(bool withAck);
// void publishPaddedCounter(int size);

int SD_INITIALISED = 0;
// ********************
// Test Vars Delete
int counter1 = 0;
// String data = "";
bool CLOUD_HAS_CONNECTED = FALSE;
// ********************

void setup() {
	Serial.begin();

// 	Particle.function("test", testHandler);

	// For testing purposes, wait 10 seconds before continuing to allow serial to connect
	// before doing publishQueue.setup() so the debug log messages can be read.
	waitFor(Serial.isConnected, 10000);
    
    // Serial.println(sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED));
    while(SD_INITIALISED!=1){
        sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED);
        Serial.print("*");
        delay(500);
        if (sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED)) {
		    SD_INITIALISED = 1;
	}
    }
    
	if (sdCard.begin(SD_CHIP_SELECT, SPI_FULL_SPEED)) {
		publishQueue.setup();
	}
	else {
		Log.info("failed to initialize sd card");
	}
	Serial.println("Setup Complete");
}

void loop() {
 // nothing happens after setup
  if(counter1>=60){
        connectCloudAndCell(5);
        if(counter1>=90){
            disconnectCloudAndCell();
            counter1=0;
    }
  } else {
      
  }
 
  String data = "Connected:  " + String(Particle.connected()) + ", count: " + String(publishQueue.getNumEvents());
  
  if(publishQueue.getNumEvents()<100){
    publishQueue.publish("testEvent", data ,PRIVATE, WITH_ACK);
  }
  Serial.println("Connected:  " + String(Particle.connected()) + ", Queued data: " + String(publishQueue.getNumEvents()));
  delay(10000);
  counter1++;
    
}

void connectCloudAndCell(int mins){
    Serial.println(" ");
    Serial.println("CONNECT FUNCTION:");
    Serial.println("----->TURNING ON CELL...");
    Cellular.on();
    Serial.println("----->CONNECTING TO CELL...");
    Cellular.connect();
    if (waitFor(Cellular.ready, mins*60000)) {
        Serial.println("----->TURNING ON CLOUD...");
        if (!Particle.connected()) Particle.connect();
        if (waitFor(Particle.connected, mins*60000)) {
            connectionStatus();
        } else {
            Serial.println("------------------------>ERROR COULD NOT CONNECT TO CLOUD");
            connectionStatus();
        }
    } else {
        Serial.println("------------------------>ERROR COULD NOT CONNECT TO CELLULAR");
        connectionStatus();
    }
    if (Particle.connected()) CLOUD_HAS_CONNECTED = TRUE;
    Serial.println(" ");
}

void disconnectCloudAndCell(){
    if (Particle.connected()) {
        Serial.println(" ");
        Serial.println("DISCONNECT FUNCTION:");
        Serial.println("-----> TURNING OFF CLOUD...");
        Serial.println("-----> TURNING OFF CELL...");
        while(Particle.connected()){
            Particle.disconnect();
            delay(500);
        }
    }
    Cellular.off();
    delay(500);
    connectionStatus();
    Serial.println(" ");
}

void connectionStatus(){
    Serial.println("CELL CONNECTED: " + String(Cellular.ready()));
    Serial.println("CLOUD CONNECTED: " + String(Particle.connected()));
}

Do you know what actual speed SPI_FULL_SPEED is for the Boron? It should be 32MHz but it might not be selecting the GEN3 maximum speed?

I remember that the SdCard begin has to happen very soon after startup - if you look at the controller specs - this is probably why a restart/repower is required.

I don’t retry the SdCard.begin() but rather record the result of the call and if OK then carry on with setup() OR if a fail then output error message, wait a period and then System.reset();

1 Like

Ok that’s great information, I’ll make a few changes with this in mind and look into the SPI speed also.

Will let you know how I get on.

This topic was automatically closed 60 days after the last reply. New replies are no longer allowed.