Update - I have now implemented the solution - slightly changed logic from above because I decided to use a software timer to trigger the process of sending stored event messages so that the sending rate should not exceed 1 per second.
Thanks and credit to @peekay123 for his ideas on how to implement a commit log on an SD card - I saw this on another thread.
I have hit a problem with the SD card readData() I get CMD17 errors. I thought to begin with this was due to conflicting attempts to open a file on the SD card (the library solution only allows for one file to be open at a time). I have implemented a volatile boolean SDInUse flag that I set and clear every time I open or close a file. This appears to work for a while but then the error comes back. Does anyone have a suggestion as to how to check there is a file open on the SD card using a low level check? Also, is there a way to check whether a software timer is running (has been started)?
I am also wondering whether I am using the SYSTEM THREAD ENABLED feature properly - that I am not blocked or swapping control between the threads in the way I am using Particle.publish(), Particle.connect() and Particle.disconnect(). My application is driving a GUI which swaps between working and standby. Standby turns off the screen and wifi.
Code with debug and test harness included if anyone can add help answer these points or spot any glaring errors!
#include "sd-card-library-photon-compat/sd-card-library-photon-compat.h"**
#include "Adafruit_ST7735/Adafruit_ST7735.h"**
#include "Adafruit_ST7735/Adafruit_mfGFX.h"**
#include "Adafruit_ST7735/fonts.h"**
#include "application.h"**
SYSTEM_MODE(SEMI_AUTOMATIC);
SYSTEM_THREAD(ENABLED);
#define BUFFPIXEL 40
> const uint8_t cs = A2; // Chip select, may use any unused A or D pin (Use A2 to start)
> const uint8_t dc = A1; // May be listed as 'A0' instead of 'dc' on the display (must use this pin)
> const uint8_t rst = 0;
> const uint8_t chipSelect = A6;
> const uint8_t lite = A7;
> int counter = 0; //just for the test harness
> int step;
> boolean connectedOnce;
> boolean sdOK;
> // declarations for the securePublish()
> boolean wasDisconnected = TRUE;
> void publishNextBacklogEvent();
> Timer publishTimer(4000, publishNextBacklogEvent); //try every 4 seconds to send
> volatile boolean publishTimer_On;
> volatile boolean SDinUse;
> // setup SD card and display
> Sd2Card card;
> SdVolume volume;
> SdFile file;
> Adafruit_ST7735 tft = Adafruit_ST7735(cs, dc, rst); // hardware spi
> void setup()
> {
> Serial.begin(9600);
> while (!Serial.available())
> pinMode(lite, OUTPUT); //PWM on A0 for backlight
> analogWrite(lite, 0); //backlight off
> sdOK = TRUE;
> if (!card.init(SPI_FULL_SPEED, chipSelect))
> {
> sdOK = FALSE;
> }
> if (!SD.begin(chipSelect))
> {
> sdOK = FALSE;
> }
> if (!volume.init(card))
> {
> sdOK = FALSE;
> }
> if (!sdOK) {Serial.println("SD card does not work"); return;}
> SDinUse = FALSE;
> tft.initR(INITR_BLACKTAB); //for the ST7735R
> tft.fillScreen(ST7735_WHITE);
> tft.setRotation(1); //portrait
> tft.setFont(3); //arial
> tft.setTextSize(1);
> tft.setTextColor(ST7735_BLACK);
> tft.setCursor(0,0);
> analogWrite(lite,127); // backlight on
> wasDisconnected = TRUE;
> publishTimer_On = FALSE;
> connectedOnce = FALSE;
> Serial.printlnf("initBacklog returned: %2i", initBacklog());
> Serial.println("bmpDraw in setup");
> (void) bmpDraw("ixlogo.bmp", 0, 0);
> tft.setCursor(0,119);
> tft.setFont(2);
> tft.setTextColor(ST7735_BLACK);
> }
> void loop()
> {
> counter++;
> step = counter%9;
> if (step == 1)
> {
> Serial.println("Step: 1 connect to cloud");
> WiFi.on(); //turn on wifi module
> WiFi.connect(); //WIFI_CONNECT_SKIP_LISTEN); //connect to stored wifi credentials
> if (waitFor(WiFi.ready, 6000))
> {
> Particle.connect(); //connect to cloud
> if (waitFor(Particle.connected, 6000))
> {
> Serial.println("Step: 1 connected");
> connectedOnce = TRUE;
> }
> }
> tft.fillScreen(ST7735_WHITE);
> (void) bmpDraw("powerhub.bmp", 0, 0);
> }
> else if (step == 2)
> {
> Serial.println("Step: 2 publish event");
> char str1[] = "case 2 and a long event name";
> char str2[255];
> String str3 = Time.timeStr() + '\0';
> str3.toCharArray(str2, str3.length());
> securePublish(str1, str2);
> }
> else if (step == 4)
> {
> Serial.println("Step: 4 bmpDraw logo");
> tft.fillScreen(ST7735_WHITE);
> (void) bmpDraw("ixlogo.bmp", 0, 0);
> }
> else if (step == 5)
> {
> Serial.print("Step: 5 just checking if...");
> if (eventsInBacklog()) Serial.println("Events in Backlog"); else Serial.println("No Events in Backlog");
> }
> else if (step == 6)
> {
> Serial.println("Step: 6 some delay");
> /*
> char str1[] = "case 6 and a very long event name";
> char str2[255];
> String str3 = Time.timeStr() + '\0';
> str3.toCharArray(str2, str3.length());
> securePublish(str1, str2);
> */
> delay(5000);
> }
> else if (step == 7)
> {
> Serial.println("Step: 7 try going offline - standby");
> if (!publishTimer_On)
> {
> if (Particle.connected())
> {
> Particle.disconnect();
> WiFi.off();
> Serial.println("Particle DISCONNECTED");
> } //disconnect
> }
> }
> }
> // function to securely publish events to the cloud, manages storing
> // events when offline and sending stored events at a controlled rate when
> // online. Takes 2 parameters; eventname and eventdata both strings
> //
> void securePublish(char *eventName, char *eventData)
> {
> boolean sent;
> if (Particle.connected()) // If Particle connected
> {
> Serial.println("*Particle connected");
> if (wasDisconnected) // there could be a backlog
> {
> Serial.println("**wasDisconnected");
> if (eventsInBacklog() && !publishTimer_On)
> {
> Serial.println("****Start publishing timer");
> publishTimer.start(); // start sending backlog at maximum steady rate
> publishTimer_On = TRUE;
> }
> sent = Particle.publish(eventName, eventData, 60, PRIVATE); // Publish current event to Particle Cloud
> Serial.printlnf("<<<< publish event %s data %s sent %i", eventName, eventData, sent);
> if (sent) // Publish of event successful
> {
> wasDisconnected = FALSE;
> }
> else
> {
> addEventToBacklog(eventName, eventData); //add event to backlog
> }
> }
> else // no backlog, publish as normal
> {
> sent = Particle.publish(eventName, eventData, 60, PRIVATE); // Publish current event to Particle Cloud
> Serial.printlnf("<<<< publish event %s data %s sent %i", eventName, eventData, sent);
> if (!sent) // publish of event unsuccessful
> {
> wasDisconnected = TRUE;
> addEventToBacklog(eventName, eventData); //add event to backlog
> }
> }
> }
> else // If not connected then store message
> {
> addEventToBacklog(eventName, eventData); //add event to backlog
> wasDisconnected = TRUE;
> }
> }
> // checks backlog and returns TRUE if there is a backlog or FALSE if no backlog
> // reads first 4 bytes = head then reads next 4 bytes = tail pointers both LSB first
> boolean eventsInBacklog()
> {
> boolean isbacklog = FALSE;
> uint32_t _head, _tail;
> File backLog = SD.open("commit.txt", FILE_READ);
> if (backLog && !SDinUse)
> {
> SDinUse = TRUE;
> _head = read32(backLog);
> _tail = read32(backLog);
> _head == _tail?isbacklog = FALSE: isbacklog = TRUE;
> backLog.close();
> SDinUse = FALSE;
> Serial.printlnf("eIB? Head: %lu Tail: %lu", _head, _tail);
> }
> return isbacklog;
> }
> // handler for timer to publish next event from backlog if none then timer stopped
> void publishNextBacklogEvent()
> {
> boolean sent;
> uint32_t _head;
> uint8_t b = '\0';
> uint16_t i;
> char eventName[64]; //maximum size of event name 63 characters plus end of string
> char eventData[256]; //maximum size of event data 255 bytes plus end of string
> if (eventsInBacklog())
> {
> Serial.println(">> publish next backlog event");
> File backLog = SD.open("commit.txt", FILE_READ);
> if (backLog && !SDinUse)
> {
> SDinUse = TRUE;
> backLog.seek(0); //ensure file pointer is at the head value
> _head = read32(backLog);
> backLog.seek(_head); //go to start of the logged and undeleted events
> i = 0;
> while(backLog.available() and b != ',') //while not end of event name and more data
> {
> b = backLog.read();
> b == ',' ? eventName[i] = '\0': eventName[i++] = b; //copy event name into buffer or add end of string if comma
> }
> i = 0;
> while(backLog.available() and b != '\n') //while not end of event data and more data
> {
> b = backLog.read();
> b == '\n' ? eventData[i] = '\0': eventData[i++] = b; //copy event name into buffer or add end of string if newline
> }
> backLog.close(); //close file
> SDinUse = FALSE;
> Serial.printlnf("Publish next event in backlog %s", eventName);
> sent = Particle.publish(eventName, eventData, 60, PRIVATE); // Publish current event to Particle Cloud
> if (sent) // publish successful
> {
> if (!deleteEventFromBacklog()) // remove published event from log
> {
> //Error message because event sent to cloud and then could not be removed from the backlog
> }
> }
> else // publish unsuccessful
> {
> wasDisconnected = TRUE; // hence items in backlog still
> }
> }
> }
> else
> {
> Serial.println("****Stop publishing timer");
> publishTimer.stop(); // can stop trying for now
> publishTimer_On = FALSE;
> }
> }
> // remove event string data from head of backlog
> // actually just move the head pointer to the next in the log
> // Return error code as follows
> // 0 = OK
> //-1 = error could not remove
> //
> int deleteEventFromBacklog()
> {
> int returnCode = -1;
> uint32_t _head;
> uint8_t b = '\0';
> File backLog = SD.open("commit.txt", FILE_WRITE);
> if (backLog && !SDinUse)
> {
> SDinUse = TRUE;
> backLog.seek(0); //ensure file pointer is at the head value
> _head = read32(backLog);
> backLog.seek(_head); //go to start of the logged and undeleted events
> while(backLog.available() and b != '\n') //while not newline and more data
> {
> b = backLog.read();
> }
> _head = backLog.position(); //read new head pointer
> backLog.seek(0); //update tail pointer
> write32(backLog,_head);
> backLog.close(); //close file
> SDinUse = FALSE;
> returnCode = 0;
> Serial.println("Deleted event from backlog");
> }
> return returnCode;
> }
> // add event string data to tail of backlog
> // Return error code as follows
> // 0 = OK
> //-1 = error not added
> //
> int addEventToBacklog(char *_eName, char *_eData)
> {
> int returnCode = -1;
> uint32_t _tail;
> File backLog = SD.open("commit.txt", FILE_WRITE);
> if (backLog && !SDinUse)
> {
> SDinUse = TRUE;
> backLog.seek(4); //ensure file pointer is at the tail value
> _tail = read32(backLog);
> backLog.seek(_tail); //go to end of the logged events
> backLog.printlnf("%s,%s",_eName, _eData); //add new event data to the end
> _tail = backLog.position(); //read new tail pointer
> backLog.seek(4); //update tail pointer
> write32(backLog,_tail);
> backLog.close(); //close file
> SDinUse = FALSE;
> returnCode = 0;
> Serial.println("Added event to backlog");
> }
> return returnCode;
> }
> // initialise backlog file to manage growth or setup new file
> // if no commit log or has no events then open (new) file and truncate to just head and tail pointers
> // truncate method not available with File object hence just reset pointer values if file exists
> // Return error code as follows
> // 0 = OK
> //-1 = error
> //
> int initBacklog()
> {
> int returnCode = -1;
> uint32_t _head, _tail;
> if (!eventsInBacklog())
> {
> File backLog = SD.open("commit.txt", FILE_WRITE);
> if (backLog)
> {
> SDinUse = TRUE;
> backLog.seek(0);
> //backLog.truncate(8);
> backLog.seek(0);
> _head = 8;
> write32(backLog,_head);
> _tail = 8;
> write32(backLog,_tail);
> backLog.close();
> SDinUse = FALSE;
> returnCode = 0;
> Serial.printlnf("Created/Truncated commit.log");
> Serial.printlnf("Head: %lu Tail: %lu", _head, _tail);
> }
> }
> else
> {
> Serial.printlnf("Existing commit.log");
> }
> return returnCode;
> }
> // This function opens a Windows Bitmap (BMP) file and
> // displays it at the given coordinates. Needs 24 bit/1 plane BMP.
> // Returns error code as follows:
> // 0 = OK
> // 1 = x or y outside screen
> // 2 = not BMP file
> // 3 = more than 1 plane
> // 4 = not 24 bits and uncompressed
> // 5 = SD card in use and timeout after 50msec
> //
> int bmpDraw(char *filename, uint8_t x, uint16_t y)
> {
> File bmpFile;
> int bmpWidth, bmpHeight; // W+H in pixels
> uint8_t bmpDepth; // Bit depth (currently must be 24)
> uint16_t bmpType;
> uint32_t bmpImageoffset; // Start of image data in file
> uint32_t rowSize; // Not always = bmpWidth; may have padding
> uint8_t sdbuffer[3*BUFFPIXEL]; // pixel buffer (R+G+B per pixel)
> uint8_t buffidx = sizeof(sdbuffer); // Current position in sdbuffer
> boolean flip = true; // BMP is stored bottom-to-top
> int w, h, row, col;
> uint8_t r, g, b;
> uint32_t pos = 0;
> uint32_t ms;
> int returnCode = 0;
> if ((x >= tft.width()) || (y >= tft.height())) return 1;
> ms = millis();
> while (SDinUse && ((millis() - ms) < 100)) //while SD file open and not timeout 100mS
> if (SDinUse) return 5; //SD file open still
> bmpFile = SD.open(filename, FILE_READ);
> SDinUse = TRUE;
> bmpType = read16(bmpFile);
> if (bmpType == 0x4D42)
> {
> (void)read32(bmpFile); // Read & ignore file size
> (void)read32(bmpFile); // Read & ignore creator bytes
> bmpImageoffset = read32(bmpFile); // Start of image data
> (void)read32(bmpFile); //Read & ignore header size
> bmpWidth = read32(bmpFile);
> bmpHeight = read32(bmpFile);
> if(read16(bmpFile) == 1) { // # planes -- must be '1'
> bmpDepth = read16(bmpFile); // bits per pixel
> if((bmpDepth == 24) && (read32(bmpFile) == 0)) { // 0 = uncompressed
> rowSize = (bmpWidth * 3 + 3) & ~3; // BMP rows are padded (if needed) to 4-byte boundary
> if(bmpHeight < 0) { // If bmpHeight is negative, image is in top-down order.
> bmpHeight = -bmpHeight;
> flip = false;
> }
> // Crop area to be loaded
> w = bmpWidth;
> h = bmpHeight;
> if((x+w-1) >= tft.width()) w = tft.width() - x;
> if((y+h-1) >= tft.height()) h = tft.height() - y;
> // Set TFT address window to clipped image bounds
> tft.setAddrWindow(x, y, x+w-1, y+h-1);
> for (row=0; row<h; row++) { // For each scanline...
> if(flip) // Bitmap is stored bottom-to-top order (normal BMP)
> pos = bmpImageoffset + (bmpHeight - 1 - row) * rowSize;
> else // Bitmap is stored top-to-bottom
> pos = bmpImageoffset + row * rowSize;
> if(bmpFile.position() != pos) { // Need seek?
> bmpFile.seek(pos);
> buffidx = sizeof(sdbuffer); // Force buffer reload
> }
> for (col=0; col<w; col++) { // For each pixel...
> // Time to read more pixel data?
> if (buffidx >= sizeof(sdbuffer)) { // Indeed
> bmpFile.read(sdbuffer, sizeof(sdbuffer));
> buffidx = 0; // Set index to beginning
> }
> // Convert pixel from BMP to TFT format, push to display
> b = sdbuffer[buffidx++];
> g = sdbuffer[buffidx++];
> r = sdbuffer[buffidx++];
> tft.pushColor(tft.Color565(r,g,b));
> } // end pixel
> } // end scanline
> } else {
> returnCode = 4;
> }
> } else {
> returnCode = 3;
> }
> } else {
> returnCode = 2;
> }
> bmpFile.close();
> SDinUse = FALSE;
> return returnCode;
> }
> // read unsigned 16 bit little endian
> uint16_t read16(File &f)
> {
> uint16_t result;
> ((uint8_t *)&result)[0] = f.read(); // LSB
> ((uint8_t *)&result)[1] = f.read(); // MSB
> return result;
> }
> // read unsigned 32 bit little endian
> uint32_t read32(File &f)
> {
> uint32_t result;
> ((uint8_t *)&result)[0] = f.read(); // LSB
> ((uint8_t *)&result)[1] = f.read();
> ((uint8_t *)&result)[2] = f.read();
> ((uint8_t *)&result)[3] = f.read(); // MSB
> return result;
> }
> // write unsigned 32 bit little endian to file
> void write32(File &f, uint32_t value)
> {
> f.write(value & 0xFF);
> f.write((value >>8) & 0xFF);
> f.write((value >>16) & 0xFF);
> f.write((value >>24) & 0xFF);
> }