Don’t think this is going to be possible via the cloud. between the 622 char limit and the 1 request/second you can’t more that much data.
I set up a photon to do high speed accelerometer logging to an SD card. I’m using an ADXL377 and reading the analog values into the Photon and using the SPI bus for the SD card. So i’m not sure what capture rates you could achieve is your had the accelerometer and the SD card on SPI. With the analog accel i’m able to log at at the follow sustained rates, i have run three channels @ 1000Hz for 3+hours continuous without problem.
- 1 channel @ 2000Hz
- 3 channels @ 1666Hz
- 6 channels @ 1111Hz
- 7 channels @1000Hz
Here is my messy code to do this, this is a modification of the “low latency logger” example from the SDFat-Particle port
#include "SdFat.h"
#include "PowerShield.h"
// Don't connect to the cloud to reduce time jitter.
// WARNING you must use safe mode for OTA flash if you use manual mode.
// Remove comment on next line to decrease time jitter.
SYSTEM_MODE(MANUAL);
PowerShield batteryMonitor;
/**
* This program logs data to a binary file. Functions are included
* to convert the binary file to a csv text file.
*
* Samples are logged at regular intervals. The maximum logging rate
* depends on the quality of your SD card and the time required to
* read sensor data. This example has been tested at 500 Hz with
* good SD card on an Uno. 4000 HZ is possible on a Due.
*
* If your SD card has a long write latency, it may be necessary to use
* slower sample rates. Using a Mega Arduino helps overcome latency
* problems since 13 512 byte buffers will be used.
*
* Data is written to the file using a SD multiple block write command.
*/
//------------------------------------------------------------------------------
// User data functions. Modify these functions for your data items.
// Read analog pins not used by primary SPI.
// You may want to connect the SD to the secondary SPI interface.
const uint8_t ADC_DIM = 3; //number of data channels limits: 7@1000Hz, 3@1666Hz, 1@2000Hz
struct data_t {
unsigned long time;
unsigned short adc[ADC_DIM];
};
unsigned long timeMillis = 0;
unsigned long closedTime = 0;
unsigned long socUpdateFreq = 43200; //how often to try and publish battery SOC in seconds
char payload[256] = "";
int wakePin = A7; //wake pin
int runStatus = 0;
LEDStatus status1(RGB_COLOR_YELLOW, LED_PATTERN_FADE, LED_SPEED_FAST); //setting up files, between button press and record
LEDStatus status2(RGB_COLOR_RED, LED_PATTERN_BLINK, LED_SPEED_SLOW); //recording
LEDStatus status3(RGB_COLOR_BLUE, LED_PATTERN_BLINK, LED_SPEED_FAST); //converting to CSV file
LEDStatus status4(RGB_COLOR_ORANGE, LED_PATTERN_FADE, LED_SPEED_FAST); //publishing SOC
//SYSTEM_THREAD(ENABLED);
// Set useSharedSpi true for use of an SPI sensor.
const bool useSharedSpi = false;
// Acquire a data record.
void acquireData(data_t* data) {
data->time = micros();
data->adc[0] = analogRead(A0); //X
data->adc[1] = analogRead(A1); //Y
data->adc[2] = analogRead(A2); //Z
//data->adc[3] = analogRead(A3);
//data->adc[4] = analogRead(A4);
//data->adc[5] = analogRead(A5);
//data->adc[3] = analogRead(D5);
//data->adc[4] = analogRead(D6);
}
// Print a data record.
void printData(Print* pr, data_t* data) {
pr->print(data->time);
for (int i = 0; i < ADC_DIM; i++) {
pr->write(',');
pr->print(data->adc[i]);
}
pr->println();
}
//map a float
float mapf(float x, float in_min, float in_max, float out_min, float out_max)
{
return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min;
}
// Print data header.
void printHeader(Print* pr) {
pr->println(F("time,X (A0),Y (A1),Z (A2)"));
}
//==============================================================================
// Start of configuration constants.
//==============================================================================
//Interval between data records in microseconds.
const uint32_t LOG_INTERVAL_USEC = 1000;
//------------------------------------------------------------------------------
// Pin definitions.
//
// SD chip select pin.
const uint8_t SD_CS_PIN = D1;
//
// Digital pin to indicate an error, set to -1 if not used.
// The led blinks for fatal errors. The led goes on solid for SD write
// overrun errors and logging continues.
const int8_t ERROR_LED_PIN = D7;
//------------------------------------------------------------------------------
// File definitions.
//
// Maximum file size in blocks.
// The program creates a contiguous file with FILE_BLOCK_COUNT 512 byte blocks.
// This file is flash erased using special SD commands. The file will be
// truncated if logging is stopped early.
const uint32_t FILE_BLOCK_COUNT = 256000;
// log file base name. Must be six characters or less.
#define FILE_BASE_NAME "3xAcel"
//------------------------------------------------------------------------------
// Buffer definitions.
//
// The logger will use SdFat's buffer plus BUFFER_BLOCK_COUNT additional
// buffers.
//
#ifndef RAMEND
// Assume ARM. Use total of nine 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 8;
//
#elif RAMEND < 0X8FF
#error Too little SRAM
//
#elif RAMEND < 0X10FF
// Use total of two 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 1;
//
#elif RAMEND < 0X20FF
// Use total of five 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 4;
//
#else // RAMEND
// Use total of 13 512 byte buffers.
const uint8_t BUFFER_BLOCK_COUNT = 12;
#endif // RAMEND
//==============================================================================
// End of configuration constants.
//==============================================================================
// Temporary log file. Will be deleted if a reset or power failure occurs.
#define TMP_FILE_NAME "tmp_log.bin"
// Size of file base name. Must not be larger than six.
const uint8_t BASE_NAME_SIZE = sizeof(FILE_BASE_NAME) - 1;
SdFat sd(1); //use secondary SPI interface
// Secondary SPI with DMA
// SCK => D4, MISO => D3, MOSI => D2, SS => D1
SdBaseFile binFile;
char binName[13] = FILE_BASE_NAME "00.bin";
// Number of data records in a block.
const uint16_t DATA_DIM = (512 - 4)/sizeof(data_t);
//Compute fill so block size is 512 bytes. FILL_DIM may be zero.
const uint16_t FILL_DIM = 512 - 4 - DATA_DIM*sizeof(data_t);
struct block_t {
uint16_t count;
uint16_t overrun;
data_t data[DATA_DIM];
uint8_t fill[FILL_DIM];
};
const uint8_t QUEUE_DIM = BUFFER_BLOCK_COUNT + 2;
block_t* emptyQueue[QUEUE_DIM];
uint8_t emptyHead;
uint8_t emptyTail;
block_t* fullQueue[QUEUE_DIM];
uint8_t fullHead;
uint8_t fullTail;
// Advance queue index.
inline uint8_t queueNext(uint8_t ht) {
return ht < (QUEUE_DIM - 1) ? ht + 1 : 0;
}
//==============================================================================
void fatalBlink() {
while (true) {
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, HIGH);
delay(200);
digitalWrite(ERROR_LED_PIN, LOW);
delay(200);
}
}
}
//------------------------------------------------------------------------------
void error(const char* msg) {
sd.errorPrint(msg);
Serial.printlnf("Switch Status = %d", runStatus);
fatalBlink();
}
//==============================================================================
// Convert binary file to csv file.
void binaryToCsv() {
Serial.println("Starting CSV Write");
uint8_t lastPct = 0;
block_t block;
uint32_t t0 = millis();
uint32_t syncCluster = 0;
SdFile csvFile;
char csvName[13];
status3.setActive(true);
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
binFile.rewind();
// Create a new csvFile.
strcpy(csvName, binName);
strcpy(&csvName[BASE_NAME_SIZE + 3], "csv");
if (!csvFile.open(csvName, O_WRITE | O_CREAT | O_TRUNC)) {
error("open csvFile failed");
}
Serial.println();
Serial.print(F("Writing: "));
Serial.print(csvName);
Serial.println(F(" - type any character to stop"));
printHeader(&csvFile);
uint32_t tPct = millis();
while (!Serial.available() && binFile.read(&block, 512) == 512) {
uint16_t i;
if (block.count == 0) {
break;
}
if (block.overrun) {
csvFile.print(F("OVERRUN,"));
csvFile.println(block.overrun);
}
for (i = 0; i < block.count; i++) {
printData(&csvFile, &block.data[i]);
}
if (csvFile.curCluster() != syncCluster) {
csvFile.sync();
syncCluster = csvFile.curCluster();
}
if ((millis() - tPct) > 1000) {
uint8_t pct = binFile.curPosition()/(binFile.fileSize()/100);
if (pct != lastPct) {
tPct = millis();
lastPct = pct;
Serial.print(pct, DEC);
Serial.println('%');
}
}
}
csvFile.close();
Serial.print(F("Done: "));
Serial.print(0.001*(millis() - t0));
Serial.println(F(" Seconds"));
Serial.println("CSV Write Done");
status3.setActive(false);
//postSOC();
Serial.printlnf("Switch Status = %d", digitalRead(wakePin));
//Serial.println("Sleeping");
//System.sleep(SLEEP_MODE_DEEP, socUpdateFreq);
//System.sleep(wakePin, RISING); //sleep untill rising edge on
}
//------------------------------------------------------------------------------
// read data file and check for overruns
void checkOverrun() {
bool headerPrinted = false;
block_t block;
uint32_t bgnBlock, endBlock;
uint32_t bn = 0;
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
if (!binFile.contiguousRange(&bgnBlock, &endBlock)) {
error("contiguousRange failed");
}
binFile.rewind();
Serial.println();
Serial.println(F("Checking overrun errors - type any character to stop"));
while (binFile.read(&block, 512) == 512) {
if (block.count == 0) {
break;
}
if (block.overrun) {
if (!headerPrinted) {
Serial.println();
Serial.println(F("Overruns:"));
Serial.println(F("fileBlockNumber,sdBlockNumber,overrunCount"));
headerPrinted = true;
}
Serial.print(bn);
Serial.print(',');
Serial.print(bgnBlock + bn);
Serial.print(',');
Serial.println(block.overrun);
}
bn++;
}
if (!headerPrinted) {
Serial.println(F("No errors found"));
} else {
Serial.println(F("Done"));
}
}
//------------------------------------------------------------------------------
// dump data file to Serial
void dumpData() {
block_t block;
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
binFile.rewind();
Serial.println();
Serial.println(F("Type any character to stop"));
delay(1000);
printHeader(&Serial);
while (!Serial.available() && binFile.read(&block , 512) == 512) {
if (block.count == 0) {
break;
}
if (block.overrun) {
Serial.print(F("OVERRUN,"));
Serial.println(block.overrun);
}
for (uint16_t i = 0; i < block.count; i++) {
printData(&Serial, &block.data[i]);
}
}
Serial.println(F("Done"));
}
//------------------------------------------------------------------------------
// log data
// max number of blocks to erase per erase call
uint32_t const ERASE_SIZE = 262144L;
void logData() {
Serial.printlnf("Switch Status = %d", runStatus);
uint32_t bgnBlock, endBlock;
// Allocate extra buffer space.
block_t block[BUFFER_BLOCK_COUNT];
block_t* curBlock = 0;
Serial.println();
// Find unused file name.
if (BASE_NAME_SIZE > 6) {
error("FILE_BASE_NAME too long");
}
while (sd.exists(binName)) {
if (binName[BASE_NAME_SIZE + 1] != '9') {
binName[BASE_NAME_SIZE + 1]++;
} else {
binName[BASE_NAME_SIZE + 1] = '0';
if (binName[BASE_NAME_SIZE] == '9') {
error("Can't create file name");
}
binName[BASE_NAME_SIZE]++;
}
}
// Delete old tmp file.
if (sd.exists(TMP_FILE_NAME)) {
Serial.println(F("Deleting tmp file"));
if (!sd.remove(TMP_FILE_NAME)) {
error("Can't remove tmp file");
}
}
// Create new file.
Serial.println(F("Creating new file"));
binFile.close();
if (!binFile.createContiguous(sd.vwd(),
TMP_FILE_NAME, 512 * FILE_BLOCK_COUNT)) {
error("createContiguous failed");
}
// Get the address of the file on the SD.
if (!binFile.contiguousRange(&bgnBlock, &endBlock)) {
error("contiguousRange failed");
}
// Use SdFat's internal buffer.
uint8_t* cache = (uint8_t*)sd.vol()->cacheClear();
if (cache == 0) {
error("cacheClear failed");
}
// Flash erase all data in the file.
Serial.println(F("Erasing all data"));
uint32_t bgnErase = bgnBlock;
uint32_t endErase;
while (bgnErase < endBlock) {
endErase = bgnErase + ERASE_SIZE;
if (endErase > endBlock) {
endErase = endBlock;
}
if (!sd.card()->erase(bgnErase, endErase)) {
error("erase failed");
}
bgnErase = endErase + 1;
}
// Start a multiple block write.
if (!sd.card()->writeStart(bgnBlock, FILE_BLOCK_COUNT)) {
error("writeBegin failed");
}
// Set chip select high if other devices use SPI.
if (useSharedSpi) {
sd.card()->chipSelectHigh();
}
// Initialize queues.
emptyHead = emptyTail = 0;
fullHead = fullTail = 0;
// Use SdFat buffer for one block.
emptyQueue[emptyHead] = (block_t*)cache;
emptyHead = queueNext(emptyHead);
// Put rest of buffers in the empty queue.
for (uint8_t i = 0; i < BUFFER_BLOCK_COUNT; i++) {
emptyQueue[emptyHead] = &block[i];
emptyHead = queueNext(emptyHead);
}
status1.setActive(false); //setup led off
status2.setActive(true); //recording led on
Serial.println(F("Logging - type any character to stop"));
// Wait for Serial Idle.
Serial.flush();
delay(10);
uint32_t bn = 0;
uint32_t t0 = millis();
uint32_t t1 = t0;
uint32_t overrun = 0;
uint32_t overrunTotal = 0;
uint32_t count = 0;
uint32_t maxDelta = 0;
uint32_t minDelta = 99999;
uint32_t maxLatency = 0;
// Start at a multiple of interval.
uint32_t logTime = micros()/LOG_INTERVAL_USEC + 1;
logTime *= LOG_INTERVAL_USEC;
bool closeFile = false;
while (1) {
// Time for next data record.
logTime += LOG_INTERVAL_USEC;
if (Serial.available()) {
closeFile = true;
}
if (closeFile) {
if (curBlock != 0) {
// Put buffer in full queue.
fullQueue[fullHead] = curBlock;
fullHead = queueNext(fullHead);
curBlock = 0;
}
} else {
if (curBlock == 0 && emptyTail != emptyHead) {
curBlock = emptyQueue[emptyTail];
emptyTail = queueNext(emptyTail);
curBlock->count = 0;
curBlock->overrun = overrun;
overrun = 0;
}
int32_t usec = logTime - micros();
if (usec < 0) {
error("Rate too fast");
}
delayMicroseconds(usec);
uint32_t delta = micros() - logTime;
if (delta > maxDelta) maxDelta = delta;
if (delta < minDelta) minDelta = delta;
if (curBlock == 0) {
overrun++;
} else {
acquireData(&curBlock->data[curBlock->count++]);
if (curBlock->count == DATA_DIM) {
fullQueue[fullHead] = curBlock;
fullHead = queueNext(fullHead);
curBlock = 0;
}
}
}
if (fullHead == fullTail) {
// Exit loop if done.
if (closeFile) {
break;
}
} else if (!sd.card()->isBusy()) {
// Get address of block to write.
block_t* pBlock = fullQueue[fullTail];
fullTail = queueNext(fullTail);
// Write block to SD.
uint32_t usec = micros();
if (!sd.card()->writeData((uint8_t*)pBlock)) {
error("write data failed");
}
usec = micros() - usec;
t1 = millis();
if (usec > maxLatency) {
maxLatency = usec;
}
count += pBlock->count;
// Add overruns and possibly light LED.
if (pBlock->overrun) {
overrunTotal += pBlock->overrun;
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, HIGH);
}
}
// Move block to empty queue.
emptyQueue[emptyHead] = pBlock;
emptyHead = queueNext(emptyHead);
bn++;
if (bn == FILE_BLOCK_COUNT) {
// File full so stop
break;
}
if (digitalRead(wakePin) == 0 && runStatus == 1) { //turn off code, switch closed and output high
runStatus = 0;
Serial.println("Stopping Log");
status2.setActive(false);
closeFile = true;
}
}
}
if (!sd.card()->writeStop()) {
error("writeStop failed");
}
// Truncate file if recording stopped early.
if (bn != FILE_BLOCK_COUNT) {
Serial.println(F("Truncating file"));
if (!binFile.truncate(512L * bn)) {
error("Can't truncate file");
}
}
if (!binFile.rename(sd.vwd(), binName)) {
error("Can't rename file");
}
Serial.print(F("File renamed: "));
Serial.println(binName);
Serial.print(F("Max block write usec: "));
Serial.println(maxLatency);
Serial.print(F("Record time sec: "));
Serial.println(0.001*(t1 - t0), 3);
Serial.print(minDelta);
Serial.print(F(" <= jitter microseconds <= "));
Serial.println(maxDelta);
Serial.print(F("Sample count: "));
Serial.println(count);
Serial.print(F("Samples/sec: "));
Serial.println((1000.0)*count/(t1-t0));
Serial.print(F("Overruns: "));
Serial.println(overrunTotal);
Serial.println(F("Done"));
Serial.println("Converting to CSV");
Serial.printlnf("Switch Status = %d", digitalRead(wakePin));
//binaryToCsv();
}
//------------------------------------------------------------------------------
void setup() {
if (ERROR_LED_PIN >= 0) {
pinMode(ERROR_LED_PIN, OUTPUT);
}
Serial.begin(9600);
pinMode(wakePin, INPUT_PULLDOWN);
//if wake with wake pin low
//don't start recording or init SD just post battery SOC and back to sleep.
if (digitalRead(wakePin) == 0) {
postSOC(3);
Serial.printlnf("Switch Status = %d", digitalRead(wakePin));
Serial.println("sleeping");
status4.setActive(false);
System.sleep(SLEEP_MODE_DEEP, socUpdateFreq);
}
if (digitalRead(wakePin) == 1) {
postSOC(1);
}
// Wait for USB Serial
while (!Serial) {
SysCall::yield();
}
Serial.println("Setup started");
Serial.print(F("FreeMemory: "));
Serial.println(System.freeMemory());
Serial.print(F("Records/block: "));
Serial.println(DATA_DIM);
if (sizeof(block_t) != 512) {
error("Invalid block size");
}
// initialize file system.
if (!sd.begin(SD_CS_PIN, SPI_FULL_SPEED)) {
sd.initErrorPrint();
fatalBlink();
}
}
//------------------------------------------------------------------------------
void loop() {
if (digitalRead(wakePin) == 1) {
//postSOC(1);
Serial.println("logging started");
status1.setActive(true);
runStatus = 1;
Serial.printlnf("Switch Status = %d", digitalRead(wakePin));
logData();
Serial.println("to csv");
binaryToCsv();
Serial.println("post");
postSOC(2);
Serial.println("sleeping");
System.sleep(SLEEP_MODE_DEEP, socUpdateFreq);
}
}
void postSOC(int reason) { //atempts to post the battery SOC
// This essentially starts the I2C bus
WiFi.on();
WiFi.connect();
batteryMonitor.begin();
Serial.println("battery mon");
// This sets up the fuel gauge
batteryMonitor.quickStart();
Serial.println("quick start");
Serial.println("process");
softDelay(3000);
status4.setActive(true);
if (waitFor(WiFi.ready, 10000)) { //if connected to WiFi upload
Particle.connect(); //this is blocking if it can't connect
Serial.println("if start");
Particle.process();
float cellVoltage = batteryMonitor.getVCell(); // Read the volatge of the LiPo
float stateOfCharge = batteryMonitor.getSoC(); // Read the State of Charge of the LiPo
Particle.process();
sprintf(payload, "{ \"Cell Voltage\":%.1f,\"SOC\":%.1f,\"Reason\":%d }", cellVoltage, stateOfCharge, reason);
//reason codes: 1 = rec start, 2 = rec finsih, 3 = SOC update only
Particle.publish("postGoogle", payload, PRIVATE, WITH_ACK);
//Particle.publish("postGCP", payload, PRIVATE);
Particle.process();
//Particle.publish(payload, NULL, PRIVATE, WITH_ACK);
softDelay(5000);
Particle.disconnect();
}
WiFi.off();
Wire.end();
while(Wire.isEnabled()) {
Serial.println("waiting for Wire.end");
softDelay(100);
}
}
//delays for X ms, should not block execution
void softDelay(unsigned long delayTime) {
unsigned long startTime = millis();
while ((millis() - startTime) < delayTime) {
//wait
}
return;
}