My application is logging 10K - 40K of data in Photon RAM while not connected to the cloud. I am interested in recommendations for the best way to upload this data to a remote server once connected to the cloud. Would just creating a tcp_connect() to the remote server and implementing FTP or some other transfer protocol be best or are there other built in mechanisms to transfer a lot of data. The publish function does not seem appropriate when working with tens of thousands of bytes of data.
@jmosk, I will be porting this library this weekend as I have a similar requirement:
http://playground.arduino.cc/Code/FTP
I’m only looking at the FTPWRITE code. 
2 Likes
@jmosk, I ported the ftp app for Particle and tested with locally hosted (free) ftp server and it worked beautifully! I’ll be adapting it for my application.
3 Likes
@peekay123 Can you post the FTP ported code here or at github?
Can you indicate what locally hosted free FTP server you tested with?
@jmosk, I used Quick’n Easy FTP Server light on windows:
http://www.pablosoftwaresolutions.com/html/quick__n_easy_ftp_server_lite.html
The Photon/Core code is a super easy adaptation of the Arduino code from the link I supplied:
/*
FTP passive client for IDE v1.0.1 and w5100/w5200
Posted October 2012 by SurferTim
Modified 6 June 2015 by SurferTim
*/
#include "SD.h"
// comment out next line to write to SD from FTP server
#define FTPWRITE
// change to your server
IPAddress server( 192, 168, 0, 158 ); //Local network connected
TCPClient client;
TCPClient dclient;
char outBuf[128];
char outCount;
#define SD_CS SS
byte doFTP();
byte eRcv();
void efail();
void readSD();
// change fileName to your file (8.3 format!)
char fileName[13] = "test.txt";
void setup()
{
Serial.begin(9600);
while(!Serial.available()) Particle.process();
if(!SD.begin(SD_CS))
{
Serial.println("SD init fail");
}
Serial.println("Ready. Press f or r");
}
void loop()
{
byte inChar;
inChar = Serial.read();
if(inChar == 'f')
{
if(doFTP()) Serial.println("FTP OK");
else Serial.println("FTP FAIL");
}
if(inChar == 'r')
{
readSD();
}
}
File fh;
byte doFTP()
{
#ifdef FTPWRITE
fh = SD.open(fileName,FILE_READ);
#else
SD.remove(fileName);
fh = SD.open(fileName,FILE_WRITE);
#endif
if(!fh)
{
Serial.println("SD open fail");
return 0;
}
#ifndef FTPWRITE
if(!fh.seek(0))
{
Serial.println("Rewind fail");
fh.close();
return 0;
}
#endif
Serial.println("SD opened");
if (client.connect(server,21)) {
Serial.println("Command connected");
}
else {
fh.close();
Serial.println("Command connection failed");
return 0;
}
if(!eRcv()) return 0;
client.println("USER particle");
if(!eRcv()) return 0;
client.println("PASS photon");
if(!eRcv()) return 0;
client.println("SYST");
if(!eRcv()) return 0;
client.println("Type I");
if(!eRcv()) return 0;
client.println("PASV");
if(!eRcv()) return 0;
char *tStr = strtok(outBuf,"(,");
int array_pasv[6];
for ( int i = 0; i < 6; i++) {
tStr = strtok(NULL,"(,");
array_pasv[i] = atoi(tStr);
if(tStr == NULL)
{
Serial.println("Bad PASV Answer");
}
}
unsigned int hiPort,loPort;
hiPort = array_pasv[4] << 8;
loPort = array_pasv[5] & 255;
Serial.print("Data port: ");
hiPort = hiPort | loPort;
Serial.println(hiPort);
if (dclient.connect(server,hiPort)) {
Serial.println("Data connected");
}
else {
Serial.println("Data connection failed");
client.stop();
fh.close();
return 0;
}
#ifdef FTPWRITE
client.print("STOR ");
client.println(fileName);
#else
client.print("RETR ");
client.println(fileName);
#endif
if(!eRcv())
{
dclient.stop();
return 0;
}
#ifdef FTPWRITE
Serial.println("Writing");
byte clientBuf[64];
int clientCount = 0;
while(fh.available())
{
clientBuf[clientCount] = fh.read();
clientCount++;
if(clientCount > 63)
{
dclient.write(clientBuf,64);
clientCount = 0;
}
}
if(clientCount > 0) dclient.write(clientBuf,clientCount);
#else
while(dclient.connected())
{
while(dclient.available())
{
char c = dclient.read();
fh.write(c);
Serial.write(c);
}
}
#endif
dclient.stop();
Serial.println("Data disconnected");
if(!eRcv()) return 0;
client.println("QUIT");
if(!eRcv()) return 0;
client.stop();
Serial.println("Command disconnected");
fh.close();
Serial.println("SD closed");
return 1;
}
byte eRcv()
{
byte respCode;
byte thisByte;
while(!client.available()) Spark.process();
respCode = client.peek();
outCount = 0;
while(client.available())
{
thisByte = client.read();
Serial.write(thisByte);
if(outCount < 127)
{
outBuf[outCount] = thisByte;
outCount++;
outBuf[outCount] = 0;
}
}
if(respCode >= '4')
{
efail();
return 0;
}
return 1;
}
void efail()
{
byte thisByte = 0;
client.println("QUIT");
while(!client.available()) Spark.process();
while(client.available())
{
thisByte = client.read();
Serial.write(thisByte);
}
client.stop();
Serial.println("Command disconnected");
fh.close();
Serial.println("SD closed");
}
void readSD()
{
fh = SD.open(fileName,FILE_READ);
if(!fh)
{
Serial.println("SD open fail");
return;
}
while(fh.available())
{
Serial.write(fh.read());
}
fh.close();
}
Of course, you need to add an SD library to that. I used a hardware SPI connected microSD. 
3 Likes
@peekay123 Thank you
1 Like
I am using this code to transfer data from SD card to an FTP server. It works partially.
- The file gets transferred only partially. The source is a text file having 838,736 bytes out of which only 203,968 get transferred. Then the connection closes normally.
- There are some errors in the file. Some characters are missing
How can I debug this? What part of the code? Any hints welcome!!
Here is the result on the serial port:
@peekay123 - just noticed this in the eRcv() function:
while(!client.available()) Spark.process();
Shouldn’t this be particle.process()?
Will try anyway and post.
@pteja, both will work but it should be Particle.process(). Which SD library are you using in your code?
SD lib from greiman
@pteja, is that the newest SDFat library? If not, I highly recommend using that one. It uses DMA and is written by the original SDFat author.
It is the latest lib. The issue is not the SD library. The data is getting stored properly.
It is the FTP part that has issues. Just rechecked with change to particle.process. Same result.
Only 204,672 bytes got transferred and missing characters again in the FTPed file on server 
Here is the link to SDFat library.
Here is a topic related to SD library.
@pteja, a small delay may need to be added between 64 byte outgoing chunks to allow the background code to catch up:
while(fh.available())
{
clientBuf[clientCount] = fh.read();
clientCount++;
if(clientCount > 63)
{
dclient.write(clientBuf,64);
clientCount = 0;
}
delay(2);
}
You may need to try different values to see if it helps or not.
1 Like
It works. But takes a VERY long time - I aborted twice thinking the transfer had stopped and then added a serial.print where delay is to view progress. It took several minutes for a 800kb file.
Need to fine-tune the transfer timing by adjusting the delay to 1ms.
The (seemingly) strange thing is nothing happens on the FTP server till the entire transfer is done. Any idea why?
Will the delay need to be changed if the available bandwidth is lower?
Many thanks @peekay123
@pteja, the bottleneck is at the tcp stack. You can reduce the delay but also increase the clientBuf size. Make it progressively larger, adjusting the clientCount threshold accordingly. You could try making it 1024 bytes to see how well that works and tuning the delay as well.
1 Like
@peekay123 Can this code be used to dump data from a FTP server to the SD card that is connected to the Photon? Or is it only the other way around?
@RWB yup. I’m not sure I posted the entire code that supports that though.
UDPATE: I did not post the entire code so that part is missing.
Your mainly using it to offload data on the Photon to the FTP server right?
Have you sent data to the SD card connected to the Photon successfully?
@RWB, I’ve only test with sd to ftp, not ftp to sd. I will work on that this weekend.
@peekay123 Sweet! Even if you don’t get to it this weekend please do let me know how it goes when you do get to it. 
1 Like