Communicating With a SD Card - "delete" Content From Text File? [Solved]

Alright,

Yesterday I made a post, which I will be redirecting here now for others who might have similar inquiries (regardless if this thread gets going or not).

In my ignorance and haste I believed that I was able to delete lines from a text file successfully, I was wrong :frowning:

Here is the code that I believed was working:

#include "application.h"
#include "sd-card-library.h"

File myFile;

// SOFTWARE SPI pin configuration - modify as required
// The default pins are the same as HARDWARE SPI
const uint8_t chipSelect = A2;    // Also used for HARDWARE SPI setup
const uint8_t mosiPin = A5;
const uint8_t misoPin = A4;
const uint8_t clockPin = A3;

char check = 0;
int i =0;
String dataString = "";

SYSTEM_MODE(SEMI_AUTOMATIC);

void setup()
{
  Serial.begin(115200);
  while (!Serial.available()) SPARK_WLAN_Loop();

  Serial.print("Initializing SD card...");
  // Initialize SOFTWARE SPI
  if (!SD.begin(mosiPin, misoPin, clockPin, chipSelect)) {
    Serial.println("initialization failed!");
    return;
}

  Serial.println("initialization done.");

  myFile = SD.open("test.txt", FILE_WRITE);

  if (myFile) {     // if the file opened okay, write to it:
    Serial.print("Writing to test.txt...");
    for (int i=0; i<1; i++){
    myFile.println("testing 1, 2, x.");
    myFile.println("testing 1, 2, z.");
    myFile.println("testing 1, 2, g.");
    myFile.println("testing 1, 2, h.");
    myFile.println("testing 1, 2, j.");
    myFile.println("testing 1, 2, n.");
    myFile.println("testing 1, 2, k.");
    myFile.println("testing 1, 2, u.");
    myFile.println("beep boop beep, 1, 2, 3.");
    myFile.println("testing 1, 2, 3 please.");
    }
    myFile.close();            // close the file: closing the file is what actually saves the data on the SD
    Serial.println("done.");   // ^^ you can .close() to save or .flush() to save
  } else {
    Serial.println("error opening test.txt");       // if the file didn't open, print an error:
  }

  myFile = SD.open("test.txt");       // re-open the file for reading:
  if (myFile) {
    Serial.print("Current File postion For reading: "); Serial.println(myFile.position());
    Serial.println("test.txt:");

    while (myFile.available()) {     // read from the file until there's nothing else in it:
      Serial.write(myFile.read());
    }
    myFile.close();
  } else {
    // if the file didn't open, print an error:
    Serial.println("error opening test.txt");
  }

  myFile = SD.open("test.txt", FILE_WRITE); // need a bitwise or ('|')  in order to use O_RDWR not the boolean operand ('||')
  if(myFile)
  {
      Serial.print("Current File postion for writing: "); Serial.println(myFile.position());
      Serial.println("\nNow delete first five lines . . .");
      myFile.seek(0); // go to the start of the file
      Serial.print("File postion after seek: "); Serial.println(myFile.position());
      while (i<5)
      {
        dataString = "";
        //myFile.seek(0);
        while (myFile.available())
          {
              check = myFile.peek();
              dataString += String (check);
              Serial.println(dataString);
              myFile.write(8);
              if (check == '\n') break;
          }
            i++;
      }
     Serial.println("Did the file close?"); //Yes it did
       myFile.close();
  }
    else {
    // if the file didn't open, print an error:
    Serial.println("error opening test.txt");
    }

myFile = SD.open("test.txt");
  if (myFile) {
    Serial.println("test.txt:");
    while (myFile.available()) {   //SEe if we deleted the lines
      Serial.write(myFile.read());
    }
    myFile.close();
  } else {
    // if the file didn't open, print an error:
    Serial.println("error opening test.txt");
  }
  //SD.remove("test.txt");   //remove the file so that on reset, we can see if 5 lines were deleted easily
}

void loop()
{
  // nothing happens after setup
}

I did varying lines of text to write in “test.txt”, not just the ones shown above; but the code SEEMED to work at first glance.

If anyone uses this code they will see that on the Serial Monitor (I was using PuTTY) that it will show10 unique lines originally in the text file, then it will print the remaining lines in the text file which were the last5. Which was the way it was intended to work. However, opening the SD card on my laptop and opening “test.txt” I saw the problem.

By putting “myFile.write(127);” I wasn’t deleting the actual characters at each cursor position, but REPLACING the original characters with DEL characters which on a Serial Monitor show up as nothing because DEL has no actual symbol so the numbers of original characters remained the same :frowning:

In hindsight this is a dumb mistake on my part. I should have realized this as I saw on the ASCII reference table I was using that DEL indeed had no symbol (aka nothing), but I thought this meant being replaced with nothing, hence deleting the text!!

SO, for anyone who might make the same mistake as me, I hope this saves you time and effort.

However, that being said . . . does anyone know a method on how to actually delete the first line in a .txt file on an SD card using the Spark Core? Workarounds less needed, but will considering of course!

As always any direction is very much appreciated :slight_smile:

Cheers,

UST

2 Likes

@UST, your idea of “deleting” a line in the file may be the issue. To actually “delete” a line, you have two choices. First, you work off the premise that items in your file are “lines” and lines can be tagged as deleted somehow. Their content is not removed but they are considered “deleted”. I don’t believe this is what you are looking for.

The second method is to create a pointer to the first char position in the line to be removed and a second pointer to the first char AFTER the line for the rest of the file. You then copy, using these pointers, the remaining content of the file over the line to be removed, effectively “deleting” the line. :smile:

Hey @peekay123 thanks for the reply!

I like the idea of your second method and I understand it at a fundamental level, but I’m not sure how I would execute it code wise?

Is there a way to do it without closing the original file at all and copying all the “keep lines” at once, then essential “paste” them back in the file at the top of the file, thus like you said deleting the line in question?
OR
Do I need to copy each line that is to be kept one at a time, put them in a String, close original file, put String into a temp file, then open original file and rinse and repeat? This method might take super long time, but I don’t think that is what you were getting at.

@UST, think of characters not lines. The only difference is a special character (\n or \r or both) defines the end of the line. I would keep away from strings if you expect to have a lot of lines. So, I assume your file is opened for READ & WRITE operations. Using the two pointer method:

Let’s call the “start of deleted line” pointer “OVERWRITE” and the “start of the keep lines” pointer “KEEP”

  1. Read the char from file at the KEEP pointer
  2. Write the char to file at the OVERWRITE pointer
  3. Repeat 1 & 2 this until you have read the last char in the file
  4. Set the OVERWRITE and KEEP pointers for the next line to delete and start over

If you are deleting a number of sequential lines then you can just set the KEEP pointer to the first char after the last line to delete.

This approach does not require that you close the file or open another. It is all done “in place”. :smile:

@peekay123 really appreciate the breakdown for dummies! Seriously.

So lets see if I got this, by trying to reiterate using an example! (AS I was typing I realized I was typing a full code, switched to Spark Dev then came back, so lets see if I got two pointer concept down!)

    myFile = SD.open("test.txt", FILE_WRITE | O_RDWR); 

char OVERWRITE = 'O';
char check = 0;
unsigned long replace = 0;
unsigned long read = 0;

myFile.seek(0); // go to beginning position of the text file

while(myFile.available()) Serial.write(myFile.read());

    /*                         // this is what is in test.txt:

    Testing 1,2,3.             // this is the line we want to delete,
                               //  so 'T' is where we want to delete
    Testing 4,5,6.             // the next two lines are our keep lines
                               // so 'T' is where we want to read
    Testing 7,8,9,10,11,12.    // both pointers are the same and cannot 
                               // be, so we replace 1st 't' with 'O'
    */

void  2pointermethod()
{
myFile.seek(0);            // go back to the beginning of the test.txt
myFile.write(OVERWRITE);   // does this move position from 0 to 1 ? if so 
                                           //then myFile.seek(0);
replace = myFile.position();  // establish our OVERWRITE pointer
while (myFile.available())
{
   check = myFile.peek();
   if (check == 'T')
   {
    read = myFile.position();  // establish our KEEP pointer  
    break;
   }
   myFile.read();
}

while (myFile.availble())  
{
  myFile.seek(read);
  check = myFile.read();
  myFile.seek(replace);
  myFile.write(check); // or should I do myFile.print(check); instead? 
  replace++; 
  read++;
}
}  

Hopefully I got the concept down, any tips to make it “cleaner” ? Or did I just miss the point completely :frowning: … hopefully not :stuck_out_tongue: !

Also just realized now I could just compile and test this dooh!

So here is the code I implemented:

#include "application.h"
#include "sd-card-library.h"

File myFile;

// SOFTWARE SPI pin configuration - modify as required
// The default pins are the same as HARDWARE SPI
const uint8_t chipSelect = A2;    // Also used for HARDWARE SPI setup
const uint8_t mosiPin = A5;
const uint8_t misoPin = A4;
const uint8_t clockPin = A3;

char OVERWRITE = 'O';
char check = 0;
unsigned long replace = 0;
unsigned long read = 0;
int i =0;

SYSTEM_MODE(SEMI_AUTOMATIC);

    void setup()
    {
      Serial.begin(115200);
      while (!Serial.available()) SPARK_WLAN_Loop();
    
      Serial.print("Initializing SD card...");
      // Initialize SOFTWARE SPI
      if (!SD.begin(mosiPin, misoPin, clockPin, chipSelect)) {
        Serial.println("initialization failed!");
        return;
    }
    
      Serial.println("initialization done.");
    
      myFile = SD.open("test.txt", FILE_WRITE | O_RDWR);
    
      myFile.seek(0); // go to beginning position of the text file
      if (myFile)
      {     // if the file opened okay, write to it:
        Serial.print("Writing to test.txt...");
        for (int i=0; i<1; i++)
        {
          myFile.println("Testing 1, 2, x.");
          myFile.println("Testing 1, 2, z.");
          myFile.println("Testing 1, 2, g.");
        }
        //myFile.close();            // close the file: closing the file is what actually saves the data on the SD
        Serial.println("done.");   // ^^ you can .close() to save or .flush() to save
      } else {
        Serial.println("error opening test.txt");       // if the file didn't open, print an error:
      }
    
    myFile.seek(0);
    
    while (myFile.available()) Serial.write(myFile.read());
    
    
    myFile.seek(0);     // go back to the beginning of the test.txt
    myFile.write(OVERWRITE);  // does this move position from 0 to 1 ? if so then myFile.seek(0);
    myFile.seek(0);
    replace = myFile.position();
    while (myFile.available())
    {
       check = myFile.peek();
       if (check == 'T')
       {
        read = myFile.position();
        break;
       }
       myFile.read();
    }
    
    while (myFile.available())
    {
      myFile.seek(read);
      check = myFile.read();
      myFile.seek(replace);
      myFile.write(check); // or should I do myFile.print(check); instead?
      replace++;
      read++;
    }
    
    myFile.seek(0);
    Serial.println();
    while(myFile.available()) Serial.write(myFile.read());
    myFile.close();
    SD.remove("test.txt");
    }
    
    void loop ()
    {
      // just loop
    }

If I am doing the 2-pointer method correctly, then you are right @peekay123 the first “line” does get “deleted” but the last “line” will always remain. So no matter what the file will be “3 lines long”, also a funny character appears in the first character of non-deleted set of characters :stuck_out_tongue:

Here are the results:

The funny character in the above picture:

▒ this translates to 226 150 146
which translates to: â–’

Thought I’d provide the info, just in case it somehow proved useful :slight_smile:

@UST, I forgot to mention that you need to write an EOF (end of file) after the copy operation, after the last “replace” position. This will get rid of the last redundant line. The weird character is really data beyond what should remain in the file. :smile:

So essentially I changed this:

    while (myFile.available())
{
  myFile.seek(read);
  check = myFile.read();
  myFile.seek(replace);
  myFile.write(check); // or should I do myFile.print(check); instead?
  replace++;
  read++;
}

To this:

    while (myFile.available())
{
  myFile.seek(read);
  check = myFile.read();
  if (myFile.peek() ==  -1)
  {
    Serial.print(check);
    myFile.seek(replace);
    myFile.write(EOF);
    //myFile.write(-1);
    break;
  }
  myFile.seek(replace);
  myFile.write(check); // or should I do myFile.print(check); instead?
  replace++;
  read++;
}

Below are my results, which seemed sort of promising. Although the \r seemed to put my “EOF” character at the front of the second “line”. So had a feeling that the last redundant “line” wasn’t actually being deleted but just not being read. Opening the text file it was sort of as I thought, but with the funny “y” character at the end of the “second line” and not at the front as PuTTY made me think.

So my # of “lines” essentially don’t change. I tried putting the “EOF” after the “\n” character in my “second line” but all I got on PuTTY was the redundant line with “T” replaced with the “EOF” fuzzy character. Any additional thoughts on the two pointer method @peekay123?

I’m going to see if I can try to poke around @BDub’s library and see if there is a method to manipulate the text file size,then use that in my sketch during this copy operation; so that maybe I can permanently delete the redundant line along with the fuzzy “EOF” character.

@UST, after doing a little more thinking on this I realize that adding EOF will not actually cut the filesize down since it does not change the FAT entry. So, to do this properly, you will need to use the truncate command with an adjusted file size:

myFile.truncate(uint32_t length);

You will need to calculate the file size minus the removed line. You could calculate that knowing the difference between the read and replace pointers and using size():

uint32_t  newSize = myFile.size() - (read - replace);
myFile.truncate(newSize);

That’s the idea, you just need to iron out the details. :smile:

BTW, are the lines all the same length (ie number of chars)?

1 Like

No sometimes the lines can be longer by one or two characters, which is by I use “\n” as my check cause they will always end with “\n”

But thanks a lot for the speedy reply, I’ll hammer this in as quickly as possible, then post full code back here once it’s working 100% :slight_smile:

1 Like

For the library I am using, it says:

"class File has no member named ‘truncate’ "

Should I be using a different SD library?

EDIT: I see it in the library I posted … under SdFat.h , I’ll see if there is a issue with the library I installed. seems to be installed correctly…

@UST, that IS the library I was looking at. I am not sure why you get an error. Can you show me your code?

Sorry for all the back and forth here’s the code:

#include "application.h"
#include "sd-card-library.h"

File myFile;

// SOFTWARE SPI pin configuration - modify as required
// The default pins are the same as HARDWARE SPI
const uint8_t chipSelect = A2;    // Also used for HARDWARE SPI setup
const uint8_t mosiPin = A5;
const uint8_t misoPin = A4;
const uint8_t clockPin = A3;

char OVERWRITE = 'O';
char check = 0;
unsigned long replace = 0;
unsigned long read = 0;
unsigned long replace2 = 0;
int i =0;

SYSTEM_MODE(SEMI_AUTOMATIC);

void setup()
{
  Serial.begin(115200);
  while (!Serial.available()) SPARK_WLAN_Loop();

  Serial.print("Initializing SD card...");
  // Initialize SOFTWARE SPI
  if (!SD.begin(mosiPin, misoPin, clockPin, chipSelect)) {
    Serial.println("initialization failed!");
    return;
}

  Serial.println("initialization done.");

  myFile = SD.open("test.txt", FILE_WRITE | O_RDWR);

  myFile.seek(0); // go to beginning position of the text file
  if (myFile)
  {     // if the file opened okay, write to it:
    Serial.print("Writing to test.txt...");
    for (int i=0; i<1; i++)
    {
      myFile.println("Testing 1, 2, x.");
      myFile.println("Testing 1, 2, z.");
      myFile.println("Testing 1, 2, g.");
    }
    //myFile.close();            // close the file: closing the file is what actually saves the data on the SD
    Serial.println("done.");   // ^^ you can .close() to save or .flush() to save
  } else {
    Serial.println("error opening test.txt");       // if the file didn't open, print an error:
  }

myFile.seek(0);

while (myFile.available()) Serial.write(myFile.read());


myFile.seek(0);     // go back to the beginning of the test.txt
myFile.write(OVERWRITE);  // does this move position from 0 to 1 ? if so then myFile.seek(0);
myFile.seek(0);
replace = myFile.position();
while (myFile.available())
{
   check = myFile.peek();
   if (check == 'T')
   {
    read = myFile.position();
    break;
   }
   myFile.read();
}


myFile.seek(read);
while (myFile.available())
{
  check = myFile.read();
  myFile.seek(replace);
  myFile.write(check); // or should I do myFile.print(check); instead?
  replace++;
  read++;
  myFile.seek(read);
  if (myFile.peek() == -1)
  {
    uint32_t newSize = myFile.size() - (read - replace);
    myFile.truncate(newSize);
    break;
  }
}


myFile.seek(0);
Serial.println();
while(myFile.available()) Serial.write(myFile.read());
myFile.close();
//SD.remove("test.txt");
}

void loop ()
{
  // just loop doe
}

EDIT: I Guess I should mention that I am using this in Spark Dev , if that makes any difference :frowning:

@UST, truncate is part of the class SDfile, not File. I will need to think about how to set this up.

1 Like

I saw that too, looking into it as well. Appreciate all the help!

EDIT: Looking into the SdFile class, it seems like I need to create a directory and then a file (or directory file) to put into that directory, had to do something else today, but still going to be plugging away at this.

EDIT: Can I use the SdFile class the same as the File class or are they fundamentally different ? Any ideas how to set this up @peekay123? :slight_smile:

Going to be tackling this today for most of the day, I’ll post once I find a solution.

But if anyone has any ideas just throw them here or PM me! :slight_smile: Thanks !

Alright,

So I made more headway than I thought I would, but now I reached a new road block. But I feel that I’m getting close.

The SdFile class has a “read” function in this form:

 * \param[out] buf Pointer to the location that will receive the data.
 * \param[in] nbyte Maximum number of bytes to read.

    int16_t SdFile::read(void* buf, uint16_t nbyte)

Now when I have this code:

while (x == 0) 
{
   char check = file.read();    //file is my SdFile object/instance
   Serial.print(check);           // I've used Serial.write as well
   if (check == EOF) break;
}

I am able to successfully read the “file” and everything in it, however, it just keeps reading. Doesn’t ever stop and the characters it prints don’t seem to be EOF (although they might be I’m not sure) the characters are fuzzy, but when copy and pasted into a word doc they are boxes with tiny “?” inside them.

So then I tried going back to the read function from the SdFile class and tried formatting it similar to that:

unsigned int size = sizeof(file);  // file is my SdFile object/instance
                                   // -- size is apparently = 40
void* bufferz[size];               //void is what threw me off the                       
              //  most, tried making it many other data types, didn't work. 
Serial.write(file.read(bufferz, size));

What comes out on the Serial Terminal (PuTTY) is “40”.

So two questions,

1.) SdFile class doesn’t have the “available” function like File class does, which makes it hard to know when I’m about to reach the end of the file. So how would I know that I’m reaching the end of my file when reading “file” similar to the way I read it in my code pasting?

2.) I know I must be doing somethign wrong in the way I’m using the read function from the SdFile class, what is it?

As always an insight or direction is always appreciated :smile:

Here is full working code, on how to delete a line using @peekay123’s 2-pointer method and using both the File and SdFile class from @BDub’s SD Card Library:

#include "application.h"
#include "sd-card-library.h"

// SOFTWARE SPI pin configuration - modify as required
// The default pins are the same as HARDWARE SPI
const uint8_t chipSelect = A2;    // Also used for HARDWARE SPI setup
const uint8_t mosiPin = A5;
const uint8_t misoPin = A4;
const uint8_t clockPin = A3;

char OVERWRITE = 'O';
uint32_t newSize = 0;
char check = 0;
unsigned long replace = 0;
unsigned long read = 0;

SdFile root;
Sd2Card card;
SdVolume volume;
SdFile file;
File myFile;

SYSTEM_MODE(SEMI_AUTOMATIC);

void setup()
{
  Serial.begin(115200);
  while (!Serial.available()) SPARK_WLAN_Loop();
  Serial.print("Initializing SD card...");
  // Initialize SOFTWARE SPI
  if (!card.init(mosiPin, misoPin, clockPin, chipSelect))
  {
    Serial.println("initialization failed! 1");
    return;
  }
  if (!SD.begin(mosiPin, misoPin, clockPin, chipSelect))
  {
    Serial.println("initialization failed! 2");
    return;
  }
  Serial.println("initialization done.");
  if (!volume.init(card))
  {
    Serial.println("Could not find FAT16/FAT32 partition.\nMake sure you've formatted the card");
    return;
  }

myFile = SD.open("delete.txt", O_RDWR);

Serial.print("myFile position is: "); Serial.println(myFile.position());
myFile.seek(0);
Serial.print("myFile position after seek is: "); Serial.println(myFile.position());

if (SD.exists("delete.txt"))
{
  Serial.println("Created file as File");
}

myFile.write("Testing 1, delete.\r\n");
myFile.write("Testing 2, delete.\r\n");
myFile.write("Testing 3, delete.\r\n");
myFile.write("Testing 4, delete.\r\n");
myFile.write("Testing 5, delete.\r\n");
myFile.write("Testing 1, 2, 3.\r\n");
myFile.write("Testing 4, 5, 6.\r\n");
myFile.write("Testing 7, 8, 9.\r\n");
myFile.write("Testing 10, 11, 12.\r\n");
myFile.write("Testing 13, 14, 15.\r\n");

myFile.seek(0);

while(myFile.available())
{
  check = myFile.read();
  Serial.write(check);
}

Serial.print("myFile position after read is: "); Serial.println(myFile.position());
myFile.seek(0);
myFile.write(OVERWRITE);  
myFile.seek(0);
replace = myFile.position();
while (myFile.available())
{
   check = myFile.peek();
   if (check == 'T')
   {
    read = myFile.position();
    break;
   }
   myFile.read();
}

myFile.seek(read);
while (myFile.available())
{
  check = myFile.read();
  myFile.seek(replace);
  myFile.write(check); 
  replace++;
  read++;
  myFile.seek(read);
  if (myFile.peek() == -1)
  {
    newSize = myFile.size() - (read - replace);
    break;
  }
}

myFile.close();

root.openRoot(volume);
boolean exists = file.open(root, "delete.txt", O_CREAT | O_WRITE | O_RDWR);
//file.seekSet(0);

if (exists)
{
Serial.println("Opened file as SdFile");
}

file.truncate(newSize);
file.close();

myFile = SD.open("delete.txt");
while(myFile.available())
{
  check = myFile.read();
  Serial.write(check);
}
Serial.println();
myFile.close();


root.ls(LS_R | LS_DATE | LS_SIZE);

}

void loop ()
{
  // just loop 
}

I used a lot of print statements for my own debugging purposes, those can be taken out obviously. Other than that, I found I needed everything else in order for it to work properly and consistently. I have it set up to just delete the first line of the text file, it works. Thanks again @peekay123 for your earlier help, if anyone has any suggestions let me know :slight_smile:

3 Likes

@UST, great work!! I am glad you figured this out for yourself since now you own that knowledge :stuck_out_tongue:

1 Like