TCPClient missing bytes

Troubleshooting
1

So I’m trying to get information from the openweathermap api (in XML), and I seem to be missing information in the reply. I’ve tried introducing longer delays, and more checks to make sure the client data is available, and nothing seems to be working. Does anyone have any ideas? Here’s the XML that it’s retrieving. Here’s the basic code I’m using:

int noCharCount = 0;


const char* server="api.openweathermap.org";
const char* url="/data/2.5/weather?lat=38&lon=-74.43&mode=xml&units=imperial";

// Start ethernet client
TCPClient client;
int bytesAvail=0;
int bytesPrevAvail=-1;


void setup()
{
    pinMode(D7,OUTPUT);         // Turn on the D7 led so we know it's time
    digitalWrite(D7,HIGH);      // to open the Serial Terminal.
    Serial.begin(9600);
    while(!Serial.available())  // Wait here until the user presses ENTER 
        SPARK_WLAN_Loop(); 
    Serial.println("connecting...");
    digitalWrite(D7,LOW);
    Serial.println(server);
    
    
    Serial.println("Version 5");
    Serial.println(url);
    
    if (sendGetRequest(server,url)) {
        Serial.println("connected");
        
    } else {
        Serial.println("connection failed");
    }  
}

void loop() {
    bytesAvail=client.available();
    while ((bytesAvail != bytesPrevAvail) || (bytesAvail<=0)) {
        Serial.println(bytesAvail);
        bytesPrevAvail=bytesAvail;
        delay(1000);
        bytesAvail=client.available();
    }
    Serial.print("bytes:");
    Serial.println(bytesAvail);
    //Serial.println(client.available());
    //delay(1000);
    // Read serial data in from web:
    while (client.connected()) {
        while (client.available()) {
            Serial.print((char) client.read()); 
            noCharCount = 0;
        }
        delay(10);
        noCharCount++;
        if(noCharCount > 3000)
        {
            Serial.println();
            Serial.println("Timeout");
            client.stop();
        }
    }
    bytesAvail=0;
    bytesPrevAvail=-1;
    if (!client.connected()) {
        //Serial.println();
        Serial.println("Disconnected");
        client.stop();
        
        // Time until next update
        Serial.println("Waiting");
        for (int t = 1; t <= 15; t++) {
             delay( 1000); 
        }
        
        if (sendGetRequest(server,url)) {
            Serial.println("Reconnected");
           // delay(2000);
        } else {
            Serial.println("Reconnect failed");
        }       
    }
}



bool sendGetRequest(const char * server, const char * url)
{
    if (client.connect(server, 80)) {
        client.print("GET ");
        client.print(url);
        client.println(" HTTP/1.1");
        //client.println("Connection: close");
        client.print("Host: ");
        client.println(server);
        client.println("User-Agent: arduino-ethernet");
        client.println("Accept: application/xml, text/xml, */*");
        client.println("Connection: close");
      // 
        client.println();
        //client.flush();
        return true;
    } 
    else return false;
}

And here’s the output that I’m getting over serial (minus the debugging stuff)

e value="59.3" min="59.3" max="59.3" unit="fahrenheit"/>
  <humidity value="93" unit="%"/>
  <pressure value="1024.92" unit="hPa"/>
  <wind>
    <speed value="6.5" name="Moderate breeze"/>
    <direction value="192.51" code="SSW" name="South-southwest"/>
  </wind>
  <clouds value="44" name="scattered clouds"/>
  <precipitation mode="no"/>
  <weather number="802" value="scattered clouds" icon="03d"/>
  <lastupdate value="2014-05-22T15:06:32"/>
</current>

Thanks for any help!

2

@mumblepins, by default the tcpclient buffer is 512 bytes and the xml response is 632 bytes total. You need to receive the data in two chunks into a larger array that will fit the entire response (if necessary). :smile:

BTW, did you see the OpenWeather library created by @Coffee? It uses JSON but it works very well.

2 Likes
3

I didn’t. Thanks for the info and the link!

1 Like
4

Huh, I’m still having the problem. I went with a simpler example (basically what’s on the doc page), and I’m still missing part of the header.

Code:

TCPClient client;

void setup()
{
  Serial.begin(9600);
  while (!Serial.available()) SPARK_WLAN_Loop();
  Serial.println("connecting...");

  if (client.connect("www.google.com", 80))
  {
    Serial.println("connected");
    client.println("GET /search?q=unicorn HTTP/1.0");
    client.println("Host: www.google.com");
    client.println();
  }
  else
  {
    Serial.println("connection failed");
  }
}

void loop()
{
  if (client.available())
  {
    char c = client.read();
    Serial.print(c);
  }

  if (!client.connected())
  {
    Serial.println();
    Serial.println("disconnecting.");
    client.stop();
    while (true) SPARK_WLAN_Loop();
  }
  
}

First section of output over serial:

connecting...
connected
le.com; HttpOnly
P3P: CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info."
Server: gws
X-XSS-Protection: 1; mode=block
X-Frame-Options: SAMEORIGIN
Alternate-Protocol: 80:quic

<!doctype html><html itemscope="" itemtype="http://schema.org/WebPage" lang="en"><head><meta content="text/html; charset=UTF-8" http-equiv="Content-Type"><meta content="/images/google_favicon_128.png" itemprop="image">

If you’ll notice the first part of the HTTP header seems to be missing. I’m using the web IDE if that’s helpful. Thanks!

5

OK, I think I see what is happening here: you need an Accept header too. Running this query with curl, I can see that Google.com is doing a chunked transfer–you want a plain text transfer. The 512 byte nature is not coming from the core per se but from the web part.

Try adding this after the Host: header:

        client.println("Accept: text/html, text/plain");

By the way, here’s the full function I am using:

void sendGetRequest(const char * server, const char * url)
{
    if (myTCP.connect(server, 80)) {
        //Serial1.print("Connected to Server");
        digitalWrite(D7,HIGH);
        myTCP.print("GET ");
        myTCP.print(url);
        myTCP.println(" HTTP/1.1");
        myTCP.println("Connection: close");
        myTCP.print("Host: ");
        myTCP.println(server);
        myTCP.println("Accept: text/html, text/plain");
        myTCP.println();
        myTCP.flush();
    } 
}
6

@bko Unfortunately, that still doesn’t seem to do the trick. I’ve also tried your function, and I still am missing part of the header. I feel like I’m missing something stupid, so sorry for all of the questions, and thanks for helping a newbie out!