Using structured data and CBOR to send bigger webhooks using Device OS 6.2.1 on Boron

Firmware Device OS
1

My webhooks have been getting cut off lately as I have grown the size of one of my webhooks to be larger than the 1024 byte limit typical for Boron platform.

I have since read about the CBOR encoding scheme used in Particle OS version 6.2.x and above, as described in the Structured Data section of the Particle OS reference.

image
image1579×1066 95.8 KB

I decided to try a unit test on Particle OS v6.2.1 on my Boron to see if I could send a 1200+ byte JSON payload, now that CBOR "automatically happens in the background" in 6.2.x and above.

Note that I am using my own JSON encoding functions that construct a buffer that is valid JSON.

Here is my test app:

unsigned int webhook_unit_test_oversized_webhook()
{
    waitFor(Particle.connected, 60000);
    static bool did_oversized_webhook = false;
    if(did_oversized_webhook){
        // myLog.info("Oversized webhook unit test already done, skipping...");
        return 0;
    }
    did_oversized_webhook = true;
    myLog.warn("testing oversized webhook...");
    startJSON();
    addLinkWebhookKeysCommon();
    //Keep adding dummy keys until the JSON buffer contains more than 1200 bytes, then close the JSON object and try to send it
    unsigned int key_count = 0;
    while(lengthJSON() < 1200){
        addkey(String::format("key%u", key_count).c_str());
        addvalue(String::format("value%u", key_count).c_str());
        key_count++;
    };
    endJSON();
    myLog.info("Oversize JSON webhook created with %u keys, length = %u bytes", key_count, lengthJSON());
    
    myLog.print(jHook);
    myLog.print("\n");
    myLog.info("Testing direct publish of oversized JSON webhook...");

    //Publish the webhook using Particle OS  directly
    Particle.publish("oversize_json_test_direct", jHook, PRIVATE | WITH_ACK);

    return true;

}

And here is the output:

0000017834 [system] INFO: Cloud connected
0000017834 [comm.coap] TRACE: Sending CoAP message
0000017835 [comm.coap] TRACE: CON 0.00  size=4 token= id=1604
0000017835 [app.webhooks] WARN: testing oversized webhook...
0000017855 [app.webhooks] INFO: Oversize JSON webhook created with 67 keys, length = 1215 bytes
{"tm":1749646737,"pid":7870,"key0":"value0","key1":"value1","key2":"value2","key3":"value3","key4":"value4","key5":"value5","key6":"value6","key7":"value7","key8":"value8","key9":"value9","key10":"value10","key11":"value11","key12":"value12","key13":"value13","key14":"value14","key15":"value15","key16":"value16","key17":"value17","key18":"value18","key19":"value19","key20":"value20","key21":"value21","key22":"value22","key23":"value23","key24":"value24","key25":"value25","key26":"value26","key27":"value27","key28":"value28","key29":"value29","key30":"value30","key31":"value31","key32":"value32","key33":"value33","key34":"value34","key35":"value35","key36":"value36","key37":"value37","key38":"value38","key39":"value39","key40":"value40","key41":"value41","key42":"value42","key43":"value43","key44":"value44","key45":"value45","key46":"value46","key47":"value47","key48":"value48","key49":"value49","key50":"value50","key51":"value51","key52":"value52","key53":"value53","key54":"value54","key55":"value55","key56":"value56","key57":"value57","key58":"value58","key59":"value59","key60":"value60","key61":"value61","key62":"value62","key63":"value63","key64":"value64","key65":"value65","key66":"value66"}
0000017865 [app.webhooks] INFO: Testing direct publish of oversized JSON webhook...
0000017867 [comm.coap] TRACE: Sending CoAP message
0000017868 [comm.coap] TRACE: CON POST /E/oversize_json_test_direct size=1058 token= id=1605
0000017948 [comm.coap] TRACE: Received CoAP message
0000017949 [comm.coap] TRACE: ACK 0.00  size=4 token= id=1604
0000018018 [comm.coap] TRACE: Received CoAP message
0000018019 [comm.coap] TRACE: ACK 0.00  size=4 token= id=1605
0000018397 [comm.coap] TRACE: Received CoAP message
0000018397 [comm.coap] TRACE: CON POST /E/particle/device/updates/pending size=47 token=01 id=7425
0000018398 [comm.coap] TRACE: Sending CoAP message
0000018399 [comm.coap] TRACE: ACK 0.00  size=4 token= id=7425

The JSON reported in the log is:

{"tm":1749646737,"pid":7870,"key0":"value0","key1":"value1","key2":"value2","key3":"value3","key4":"value4","key5":"value5","key6":"value6","key7":"value7","key8":"value8","key9":"value9","key10":"value10","key11":"value11","key12":"value12","key13":"value13","key14":"value14","key15":"value15","key16":"value16","key17":"value17","key18":"value18","key19":"value19","key20":"value20","key21":"value21","key22":"value22","key23":"value23","key24":"value24","key25":"value25","key26":"value26","key27":"value27","key28":"value28","key29":"value29","key30":"value30","key31":"value31","key32":"value32","key33":"value33","key34":"value34","key35":"value35","key36":"value36","key37":"value37","key38":"value38","key39":"value39","key40":"value40","key41":"value41","key42":"value42","key43":"value43","key44":"value44","key45":"value45","key46":"value46","key47":"value47","key48":"value48","key49":"value49","key50":"value50","key51":"value51","key52":"value52","key53":"value53","key54":"value54","key55":"value55","key56":"value56","key57":"value57","key58":"value58","key59":"value59","key60":"value60","key61":"value61","key62":"value62","key63":"value63","key64":"value64","key65":"value65","key66":"value66"}

However the value that I get on the event log on the particle dashboard is cutoff at 1024 bytes:

{"tm":1749645020,"pid":7870,"key0":"value0","key1":"value1","key2":"value2","key3":"value3","key4":"value4","key5":"value5","key6":"value6","key7":"value7","key8":"value8","key9":"value9","key10":"value10","key11":"value11","key12":"value12","key13":"value13","key14":"value14","key15":"value15","key16":"value16","key17":"value17","key18":"value18","key19":"value19","key20":"value20","key21":"value21","key22":"value22","key23":"value23","key24":"value24","key25":"value25","key26":"value26","key27":"value27","key28":"value28","key29":"value29","key30":"value30","key31":"value31","key32":"value32","key33":"value33","key34":"value34","key35":"value35","key36":"value36","key37":"value37","key38":"value38","key39":"value39","key40":"value40","key41":"value41","key42":"value42","key43":"value43","key44":"value44","key45":"value45","key46":"value46","key47":"value47","key48":"value48","key49":"value49","key50":"value50","key51":"value51","key52":"value52","key53":"value53","key54":"value54","key55":"value55","key56":

Here is a screenshot from the device's particle dashboard page:

image
image588×423 18.1 KB

According to the JSON Tools particle page,
my 1214 byte JSON payload should be converted to 935 bytes of CBOR (which is less than the 1024 byte limit).

Here is a screenshot:

image
image1628×874 87 KB

And here is the converted CBOR:

b84562746d1a68497d9163706964191ebe646b6579306676616c756530646b6579316676616c756531646b6579326676616c756532646b6579336676616c756533646b6579346676616c756534646b6579356676616c756535646b6579366676616c756536646b6579376676616c756537646b6579386676616c756538646b6579396676616c756539656b657931306776616c75653130656b657931316776616c75653131656b657931326776616c75653132656b657931336776616c75653133656b657931346776616c75653134656b657931356776616c75653135656b657931366776616c75653136656b657931376776616c75653137656b657931386776616c75653138656b657931396776616c75653139656b657932306776616c75653230656b657932316776616c75653231656b657932326776616c75653232656b657932336776616c75653233656b657932346776616c75653234656b657932356776616c75653235656b657932366776616c75653236656b657932376776616c75653237656b657932386776616c75653238656b657932396776616c75653239656b657933306776616c75653330656b657933316776616c75653331656b657933326776616c75653332656b657933336776616c75653333656b657933346776616c75653334656b657933356776616c75653335656b657933366776616c75653336656b657933376776616c75653337656b657933386776616c75653338656b657933396776616c75653339656b657934306776616c75653430656b657934316776616c75653431656b657934326776616c75653432656b657934336776616c75653433656b657934346776616c75653434656b657934356776616c75653435656b657934366776616c75653436656b657934376776616c75653437656b657934386776616c75653438656b657934396776616c75653439656b657935306776616c75653530656b657935316776616c75653531656b657935326776616c75653532656b657935336776616c75653533656b657935346776616c75653534656b657935356776616c75653535656b657935366776616c75653536656b657935376776616c75653537656b657935386776616c75653538656b657935396776616c75653539656b657936306776616c75653630656b657936316776616c75653631656b657936326776616c75653632656b657936336776616c75653633656b657936346776616c75653634656b657936356776616c75653635656b657936366776616c75653636

Can anyone help me to get by 1214 byte JSON payload to be successfully sent using device OS 6.2.1 on a Boron?

Is there some sort of special syntax to use when calling Particle.publish() in order to "activate" the automatic CBOR conversion?

@rickkas7 this seems like it may be in your wheelhouse?

2

It only works if you use structured data using Variant and use the CloudEvent version of Particle.publish.

It won't convert a pre-built JSON object passed to the overload of Particle.publish that takes a string.

And once you've converted to Variant and CloudEvent, the maximum event size will be 16 Kbytes as well.

3

OK thanks.

One problem however: it appears that the CloudEvent API is only available in Device OS 6.3.x and higher.

So that means, the documentation is misleading then when it says:

Furthermore, the 1024 byte publish limit in 6.2.x and earlier occurs after the conversion to CBOR, so you can publish significantly larger JSON data than before.

It seems to be there should be no references to OS version 6.2.x if that line of Particle OS is not able to publish beyond its 1024 limit.

image
image1579×1066 95.8 KB

4

I think 6.2.x does support structured data with the overload of Particle.publish that takes an EventData parameter, which is a Variant but you only get the CBOR size reduction. You need to use 6.3.x to get 16K payload.

5

Right you are!

The following is able to send a 1205 byte JSON string:

#if SYSTEM_VERSION >= SYSTEM_VERSION_DEFAULT(6,2,0)
unsigned int webhook_unit_test_oversized_webhook_CBOR_621()
{
    waitFor(Particle.connected, 60000);
    myLog.info("webhook_unit_test_oversized_webhook_CBOR_621()");

    particle::Variant obj;
    while(obj.toJSON().length() < 1200) {
        // Add dummy keys until the JSON buffer contains more than 1200 bytes
        obj.set(String::format("key%u", obj.size()).c_str(), String::format("value%u", obj.size()).c_str());
    }
    myLog.info("Oversize JSON object created with %u keys, length = %u bytes", obj.size(), obj.toJSON().length());

    myLog.info("publishing %s", obj.toJSON().c_str());

    EventData ev = obj;
    Particle.publish("test-cbor_publish", ev);

    // Wait while sending
    return 1;

}
#endif

And here is what is received on the Particle Dashboard:

{"key0":"value0","key1":"value1","key10":"value10","key11":"value11","key12":"value12","key13":"value13","key14":"value14","key15":"value15","key16":"value16","key17":"value17","key18":"value18","key19":"value19","key2":"value2","key20":"value20","key21":"value21","key22":"value22","key23":"value23","key24":"value24","key25":"value25","key26":"value26","key27":"value27","key28":"value28","key29":"value29","key3":"value3","key30":"value30","key31":"value31","key32":"value32","key33":"value33","key34":"value34","key35":"value35","key36":"value36","key37":"value37","key38":"value38","key39":"value39","key4":"value4","key40":"value40","key41":"value41","key42":"value42","key43":"value43","key44":"value44","key45":"value45","key46":"value46","key47":"value47","key48":"value48","key49":"value49","key5":"value5","key50":"value50","key51":"value51","key52":"value52","key53":"value53","key54":"value54","key55":"value55","key56":"value56","key57":"value57","key58":"value58","key59":"value59","key6":"value6","key60":"value60","key61":"value61","key62":"value62","key63":"value63","key64":"value64","key65":"value65","key66":"value66","key67":"value67","key7":"value7","key8":"value8","key9":"value9"}
1 Like