From cee29ea0c379d936a583eabfc1b29d9713c26c5a Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Fri, 16 Aug 2024 13:57:40 +0200 Subject: [PATCH 1/7] feat(utils): add bloat control utility to ensure JSON strings don't get too long --- src/utils.mjs | 35 +++++++++++++++++++++++++++++++++++ test/utils.test.mjs | 23 +++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/utils.mjs b/src/utils.mjs index 1ba1cec..d74359b 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -276,3 +276,38 @@ export function getSubsystem(req) { } return 'undefined'; } + +/** + * Cloudflare applies a 16kb limit to all log messages, so we need to make sure + * that the JSON we send to the logger is not too large. This function will + * apply a couple of tricks to make this happen + * 1. limit the length of any string value to 1024 characters + * 2. cast LCP, INP, TTFB to integers + * @param {Object} obj an object to be logged + * @returns {String} the sanitized object as a JSON string + */ +export function bloatControl(obj) { + // the object can have nested objects, so we may need recursion + const sanitize = (o, k) => { + if (typeof o === 'string' && o.length > 1024) { + return `${o.substring(0, 1024)}…`; + } + if (typeof o === 'number') { + if (['LCP', 'INP', 'TTFB'].includes(k)) { + return Math.floor(o); + } + return o; + } + if (Array.isArray(o)) { + return o.map(sanitize); + } + if (typeof o === 'object') { + return Object.entries(o).reduce((acc, [key, value]) => { + acc[key] = sanitize(value, key); + return acc; + }, {}); + } + return o; + }; + return JSON.stringify(sanitize(obj)); +} diff --git a/test/utils.test.mjs b/test/utils.test.mjs index 0ac10b7..0f0d4f2 100644 --- a/test/utils.test.mjs +++ b/test/utils.test.mjs @@ -20,9 +20,32 @@ import { getSubsystem, isValidId, maskTime, + bloatControl, } from '../src/utils.mjs'; describe('Test Utils', () => { + it('Bloat control', () => { + assert.equal('{}', bloatControl({})); + assert.equal('[]', bloatControl([])); + assert.equal('{"lorem":"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent ac molestie ex. Suspendisse a imperdiet enim. Nulla tempor, tellus volutpat dictum auctor, purus justo consequat felis, sit amet condimentum odio urna ornare dolor. Pellentesque eget ultrices libero. Suspendisse quis diam eu augue consectetur lobortis at et leo. Quisque efficitur sit amet sem id aliquet. In hac habitasse platea dictumst. Maecenas sed orci tincidunt, tempus diam lobortis, egestas nibh. Nulla arcu purus, fermentum vitae augue vel, sodales porttitor arcu. Quisque iaculis porttitor lectus, id rhoncus arcu sollicitudin lobortis.\\n\\nMauris massa leo, feugiat ac congue sit amet, auctor id arcu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vestibulum hendrerit urna sit amet quam accumsan consequat quis id diam. Morbi scelerisque a diam in mollis. Ut at convallis diam, a accumsan sem. Curabitur molestie sem nec orci mattis, ut convallis metus pellentesque. Morbi vitae erat felis. Curabitur purus n…"}', bloatControl({ + lorem: `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent ac molestie ex. Suspendisse a imperdiet enim. Nulla tempor, tellus volutpat dictum auctor, purus justo consequat felis, sit amet condimentum odio urna ornare dolor. Pellentesque eget ultrices libero. Suspendisse quis diam eu augue consectetur lobortis at et leo. Quisque efficitur sit amet sem id aliquet. In hac habitasse platea dictumst. Maecenas sed orci tincidunt, tempus diam lobortis, egestas nibh. Nulla arcu purus, fermentum vitae augue vel, sodales porttitor arcu. Quisque iaculis porttitor lectus, id rhoncus arcu sollicitudin lobortis. + +Mauris massa leo, feugiat ac congue sit amet, auctor id arcu. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vestibulum hendrerit urna sit amet quam accumsan consequat quis id diam. Morbi scelerisque a diam in mollis. Ut at convallis diam, a accumsan sem. Curabitur molestie sem nec orci mattis, ut convallis metus pellentesque. Morbi vitae erat felis. Curabitur purus nulla, tempus non arcu ut, convallis euismod justo. Etiam hendrerit risus ligula, a mattis libero placerat eu. Proin interdum elit quam. In sit amet dictum sapien. Duis tempor vulputate tellus. Nam fermentum ligula a nibh facilisis, et eleifend mi euismod. + +Pellentesque viverra id magna vel varius. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec porta quis mauris sit amet aliquet. Duis non nulla sed metus sagittis condimentum. Nam rhoncus, risus et gravida tempus, nibh diam pellentesque tellus, vel accumsan arcu nibh in lorem. Pellentesque eu semper ipsum, ac lacinia ante. Phasellus neque urna, laoreet eu purus id, interdum tristique turpis. Nulla pretium fermentum elit non tristique. Aliquam ut orci elit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed a pulvinar urna. Nunc vel augue ultrices, aliquet sapien eget, ornare sem. Etiam sagittis porttitor arcu vitae tristique. Praesent tempus neque ac vehicula viverra. Donec justo nibh, faucibus a rhoncus nec.`, + })); + + assert.equal('{"LCP":1234,"INP":2345,"CLS":0.789,"TTFB":1234}', bloatControl({ + LCP: 1234.5678, + INP: 2345.6789, + CLS: 0.7890, + TTFB: 1234.5678, + })); + + // what is neither a string nor a number nor an array nor an object gets stringified as is + assert.equal('true', bloatControl(true)); + }); + it('Mask the time', () => { const now = Date.now(); const nearestHour = Math.floor(now / 3600000) * 3600000; From b872d3bfce70ebaa2d1598217b26571c4a57e0c0 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Fri, 16 Aug 2024 14:01:48 +0200 Subject: [PATCH 2/7] feat(logger): control bloat of JSON logs for Cloudflare --- src/coralogix-logger.mjs | 3 ++- src/google-logger.mjs | 3 ++- src/s3-logger.mjs | 3 ++- test/index.test.mjs | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/coralogix-logger.mjs b/src/coralogix-logger.mjs index d69846a..75132c1 100644 --- a/src/coralogix-logger.mjs +++ b/src/coralogix-logger.mjs @@ -11,6 +11,7 @@ */ import { Logger } from './logger.mjs'; import { + bloatControl, cleanurl, getMaskedTime, getMaskedUserAgent, getSubsystem, isReasonableWeight, isValidCheckpoint, @@ -89,7 +90,7 @@ export class CoralogixLogger { }; console.log('ready to log (coralogix)'); // console.log(JSON.stringify(data)); - this.logger.log(JSON.stringify(data)); + this.logger.log(bloatControl(data)); console.log('done'); } } diff --git a/src/google-logger.mjs b/src/google-logger.mjs index 45fc017..b790d4b 100644 --- a/src/google-logger.mjs +++ b/src/google-logger.mjs @@ -11,6 +11,7 @@ */ import { Logger } from './logger.mjs'; import { + bloatControl, cleanurl, getMaskedTime, getMaskedUserAgent, @@ -69,6 +70,6 @@ export class GoogleLogger { url: cleanurl(data.url), }; - this.clusterlogger.log(JSON.stringify(clusterdata)); + this.clusterlogger.log(bloatControl(clusterdata)); } } diff --git a/src/s3-logger.mjs b/src/s3-logger.mjs index 899b07d..dc6a485 100644 --- a/src/s3-logger.mjs +++ b/src/s3-logger.mjs @@ -11,6 +11,7 @@ */ import { Logger } from './logger.mjs'; import { + bloatControl, cleanurl, getMaskedTime, getMaskedUserAgent, @@ -51,6 +52,6 @@ export class S3Logger { ...json, }; - this.logger.log(JSON.stringify(data)); + this.logger.log(bloatControl(data)); } } diff --git a/test/index.test.mjs b/test/index.test.mjs index 302f0fd..3acf67a 100644 --- a/test/index.test.mjs +++ b/test/index.test.mjs @@ -334,7 +334,7 @@ describe('Test index', () => { const logged = JSON.parse(lastLogMessage); assert.equal(0.06, logged.CLS); - assert.equal(1.1, logged.LCP); + assert.equal(1, logged.LCP); assert.equal(0.9, logged.FCP); assert.equal(800, logged.TTFB); }); From a426b70600bc71c95b383455140b2568b75f0dec Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Mon, 26 Aug 2024 12:53:44 +0200 Subject: [PATCH 3/7] fix(bloat): clip time back to integers --- src/utils.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.mjs b/src/utils.mjs index d74359b..876be52 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -293,7 +293,7 @@ export function bloatControl(obj) { return `${o.substring(0, 1024)}…`; } if (typeof o === 'number') { - if (['LCP', 'INP', 'TTFB'].includes(k)) { + if (['LCP', 'INP', 'TTFB', 'time'].includes(k)) { return Math.floor(o); } return o; From efc431a7c075f77be50cadaa4548b0e1b963af65 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Mon, 26 Aug 2024 13:16:19 +0200 Subject: [PATCH 4/7] chore(logging): log more error details for fastly --- src/index.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.mjs b/src/index.mjs index 78c2c23..ddc1091 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -34,7 +34,7 @@ function respondError(message, status, e, req) { status, headers, }); - console.error(msg); + console.error('Loggable error:', msg, e && e.stack); try { const c = new CoralogixErrorLogger(req); c.logError(status, message); From c6514ab24b07c2c5196b7e68818cc6fdf181bef5 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Mon, 26 Aug 2024 13:20:39 +0200 Subject: [PATCH 5/7] Revert "fix(bloat): clip time back to integers" This reverts commit a426b70600bc71c95b383455140b2568b75f0dec. --- src/utils.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.mjs b/src/utils.mjs index 876be52..d74359b 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -293,7 +293,7 @@ export function bloatControl(obj) { return `${o.substring(0, 1024)}…`; } if (typeof o === 'number') { - if (['LCP', 'INP', 'TTFB', 'time'].includes(k)) { + if (['LCP', 'INP', 'TTFB'].includes(k)) { return Math.floor(o); } return o; From dcbdfeb0ec31fe74bf6464db3ac477f85ae381ca Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Mon, 26 Aug 2024 13:26:40 +0200 Subject: [PATCH 6/7] test(node): exclude post deploy tests in ordinary test run --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8a5d1e1..7c96a0d 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "fastly-build": "hedy --build --verbose", "deploy": "hedy --build --verbose --deploy --fastly-auth $HLX_FASTLY_AUTH --compute-service-id ${HLX_FASTLY_SVC:-5Qir8H8bLeaRuldIp9TWq4} --cloudflare-email $HLX_CLOUDFLARE_EMAIL --cloudflare-account-id $HLX_CLOUDFLARE_ACCOUNT --cloudflare-auth $HLX_CLOUDFLARE_AUTH --name ${HLX_CLOUDFLARE_NAME:-helix-rum-collector-prod}", "predeploy": "node tools/spider-list.js", - "test": "node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml", + "test": "node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml --test-skip-pattern=Post-Deploy", "pretest": "node tools/spider-list.js", "test-postdeploy": "TEST_INTEGRATION=true node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml --test-name-pattern=Post-Deploy", "prelint": "node tools/spider-list.js", From 61b381bb7acc40a188ed5b50666a2419532d5fae Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Mon, 26 Aug 2024 13:33:00 +0200 Subject: [PATCH 7/7] fix(sanitize): guard against empty objects --- src/utils.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.mjs b/src/utils.mjs index d74359b..c362380 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -301,7 +301,7 @@ export function bloatControl(obj) { if (Array.isArray(o)) { return o.map(sanitize); } - if (typeof o === 'object') { + if (typeof o === 'object' && o !== null) { return Object.entries(o).reduce((acc, [key, value]) => { acc[key] = sanitize(value, key); return acc;