tests(smoke): fix devtools test runner & a11y errors

cli/test/smokehouse/test-definitions/a11y.js

@@ -419,7 +419,7 @@ const expectations = {
                 'type': 'node',
                 'selector': 'body > section > button#button-name',
                 'snippet': '<button id="button-name">',
-                'explanation': 'Fix any of the following:\n  Element does not have inner text that is visible to screen readers\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
+                'explanation': 'Fix any of the following:\n  Element does not have inner text that is visible to screen readers\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element does not have an implicit (wrapped) <label>\n  Element does not have an explicit <label>\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
                 'nodeLabel': 'body > section > button#button-name',
               },
             },
@@ -688,7 +688,7 @@ const expectations = {
                 'type': 'node',
                 'selector': 'body > section > form > input#input-button-name',
                 'snippet': '<input type="button" id="input-button-name">',
-                'explanation': 'Fix any of the following:\n  Element has a value attribute and the value attribute is empty\n  Element has no value attribute\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
+                'explanation': 'Fix any of the following:\n  Element has a value attribute and the value attribute is empty\n  Element has no value attribute\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element does not have an implicit (wrapped) <label>\n  Element does not have an explicit <label>\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
                 'nodeLabel': 'body > section > form > input#input-button-name',
               },
             },
@@ -704,7 +704,7 @@ const expectations = {
                 'type': 'node',
                 'selector': 'body > section > input#input-image-alt',
                 'snippet': '<input type="image" id="input-image-alt">',
-                'explanation': 'Fix any of the following:\n  Element has no alt attribute\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute',
+                'explanation': 'Fix any of the following:\n  Element has no alt attribute\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element does not have an implicit (wrapped) <label>\n  Element does not have an explicit <label>',
                 'nodeLabel': 'body > section > input#input-image-alt',
               },
             },
@@ -866,7 +866,7 @@ const expectations = {
                 'type': 'node',
                 'selector': 'body > section > select#select-name',
                 'snippet': '<select id="select-name">',
-                'explanation': 'Fix any of the following:\n  Form element does not have an implicit (wrapped) <label>\n  Form element does not have an explicit <label>\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
+                'explanation': 'Fix any of the following:\n  Element does not have an implicit (wrapped) <label>\n  Element does not have an explicit <label>\n  aria-label attribute does not exist or is empty\n  aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n  Element has no title attribute\n  Element\'s default semantics were not overridden with role="none" or role="presentation"',
                 'nodeLabel': 'body > section > select#select-name',
               },
             },

core/scripts/pptr-run-devtools.js

@@ -100,12 +100,22 @@ async function evaluateInSession(session, fn, deps) {
  * @return {Promise<R>}
  */
 async function waitForFunction(session, fn, deps) {
+  let iterations = 0;
   // eslint-disable-next-line no-constant-condition
   while (true) {
     try {
       return await evaluateInSession(session, fn, deps);
-    } catch {
+    } catch (err) {
+      // Random transient errors are common when first booting up.
+      // Only surface errors if this fails 10 times in a row (~5s)
+      if (iterations > 10) {
+        console.error(`Error waiting for function (#${iterations}):`);
+        console.error(err);
+        console.error('Retrying...');
+      }
       await new Promise(r => setTimeout(r, 500));
+    } finally {
+      ++iterations;
     }
   }
 }
@@ -161,7 +171,7 @@ async function waitForLighthouseReady() {
 
   const panel = LighthousePanel.LighthousePanel.instance();
 
-  const button = panel.contentElement.querySelector('devtools-button,button');
+  const button = panel.contentElement.querySelector('.vbox.flex-auto').shadowRoot.querySelector('devtools-button,button');
   if (button.disabled) throw new Error('Start button disabled');
 
   const targetManager = TargetManager.TargetManager.instance();
@@ -213,7 +223,7 @@ async function runLighthouse() {
   // In CI clicking the start button just once is flaky and can cause a timeout.
   // Therefore, keep clicking the button until we detect that the run started.
   const intervalHandle = setInterval(() => {
-    const button = panel.contentElement.querySelector('devtools-button,button');
+    const button = panel.contentElement.querySelector('.vbox.flex-auto').shadowRoot.querySelector('devtools-button,button');
     button.click();
   }, 100);
 

package.json

@@ -183,7 +183,7 @@
   "dependencies": {
     "@paulirish/trace_engine": "0.0.32",
     "@sentry/node": "^7.0.0",
-    "axe-core": "^4.10.0",
+    "axe-core": "^4.10.2",
     "chrome-launcher": "^1.1.2",
     "configstore": "^5.0.1",
     "csp_evaluator": "1.1.1",

yarn.lock

@@ -1964,10 +1964,10 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
   integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==
 
-axe-core@^4.10.0:
-  version "4.10.0"
-  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.10.0.tgz#d9e56ab0147278272739a000880196cdfe113b59"
-  integrity sha512-Mr2ZakwQ7XUAjp7pAwQWRhhK8mQQ6JAaNWSjmjxil0R8BPioMtQsTLOolGYkji1rcL++3dCqZA3zWqpT+9Ew6g==
+axe-core@^4.10.2:
+  version "4.10.2"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.10.2.tgz#85228e3e1d8b8532a27659b332e39b7fa0e022df"
+  integrity sha512-RE3mdQ7P3FRSe7eqCWoeQ/Z9QXrtniSjp1wUjt5nRC3WIpz5rSCve6o3fsZ2aCpJtrZjSZgjwXAoTO5k4tEI0w==
 
 b4a@^1.6.4:
   version "1.6.4"