diff --git a/examples/web-navigation/mark_page.js b/examples/web-navigation/mark_page.js index 688c46b83..e42270b75 100644 --- a/examples/web-navigation/mark_page.js +++ b/examples/web-navigation/mark_page.js @@ -14,75 +14,94 @@ const customCSS = ` } `; -const styleTag = document.createElement('style'); +const styleTag = document.createElement("style"); styleTag.textContent = customCSS; document.head.append(styleTag); let labels = []; function unmarkPage() { - // Unmark page logic - for(const label of labels) { - document.body.removeChild(label); - } - labels = []; + // Unmark page logic + for (const label of labels) { + document.body.removeChild(label); + } + labels = []; } function markPage() { unmarkPage(); - + var bodyRect = document.body.getBoundingClientRect(); - var items = Array.prototype.slice.call( - document.querySelectorAll('*') - ).map(function(element) { - var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0); - var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0); - - var rects = [...element.getClientRects()].filter(bb => { - var center_x = bb.left + bb.width / 2; - var center_y = bb.top + bb.height / 2; - var elAtCenter = document.elementFromPoint(center_x, center_y); + var items = Array.prototype.slice + .call(document.querySelectorAll("*")) + .map(function (element) { + var vw = Math.max( + document.documentElement.clientWidth || 0, + window.innerWidth || 0 + ); + var vh = Math.max( + document.documentElement.clientHeight || 0, + window.innerHeight || 0 + ); + var textualContent = element.textContent.trim().replace(/\s{2,}/g, " "); + var elementType = element.tagName.toLowerCase(); + var ariaLabel = element.getAttribute("aria-label") || ""; + + var rects = [...element.getClientRects()] + .filter((bb) => { + var center_x = bb.left + bb.width / 2; + var center_y = bb.top + bb.height / 2; + var elAtCenter = document.elementFromPoint(center_x, center_y); + + return elAtCenter === element || element.contains(elAtCenter); + }) + .map((bb) => { + const rect = { + left: Math.max(0, bb.left), + top: Math.max(0, bb.top), + right: Math.min(vw, bb.right), + bottom: Math.min(vh, bb.bottom), + }; + return { + ...rect, + width: rect.right - rect.left, + height: rect.bottom - rect.top, + }; + }); + + var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0); - return elAtCenter === element || element.contains(elAtCenter) - }).map(bb => { - const rect = { - left: Math.max(0, bb.left), - top: Math.max(0, bb.top), - right: Math.min(vw, bb.right), - bottom: Math.min(vh, bb.bottom) - }; return { - ...rect, - width: rect.right - rect.left, - height: rect.bottom - rect.top - } - }); - - var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0); - - return { - element: element, - include: - (element.tagName === "INPUT" || element.tagName === "TEXTAREA" || element.tagName === "SELECT") || - (element.tagName === "BUTTON" || element.tagName === "A" || (element.onclick != null) || window.getComputedStyle(element).cursor == "pointer") || - (element.tagName === "IFRAME" || element.tagName === "VIDEO") - , - area, - rects, - text: element.textContent.trim().replace(/\s{2,}/g, ' ') - }; - }).filter(item => - item.include && (item.area >= 20) - ); + element: element, + include: + element.tagName === "INPUT" || + element.tagName === "TEXTAREA" || + element.tagName === "SELECT" || + element.tagName === "BUTTON" || + element.tagName === "A" || + element.onclick != null || + window.getComputedStyle(element).cursor == "pointer" || + element.tagName === "IFRAME" || + element.tagName === "VIDEO", + area, + rects, + text: textualContent, + type: elementType, + ariaLabel: ariaLabel, + }; + }) + .filter((item) => item.include && item.area >= 20); // Only keep inner clickable items - items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y))) + items = items.filter( + (x) => !items.some((y) => x.element.contains(y.element) && !(x == y)) + ); // Function to generate random colors function getRandomColor() { - var letters = '0123456789ABCDEF'; - var color = '#'; + var letters = "0123456789ABCDEF"; + var color = "#"; for (var i = 0; i < 6; i++) { color += letters[Math.floor(Math.random() * 16)]; } @@ -90,7 +109,7 @@ function markPage() { } // Lets create a floating border on top of these elements that will always be visible - items.forEach(function(item, index) { + items.forEach(function (item, index) { item.rects.forEach((bbox) => { newElement = document.createElement("div"); var borderColor = getRandomColor(); @@ -104,7 +123,7 @@ function markPage() { newElement.style.boxSizing = "border-box"; newElement.style.zIndex = 2147483647; // newElement.style.background = `${borderColor}80`; - + // Add floating label at the corner var label = document.createElement("span"); label.textContent = index; @@ -119,15 +138,20 @@ function markPage() { label.style.fontSize = "12px"; label.style.borderRadius = "2px"; newElement.appendChild(label); - + document.body.appendChild(newElement); labels.push(newElement); // item.element.setAttribute("-ai-label", label.textContent); }); }); - const coordinates = items.flatMap(item => item.rects.map(({ left, top, width, height }) => ({ - x: (left + left + width) / 2, - y: (top + top + height) / 2, - }))); + const coordinates = items.flatMap((item) => + item.rects.map(({ left, top, width, height }) => ({ + x: (left + left + width) / 2, + y: (top + top + height) / 2, + type: item.type, + text: item.text, + ariaLabel: item.ariaLabel, + })) + ); return coordinates; -} \ No newline at end of file +}