WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-17 21:25:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update annotation script

Browse Source
This commit is contained in:
William Fu-Hinthorn
2024-02-06 06:18:31 -08:00
parent 84ac02d1f0
commit 200403f8ed
1 changed files with 82 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/web-navigation/mark_page.js
+82 -58
View File
@@ -14,75 +14,94 @@ const customCSS = `
}
`;
const styleTag = document.createElement('style');
const styleTag = document.createElement("style");
styleTag.textContent = customCSS;
document.head.append(styleTag);
let labels = [];
function unmarkPage() {
// Unmark page logic
for(const label of labels) {
document.body.removeChild(label);
}
labels = [];
// Unmark page logic
for (const label of labels) {
document.body.removeChild(label);
}
labels = [];
}
function markPage() {
unmarkPage();
var bodyRect = document.body.getBoundingClientRect();
var items = Array.prototype.slice.call(
document.querySelectorAll('*')
).map(function(element) {
var vw = Math.max(document.documentElement.clientWidth || 0, window.innerWidth || 0);
var vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);
var rects = [...element.getClientRects()].filter(bb => {
var center_x = bb.left + bb.width / 2;
var center_y = bb.top + bb.height / 2;
var elAtCenter = document.elementFromPoint(center_x, center_y);
var items = Array.prototype.slice
.call(document.querySelectorAll("*"))
.map(function (element) {
var vw = Math.max(
document.documentElement.clientWidth || 0,
window.innerWidth || 0
);
var vh = Math.max(
document.documentElement.clientHeight || 0,
window.innerHeight || 0
);
var textualContent = element.textContent.trim().replace(/\s{2,}/g, " ");
var elementType = element.tagName.toLowerCase();
var ariaLabel = element.getAttribute("aria-label") || "";
var rects = [...element.getClientRects()]
.filter((bb) => {
var center_x = bb.left + bb.width / 2;
var center_y = bb.top + bb.height / 2;
var elAtCenter = document.elementFromPoint(center_x, center_y);
return elAtCenter === element || element.contains(elAtCenter);
})
.map((bb) => {
const rect = {
left: Math.max(0, bb.left),
top: Math.max(0, bb.top),
right: Math.min(vw, bb.right),
bottom: Math.min(vh, bb.bottom),
};
return {
...rect,
width: rect.right - rect.left,
height: rect.bottom - rect.top,
};
});
var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
return elAtCenter === element || element.contains(elAtCenter)
}).map(bb => {
const rect = {
left: Math.max(0, bb.left),
top: Math.max(0, bb.top),
right: Math.min(vw, bb.right),
bottom: Math.min(vh, bb.bottom)
};
return {
...rect,
width: rect.right - rect.left,
height: rect.bottom - rect.top
}
});
var area = rects.reduce((acc, rect) => acc + rect.width * rect.height, 0);
return {
element: element,
include:
(element.tagName === "INPUT" || element.tagName === "TEXTAREA" || element.tagName === "SELECT") ||
(element.tagName === "BUTTON" || element.tagName === "A" || (element.onclick != null) || window.getComputedStyle(element).cursor == "pointer") ||
(element.tagName === "IFRAME" || element.tagName === "VIDEO")
,
area,
rects,
text: element.textContent.trim().replace(/\s{2,}/g, ' ')
};
}).filter(item =>
item.include && (item.area >= 20)
);
element: element,
include:
element.tagName === "INPUT" ||
element.tagName === "TEXTAREA" ||
element.tagName === "SELECT" ||
element.tagName === "BUTTON" ||
element.tagName === "A" ||
element.onclick != null ||
window.getComputedStyle(element).cursor == "pointer" ||
element.tagName === "IFRAME" ||
element.tagName === "VIDEO",
area,
rects,
text: textualContent,
type: elementType,
ariaLabel: ariaLabel,
};
})
.filter((item) => item.include && item.area >= 20);
// Only keep inner clickable items
items = items.filter(x => !items.some(y => x.element.contains(y.element) && !(x == y)))
items = items.filter(
(x) => !items.some((y) => x.element.contains(y.element) && !(x == y))
);
// Function to generate random colors
function getRandomColor() {
var letters = '0123456789ABCDEF';
var color = '#';
var letters = "0123456789ABCDEF";
var color = "#";
for (var i = 0; i < 6; i++) {
color += letters[Math.floor(Math.random() * 16)];
}
@@ -90,7 +109,7 @@ function markPage() {
}
// Lets create a floating border on top of these elements that will always be visible
items.forEach(function(item, index) {
items.forEach(function (item, index) {
item.rects.forEach((bbox) => {
newElement = document.createElement("div");
var borderColor = getRandomColor();
@@ -104,7 +123,7 @@ function markPage() {
newElement.style.boxSizing = "border-box";
newElement.style.zIndex = 2147483647;
// newElement.style.background = `${borderColor}80`;
// Add floating label at the corner
var label = document.createElement("span");
label.textContent = index;
@@ -119,15 +138,20 @@ function markPage() {
label.style.fontSize = "12px";
label.style.borderRadius = "2px";
newElement.appendChild(label);
document.body.appendChild(newElement);
labels.push(newElement);
// item.element.setAttribute("-ai-label", label.textContent);
});
});
const coordinates = items.flatMap(item => item.rects.map(({ left, top, width, height }) => ({
x: (left + left + width) / 2,
y: (top + top + height) / 2,
})));
const coordinates = items.flatMap((item) =>
item.rects.map(({ left, top, width, height }) => ({
x: (left + left + width) / 2,
y: (top + top + height) / 2,
type: item.type,
text: item.text,
ariaLabel: item.ariaLabel,
}))
);
return coordinates;
}
}