diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 612678b6f4..db9763f98f 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -30444,6 +30444,115 @@ importers: specifier: ^5.9.3 version: 5.9.3 + ../../pods/link-preview: + dependencies: + '@hcengineering/analytics': + specifier: workspace:^0.7.17 + version: link:../../foundations/core/packages/analytics + '@hcengineering/analytics-service': + specifier: workspace:^0.7.17 + version: link:../../foundations/core/packages/analytics-service + '@hcengineering/core': + specifier: workspace:^0.7.24 + version: link:../../foundations/core/packages/core + '@hcengineering/platform': + specifier: workspace:^0.7.19 + version: link:../../foundations/core/packages/platform + '@hcengineering/server-client': + specifier: workspace:^0.7.16 + version: link:../../foundations/server/packages/client + '@hcengineering/server-core': + specifier: workspace:^0.7.18 + version: link:../../foundations/server/packages/core + '@hcengineering/server-token': + specifier: workspace:^0.7.17 + version: link:../../foundations/core/packages/token + cheerio: + specifier: ^1.1.2 + version: 1.1.2 + cors: + specifier: ^2.8.5 + version: 2.8.5 + dotenv: + specifier: ^16.4.5 + version: 16.6.1 + express: + specifier: ^4.21.2 + version: 4.21.2 + image-size: + specifier: ^1.1.1 + version: 1.2.1 + morgan: + specifier: ^1.10.0 + version: 1.10.1 + oembed-providers: + specifier: ~1.0.20260106 + version: 1.0.20260106 + devDependencies: + '@hcengineering/platform-rig': + specifier: workspace:^0.7.19 + version: link:../../foundations/utils/packages/platform-rig + '@types/cors': + specifier: ^2.8.12 + version: 2.8.19 + '@types/express': + specifier: ^4.17.13 + version: 4.17.25 + '@types/jest': + specifier: ^29.5.5 + version: 29.5.14 + '@types/morgan': + specifier: ~1.9.9 + version: 1.9.10 + '@types/node': + specifier: ^22.18.1 + version: 22.19.0 + '@typescript-eslint/eslint-plugin': + specifier: ^6.21.0 + version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3) + '@typescript-eslint/parser': + specifier: ^6.21.0 + version: 6.21.0(eslint@8.57.1)(typescript@5.9.3) + cross-env: + specifier: ~7.0.3 + version: 7.0.3 + esbuild: + specifier: ^0.25.10 + version: 0.25.12 + eslint: + specifier: ^8.54.0 + version: 8.57.1 + eslint-config-standard-with-typescript: + specifier: ^40.0.0 + version: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint-plugin-import@2.32.0(eslint@8.57.1))(eslint-plugin-n@15.7.0(eslint@8.57.1))(eslint-plugin-promise@6.6.0(eslint@8.57.1))(eslint@8.57.1)(typescript@5.9.3) + eslint-plugin-import: + specifier: ^2.26.0 + version: 2.32.0(eslint@8.57.1) + eslint-plugin-n: + specifier: ^15.4.0 + version: 15.7.0(eslint@8.57.1) + eslint-plugin-node: + specifier: ^11.1.0 + version: 11.1.0(eslint@8.57.1) + eslint-plugin-promise: + specifier: ^6.1.1 + version: 6.6.0(eslint@8.57.1) + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3)) + prettier: + specifier: ^3.6.2 + version: 3.6.2 + ts-jest: + specifier: ^29.1.1 + version: 29.4.5(@babel/core@7.28.5)(@jest/transform@29.7.0)(@jest/types@30.2.0)(babel-jest@29.7.0(@babel/core@7.28.5))(esbuild@0.25.12)(jest-util@30.2.0)(jest@29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3)))(typescript@5.9.3) + ts-node: + specifier: ^10.9.2 + version: 10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3) + typescript: + specifier: ^5.9.3 + version: 5.9.3 + ../../pods/media: dependencies: '@hcengineering/account-client': @@ -45554,10 +45663,17 @@ packages: charenc@0.0.2: resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + cheerio@0.22.0: resolution: {integrity: sha512-8/MzidM6G/TgRelkzDG13y3Y9LxBjCb+8yOEZ9+wwq5gVF2w2pV0wmHvjfT0RvuxGyR7UEuK36r+yYMbT4uKgA==} engines: {node: '>= 0.6'} + cheerio@1.1.2: + resolution: {integrity: sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==} + engines: {node: '>=20.18.1'} + chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -45940,6 +46056,9 @@ packages: css-select@4.3.0: resolution: {integrity: sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==} + css-select@5.2.2: + resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} + css-tree@1.1.3: resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==} engines: {node: '>=8.0.0'} @@ -46595,6 +46714,9 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + encoding-sniffer@0.2.1: + resolution: {integrity: sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==} + encoding@0.1.13: resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} @@ -47564,6 +47686,9 @@ packages: webpack: optional: true + htmlparser2@10.0.0: + resolution: {integrity: sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==} + htmlparser2@3.10.1: resolution: {integrity: sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==} @@ -49197,6 +49322,9 @@ packages: resolution: {integrity: sha512-7Abo3nADdja8l/aglU6Y3lpnHSfv0tw7gFPiqzry/yCU+2gTAX7R1roJ8hJrxIK+S1j+7iqRJXtmuHJ/UDsBhQ==} engines: {node: '>= 18'} + oembed-providers@1.0.20260106: + resolution: {integrity: sha512-u1mNQgfTuuY3ydQDN7G7MnlCVlqM5fhgVxNc6VQTnEIq/OkLPtBaFK0ADvkRciVNLEa/j925jUR30kFVHN2WtQ==} + oidc-token-hash@5.2.0: resolution: {integrity: sha512-6gj2m8cJZ+iSW8bm0FXdGF0YhIQbKrfP4yWTNzxc31U6MOjfEmB1rHvlYvxI1B7t7BCi1F2vYTT6YhtQRG4hxw==} engines: {node: ^10.13.0 || >=12.0.0} @@ -49384,6 +49512,12 @@ packages: parse-srcset@1.0.2: resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==} + parse5-htmlparser2-tree-adapter@7.1.0: + resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} + + parse5-parser-stream@7.1.2: + resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} @@ -51167,6 +51301,10 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@7.18.2: + resolution: {integrity: sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==} + engines: {node: '>=20.18.1'} + unionfs@4.6.0: resolution: {integrity: sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA==} @@ -51440,6 +51578,7 @@ packages: whatwg-encoding@3.1.1: resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} engines: {node: '>=18'} + deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation whatwg-mimetype@4.0.0: resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} @@ -57038,6 +57177,15 @@ snapshots: charenc@0.0.2: {} + cheerio-select@2.1.0: + dependencies: + boolbase: 1.0.0 + css-select: 5.2.2 + css-what: 6.2.2 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + cheerio@0.22.0: dependencies: css-select: 1.2.0 @@ -57057,6 +57205,20 @@ snapshots: lodash.reject: 4.6.0 lodash.some: 4.6.0 + cheerio@1.1.2: + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.2.2 + encoding-sniffer: 0.2.1 + htmlparser2: 10.0.0 + parse5: 7.3.0 + parse5-htmlparser2-tree-adapter: 7.1.0 + parse5-parser-stream: 7.1.2 + undici: 7.18.2 + whatwg-mimetype: 4.0.0 + chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -57474,6 +57636,14 @@ snapshots: domutils: 2.8.0 nth-check: 2.1.1 + css-select@5.2.2: + dependencies: + boolbase: 1.0.0 + css-what: 6.2.2 + domhandler: 5.0.3 + domutils: 3.2.2 + nth-check: 2.1.1 + css-tree@1.1.3: dependencies: mdn-data: 2.0.14 @@ -57979,7 +58149,7 @@ snapshots: domutils@1.5.1: dependencies: - dom-serializer: 0.1.1 + dom-serializer: 0.2.2 domelementtype: 1.3.1 domutils@1.7.0: @@ -58164,6 +58334,11 @@ snapshots: encodeurl@2.0.0: {} + encoding-sniffer@0.2.1: + dependencies: + iconv-lite: 0.6.3 + whatwg-encoding: 3.1.1 + encoding@0.1.13: dependencies: iconv-lite: 0.6.3 @@ -59429,6 +59604,13 @@ snapshots: optionalDependencies: webpack: 5.102.1(@swc/core@1.15.1)(esbuild@0.25.12)(webpack-cli@5.1.4) + htmlparser2@10.0.0: + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.2.2 + entities: 6.0.1 + htmlparser2@3.10.1: dependencies: domelementtype: 1.3.1 @@ -61389,6 +61571,8 @@ snapshots: '@octokit/types': 13.10.0 '@octokit/webhooks': 12.3.2 + oembed-providers@1.0.20260106: {} + oidc-token-hash@5.2.0: {} omggif@1.0.10: {} @@ -61589,6 +61773,15 @@ snapshots: parse-srcset@1.0.2: {} + parse5-htmlparser2-tree-adapter@7.1.0: + dependencies: + domhandler: 5.0.3 + parse5: 7.3.0 + + parse5-parser-stream@7.1.2: + dependencies: + parse5: 7.3.0 + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -63686,6 +63879,8 @@ snapshots: undici-types@6.21.0: {} + undici@7.18.2: {} + unionfs@4.6.0: dependencies: fs-monkey: 1.1.0 diff --git a/common/scripts/docker.sh b/common/scripts/docker.sh index 83f141f8b1..5263a290ad 100755 --- a/common/scripts/docker.sh +++ b/common/scripts/docker.sh @@ -21,6 +21,7 @@ rush docker:build -p 20 \ --to @hcengineering/pod-export \ --to @hcengineering/pod-media \ --to @hcengineering/pod-preview \ +--to @hcengineering/pod-link-preview \ --to @hcengineering/pod-external \ --to @hcengineering/pod-backup \ --to @hcengineering/backup-api-pod \ diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 66beca9555..ad14db6e43 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -40,6 +40,17 @@ services: ports: - 4040:4040 restart: unless-stopped + link-preview: + image: 'hardcoreeng/link-preview' + extra_hosts: + - 'huly.local:host-gateway' + container_name: link-preview + environment: + - SECRET=secret + - PORT=4041 + ports: + - 4041:4041 + restart: unless-stopped cockroach: image: cockroachdb/cockroach:latest-v24.3 extra_hosts: @@ -306,6 +317,7 @@ services: - DATALAKE_URL=http://huly.local:4030 - HULYLAKE_URL=http://huly.local:8096 - EXPORT_URL=http://huly.local:4009 + - LINK_PREVIEW_URL=http://huly.local:4041 restart: unless-stopped transactor_cockroach: image: hardcoreeng/transactor diff --git a/dev/prod/public/config.json b/dev/prod/public/config.json index b2c8376eef..c339391708 100644 --- a/dev/prod/public/config.json +++ b/dev/prod/public/config.json @@ -31,5 +31,6 @@ "PULSE_URL": "ws://huly.local:8099/ws", "HULYLAKE_URL": "http://huly.local:8096", "EXCLUDED_APPLICATIONS_FOR_ANONYMOUS": "[\"chunter\", \"notification\"]", - "ANALYTICS_COLLECTOR_URL": "http://huly.local:4017" + "ANALYTICS_COLLECTOR_URL": "http://huly.local:4017", + "LINK_PREVIEW_URL": "http://huly.local:4041" } diff --git a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte index c86557077f..f60068fc63 100644 --- a/plugins/attachment-resources/src/components/AttachmentRefInput.svelte +++ b/plugins/attachment-resources/src/components/AttachmentRefInput.svelte @@ -39,7 +39,8 @@ getClient, isLinkPreviewEnabled, uploadFile, - LinkPreviewAttachmentMetadata + LinkPreviewAttachmentMetadata, + generateFileId } from '@hcengineering/presentation' import { EmptyMarkup, isEmptyMarkup } from '@hcengineering/text' import textEditor, { type RefAction } from '@hcengineering/text-editor' @@ -383,9 +384,10 @@ for (const url of urls) { try { const meta = await fetchLinkPreviewDetails(url) - if (canDisplayLinkPreview(meta) && meta.url !== undefined) { + if (canDisplayLinkPreview(meta)) { + const uuid = generateFileId() const blob = new Blob([JSON.stringify(meta)]) - const file = new File([blob], meta.url, { type: 'application/link-preview' }) + const file = new File([blob], uuid, { type: 'application/link-preview' }) const metadata: LinkPreviewAttachmentMetadata = { title: meta.title, image: meta.image, diff --git a/pods/link-preview/.eslintrc.js b/pods/link-preview/.eslintrc.js new file mode 100644 index 0000000000..297e1a11f7 --- /dev/null +++ b/pods/link-preview/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} \ No newline at end of file diff --git a/pods/link-preview/Dockerfile b/pods/link-preview/Dockerfile new file mode 100644 index 0000000000..5592351652 --- /dev/null +++ b/pods/link-preview/Dockerfile @@ -0,0 +1,9 @@ +FROM hardcoreeng/base-slim:v20250916 + +WORKDIR /app + +COPY bundle/bundle.js ./ +COPY bundle/bundle.js.map ./ + +EXPOSE 4041 +CMD ["dumb-init", "node", "bundle.js"] \ No newline at end of file diff --git a/pods/link-preview/jest.config.js b/pods/link-preview/jest.config.js new file mode 100644 index 0000000000..1229bb954c --- /dev/null +++ b/pods/link-preview/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} \ No newline at end of file diff --git a/pods/link-preview/package.json b/pods/link-preview/package.json new file mode 100644 index 0000000000..2e38b6030d --- /dev/null +++ b/pods/link-preview/package.json @@ -0,0 +1,72 @@ +{ + "name": "@hcengineering/pod-link-preview", + "version": "0.7.0", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Hardcore Engineering Inc.", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "jest --passWithNoTests --silent", + "_phase:bundle": "rushx bundle", + "_phase:docker-build": "rushx docker:build", + "_phase:docker-staging": "rushx docker:staging", + "bundle": "node ../../common/scripts/esbuild.js --keep-names=true --sourcemap=external", + "docker:build": "../../common/scripts/docker_build.sh hardcoreeng/link-preview", + "docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/link-preview staging", + "docker:abuild": "docker build -t hardcoreeng/link-preview . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/link-preview", + "docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/link-preview", + "run-bundle": "rushx bundle --to @hcengineering/pod-link-preview && cross-env NODE_ENV=production node --inspect bundle/bundle.js", + "run-local": "ts-node src/index.ts", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "workspace:^0.7.19", + "@types/jest": "^29.5.5", + "@types/node": "^22.18.1", + "@typescript-eslint/eslint-plugin": "^6.21.0", + "@typescript-eslint/parser": "^6.21.0", + "cross-env": "~7.0.3", + "esbuild": "^0.25.10", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^6.1.1", + "jest": "^29.7.0", + "prettier": "^3.6.2", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3", + "@types/express": "^4.17.13", + "@types/cors": "^2.8.12", + "@types/morgan": "~1.9.9" + }, + "dependencies": { + "@hcengineering/core": "workspace:^0.7.24", + "@hcengineering/platform": "workspace:^0.7.19", + "@hcengineering/server-core": "workspace:^0.7.18", + "@hcengineering/analytics-service": "workspace:^0.7.17", + "@hcengineering/analytics": "workspace:^0.7.17", + "@hcengineering/server-client": "workspace:^0.7.16", + "@hcengineering/server-token": "workspace:^0.7.17", + "dotenv": "^16.4.5", + "express": "^4.21.2", + "cors": "^2.8.5", + "morgan": "^1.10.0", + "cheerio": "^1.1.2", + "image-size": "^1.1.1", + "oembed-providers": "~1.0.20260106" + } +} diff --git a/pods/link-preview/src/__tests__/parse.test.ts b/pods/link-preview/src/__tests__/parse.test.ts new file mode 100644 index 0000000000..38921b3507 --- /dev/null +++ b/pods/link-preview/src/__tests__/parse.test.ts @@ -0,0 +1,828 @@ +import { MeasureContext } from '@hcengineering/core' +import { Config } from '../config' +import { parseLinkPreviewDetails, LinkPreviewError } from '../parse' + +// ============================================================================ +// Test Utilities & Mocks +// ============================================================================ + +const createMockContext = (): MeasureContext => + ({ + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + measure: jest.fn(), + newChild: jest.fn(() => createMockContext()) + }) as unknown as MeasureContext + +const defaultConfig: Config = { + Port: 4041, + Secret: 'secret', + ServiceID: 'link-preview', + UserAgent: 'TestBot/1.0', + DescriptionMaxSentences: 3, + DescriptionMaxLength: 200, + TimeoutMs: 5000, + MaxImageBytes: 1024 * 1024 +} + +const createHtmlResponse = (html: string, headers: Record = {}): Response => { + return new Response(html, { + status: 200, + headers: { + 'Content-Type': 'text/html; charset=utf-8', + ...headers + } + }) +} + +const createJsonResponse = (data: object): Response => { + return new Response(JSON.stringify(data), { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) +} + +// ============================================================================ +// URL Validation Tests +// ============================================================================ + +describe('URL Validation', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + describe('protocol validation', () => { + it('should accept https URLs', async () => { + const html = 'Test' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.title).toBe('Test') + }) + + it('should accept http URLs', async () => { + const html = 'Test' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'http://example.com') + expect(result.title).toBe('Test') + }) + + it('should reject file:// protocol', async () => { + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'file:///etc/passwd')).rejects.toThrow(LinkPreviewError) + }) + + it('should reject javascript: protocol', async () => { + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'javascript:alert(1)')).rejects.toThrow(LinkPreviewError) + }) + + it('should reject data: protocol', async () => { + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'data:text/html,

test

')).rejects.toThrow( + LinkPreviewError + ) + }) + + it('should reject invalid URLs', async () => { + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'not-a-valid-url')).rejects.toThrow(LinkPreviewError) + }) + }) + + describe('SSRF protection', () => { + const blockedAddresses = [ + '127.0.0.1', + '127.0.0.2', + '10.0.0.1', + '10.255.255.255', + '172.16.0.1', + '172.31.255.255', + '192.168.0.1', + '192.168.255.255', + '169.254.0.1', + '0.0.0.0', + 'localhost' + ] + + it.each(blockedAddresses)('should block access to %s', async (host) => { + await expect(parseLinkPreviewDetails(ctx, defaultConfig, `https://${host}/path`)).rejects.toThrow( + LinkPreviewError + ) + }) + + it('should allow public IP addresses', async () => { + const html = 'Public' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + // These should not throw BLOCKED_URL + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://8.8.8.8') + expect(result).toBeDefined() + }) + }) +}) + +// ============================================================================ +// Open Graph Parsing Tests +// ============================================================================ + +describe('Open Graph Parsing', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should parse basic OG tags', async () => { + const html = ` + + + + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page') + + expect(result.title).toBe('OG Title') + expect(result.description).toBe('OG Description') + expect(result.image).toBe('https://example.com/image.jpg') + expect(result.hostname).toBe('Example Site') + }) + + it('should fall back to title tag when OG title is missing', async () => { + const html = ` + + + Page Title + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.title).toBe('Page Title') + }) + + it('should fall back to meta description when OG description is missing', async () => { + const html = ` + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.description).toBe('Meta Description') + }) + + it('should parse Twitter card tags as fallback', async () => { + const html = ` + + + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.title).toBe('Twitter Title') + expect(result.description).toBe('Twitter Description') + expect(result.image).toBe('https://example.com/twitter.jpg') + }) + + it('should resolve relative image URLs', async () => { + const html = ` + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page') + expect(result.image).toBe('https://example.com/images/photo.jpg') + }) + + it('should parse canonical URL', async () => { + const html = ` + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page?utm=123') + expect(result.url).toBe('https://example.com/canonical-page') + }) + + it('should parse favicon', async () => { + const html = ` + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.icon).toBe('https://example.com/favicon.ico') + }) + + it('should parse image dimensions from OG tags', async () => { + const html = ` + + + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.imageWidth).toBe(1200) + expect(result.imageHeight).toBe(630) + }) +}) + +// ============================================================================ +// Description Extraction Tests +// ============================================================================ + +describe('Description Extraction', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should extract description from paragraphs when meta tags are missing', async () => { + const html = ` + + +

This is the first sentence. This is the second sentence.

+

This is the third sentence.

+ + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.description).toBe( + 'This is the first sentence. This is the second sentence. This is the third sentence.' + ) + }) + + it('should limit description to configured max sentences', async () => { + const html = ` + + +

Sentence one. Sentence two. Sentence three. Sentence four. Sentence five.

+ + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const config = { ...defaultConfig, DescriptionMaxSentences: 2 } + const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com') + + expect(result.description).toBe('Sentence one. Sentence two.') + }) + + it('should truncate description to configured max length', async () => { + const html = ` + + +

This is a very long sentence that exceeds the maximum length configured for descriptions in the system.

+ + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const config = { ...defaultConfig, DescriptionMaxLength: 50 } + const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com') + + expect(result.description?.length).toBeLessThanOrEqual(50) + expect(result.description).toMatch(/\.\.\.$/) + }) + + it('should handle paragraphs with question marks and exclamation points', async () => { + const html = ` + + +

Is this a question? Yes it is! And here's more.

+ + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.description).toBe("Is this a question? Yes it is! And here's more.") + }) +}) + +// ============================================================================ +// oEmbed Tests +// ============================================================================ + +describe('oEmbed Integration', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should use oEmbed data when available via discovery', async () => { + const oembedEndpoint = 'https://example.com/oembed?url=test' + const html = ` + + + + + + + ` + const oembedData = { + type: 'video', + version: '1.0', + title: 'oEmbed Title', + author_name: 'Author Name', + provider_name: 'Provider', + thumbnail_url: 'https://example.com/thumb.jpg', + thumbnail_width: 480, + thumbnail_height: 360 + } + + global.fetch = jest.fn().mockImplementation((url: string) => { + if (url === oembedEndpoint) { + return Promise.resolve(createJsonResponse(oembedData)) + } + return Promise.resolve(createHtmlResponse(html)) + }) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/video') + + expect(result.title).toBe('oEmbed Title') + expect(result.description).toBe('Provider - Author Name') + expect(result.image).toBe('https://example.com/thumb.jpg') + expect(result.imageWidth).toBe(480) + expect(result.imageHeight).toBe(360) + }) + + it('should fall back to OG when oEmbed fails', async () => { + const html = ` + + + + + + + ` + + global.fetch = jest.fn().mockImplementation((url: string) => { + if (url.includes('/oembed')) { + return Promise.resolve(new Response(null, { status: 404 })) + } + return Promise.resolve(createHtmlResponse(html)) + }) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.title).toBe('Fallback OG Title') + }) + + it('should reject invalid oEmbed responses', async () => { + const html = ` + + + + + + + ` + // Missing required 'type' and 'version' fields + const invalidOembed = { title: 'Invalid' } + + global.fetch = jest.fn().mockImplementation((url: string) => { + if (url.includes('/oembed')) { + return Promise.resolve(createJsonResponse(invalidOembed)) + } + return Promise.resolve(createHtmlResponse(html)) + }) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + // Should fall back to OG + expect(result.title).toBe('OG Title') + }) + + it('should handle oEmbed photo type', async () => { + const html = ` + + + + + + ` + const oembedData = { + type: 'photo', + version: '1.0', + title: 'Photo Title', + url: 'https://flickr.com/photo.jpg', + width: 1920, + height: 1080 + } + + global.fetch = jest.fn().mockImplementation((url: string) => { + if (url.includes('/oembed')) { + return Promise.resolve(createJsonResponse(oembedData)) + } + return Promise.resolve(createHtmlResponse(html)) + }) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://flickr.com/photo/123') + + expect(result.image).toBe('https://flickr.com/photo.jpg') + expect(result.imageWidth).toBe(1920) + expect(result.imageHeight).toBe(1080) + }) +}) + +// ============================================================================ +// Charset Handling Tests +// ============================================================================ + +describe('Charset Handling', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should detect charset from Content-Type header', async () => { + const html = 'Test' + global.fetch = jest + .fn() + .mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html; charset=iso-8859-1' })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.charset).toBe('iso-8859-1') + }) + + it('should detect charset from meta charset tag', async () => { + const html = ` + + + + 日本語 + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.jp') + expect(result.charset).toBe('shift_jis') + }) + + it('should detect charset from http-equiv meta tag', async () => { + const html = ` + + + + 한국어 + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.kr') + expect(result.charset).toBe('euc-kr') + }) + + it('should default to utf-8 when no charset specified', async () => { + const html = 'Test' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.charset).toBe('utf-8') + }) + + it('should handle quoted charset values', async () => { + const html = 'Test' + global.fetch = jest + .fn() + .mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html; charset="UTF-8"' })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.charset).toBe('utf-8') + }) +}) + +// ============================================================================ +// Error Handling Tests +// ============================================================================ + +describe('Error Handling', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should throw on HTTP error responses', async () => { + global.fetch = jest.fn().mockResolvedValue(new Response(null, { status: 404, statusText: 'Not Found' })) + + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/missing')).rejects.toThrow( + LinkPreviewError + ) + }) + + it('should throw on network errors', async () => { + global.fetch = jest.fn().mockRejectedValue(new Error('Network error')) + + await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')).rejects.toThrow(LinkPreviewError) + }) + + it('should handle malformed HTML gracefully', async () => { + const html = 'Broken<<<<' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + // Should not throw + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result).toBeDefined() + expect(result.host).toBe('https://example.com') + }) + + it('should continue when image size fetch fails', async () => { + const html = ` + + + + + + + ` + global.fetch = jest + .fn() + .mockResolvedValueOnce(createHtmlResponse(html)) + .mockResolvedValueOnce(new Response(null, { status: 404 })) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.title).toBe('Test') + expect(result.image).toBe('https://example.com/broken.jpg') + expect(result.imageWidth).toStrictEqual(undefined) + expect(result.imageHeight).toStrictEqual(undefined) + }) +}) + +// ============================================================================ +// Image Handling Tests +// ============================================================================ + +describe('Image Handling', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should handle direct image URLs', async () => { + // 1x1 transparent PNG + const pngData = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', + 'base64' + ) + + global.fetch = jest.fn().mockResolvedValue( + new Response(pngData, { + status: 200, + headers: { 'Content-Type': 'image/png' } + }) + ) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/image.png') + + expect(result.image).toBe('https://example.com/image.png') + expect(result.url).toBe('https://example.com/image.png') + }) + + it('should reject images exceeding max size', async () => { + const html = ` + + + + + + ` + global.fetch = jest + .fn() + .mockResolvedValueOnce(createHtmlResponse(html)) + .mockResolvedValueOnce( + new Response(null, { + status: 200, + headers: { + 'Content-Type': 'image/jpeg', + 'Content-Length': '999999999' + } + }) + ) + + const config = { ...defaultConfig, MaxImageBytes: 1000 } + const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com') + + // Should not have image dimensions since fetch was skipped + expect(result.imageWidth).toBeUndefined() + expect(ctx.warn).toHaveBeenCalledWith('image too large', expect.any(Object)) + }) +}) + +// ============================================================================ +// Edge Cases +// ============================================================================ + +describe('Edge Cases', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should handle empty HTML document', async () => { + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse('')) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + + expect(result.host).toBe('https://example.com') + expect(result.hostname).toBe('example.com') + expect(result.title).toStrictEqual(undefined) + }) + + it('should handle URLs with special characters', async () => { + const html = 'Special' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/path?q=hello%20world&foo=bar') + + expect(result.title).toBe('Special') + }) + + it('should handle internationalized domain names', async () => { + const html = 'IDN Test' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://例え.jp/page') + + expect(result.title).toBe('IDN Test') + }) + + it('should handle multiple OG images (use first)', async () => { + const html = ` + + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.image).toBe('https://example.com/first.jpg') + }) + + it('should handle redirect responses', async () => { + const html = 'Redirected' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/redirect') + expect(result.title).toBe('Redirected') + }) + + it('should sanitize HTML entities in titles', async () => { + const html = 'Rock & Roll' + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + expect(result.title).toBe('Rock & Roll') + }) + + it('should trim whitespace from extracted values', async () => { + const html = ` + + + + + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com') + // Note: cheerio may or may not trim - this tests current behavior + expect(result.title).toBeDefined() + }) +}) + +// ============================================================================ +// Integration-style Tests +// ============================================================================ + +describe('Real-world Scenarios', () => { + let ctx: MeasureContext + + beforeEach(() => { + ctx = createMockContext() + jest.clearAllMocks() + }) + + it('should parse a typical blog post page', async () => { + const html = ` + + + + + 10 Tips for Better Code - Dev Blog + + + + + + + + + + + + +
+

10 Tips for Better Code

+

Writing clean code is essential for maintainability.

+
+ + + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails( + ctx, + defaultConfig, + 'https://blog.example.com/posts/10-tips?utm_source=twitter' + ) + + expect(result).toMatchObject({ + title: '10 Tips for Better Code', + description: 'Learn the top 10 tips for writing better, cleaner code.', + url: 'https://blog.example.com/posts/10-tips', + hostname: 'Dev Blog', + host: 'https://blog.example.com', + image: 'https://blog.example.com/images/code-tips.jpg', + imageWidth: 1200, + imageHeight: 630, + icon: 'https://blog.example.com/favicon.ico', + charset: 'utf-8' + }) + }) + + it('should handle minimal HTML pages', async () => { + const html = ` + + Simple Page +

Just some content here.

+ + ` + global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html)) + + const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://simple.example.com') + + expect(result.title).toBe('Simple Page') + expect(result.description).toBe('Just some content here.') + expect(result.hostname).toBe('simple.example.com') + }) +}) diff --git a/pods/link-preview/src/config.ts b/pods/link-preview/src/config.ts new file mode 100644 index 0000000000..eabe63eedf --- /dev/null +++ b/pods/link-preview/src/config.ts @@ -0,0 +1,50 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { config as dotenv } from 'dotenv' + +dotenv() + +export interface Config { + Port: number + Secret: string + ServiceID: string + UserAgent: string + DescriptionMaxSentences: number + DescriptionMaxLength: number + TimeoutMs?: number + MaxImageBytes?: number +} + +const config: Config = (() => { + const params: Partial = { + Port: parseInt(process.env.PORT ?? '4041'), + Secret: process.env.SECRET, + ServiceID: process.env.SERVICE_ID ?? 'link-preview', + UserAgent: process.env.USER_AGENT ?? 'Huly Link Preview Service/1.0', + DescriptionMaxSentences: parseInt(process.env.DESCRIPTION_MAX_SENTENCES ?? '3'), + DescriptionMaxLength: parseInt(process.env.DESCRIPTION_MAX_LENGTH ?? '200'), + TimeoutMs: parseInt(process.env.TIMEOUT ?? '5') * 1000 + } + + const missingEnv = (Object.keys(params) as Array).filter((key) => params[key] === undefined) + + if (missingEnv.length > 0) { + throw Error(`Missing config for attributes: ${missingEnv.join(', ')}`) + } + + return params as Config +})() + +export default config diff --git a/pods/link-preview/src/error.ts b/pods/link-preview/src/error.ts new file mode 100644 index 0000000000..2ccd1b9b22 --- /dev/null +++ b/pods/link-preview/src/error.ts @@ -0,0 +1,23 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export class HttpError extends Error { + code: number + + constructor (code: number, message: string) { + super(message) + this.code = code + } +} diff --git a/pods/link-preview/src/index.ts b/pods/link-preview/src/index.ts new file mode 100644 index 0000000000..e3ebadc5ad --- /dev/null +++ b/pods/link-preview/src/index.ts @@ -0,0 +1,82 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { configureAnalytics, createOpenTelemetryMetricsContext, SplitLogger } from '@hcengineering/analytics-service' +import { newMetrics } from '@hcengineering/core' +import { setMetadata } from '@hcengineering/platform' +import { initStatisticsContext } from '@hcengineering/server-core' +import serverToken from '@hcengineering/server-token' +import { join } from 'path' + +import config from './config' +import { createServer, listen } from './server' + +const application = config.ServiceID + +const setupMetadata = (): void => { + setMetadata(serverToken.metadata.Secret, config.Secret) + setMetadata(serverToken.metadata.Service, config.ServiceID) +} + +async function main (): Promise { + setupMetadata() + + configureAnalytics(application, process.env.VERSION ?? '0.7.0') + Analytics.setTag('application', application) + + const ctx = initStatisticsContext(application, { + factory: () => + createOpenTelemetryMetricsContext( + application, + {}, + {}, + newMetrics(), + new SplitLogger(application, { + root: join(process.cwd(), 'logs'), + enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true' + }) + ) + }) + + const { app, close } = await createServer(ctx, config) + const server = listen(app, config.Port) + + const shutdown = (): void => { + try { + server.close() + close() + + ctx.info('Shutdown complete') + process.exit(0) + } catch (error) { + ctx.error('Error during shutdown', { error }) + process.exit(1) + } + } + + process.on('SIGINT', shutdown) + process.on('SIGTERM', shutdown) + process.on('uncaughtException', (error: any) => { + ctx.error('Uncaught exception', { error }) + }) + process.on('unhandledRejection', (error: any) => { + ctx.error('Unhandled rejection', { error }) + }) +} + +void main().catch((err) => { + console.error(err) +}) diff --git a/pods/link-preview/src/middleware.ts b/pods/link-preview/src/middleware.ts new file mode 100644 index 0000000000..5f07c24504 --- /dev/null +++ b/pods/link-preview/src/middleware.ts @@ -0,0 +1,74 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { MeasureContext } from '@hcengineering/core' +import { extractToken } from '@hcengineering/server-client' +import { Token } from '@hcengineering/server-token' +import { type Response, type Request, type NextFunction, type RequestHandler, type ErrorRequestHandler } from 'express' +import { HttpError } from './error' + +export interface KeepAliveOptions { + timeout: number + max: number +} + +export interface RequestWithAuth extends Request { + token?: Token +} + +export const keepAlive = (options: KeepAliveOptions): RequestHandler => { + const { timeout, max } = options + return (req: Request, res: Response, next: NextFunction) => { + res.setHeader('Connection', 'keep-alive') + res.setHeader('Keep-Alive', `timeout=${timeout}, max=${max}`) + next() + } +} + +export const withAuthorization = (req: RequestWithAuth, res: Response, next: NextFunction): void => { + try { + const token = extractToken(req.headers) + if (token == null) { + throw new HttpError(401, 'Unauthorized') + } + req.token = token + + next() + } catch (err: any) { + next(err) + } +} + +export interface ErrorHandlerOptions { + ctx: MeasureContext +} + +export const errorHandler = (options: ErrorHandlerOptions): ErrorRequestHandler => { + const { ctx } = options + + return (err: any, req: Request, res: Response, _next: NextFunction): void => { + ctx.error(err.message, { code: err.code, message: err.message }) + + if (err instanceof HttpError) { + res.status(err.code).json({ message: err.message }) + return + } + + Analytics.handleError(err) + + res.status(500).json({ message: err.message ?? 'Internal Server Error' }) + } +} diff --git a/pods/link-preview/src/parse.ts b/pods/link-preview/src/parse.ts new file mode 100644 index 0000000000..2078a50b56 --- /dev/null +++ b/pods/link-preview/src/parse.ts @@ -0,0 +1,696 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { MeasureContext } from '@hcengineering/core' +import * as cheerio from 'cheerio' +import { imageSize } from 'image-size' +import oembedProviders from 'oembed-providers' + +// ============================================================================ +// Types and Interfaces +// ============================================================================ + +export interface LinkPreviewDetails { + title?: string + description?: string + url?: string + icon?: string + image?: string + imageWidth?: number + imageHeight?: number + charset?: string + hostname?: string + host: string +} + +export interface OEmbedProvider { + provider_name: string + provider_url: string + endpoints: Array<{ + url: string + schemes?: string[] + formats?: string[] + discovery?: boolean + }> +} + +export interface OEmbedResponse { + type: 'photo' | 'video' | 'link' | 'rich' + version: string + title?: string + author_name?: string + author_url?: string + provider_name?: string + provider_url?: string + cache_age?: number + thumbnail_url?: string + thumbnail_width?: number + thumbnail_height?: number + url?: string + html?: string + width?: number + height?: number +} + +export interface Config { + UserAgent: string + DescriptionMaxSentences: number + DescriptionMaxLength: number + /** Request timeout in milliseconds (default: 10000) */ + TimeoutMs?: number + /** Maximum image size to fetch in bytes (default: 10MB) */ + MaxImageBytes?: number + /** Blocked IP ranges for SSRF protection */ + BlockedIpRanges?: string[] +} + +interface ImageDimensions { + width: number + height: number +} + +// ============================================================================ +// Constants +// ============================================================================ + +const DEFAULT_TIMEOUT_MS = 10_000 +const DEFAULT_MAX_IMAGE_BYTES = 10 * 1024 * 1024 // 10MB +const OEMBED_SERVICE_NAME = 'Huly Link Preview Service/1.0' + +// Private IP ranges to block for SSRF protection +const BLOCKED_IP_PATTERNS = [ + /^127\./, // Loopback + /^10\./, // Private Class A + /^172\.(1[6-9]|2\d|3[01])\./, // Private Class B + /^192\.168\./, // Private Class C + /^169\.254\./, // Link-local + /^0\./, // Current network + /^localhost$/i, + /^::1$/, // IPv6 loopback + /^fc00:/i, // IPv6 private + /^fe80:/i // IPv6 link-local +] + +// ============================================================================ +// Error Classes +// ============================================================================ + +export class LinkPreviewError extends Error { + constructor ( + message: string, + public readonly cause?: unknown + ) { + super(message) + this.name = 'LinkPreviewError' + } +} + +// ============================================================================ +// URL Validation +// ============================================================================ + +function validateUrl (urlString: string): URL { + let url: URL + try { + url = new URL(urlString) + } catch { + throw new LinkPreviewError(`Invalid URL: ${urlString}`) + } + + // Only allow HTTP(S) protocols + if (!['http:', 'https:'].includes(url.protocol)) { + throw new LinkPreviewError(`Invalid protocol: ${url.protocol}. Only HTTP and HTTPS are allowed.`) + } + + // SSRF protection: block private/internal IPs + const hostname = url.hostname + for (const pattern of BLOCKED_IP_PATTERNS) { + if (pattern.test(hostname)) { + throw new LinkPreviewError('Blocked URL: Access to internal addresses is not allowed.') + } + } + + return url +} + +// ============================================================================ +// Fetch Utilities +// ============================================================================ + +async function fetchWithTimeout (url: string, options: RequestInit, timeoutMs: number): Promise { + const controller = new AbortController() + const timeoutId = setTimeout(() => { + controller.abort() + }, timeoutMs) + + try { + const response = await fetch(url, { + ...options, + signal: controller.signal + }) + return response + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new LinkPreviewError(`Request timed out after ${timeoutMs}ms`, error) + } + throw new LinkPreviewError( + `Failed to fetch URL: ${error instanceof Error ? error.message : 'Unknown error'}`, + error + ) + } finally { + clearTimeout(timeoutId) + } +} + +// ============================================================================ +// oEmbed Functions +// ============================================================================ + +function findOEmbedProviderUrl (targetUrl: string): string | null { + for (const provider of oembedProviders as OEmbedProvider[]) { + const endpoints = provider.endpoints ?? [] + + for (const endpoint of endpoints) { + const schemes = endpoint.schemes ?? [] + if (schemes.length === 0) continue + + for (const scheme of schemes) { + const regex = oembedSchemeToRegex(scheme) + if (regex.test(targetUrl)) { + const endpointUrl = endpoint.url + if (endpointUrl != null) { + return endpointUrl.replace('{format}', 'json').replace('{url}', encodeURIComponent(targetUrl)) + } + } + } + } + } + return null +} + +function findOEmbedDiscoveryUrl ($: ReturnType, baseUrl: string): string | null { + const jsonOembedLink = $('link[type="application/json+oembed"]').attr('href') + const xmlOembedLink = $('link[type="text/xml+oembed"]').attr('href') + + let oembedLink: string | null = null + if (isNonEmptyString(jsonOembedLink)) { + oembedLink = jsonOembedLink + } else if (isNonEmptyString(xmlOembedLink)) { + oembedLink = xmlOembedLink + } + + if (oembedLink === null) return null + + return oembedLink.startsWith('http') ? oembedLink : new URL(oembedLink, baseUrl).href +} + +async function fetchOEmbedData ( + ctx: MeasureContext, + $: ReturnType, + baseUrl: string, + timeoutMs: number +): Promise { + try { + // Try provider registry first, then discovery + const providerUrl = findOEmbedProviderUrl(baseUrl) + const discoveryUrl = findOEmbedDiscoveryUrl($, baseUrl) + const oembedUrl = providerUrl ?? discoveryUrl + + if (oembedUrl === null) return null + + ctx.info('fetching oEmbed data', { oembedUrl }) + + const response = await fetchWithTimeout(oembedUrl, { headers: { 'User-Agent': OEMBED_SERVICE_NAME } }, timeoutMs) + + if (!response.ok) { + ctx.warn('oEmbed fetch failed', { + status: response.status, + statusText: response.statusText + }) + return null + } + + const contentType = response.headers.get('content-type') + if (contentType === null || !contentType.includes('json')) { + ctx.warn('oEmbed response is not JSON', { contentType: contentType ?? 'null' }) + return null + } + + const data = (await response.json()) as OEmbedResponse + + // Validate required fields per oEmbed spec + if (data.type == null || data.version == null) { + ctx.warn('Invalid oEmbed response: missing required fields') + return null + } + + ctx.info('successfully fetched oEmbed data', { type: data.type }) + return data + } catch (error) { + // Don't throw - oEmbed failure should fall back to OG parsing + ctx.warn('failed to fetch oEmbed data', { + error: error instanceof Error ? error.message : String(error) + }) + // return null + throw error + } +} + +function convertOEmbedToPreview ( + oembed: OEmbedResponse, + baseUrl: string, + hostname: string, + host: string +): LinkPreviewDetails { + const descriptionParts: string[] = [] + if (oembed.provider_name !== undefined && oembed.provider_name.length > 0) { + descriptionParts.push(oembed.provider_name) + } + if (oembed.author_name !== undefined && oembed.author_name.length > 0) { + descriptionParts.push(oembed.author_name) + } + const description = descriptionParts.length > 0 ? descriptionParts.join(' - ') : undefined + + let image: string | undefined + if (oembed.thumbnail_url !== undefined && oembed.thumbnail_url.length > 0) { + image = oembed.thumbnail_url + } else if (oembed.type === 'photo' && oembed.url !== undefined && oembed.url.length > 0) { + image = oembed.url + } + + let imageWidth: number | undefined + if (oembed.thumbnail_width !== undefined) { + imageWidth = oembed.thumbnail_width + } else if (oembed.type === 'photo' && oembed.width !== undefined) { + imageWidth = oembed.width + } + + let imageHeight: number | undefined + if (oembed.thumbnail_height !== undefined) { + imageHeight = oembed.thumbnail_height + } else if (oembed.type === 'photo' && oembed.height !== undefined) { + imageHeight = oembed.height + } + + return { + title: oembed.title, + description, + url: baseUrl, + hostname, + host, + image, + imageWidth, + imageHeight + } +} + +// ============================================================================ +// Charset Parsing +// ============================================================================ + +function parseCharset (headers: Headers, $: ReturnType): string { + // Try Content-Type header first + const headerContentType = headers.get('content-type') + const headerCharset = extractCharsetFromContentType(headerContentType) + if (isNonEmptyString(headerCharset)) return headerCharset + + // Try + const metaCharset = $('meta[charset]').attr('charset') + if (isNonEmptyString(metaCharset)) { + return normalizeCharset(metaCharset) + } + + // Try + const metaContentType = $('meta[http-equiv="Content-Type" i]').attr('content') + const bodyCharset = extractCharsetFromContentType(metaContentType) + if (isNonEmptyString(bodyCharset)) return bodyCharset + + return 'utf-8' +} + +function extractCharsetFromContentType (contentType: string | null | undefined): string | null { + if (contentType === null || contentType === undefined) return null + if (!contentType.includes('charset=')) return null + + const parts = contentType.split('charset=') + if (parts.length < 2) return null + + const charsetPart = parts[1] + if (!isNonEmptyString(charsetPart)) return null + + // Handle quoted values and trim; also handle trailing parameters + const cleanedCharset = charsetPart.split(';')[0].split(',')[0] + if (!isNonEmptyString(cleanedCharset)) return null + + return normalizeCharset(cleanedCharset) +} + +function normalizeCharset (charset: string): string { + return charset.toLowerCase().trim().replace(/^["']/, '').replace(/["']$/, '') +} + +// ============================================================================ +// Description Parsing +// ============================================================================ + +function parseDescription ($: ReturnType, config: Config): string | undefined { + const paragraphs = $('p') + if (paragraphs.length === 0) return undefined + + const sentences: string[] = [] + const maxSentences = config.DescriptionMaxSentences + + for (const p of paragraphs.toArray()) { + const rawText = $(p).text() + if (rawText === undefined || rawText === null) continue + + const text = rawText.trim() + if (text.length === 0) continue + + // Split on sentence-ending punctuation followed by space or end + const extracted = extractSentences(text) + sentences.push(...extracted) + + if (sentences.length >= maxSentences) break + } + + if (sentences.length === 0) return undefined + + let result = sentences.slice(0, maxSentences).join(' ') + + const maxLength = config.DescriptionMaxLength + if (result.length > maxLength && maxLength >= 3) { + result = result.substring(0, maxLength - 3) + '...' + } + + return result +} + +function extractSentences (text: string): string[] { + // Match sentences ending with . ? or ! + const sentenceRegex = /[^.!?]*[.!?]+/g + const matches = text.match(sentenceRegex) + + if (matches === null || matches.length === 0) { + // No sentence-ending punctuation, return whole text as one "sentence" + const trimmed = text.trim() + return trimmed.length > 0 ? [trimmed] : [] + } + + return matches.map((s) => s.trim()).filter((s) => s.length > 0) +} + +// ============================================================================ +// Image Size Loading +// ============================================================================ + +async function loadImageSize (ctx: MeasureContext, url: string, config: Config): Promise { + const timeoutMs = config.TimeoutMs ?? DEFAULT_TIMEOUT_MS + const maxBytes = config.MaxImageBytes ?? DEFAULT_MAX_IMAGE_BYTES + + try { + // Validate the image URL too + validateUrl(url) + + const response = await fetchWithTimeout(url, { headers: { 'User-Agent': config.UserAgent } }, timeoutMs) + + if (!response.ok) { + ctx.warn('failed to fetch image', { status: response.status, url }) + return undefined + } + + // Check content length before downloading + const contentLengthHeader = response.headers.get('content-length') + if (isNonEmptyString(contentLengthHeader)) { + const contentLength = parseInt(contentLengthHeader, 10) + if (!Number.isNaN(contentLength) && contentLength > maxBytes) { + ctx.warn('image too large', { contentLength, maxBytes, url }) + return undefined + } + } + + // Verify it's actually an image + const contentType = response.headers.get('content-type') + if (contentType === null || !contentType.startsWith('image/')) { + ctx.warn('URL is not an image', { contentType: contentType ?? 'null', url }) + return undefined + } + + const arrayBuffer = await response.arrayBuffer() + + // Double-check size after download (in case Content-Length was missing/wrong) + if (arrayBuffer.byteLength > maxBytes) { + ctx.warn('downloaded image too large', { size: arrayBuffer.byteLength, maxBytes, url }) + return undefined + } + + const uint8Array = new Uint8Array(arrayBuffer) + const size = imageSize(uint8Array) + + if (typeof size.width === 'number' && typeof size.height === 'number') { + return { width: size.width, height: size.height } + } + + return undefined + } catch (error) { + // Don't propagate image size errors - they're non-critical + ctx.warn('failed to get image size', { + error: error instanceof Error ? error.message : String(error), + url + }) + return undefined + } +} + +// ============================================================================ +// Open Graph Parsing +// ============================================================================ + +function parseOpenGraphData ( + $: ReturnType, + config: Config, + parsedUrl: URL, + finalUrl: string +): Omit { + const host = `${parsedUrl.protocol}//${parsedUrl.host}` + + const ogSiteName = $('meta[property="og:site_name"]').attr('content') + const hostname = isNonEmptyString(ogSiteName) ? ogSiteName : parsedUrl.hostname + + // Title: OG > Twitter > title tag + const ogTitle = $('meta[property="og:title"]').attr('content') + const twitterTitle = $('meta[name="twitter:title"]').attr('content') + const htmlTitle = $('title').text().trim() + + let title: string | undefined + if (isNonEmptyString(ogTitle)) { + title = ogTitle + } else if (isNonEmptyString(twitterTitle)) { + title = twitterTitle + } else if (htmlTitle.length > 0) { + title = htmlTitle + } + + // Description: OG > Twitter > meta description > paragraph extraction + const ogDescription = $('meta[property="og:description"]').attr('content') + const twitterDescription = $('meta[name="twitter:description"]').attr('content') + const metaDescription = $('meta[name="description"]').attr('content') + + let description: string | undefined + if (isNonEmptyString(ogDescription)) { + description = ogDescription + } else if (isNonEmptyString(twitterDescription)) { + description = twitterDescription + } else if (isNonEmptyString(metaDescription)) { + description = metaDescription + } else { + description = parseDescription($, config) + } + + // URL: canonical > OG URL > original + const canonicalHref = $('link[rel="canonical"]').attr('href') + const ogUrl = $('meta[property="og:url"]').attr('content') + + let url = finalUrl + if (isNonEmptyString(canonicalHref)) { + url = resolveUrl(canonicalHref, finalUrl) + } else if (isNonEmptyString(ogUrl)) { + url = resolveUrl(ogUrl, finalUrl) + } + + // Icon: shortcut icon > icon > apple-touch-icon + const shortcutIcon = $('link[rel="shortcut icon"]').attr('href') + const iconLink = $('link[rel="icon"]').attr('href') + const appleTouchIcon = $('link[rel="apple-touch-icon"]').attr('href') + + let icon: string | undefined + if (isNonEmptyString(shortcutIcon)) { + icon = shortcutIcon + } else if (isNonEmptyString(iconLink)) { + icon = iconLink + } else if (isNonEmptyString(appleTouchIcon)) { + icon = appleTouchIcon + } + + if (icon !== undefined && !icon.startsWith('http')) { + icon = new URL(icon, finalUrl).href + } + + // Image: OG > Twitter + const ogImage = $('meta[property="og:image"]').attr('content') + const ogImageUrl = $('meta[property="og:image:url"]').attr('content') + const twitterImage = $('meta[name="twitter:image"]').attr('content') + const twitterImageSrc = $('meta[name="twitter:image:src"]').attr('content') + + let image: string | undefined + if (isNonEmptyString(ogImage)) { + image = resolveUrl(ogImage, finalUrl) + } else if (isNonEmptyString(ogImageUrl)) { + image = resolveUrl(ogImageUrl, finalUrl) + } else if (isNonEmptyString(twitterImage)) { + image = resolveUrl(twitterImage, finalUrl) + } else if (isNonEmptyString(twitterImageSrc)) { + image = resolveUrl(twitterImageSrc, finalUrl) + } + + return { title, description, url, hostname, host, icon, image } +} + +function resolveUrl (url: string, base: string): string { + if (url.startsWith('https://') || url.startsWith('http://')) { + return url + } + try { + return new URL(url, base).href + } catch { + return base + } +} + +function isNonEmptyString (value: string | undefined | null): value is string { + return value !== undefined && value !== null && value.length > 0 +} + +// ============================================================================ +// Main Export +// ============================================================================ + +export async function parseLinkPreviewDetails ( + ctx: MeasureContext, + config: Config, + query: string +): Promise { + const timeoutMs = config.TimeoutMs ?? DEFAULT_TIMEOUT_MS + + // Validate URL first (SSRF protection) + const parsedUrl = validateUrl(query) + + // Fetch the page + const response = await fetchWithTimeout( + query, + { + redirect: 'follow', + headers: { + Accept: 'text/html,application/xhtml+xml', + 'User-Agent': config.UserAgent + } + }, + timeoutMs + ) + + if (!response.ok) { + throw new LinkPreviewError(`Failed to fetch URL: ${response.status} ${response.statusText}`) + } + + // Check if response is an image + const contentType = response.headers.get('content-type') ?? '' + if (contentType.startsWith('image/')) { + const size = await loadImageSize(ctx, query, config) + return { + url: query, + image: query, + host: `${parsedUrl.protocol}//${parsedUrl.host}`, + hostname: parsedUrl.hostname, + imageWidth: size?.width, + imageHeight: size?.height + } + } + + // Parse HTML + const rawBody = await response.arrayBuffer() + const utf8Body = new TextDecoder('utf-8').decode(rawBody) + const $utf8 = cheerio.load(utf8Body) + + const detectedCharset = parseCharset(response.headers, $utf8) + const $ = detectedCharset === 'utf-8' ? $utf8 : cheerio.load(new TextDecoder(detectedCharset).decode(rawBody)) + + const host = `${parsedUrl.protocol}//${parsedUrl.host}` + + // Try oEmbed first + const oembedData = await fetchOEmbedData(ctx, $, query, timeoutMs) + if (oembedData !== null) { + const ogSiteName = $('meta[property="og:site_name"]').attr('content') + const hostname = isNonEmptyString(ogSiteName) ? ogSiteName : parsedUrl.hostname + ctx.info('using oEmbed data', { url: query }) + return convertOEmbedToPreview(oembedData, query, hostname, host) + } + + // Fall back to Open Graph / meta tag parsing + ctx.info('using Open Graph data', { url: query }) + const preview = parseOpenGraphData($, config, parsedUrl, query) + + // Get image dimensions if we have an image but no dimensions + let imageWidth: number | undefined + let imageHeight: number | undefined + + const ogWidth = $('meta[property="og:image:width"]').attr('content') + const ogHeight = $('meta[property="og:image:height"]').attr('content') + + if (isNonEmptyString(ogWidth) && isNonEmptyString(ogHeight)) { + const width = parseInt(ogWidth, 10) + const height = parseInt(ogHeight, 10) + if (!Number.isNaN(width) && !Number.isNaN(height) && width > 0 && height > 0) { + imageWidth = width + imageHeight = height + } + } + + if (preview.image !== undefined && imageWidth === undefined) { + const size = await loadImageSize(ctx, preview.image, config) + if (size !== undefined) { + imageWidth = size.width + imageHeight = size.height + } + } + + return { + ...preview, + charset: detectedCharset, + imageWidth, + imageHeight + } +} + +function oembedSchemeToRegex (scheme: string): RegExp { + // Escape special regex characters except * + const regex = scheme + .replace(/[.+?^${}()|[\]\\]/g, '\\$&') + // ** matches anything (including slashes) + .replace(/\*\*/g, '.*') + // * matches anything except slashes + .replace(/\*/g, '[^/]*') + + return new RegExp(`^${regex}$`) +} diff --git a/pods/link-preview/src/server.ts b/pods/link-preview/src/server.ts new file mode 100644 index 0000000000..4a8512988d --- /dev/null +++ b/pods/link-preview/src/server.ts @@ -0,0 +1,165 @@ +// +// Copyright © 2026 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import { Analytics } from '@hcengineering/analytics' +import { MeasureContext, metricsAggregate } from '@hcengineering/core' +import { getCPUInfo, getMemoryInfo } from '@hcengineering/server-core' +import { decodeToken, TokenError } from '@hcengineering/server-token' + +import cors from 'cors' +import express, { type Express, type NextFunction, type Response } from 'express' +import { type Server } from 'http' +import morgan from 'morgan' + +import { type Config } from './config' +import { type RequestWithAuth, errorHandler, keepAlive, withAuthorization } from './middleware' +import { parseLinkPreviewDetails } from './parse' + +const KEEP_ALIVE_TIMEOUT = 5 // seconds +const KEEP_ALIVE_MAX = 1000 + +const cacheControlNoCache = 'public, no-store, no-cache, must-revalidate, max-age=0' + +type AsyncRequestHandler = (ctx: MeasureContext, req: RequestWithAuth, res: Response) => Promise + +const handleRequest = async ( + ctx: MeasureContext, + name: string, + fn: AsyncRequestHandler, + req: RequestWithAuth, + res: Response, + next: NextFunction +): Promise => { + try { + const source = req.token?.extra?.service ?? '🤦‍♂️user' + await ctx.with(name, { source }, (ctx) => { + return fn(ctx, req, res) + }) + } catch (err: unknown) { + next(err) + } +} + +export async function createServer (ctx: MeasureContext, config: Config): Promise<{ app: Express, close: () => void }> { + const app = express() + + app.use( + cors({ + maxAge: 86400 + }) + ) + app.use(express.json({ limit: '1mb' })) + app.use(keepAlive({ timeout: KEEP_ALIVE_TIMEOUT, max: KEEP_ALIVE_MAX })) + + const childLogger = ctx.logger.childLogger?.('requests', { enableConsole: 'true' }) + const requests = ctx.newChild('requests', {}, { logger: childLogger, span: false }) + class LogStream { + write (text: string): void { + requests.info(text) + } + } + + const wrapRequest = + (ctx: MeasureContext, name: string, fn: AsyncRequestHandler) => + (req: RequestWithAuth, res: Response, next: NextFunction) => { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + handleRequest(ctx, name, fn, req, res, next) + } + + app.use(morgan('short', { stream: new LogStream() })) + + app.get( + '/details', + withAuthorization, + wrapRequest(ctx, 'getLinkPreviewDetails', async (ctx, req, res) => { + if (req.query === undefined) { + res.status(400).json({ message: 'Bad Request' }) + return + } + if (req.query.q === undefined) { + res.status(400).json({ message: 'Bad Request' }) + return + } + const url = req.query.q as string + try { + const result = await parseLinkPreviewDetails(ctx, config, url) + res.status(200).json(result) + } catch (err) { + console.error({ message: 'failed to parse link preview details', url, err }) + res.status(422).json({ message: 'Unprocessable Entity' }) + } + }) + ) + + app.get('/api/v1/statistics', (req, res) => { + try { + const token = req.query.token as string + const payload = decodeToken(token) + const admin = payload.extra?.admin === 'true' + res.setHeader('Content-Type', 'application/json') + res.setHeader('Cache-Control', cacheControlNoCache) + + const json = JSON.stringify({ + metrics: metricsAggregate((ctx as any).metrics), + statistics: { + cpu: getCPUInfo(), + memory: getMemoryInfo() + }, + admin + }) + res.status(200).send(json) + } catch (err: any) { + if (err instanceof TokenError) { + res.status(401).send() + return + } + ctx.error('statistics error', { err }) + Analytics.handleError(err) + res.status(404).send() + } + }) + + app.get('/', (_req, res) => { + res.send(` + Huly® Link Preview Service™ https://huly.io + © 2026 Huly Labs + `) + }) + + app.use((req, res) => { + res.status(404).json({ message: 'Not Found' }) + }) + + app.use(errorHandler({ ctx })) + + return { + app, + close: () => { + // No cleanup needed for this service + } + } +} + +export function listen (e: Express, port: number, host?: string): Server { + const cb = (): void => { + console.log(`Service started at ${host ?? '*'}:${port}`) + } + + const server = host !== undefined ? e.listen(port, host, cb) : e.listen(port, cb) + server.keepAliveTimeout = KEEP_ALIVE_TIMEOUT * 1000 + 1000 + server.headersTimeout = KEEP_ALIVE_TIMEOUT * 1000 + 2000 + + return server +} diff --git a/pods/link-preview/tsconfig.json b/pods/link-preview/tsconfig.json new file mode 100644 index 0000000000..c6a877cf6c --- /dev/null +++ b/pods/link-preview/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "lib", "dist", "types", "bundle"] +} \ No newline at end of file diff --git a/rush.json b/rush.json index f9b61dea0c..aeca18c9d4 100644 --- a/rush.json +++ b/rush.json @@ -1606,6 +1606,11 @@ "projectFolder": "pods/preview", "shouldPublish": false }, + { + "packageName": "@hcengineering/pod-link-preview", + "projectFolder": "pods/link-preview", + "shouldPublish": false + }, { "packageName": "@hcengineering/hr", "projectFolder": "plugins/hr",