mirror of
https://github.com/hcengineering/platform.git
synced 2026-08-17 18:05:42 +02:00
feat: link preview service (#10424)
Signed-off-by: Alexander Onnikov <Alexander.Onnikov@xored.com>
This commit is contained in:
Generated
+196
-1
@@ -30444,6 +30444,115 @@ importers:
|
||||
specifier: ^5.9.3
|
||||
version: 5.9.3
|
||||
|
||||
../../pods/link-preview:
|
||||
dependencies:
|
||||
'@hcengineering/analytics':
|
||||
specifier: workspace:^0.7.17
|
||||
version: link:../../foundations/core/packages/analytics
|
||||
'@hcengineering/analytics-service':
|
||||
specifier: workspace:^0.7.17
|
||||
version: link:../../foundations/core/packages/analytics-service
|
||||
'@hcengineering/core':
|
||||
specifier: workspace:^0.7.24
|
||||
version: link:../../foundations/core/packages/core
|
||||
'@hcengineering/platform':
|
||||
specifier: workspace:^0.7.19
|
||||
version: link:../../foundations/core/packages/platform
|
||||
'@hcengineering/server-client':
|
||||
specifier: workspace:^0.7.16
|
||||
version: link:../../foundations/server/packages/client
|
||||
'@hcengineering/server-core':
|
||||
specifier: workspace:^0.7.18
|
||||
version: link:../../foundations/server/packages/core
|
||||
'@hcengineering/server-token':
|
||||
specifier: workspace:^0.7.17
|
||||
version: link:../../foundations/core/packages/token
|
||||
cheerio:
|
||||
specifier: ^1.1.2
|
||||
version: 1.1.2
|
||||
cors:
|
||||
specifier: ^2.8.5
|
||||
version: 2.8.5
|
||||
dotenv:
|
||||
specifier: ^16.4.5
|
||||
version: 16.6.1
|
||||
express:
|
||||
specifier: ^4.21.2
|
||||
version: 4.21.2
|
||||
image-size:
|
||||
specifier: ^1.1.1
|
||||
version: 1.2.1
|
||||
morgan:
|
||||
specifier: ^1.10.0
|
||||
version: 1.10.1
|
||||
oembed-providers:
|
||||
specifier: ~1.0.20260106
|
||||
version: 1.0.20260106
|
||||
devDependencies:
|
||||
'@hcengineering/platform-rig':
|
||||
specifier: workspace:^0.7.19
|
||||
version: link:../../foundations/utils/packages/platform-rig
|
||||
'@types/cors':
|
||||
specifier: ^2.8.12
|
||||
version: 2.8.19
|
||||
'@types/express':
|
||||
specifier: ^4.17.13
|
||||
version: 4.17.25
|
||||
'@types/jest':
|
||||
specifier: ^29.5.5
|
||||
version: 29.5.14
|
||||
'@types/morgan':
|
||||
specifier: ~1.9.9
|
||||
version: 1.9.10
|
||||
'@types/node':
|
||||
specifier: ^22.18.1
|
||||
version: 22.19.0
|
||||
'@typescript-eslint/eslint-plugin':
|
||||
specifier: ^6.21.0
|
||||
version: 6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3)
|
||||
'@typescript-eslint/parser':
|
||||
specifier: ^6.21.0
|
||||
version: 6.21.0(eslint@8.57.1)(typescript@5.9.3)
|
||||
cross-env:
|
||||
specifier: ~7.0.3
|
||||
version: 7.0.3
|
||||
esbuild:
|
||||
specifier: ^0.25.10
|
||||
version: 0.25.12
|
||||
eslint:
|
||||
specifier: ^8.54.0
|
||||
version: 8.57.1
|
||||
eslint-config-standard-with-typescript:
|
||||
specifier: ^40.0.0
|
||||
version: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@8.57.1)(typescript@5.9.3))(eslint@8.57.1)(typescript@5.9.3))(eslint-plugin-import@2.32.0(eslint@8.57.1))(eslint-plugin-n@15.7.0(eslint@8.57.1))(eslint-plugin-promise@6.6.0(eslint@8.57.1))(eslint@8.57.1)(typescript@5.9.3)
|
||||
eslint-plugin-import:
|
||||
specifier: ^2.26.0
|
||||
version: 2.32.0(eslint@8.57.1)
|
||||
eslint-plugin-n:
|
||||
specifier: ^15.4.0
|
||||
version: 15.7.0(eslint@8.57.1)
|
||||
eslint-plugin-node:
|
||||
specifier: ^11.1.0
|
||||
version: 11.1.0(eslint@8.57.1)
|
||||
eslint-plugin-promise:
|
||||
specifier: ^6.1.1
|
||||
version: 6.6.0(eslint@8.57.1)
|
||||
jest:
|
||||
specifier: ^29.7.0
|
||||
version: 29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3))
|
||||
prettier:
|
||||
specifier: ^3.6.2
|
||||
version: 3.6.2
|
||||
ts-jest:
|
||||
specifier: ^29.1.1
|
||||
version: 29.4.5(@babel/core@7.28.5)(@jest/transform@29.7.0)(@jest/types@30.2.0)(babel-jest@29.7.0(@babel/core@7.28.5))(esbuild@0.25.12)(jest-util@30.2.0)(jest@29.7.0(@types/node@22.19.0)(ts-node@10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3)))(typescript@5.9.3)
|
||||
ts-node:
|
||||
specifier: ^10.9.2
|
||||
version: 10.9.2(@swc/core@1.15.1)(@types/node@22.19.0)(typescript@5.9.3)
|
||||
typescript:
|
||||
specifier: ^5.9.3
|
||||
version: 5.9.3
|
||||
|
||||
../../pods/media:
|
||||
dependencies:
|
||||
'@hcengineering/account-client':
|
||||
@@ -45554,10 +45663,17 @@ packages:
|
||||
charenc@0.0.2:
|
||||
resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}
|
||||
|
||||
cheerio-select@2.1.0:
|
||||
resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}
|
||||
|
||||
cheerio@0.22.0:
|
||||
resolution: {integrity: sha512-8/MzidM6G/TgRelkzDG13y3Y9LxBjCb+8yOEZ9+wwq5gVF2w2pV0wmHvjfT0RvuxGyR7UEuK36r+yYMbT4uKgA==}
|
||||
engines: {node: '>= 0.6'}
|
||||
|
||||
cheerio@1.1.2:
|
||||
resolution: {integrity: sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
chevrotain-allstar@0.3.1:
|
||||
resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==}
|
||||
peerDependencies:
|
||||
@@ -45940,6 +46056,9 @@ packages:
|
||||
css-select@4.3.0:
|
||||
resolution: {integrity: sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==}
|
||||
|
||||
css-select@5.2.2:
|
||||
resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==}
|
||||
|
||||
css-tree@1.1.3:
|
||||
resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
@@ -46595,6 +46714,9 @@ packages:
|
||||
resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==}
|
||||
engines: {node: '>= 0.8'}
|
||||
|
||||
encoding-sniffer@0.2.1:
|
||||
resolution: {integrity: sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==}
|
||||
|
||||
encoding@0.1.13:
|
||||
resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==}
|
||||
|
||||
@@ -47564,6 +47686,9 @@ packages:
|
||||
webpack:
|
||||
optional: true
|
||||
|
||||
htmlparser2@10.0.0:
|
||||
resolution: {integrity: sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==}
|
||||
|
||||
htmlparser2@3.10.1:
|
||||
resolution: {integrity: sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==}
|
||||
|
||||
@@ -49197,6 +49322,9 @@ packages:
|
||||
resolution: {integrity: sha512-7Abo3nADdja8l/aglU6Y3lpnHSfv0tw7gFPiqzry/yCU+2gTAX7R1roJ8hJrxIK+S1j+7iqRJXtmuHJ/UDsBhQ==}
|
||||
engines: {node: '>= 18'}
|
||||
|
||||
oembed-providers@1.0.20260106:
|
||||
resolution: {integrity: sha512-u1mNQgfTuuY3ydQDN7G7MnlCVlqM5fhgVxNc6VQTnEIq/OkLPtBaFK0ADvkRciVNLEa/j925jUR30kFVHN2WtQ==}
|
||||
|
||||
oidc-token-hash@5.2.0:
|
||||
resolution: {integrity: sha512-6gj2m8cJZ+iSW8bm0FXdGF0YhIQbKrfP4yWTNzxc31U6MOjfEmB1rHvlYvxI1B7t7BCi1F2vYTT6YhtQRG4hxw==}
|
||||
engines: {node: ^10.13.0 || >=12.0.0}
|
||||
@@ -49384,6 +49512,12 @@ packages:
|
||||
parse-srcset@1.0.2:
|
||||
resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==}
|
||||
|
||||
parse5-htmlparser2-tree-adapter@7.1.0:
|
||||
resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==}
|
||||
|
||||
parse5-parser-stream@7.1.2:
|
||||
resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==}
|
||||
|
||||
parse5@7.3.0:
|
||||
resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==}
|
||||
|
||||
@@ -51167,6 +51301,10 @@ packages:
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
undici@7.18.2:
|
||||
resolution: {integrity: sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
|
||||
unionfs@4.6.0:
|
||||
resolution: {integrity: sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA==}
|
||||
|
||||
@@ -51440,6 +51578,7 @@ packages:
|
||||
whatwg-encoding@3.1.1:
|
||||
resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==}
|
||||
engines: {node: '>=18'}
|
||||
deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation
|
||||
|
||||
whatwg-mimetype@4.0.0:
|
||||
resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==}
|
||||
@@ -57038,6 +57177,15 @@ snapshots:
|
||||
|
||||
charenc@0.0.2: {}
|
||||
|
||||
cheerio-select@2.1.0:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-select: 5.2.2
|
||||
css-what: 6.2.2
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
|
||||
cheerio@0.22.0:
|
||||
dependencies:
|
||||
css-select: 1.2.0
|
||||
@@ -57057,6 +57205,20 @@ snapshots:
|
||||
lodash.reject: 4.6.0
|
||||
lodash.some: 4.6.0
|
||||
|
||||
cheerio@1.1.2:
|
||||
dependencies:
|
||||
cheerio-select: 2.1.0
|
||||
dom-serializer: 2.0.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
encoding-sniffer: 0.2.1
|
||||
htmlparser2: 10.0.0
|
||||
parse5: 7.3.0
|
||||
parse5-htmlparser2-tree-adapter: 7.1.0
|
||||
parse5-parser-stream: 7.1.2
|
||||
undici: 7.18.2
|
||||
whatwg-mimetype: 4.0.0
|
||||
|
||||
chevrotain-allstar@0.3.1(chevrotain@11.0.3):
|
||||
dependencies:
|
||||
chevrotain: 11.0.3
|
||||
@@ -57474,6 +57636,14 @@ snapshots:
|
||||
domutils: 2.8.0
|
||||
nth-check: 2.1.1
|
||||
|
||||
css-select@5.2.2:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-what: 6.2.2
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
nth-check: 2.1.1
|
||||
|
||||
css-tree@1.1.3:
|
||||
dependencies:
|
||||
mdn-data: 2.0.14
|
||||
@@ -57979,7 +58149,7 @@ snapshots:
|
||||
|
||||
domutils@1.5.1:
|
||||
dependencies:
|
||||
dom-serializer: 0.1.1
|
||||
dom-serializer: 0.2.2
|
||||
domelementtype: 1.3.1
|
||||
|
||||
domutils@1.7.0:
|
||||
@@ -58164,6 +58334,11 @@ snapshots:
|
||||
|
||||
encodeurl@2.0.0: {}
|
||||
|
||||
encoding-sniffer@0.2.1:
|
||||
dependencies:
|
||||
iconv-lite: 0.6.3
|
||||
whatwg-encoding: 3.1.1
|
||||
|
||||
encoding@0.1.13:
|
||||
dependencies:
|
||||
iconv-lite: 0.6.3
|
||||
@@ -59429,6 +59604,13 @@ snapshots:
|
||||
optionalDependencies:
|
||||
webpack: 5.102.1(@swc/core@1.15.1)(esbuild@0.25.12)(webpack-cli@5.1.4)
|
||||
|
||||
htmlparser2@10.0.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.2.2
|
||||
entities: 6.0.1
|
||||
|
||||
htmlparser2@3.10.1:
|
||||
dependencies:
|
||||
domelementtype: 1.3.1
|
||||
@@ -61389,6 +61571,8 @@ snapshots:
|
||||
'@octokit/types': 13.10.0
|
||||
'@octokit/webhooks': 12.3.2
|
||||
|
||||
oembed-providers@1.0.20260106: {}
|
||||
|
||||
oidc-token-hash@5.2.0: {}
|
||||
|
||||
omggif@1.0.10: {}
|
||||
@@ -61589,6 +61773,15 @@ snapshots:
|
||||
|
||||
parse-srcset@1.0.2: {}
|
||||
|
||||
parse5-htmlparser2-tree-adapter@7.1.0:
|
||||
dependencies:
|
||||
domhandler: 5.0.3
|
||||
parse5: 7.3.0
|
||||
|
||||
parse5-parser-stream@7.1.2:
|
||||
dependencies:
|
||||
parse5: 7.3.0
|
||||
|
||||
parse5@7.3.0:
|
||||
dependencies:
|
||||
entities: 6.0.1
|
||||
@@ -63686,6 +63879,8 @@ snapshots:
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
undici@7.18.2: {}
|
||||
|
||||
unionfs@4.6.0:
|
||||
dependencies:
|
||||
fs-monkey: 1.1.0
|
||||
|
||||
@@ -21,6 +21,7 @@ rush docker:build -p 20 \
|
||||
--to @hcengineering/pod-export \
|
||||
--to @hcengineering/pod-media \
|
||||
--to @hcengineering/pod-preview \
|
||||
--to @hcengineering/pod-link-preview \
|
||||
--to @hcengineering/pod-external \
|
||||
--to @hcengineering/pod-backup \
|
||||
--to @hcengineering/backup-api-pod \
|
||||
|
||||
@@ -40,6 +40,17 @@ services:
|
||||
ports:
|
||||
- 4040:4040
|
||||
restart: unless-stopped
|
||||
link-preview:
|
||||
image: 'hardcoreeng/link-preview'
|
||||
extra_hosts:
|
||||
- 'huly.local:host-gateway'
|
||||
container_name: link-preview
|
||||
environment:
|
||||
- SECRET=secret
|
||||
- PORT=4041
|
||||
ports:
|
||||
- 4041:4041
|
||||
restart: unless-stopped
|
||||
cockroach:
|
||||
image: cockroachdb/cockroach:latest-v24.3
|
||||
extra_hosts:
|
||||
@@ -306,6 +317,7 @@ services:
|
||||
- DATALAKE_URL=http://huly.local:4030
|
||||
- HULYLAKE_URL=http://huly.local:8096
|
||||
- EXPORT_URL=http://huly.local:4009
|
||||
- LINK_PREVIEW_URL=http://huly.local:4041
|
||||
restart: unless-stopped
|
||||
transactor_cockroach:
|
||||
image: hardcoreeng/transactor
|
||||
|
||||
@@ -31,5 +31,6 @@
|
||||
"PULSE_URL": "ws://huly.local:8099/ws",
|
||||
"HULYLAKE_URL": "http://huly.local:8096",
|
||||
"EXCLUDED_APPLICATIONS_FOR_ANONYMOUS": "[\"chunter\", \"notification\"]",
|
||||
"ANALYTICS_COLLECTOR_URL": "http://huly.local:4017"
|
||||
"ANALYTICS_COLLECTOR_URL": "http://huly.local:4017",
|
||||
"LINK_PREVIEW_URL": "http://huly.local:4041"
|
||||
}
|
||||
|
||||
@@ -39,7 +39,8 @@
|
||||
getClient,
|
||||
isLinkPreviewEnabled,
|
||||
uploadFile,
|
||||
LinkPreviewAttachmentMetadata
|
||||
LinkPreviewAttachmentMetadata,
|
||||
generateFileId
|
||||
} from '@hcengineering/presentation'
|
||||
import { EmptyMarkup, isEmptyMarkup } from '@hcengineering/text'
|
||||
import textEditor, { type RefAction } from '@hcengineering/text-editor'
|
||||
@@ -383,9 +384,10 @@
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const meta = await fetchLinkPreviewDetails(url)
|
||||
if (canDisplayLinkPreview(meta) && meta.url !== undefined) {
|
||||
if (canDisplayLinkPreview(meta)) {
|
||||
const uuid = generateFileId()
|
||||
const blob = new Blob([JSON.stringify(meta)])
|
||||
const file = new File([blob], meta.url, { type: 'application/link-preview' })
|
||||
const file = new File([blob], uuid, { type: 'application/link-preview' })
|
||||
const metadata: LinkPreviewAttachmentMetadata = {
|
||||
title: meta.title,
|
||||
image: meta.image,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
module.exports = {
|
||||
extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
|
||||
parserOptions: {
|
||||
tsconfigRootDir: __dirname,
|
||||
project: './tsconfig.json'
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM hardcoreeng/base-slim:v20250916
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY bundle/bundle.js ./
|
||||
COPY bundle/bundle.js.map ./
|
||||
|
||||
EXPOSE 4041
|
||||
CMD ["dumb-init", "node", "bundle.js"]
|
||||
@@ -0,0 +1,7 @@
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
|
||||
roots: ["./src"],
|
||||
coverageReporters: ["text-summary", "html"]
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"name": "@hcengineering/pod-link-preview",
|
||||
"version": "0.7.0",
|
||||
"main": "lib/index.js",
|
||||
"svelte": "src/index.ts",
|
||||
"types": "types/index.d.ts",
|
||||
"files": [
|
||||
"lib/**/*",
|
||||
"types/**/*",
|
||||
"tsconfig.json"
|
||||
],
|
||||
"author": "Hardcore Engineering Inc.",
|
||||
"scripts": {
|
||||
"build": "compile",
|
||||
"build:watch": "compile",
|
||||
"test": "jest --passWithNoTests --silent",
|
||||
"_phase:bundle": "rushx bundle",
|
||||
"_phase:docker-build": "rushx docker:build",
|
||||
"_phase:docker-staging": "rushx docker:staging",
|
||||
"bundle": "node ../../common/scripts/esbuild.js --keep-names=true --sourcemap=external",
|
||||
"docker:build": "../../common/scripts/docker_build.sh hardcoreeng/link-preview",
|
||||
"docker:staging": "../../common/scripts/docker_tag.sh hardcoreeng/link-preview staging",
|
||||
"docker:abuild": "docker build -t hardcoreeng/link-preview . --platform=linux/arm64 && ../../common/scripts/docker_tag_push.sh hardcoreeng/link-preview",
|
||||
"docker:push": "../../common/scripts/docker_tag.sh hardcoreeng/link-preview",
|
||||
"run-bundle": "rushx bundle --to @hcengineering/pod-link-preview && cross-env NODE_ENV=production node --inspect bundle/bundle.js",
|
||||
"run-local": "ts-node src/index.ts",
|
||||
"format": "format src",
|
||||
"_phase:build": "compile transpile src",
|
||||
"_phase:test": "jest --passWithNoTests --silent",
|
||||
"_phase:format": "format src",
|
||||
"_phase:validate": "compile validate"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@hcengineering/platform-rig": "workspace:^0.7.19",
|
||||
"@types/jest": "^29.5.5",
|
||||
"@types/node": "^22.18.1",
|
||||
"@typescript-eslint/eslint-plugin": "^6.21.0",
|
||||
"@typescript-eslint/parser": "^6.21.0",
|
||||
"cross-env": "~7.0.3",
|
||||
"esbuild": "^0.25.10",
|
||||
"eslint": "^8.54.0",
|
||||
"eslint-config-standard-with-typescript": "^40.0.0",
|
||||
"eslint-plugin-import": "^2.26.0",
|
||||
"eslint-plugin-n": "^15.4.0",
|
||||
"eslint-plugin-node": "^11.1.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"jest": "^29.7.0",
|
||||
"prettier": "^3.6.2",
|
||||
"ts-jest": "^29.1.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/cors": "^2.8.12",
|
||||
"@types/morgan": "~1.9.9"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hcengineering/core": "workspace:^0.7.24",
|
||||
"@hcengineering/platform": "workspace:^0.7.19",
|
||||
"@hcengineering/server-core": "workspace:^0.7.18",
|
||||
"@hcengineering/analytics-service": "workspace:^0.7.17",
|
||||
"@hcengineering/analytics": "workspace:^0.7.17",
|
||||
"@hcengineering/server-client": "workspace:^0.7.16",
|
||||
"@hcengineering/server-token": "workspace:^0.7.17",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.21.2",
|
||||
"cors": "^2.8.5",
|
||||
"morgan": "^1.10.0",
|
||||
"cheerio": "^1.1.2",
|
||||
"image-size": "^1.1.1",
|
||||
"oembed-providers": "~1.0.20260106"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,828 @@
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import { Config } from '../config'
|
||||
import { parseLinkPreviewDetails, LinkPreviewError } from '../parse'
|
||||
|
||||
// ============================================================================
|
||||
// Test Utilities & Mocks
|
||||
// ============================================================================
|
||||
|
||||
const createMockContext = (): MeasureContext =>
|
||||
({
|
||||
info: jest.fn(),
|
||||
warn: jest.fn(),
|
||||
error: jest.fn(),
|
||||
measure: jest.fn(),
|
||||
newChild: jest.fn(() => createMockContext())
|
||||
}) as unknown as MeasureContext
|
||||
|
||||
const defaultConfig: Config = {
|
||||
Port: 4041,
|
||||
Secret: 'secret',
|
||||
ServiceID: 'link-preview',
|
||||
UserAgent: 'TestBot/1.0',
|
||||
DescriptionMaxSentences: 3,
|
||||
DescriptionMaxLength: 200,
|
||||
TimeoutMs: 5000,
|
||||
MaxImageBytes: 1024 * 1024
|
||||
}
|
||||
|
||||
const createHtmlResponse = (html: string, headers: Record<string, string> = {}): Response => {
|
||||
return new Response(html, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
...headers
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const createJsonResponse = (data: object): Response => {
|
||||
return new Response(JSON.stringify(data), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
})
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// URL Validation Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('URL Validation', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
describe('protocol validation', () => {
|
||||
it('should accept https URLs', async () => {
|
||||
const html = '<html><head><title>Test</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.title).toBe('Test')
|
||||
})
|
||||
|
||||
it('should accept http URLs', async () => {
|
||||
const html = '<html><head><title>Test</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'http://example.com')
|
||||
expect(result.title).toBe('Test')
|
||||
})
|
||||
|
||||
it('should reject file:// protocol', async () => {
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'file:///etc/passwd')).rejects.toThrow(LinkPreviewError)
|
||||
})
|
||||
|
||||
it('should reject javascript: protocol', async () => {
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'javascript:alert(1)')).rejects.toThrow(LinkPreviewError)
|
||||
})
|
||||
|
||||
it('should reject data: protocol', async () => {
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'data:text/html,<h1>test</h1>')).rejects.toThrow(
|
||||
LinkPreviewError
|
||||
)
|
||||
})
|
||||
|
||||
it('should reject invalid URLs', async () => {
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'not-a-valid-url')).rejects.toThrow(LinkPreviewError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('SSRF protection', () => {
|
||||
const blockedAddresses = [
|
||||
'127.0.0.1',
|
||||
'127.0.0.2',
|
||||
'10.0.0.1',
|
||||
'10.255.255.255',
|
||||
'172.16.0.1',
|
||||
'172.31.255.255',
|
||||
'192.168.0.1',
|
||||
'192.168.255.255',
|
||||
'169.254.0.1',
|
||||
'0.0.0.0',
|
||||
'localhost'
|
||||
]
|
||||
|
||||
it.each(blockedAddresses)('should block access to %s', async (host) => {
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, `https://${host}/path`)).rejects.toThrow(
|
||||
LinkPreviewError
|
||||
)
|
||||
})
|
||||
|
||||
it('should allow public IP addresses', async () => {
|
||||
const html = '<html><head><title>Public</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
// These should not throw BLOCKED_URL
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://8.8.8.8')
|
||||
expect(result).toBeDefined()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Open Graph Parsing Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Open Graph Parsing', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should parse basic OG tags', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:title" content="OG Title">
|
||||
<meta property="og:description" content="OG Description">
|
||||
<meta property="og:image" content="https://example.com/image.jpg">
|
||||
<meta property="og:site_name" content="Example Site">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page')
|
||||
|
||||
expect(result.title).toBe('OG Title')
|
||||
expect(result.description).toBe('OG Description')
|
||||
expect(result.image).toBe('https://example.com/image.jpg')
|
||||
expect(result.hostname).toBe('Example Site')
|
||||
})
|
||||
|
||||
it('should fall back to title tag when OG title is missing', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<title>Page Title</title>
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.title).toBe('Page Title')
|
||||
})
|
||||
|
||||
it('should fall back to meta description when OG description is missing', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta name="description" content="Meta Description">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.description).toBe('Meta Description')
|
||||
})
|
||||
|
||||
it('should parse Twitter card tags as fallback', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta name="twitter:title" content="Twitter Title">
|
||||
<meta name="twitter:description" content="Twitter Description">
|
||||
<meta name="twitter:image" content="https://example.com/twitter.jpg">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.title).toBe('Twitter Title')
|
||||
expect(result.description).toBe('Twitter Description')
|
||||
expect(result.image).toBe('https://example.com/twitter.jpg')
|
||||
})
|
||||
|
||||
it('should resolve relative image URLs', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="/images/photo.jpg">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page')
|
||||
expect(result.image).toBe('https://example.com/images/photo.jpg')
|
||||
})
|
||||
|
||||
it('should parse canonical URL', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link rel="canonical" href="https://example.com/canonical-page">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/page?utm=123')
|
||||
expect(result.url).toBe('https://example.com/canonical-page')
|
||||
})
|
||||
|
||||
it('should parse favicon', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link rel="icon" href="/favicon.ico">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.icon).toBe('https://example.com/favicon.ico')
|
||||
})
|
||||
|
||||
it('should parse image dimensions from OG tags', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="https://example.com/image.jpg">
|
||||
<meta property="og:image:width" content="1200">
|
||||
<meta property="og:image:height" content="630">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.imageWidth).toBe(1200)
|
||||
expect(result.imageHeight).toBe(630)
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Description Extraction Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Description Extraction', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should extract description from paragraphs when meta tags are missing', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<body>
|
||||
<p>This is the first sentence. This is the second sentence.</p>
|
||||
<p>This is the third sentence.</p>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.description).toBe(
|
||||
'This is the first sentence. This is the second sentence. This is the third sentence.'
|
||||
)
|
||||
})
|
||||
|
||||
it('should limit description to configured max sentences', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<body>
|
||||
<p>Sentence one. Sentence two. Sentence three. Sentence four. Sentence five.</p>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const config = { ...defaultConfig, DescriptionMaxSentences: 2 }
|
||||
const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com')
|
||||
|
||||
expect(result.description).toBe('Sentence one. Sentence two.')
|
||||
})
|
||||
|
||||
it('should truncate description to configured max length', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<body>
|
||||
<p>This is a very long sentence that exceeds the maximum length configured for descriptions in the system.</p>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const config = { ...defaultConfig, DescriptionMaxLength: 50 }
|
||||
const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com')
|
||||
|
||||
expect(result.description?.length).toBeLessThanOrEqual(50)
|
||||
expect(result.description).toMatch(/\.\.\.$/)
|
||||
})
|
||||
|
||||
it('should handle paragraphs with question marks and exclamation points', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<body>
|
||||
<p>Is this a question? Yes it is! And here's more.</p>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.description).toBe("Is this a question? Yes it is! And here's more.")
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// oEmbed Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('oEmbed Integration', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should use oEmbed data when available via discovery', async () => {
|
||||
const oembedEndpoint = 'https://example.com/oembed?url=test'
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link type="application/json+oembed" href="https://example.com/oembed?url=test">
|
||||
<meta property="og:title" content="OG Title">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
const oembedData = {
|
||||
type: 'video',
|
||||
version: '1.0',
|
||||
title: 'oEmbed Title',
|
||||
author_name: 'Author Name',
|
||||
provider_name: 'Provider',
|
||||
thumbnail_url: 'https://example.com/thumb.jpg',
|
||||
thumbnail_width: 480,
|
||||
thumbnail_height: 360
|
||||
}
|
||||
|
||||
global.fetch = jest.fn().mockImplementation((url: string) => {
|
||||
if (url === oembedEndpoint) {
|
||||
return Promise.resolve(createJsonResponse(oembedData))
|
||||
}
|
||||
return Promise.resolve(createHtmlResponse(html))
|
||||
})
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/video')
|
||||
|
||||
expect(result.title).toBe('oEmbed Title')
|
||||
expect(result.description).toBe('Provider - Author Name')
|
||||
expect(result.image).toBe('https://example.com/thumb.jpg')
|
||||
expect(result.imageWidth).toBe(480)
|
||||
expect(result.imageHeight).toBe(360)
|
||||
})
|
||||
|
||||
it('should fall back to OG when oEmbed fails', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link type="application/json+oembed" href="https://example.com/oembed">
|
||||
<meta property="og:title" content="Fallback OG Title">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
|
||||
global.fetch = jest.fn().mockImplementation((url: string) => {
|
||||
if (url.includes('/oembed')) {
|
||||
return Promise.resolve(new Response(null, { status: 404 }))
|
||||
}
|
||||
return Promise.resolve(createHtmlResponse(html))
|
||||
})
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.title).toBe('Fallback OG Title')
|
||||
})
|
||||
|
||||
it('should reject invalid oEmbed responses', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link type="application/json+oembed" href="https://example.com/oembed">
|
||||
<meta property="og:title" content="OG Title">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
// Missing required 'type' and 'version' fields
|
||||
const invalidOembed = { title: 'Invalid' }
|
||||
|
||||
global.fetch = jest.fn().mockImplementation((url: string) => {
|
||||
if (url.includes('/oembed')) {
|
||||
return Promise.resolve(createJsonResponse(invalidOembed))
|
||||
}
|
||||
return Promise.resolve(createHtmlResponse(html))
|
||||
})
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
// Should fall back to OG
|
||||
expect(result.title).toBe('OG Title')
|
||||
})
|
||||
|
||||
it('should handle oEmbed photo type', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<link type="application/json+oembed" href="https://flickr.com/oembed">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
const oembedData = {
|
||||
type: 'photo',
|
||||
version: '1.0',
|
||||
title: 'Photo Title',
|
||||
url: 'https://flickr.com/photo.jpg',
|
||||
width: 1920,
|
||||
height: 1080
|
||||
}
|
||||
|
||||
global.fetch = jest.fn().mockImplementation((url: string) => {
|
||||
if (url.includes('/oembed')) {
|
||||
return Promise.resolve(createJsonResponse(oembedData))
|
||||
}
|
||||
return Promise.resolve(createHtmlResponse(html))
|
||||
})
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://flickr.com/photo/123')
|
||||
|
||||
expect(result.image).toBe('https://flickr.com/photo.jpg')
|
||||
expect(result.imageWidth).toBe(1920)
|
||||
expect(result.imageHeight).toBe(1080)
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Charset Handling Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Charset Handling', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should detect charset from Content-Type header', async () => {
|
||||
const html = '<html><head><title>Test</title></head></html>'
|
||||
global.fetch = jest
|
||||
.fn()
|
||||
.mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html; charset=iso-8859-1' }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.charset).toBe('iso-8859-1')
|
||||
})
|
||||
|
||||
it('should detect charset from meta charset tag', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="shift_jis">
|
||||
<title>日本語</title>
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.jp')
|
||||
expect(result.charset).toBe('shift_jis')
|
||||
})
|
||||
|
||||
it('should detect charset from http-equiv meta tag', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=euc-kr">
|
||||
<title>한국어</title>
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.kr')
|
||||
expect(result.charset).toBe('euc-kr')
|
||||
})
|
||||
|
||||
it('should default to utf-8 when no charset specified', async () => {
|
||||
const html = '<html><head><title>Test</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html' }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.charset).toBe('utf-8')
|
||||
})
|
||||
|
||||
it('should handle quoted charset values', async () => {
|
||||
const html = '<html><head><title>Test</title></head></html>'
|
||||
global.fetch = jest
|
||||
.fn()
|
||||
.mockResolvedValue(createHtmlResponse(html, { 'Content-Type': 'text/html; charset="UTF-8"' }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.charset).toBe('utf-8')
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Error Handling Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Error Handling', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should throw on HTTP error responses', async () => {
|
||||
global.fetch = jest.fn().mockResolvedValue(new Response(null, { status: 404, statusText: 'Not Found' }))
|
||||
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/missing')).rejects.toThrow(
|
||||
LinkPreviewError
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw on network errors', async () => {
|
||||
global.fetch = jest.fn().mockRejectedValue(new Error('Network error'))
|
||||
|
||||
await expect(parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')).rejects.toThrow(LinkPreviewError)
|
||||
})
|
||||
|
||||
it('should handle malformed HTML gracefully', async () => {
|
||||
const html = '<html><head><title>Broken<<<<</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
// Should not throw
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result).toBeDefined()
|
||||
expect(result.host).toBe('https://example.com')
|
||||
})
|
||||
|
||||
it('should continue when image size fetch fails', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:title" content="Test">
|
||||
<meta property="og:image" content="https://example.com/broken.jpg">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(createHtmlResponse(html))
|
||||
.mockResolvedValueOnce(new Response(null, { status: 404 }))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.title).toBe('Test')
|
||||
expect(result.image).toBe('https://example.com/broken.jpg')
|
||||
expect(result.imageWidth).toStrictEqual(undefined)
|
||||
expect(result.imageHeight).toStrictEqual(undefined)
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Image Handling Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Image Handling', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should handle direct image URLs', async () => {
|
||||
// 1x1 transparent PNG
|
||||
const pngData = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
|
||||
'base64'
|
||||
)
|
||||
|
||||
global.fetch = jest.fn().mockResolvedValue(
|
||||
new Response(pngData, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'image/png' }
|
||||
})
|
||||
)
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/image.png')
|
||||
|
||||
expect(result.image).toBe('https://example.com/image.png')
|
||||
expect(result.url).toBe('https://example.com/image.png')
|
||||
})
|
||||
|
||||
it('should reject images exceeding max size', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="https://example.com/huge.jpg">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce(createHtmlResponse(html))
|
||||
.mockResolvedValueOnce(
|
||||
new Response(null, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Content-Length': '999999999'
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
const config = { ...defaultConfig, MaxImageBytes: 1000 }
|
||||
const result = await parseLinkPreviewDetails(ctx, config, 'https://example.com')
|
||||
|
||||
// Should not have image dimensions since fetch was skipped
|
||||
expect(result.imageWidth).toBeUndefined()
|
||||
expect(ctx.warn).toHaveBeenCalledWith('image too large', expect.any(Object))
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
describe('Edge Cases', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should handle empty HTML document', async () => {
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(''))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
|
||||
expect(result.host).toBe('https://example.com')
|
||||
expect(result.hostname).toBe('example.com')
|
||||
expect(result.title).toStrictEqual(undefined)
|
||||
})
|
||||
|
||||
it('should handle URLs with special characters', async () => {
|
||||
const html = '<html><head><title>Special</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/path?q=hello%20world&foo=bar')
|
||||
|
||||
expect(result.title).toBe('Special')
|
||||
})
|
||||
|
||||
it('should handle internationalized domain names', async () => {
|
||||
const html = '<html><head><title>IDN Test</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://例え.jp/page')
|
||||
|
||||
expect(result.title).toBe('IDN Test')
|
||||
})
|
||||
|
||||
it('should handle multiple OG images (use first)', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="https://example.com/first.jpg">
|
||||
<meta property="og:image" content="https://example.com/second.jpg">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.image).toBe('https://example.com/first.jpg')
|
||||
})
|
||||
|
||||
it('should handle redirect responses', async () => {
|
||||
const html = '<html><head><title>Redirected</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com/redirect')
|
||||
expect(result.title).toBe('Redirected')
|
||||
})
|
||||
|
||||
it('should sanitize HTML entities in titles', async () => {
|
||||
const html = '<html><head><title>Rock & Roll</title></head></html>'
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
expect(result.title).toBe('Rock & Roll')
|
||||
})
|
||||
|
||||
it('should trim whitespace from extracted values', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:title" content=" Padded Title ">
|
||||
</head>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://example.com')
|
||||
// Note: cheerio may or may not trim - this tests current behavior
|
||||
expect(result.title).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
// ============================================================================
|
||||
// Integration-style Tests
|
||||
// ============================================================================
|
||||
|
||||
describe('Real-world Scenarios', () => {
|
||||
let ctx: MeasureContext
|
||||
|
||||
beforeEach(() => {
|
||||
ctx = createMockContext()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should parse a typical blog post page', async () => {
|
||||
const html = `
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>10 Tips for Better Code - Dev Blog</title>
|
||||
<meta name="description" content="Learn the top 10 tips for writing better code.">
|
||||
<meta property="og:title" content="10 Tips for Better Code">
|
||||
<meta property="og:description" content="Learn the top 10 tips for writing better, cleaner code.">
|
||||
<meta property="og:image" content="https://blog.example.com/images/code-tips.jpg">
|
||||
<meta property="og:image:width" content="1200">
|
||||
<meta property="og:image:height" content="630">
|
||||
<meta property="og:site_name" content="Dev Blog">
|
||||
<meta property="og:url" content="https://blog.example.com/posts/10-tips">
|
||||
<link rel="canonical" href="https://blog.example.com/posts/10-tips">
|
||||
<link rel="icon" href="/favicon.ico">
|
||||
</head>
|
||||
<body>
|
||||
<article>
|
||||
<h1>10 Tips for Better Code</h1>
|
||||
<p>Writing clean code is essential for maintainability.</p>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(
|
||||
ctx,
|
||||
defaultConfig,
|
||||
'https://blog.example.com/posts/10-tips?utm_source=twitter'
|
||||
)
|
||||
|
||||
expect(result).toMatchObject({
|
||||
title: '10 Tips for Better Code',
|
||||
description: 'Learn the top 10 tips for writing better, cleaner code.',
|
||||
url: 'https://blog.example.com/posts/10-tips',
|
||||
hostname: 'Dev Blog',
|
||||
host: 'https://blog.example.com',
|
||||
image: 'https://blog.example.com/images/code-tips.jpg',
|
||||
imageWidth: 1200,
|
||||
imageHeight: 630,
|
||||
icon: 'https://blog.example.com/favicon.ico',
|
||||
charset: 'utf-8'
|
||||
})
|
||||
})
|
||||
|
||||
it('should handle minimal HTML pages', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<head><title>Simple Page</title></head>
|
||||
<body><p>Just some content here.</p></body>
|
||||
</html>
|
||||
`
|
||||
global.fetch = jest.fn().mockResolvedValue(createHtmlResponse(html))
|
||||
|
||||
const result = await parseLinkPreviewDetails(ctx, defaultConfig, 'https://simple.example.com')
|
||||
|
||||
expect(result.title).toBe('Simple Page')
|
||||
expect(result.description).toBe('Just some content here.')
|
||||
expect(result.hostname).toBe('simple.example.com')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,50 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
import { config as dotenv } from 'dotenv'
|
||||
|
||||
dotenv()
|
||||
|
||||
export interface Config {
|
||||
Port: number
|
||||
Secret: string
|
||||
ServiceID: string
|
||||
UserAgent: string
|
||||
DescriptionMaxSentences: number
|
||||
DescriptionMaxLength: number
|
||||
TimeoutMs?: number
|
||||
MaxImageBytes?: number
|
||||
}
|
||||
|
||||
const config: Config = (() => {
|
||||
const params: Partial<Config> = {
|
||||
Port: parseInt(process.env.PORT ?? '4041'),
|
||||
Secret: process.env.SECRET,
|
||||
ServiceID: process.env.SERVICE_ID ?? 'link-preview',
|
||||
UserAgent: process.env.USER_AGENT ?? 'Huly Link Preview Service/1.0',
|
||||
DescriptionMaxSentences: parseInt(process.env.DESCRIPTION_MAX_SENTENCES ?? '3'),
|
||||
DescriptionMaxLength: parseInt(process.env.DESCRIPTION_MAX_LENGTH ?? '200'),
|
||||
TimeoutMs: parseInt(process.env.TIMEOUT ?? '5') * 1000
|
||||
}
|
||||
|
||||
const missingEnv = (Object.keys(params) as Array<keyof Config>).filter((key) => params[key] === undefined)
|
||||
|
||||
if (missingEnv.length > 0) {
|
||||
throw Error(`Missing config for attributes: ${missingEnv.join(', ')}`)
|
||||
}
|
||||
|
||||
return params as Config
|
||||
})()
|
||||
|
||||
export default config
|
||||
@@ -0,0 +1,23 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
export class HttpError extends Error {
|
||||
code: number
|
||||
|
||||
constructor (code: number, message: string) {
|
||||
super(message)
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { Analytics } from '@hcengineering/analytics'
|
||||
import { configureAnalytics, createOpenTelemetryMetricsContext, SplitLogger } from '@hcengineering/analytics-service'
|
||||
import { newMetrics } from '@hcengineering/core'
|
||||
import { setMetadata } from '@hcengineering/platform'
|
||||
import { initStatisticsContext } from '@hcengineering/server-core'
|
||||
import serverToken from '@hcengineering/server-token'
|
||||
import { join } from 'path'
|
||||
|
||||
import config from './config'
|
||||
import { createServer, listen } from './server'
|
||||
|
||||
const application = config.ServiceID
|
||||
|
||||
const setupMetadata = (): void => {
|
||||
setMetadata(serverToken.metadata.Secret, config.Secret)
|
||||
setMetadata(serverToken.metadata.Service, config.ServiceID)
|
||||
}
|
||||
|
||||
async function main (): Promise<void> {
|
||||
setupMetadata()
|
||||
|
||||
configureAnalytics(application, process.env.VERSION ?? '0.7.0')
|
||||
Analytics.setTag('application', application)
|
||||
|
||||
const ctx = initStatisticsContext(application, {
|
||||
factory: () =>
|
||||
createOpenTelemetryMetricsContext(
|
||||
application,
|
||||
{},
|
||||
{},
|
||||
newMetrics(),
|
||||
new SplitLogger(application, {
|
||||
root: join(process.cwd(), 'logs'),
|
||||
enableConsole: (process.env.ENABLE_CONSOLE ?? 'true') === 'true'
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
const { app, close } = await createServer(ctx, config)
|
||||
const server = listen(app, config.Port)
|
||||
|
||||
const shutdown = (): void => {
|
||||
try {
|
||||
server.close()
|
||||
close()
|
||||
|
||||
ctx.info('Shutdown complete')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
ctx.error('Error during shutdown', { error })
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
process.on('uncaughtException', (error: any) => {
|
||||
ctx.error('Uncaught exception', { error })
|
||||
})
|
||||
process.on('unhandledRejection', (error: any) => {
|
||||
ctx.error('Unhandled rejection', { error })
|
||||
})
|
||||
}
|
||||
|
||||
void main().catch((err) => {
|
||||
console.error(err)
|
||||
})
|
||||
@@ -0,0 +1,74 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { Analytics } from '@hcengineering/analytics'
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import { extractToken } from '@hcengineering/server-client'
|
||||
import { Token } from '@hcengineering/server-token'
|
||||
import { type Response, type Request, type NextFunction, type RequestHandler, type ErrorRequestHandler } from 'express'
|
||||
import { HttpError } from './error'
|
||||
|
||||
export interface KeepAliveOptions {
|
||||
timeout: number
|
||||
max: number
|
||||
}
|
||||
|
||||
export interface RequestWithAuth extends Request {
|
||||
token?: Token
|
||||
}
|
||||
|
||||
export const keepAlive = (options: KeepAliveOptions): RequestHandler => {
|
||||
const { timeout, max } = options
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
res.setHeader('Connection', 'keep-alive')
|
||||
res.setHeader('Keep-Alive', `timeout=${timeout}, max=${max}`)
|
||||
next()
|
||||
}
|
||||
}
|
||||
|
||||
export const withAuthorization = (req: RequestWithAuth, res: Response, next: NextFunction): void => {
|
||||
try {
|
||||
const token = extractToken(req.headers)
|
||||
if (token == null) {
|
||||
throw new HttpError(401, 'Unauthorized')
|
||||
}
|
||||
req.token = token
|
||||
|
||||
next()
|
||||
} catch (err: any) {
|
||||
next(err)
|
||||
}
|
||||
}
|
||||
|
||||
export interface ErrorHandlerOptions {
|
||||
ctx: MeasureContext
|
||||
}
|
||||
|
||||
export const errorHandler = (options: ErrorHandlerOptions): ErrorRequestHandler => {
|
||||
const { ctx } = options
|
||||
|
||||
return (err: any, req: Request, res: Response, _next: NextFunction): void => {
|
||||
ctx.error(err.message, { code: err.code, message: err.message })
|
||||
|
||||
if (err instanceof HttpError) {
|
||||
res.status(err.code).json({ message: err.message })
|
||||
return
|
||||
}
|
||||
|
||||
Analytics.handleError(err)
|
||||
|
||||
res.status(500).json({ message: err.message ?? 'Internal Server Error' })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { MeasureContext } from '@hcengineering/core'
|
||||
import * as cheerio from 'cheerio'
|
||||
import { imageSize } from 'image-size'
|
||||
import oembedProviders from 'oembed-providers'
|
||||
|
||||
// ============================================================================
|
||||
// Types and Interfaces
|
||||
// ============================================================================
|
||||
|
||||
export interface LinkPreviewDetails {
|
||||
title?: string
|
||||
description?: string
|
||||
url?: string
|
||||
icon?: string
|
||||
image?: string
|
||||
imageWidth?: number
|
||||
imageHeight?: number
|
||||
charset?: string
|
||||
hostname?: string
|
||||
host: string
|
||||
}
|
||||
|
||||
export interface OEmbedProvider {
|
||||
provider_name: string
|
||||
provider_url: string
|
||||
endpoints: Array<{
|
||||
url: string
|
||||
schemes?: string[]
|
||||
formats?: string[]
|
||||
discovery?: boolean
|
||||
}>
|
||||
}
|
||||
|
||||
export interface OEmbedResponse {
|
||||
type: 'photo' | 'video' | 'link' | 'rich'
|
||||
version: string
|
||||
title?: string
|
||||
author_name?: string
|
||||
author_url?: string
|
||||
provider_name?: string
|
||||
provider_url?: string
|
||||
cache_age?: number
|
||||
thumbnail_url?: string
|
||||
thumbnail_width?: number
|
||||
thumbnail_height?: number
|
||||
url?: string
|
||||
html?: string
|
||||
width?: number
|
||||
height?: number
|
||||
}
|
||||
|
||||
export interface Config {
|
||||
UserAgent: string
|
||||
DescriptionMaxSentences: number
|
||||
DescriptionMaxLength: number
|
||||
/** Request timeout in milliseconds (default: 10000) */
|
||||
TimeoutMs?: number
|
||||
/** Maximum image size to fetch in bytes (default: 10MB) */
|
||||
MaxImageBytes?: number
|
||||
/** Blocked IP ranges for SSRF protection */
|
||||
BlockedIpRanges?: string[]
|
||||
}
|
||||
|
||||
interface ImageDimensions {
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 10_000
|
||||
const DEFAULT_MAX_IMAGE_BYTES = 10 * 1024 * 1024 // 10MB
|
||||
const OEMBED_SERVICE_NAME = 'Huly Link Preview Service/1.0'
|
||||
|
||||
// Private IP ranges to block for SSRF protection
|
||||
const BLOCKED_IP_PATTERNS = [
|
||||
/^127\./, // Loopback
|
||||
/^10\./, // Private Class A
|
||||
/^172\.(1[6-9]|2\d|3[01])\./, // Private Class B
|
||||
/^192\.168\./, // Private Class C
|
||||
/^169\.254\./, // Link-local
|
||||
/^0\./, // Current network
|
||||
/^localhost$/i,
|
||||
/^::1$/, // IPv6 loopback
|
||||
/^fc00:/i, // IPv6 private
|
||||
/^fe80:/i // IPv6 link-local
|
||||
]
|
||||
|
||||
// ============================================================================
|
||||
// Error Classes
|
||||
// ============================================================================
|
||||
|
||||
export class LinkPreviewError extends Error {
|
||||
constructor (
|
||||
message: string,
|
||||
public readonly cause?: unknown
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'LinkPreviewError'
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// URL Validation
|
||||
// ============================================================================
|
||||
|
||||
function validateUrl (urlString: string): URL {
|
||||
let url: URL
|
||||
try {
|
||||
url = new URL(urlString)
|
||||
} catch {
|
||||
throw new LinkPreviewError(`Invalid URL: ${urlString}`)
|
||||
}
|
||||
|
||||
// Only allow HTTP(S) protocols
|
||||
if (!['http:', 'https:'].includes(url.protocol)) {
|
||||
throw new LinkPreviewError(`Invalid protocol: ${url.protocol}. Only HTTP and HTTPS are allowed.`)
|
||||
}
|
||||
|
||||
// SSRF protection: block private/internal IPs
|
||||
const hostname = url.hostname
|
||||
for (const pattern of BLOCKED_IP_PATTERNS) {
|
||||
if (pattern.test(hostname)) {
|
||||
throw new LinkPreviewError('Blocked URL: Access to internal addresses is not allowed.')
|
||||
}
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Fetch Utilities
|
||||
// ============================================================================
|
||||
|
||||
async function fetchWithTimeout (url: string, options: RequestInit, timeoutMs: number): Promise<Response> {
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => {
|
||||
controller.abort()
|
||||
}, timeoutMs)
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
signal: controller.signal
|
||||
})
|
||||
return response
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new LinkPreviewError(`Request timed out after ${timeoutMs}ms`, error)
|
||||
}
|
||||
throw new LinkPreviewError(
|
||||
`Failed to fetch URL: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
error
|
||||
)
|
||||
} finally {
|
||||
clearTimeout(timeoutId)
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// oEmbed Functions
|
||||
// ============================================================================
|
||||
|
||||
function findOEmbedProviderUrl (targetUrl: string): string | null {
|
||||
for (const provider of oembedProviders as OEmbedProvider[]) {
|
||||
const endpoints = provider.endpoints ?? []
|
||||
|
||||
for (const endpoint of endpoints) {
|
||||
const schemes = endpoint.schemes ?? []
|
||||
if (schemes.length === 0) continue
|
||||
|
||||
for (const scheme of schemes) {
|
||||
const regex = oembedSchemeToRegex(scheme)
|
||||
if (regex.test(targetUrl)) {
|
||||
const endpointUrl = endpoint.url
|
||||
if (endpointUrl != null) {
|
||||
return endpointUrl.replace('{format}', 'json').replace('{url}', encodeURIComponent(targetUrl))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function findOEmbedDiscoveryUrl ($: ReturnType<typeof cheerio.load>, baseUrl: string): string | null {
|
||||
const jsonOembedLink = $('link[type="application/json+oembed"]').attr('href')
|
||||
const xmlOembedLink = $('link[type="text/xml+oembed"]').attr('href')
|
||||
|
||||
let oembedLink: string | null = null
|
||||
if (isNonEmptyString(jsonOembedLink)) {
|
||||
oembedLink = jsonOembedLink
|
||||
} else if (isNonEmptyString(xmlOembedLink)) {
|
||||
oembedLink = xmlOembedLink
|
||||
}
|
||||
|
||||
if (oembedLink === null) return null
|
||||
|
||||
return oembedLink.startsWith('http') ? oembedLink : new URL(oembedLink, baseUrl).href
|
||||
}
|
||||
|
||||
async function fetchOEmbedData (
|
||||
ctx: MeasureContext,
|
||||
$: ReturnType<typeof cheerio.load>,
|
||||
baseUrl: string,
|
||||
timeoutMs: number
|
||||
): Promise<OEmbedResponse | null> {
|
||||
try {
|
||||
// Try provider registry first, then discovery
|
||||
const providerUrl = findOEmbedProviderUrl(baseUrl)
|
||||
const discoveryUrl = findOEmbedDiscoveryUrl($, baseUrl)
|
||||
const oembedUrl = providerUrl ?? discoveryUrl
|
||||
|
||||
if (oembedUrl === null) return null
|
||||
|
||||
ctx.info('fetching oEmbed data', { oembedUrl })
|
||||
|
||||
const response = await fetchWithTimeout(oembedUrl, { headers: { 'User-Agent': OEMBED_SERVICE_NAME } }, timeoutMs)
|
||||
|
||||
if (!response.ok) {
|
||||
ctx.warn('oEmbed fetch failed', {
|
||||
status: response.status,
|
||||
statusText: response.statusText
|
||||
})
|
||||
return null
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type')
|
||||
if (contentType === null || !contentType.includes('json')) {
|
||||
ctx.warn('oEmbed response is not JSON', { contentType: contentType ?? 'null' })
|
||||
return null
|
||||
}
|
||||
|
||||
const data = (await response.json()) as OEmbedResponse
|
||||
|
||||
// Validate required fields per oEmbed spec
|
||||
if (data.type == null || data.version == null) {
|
||||
ctx.warn('Invalid oEmbed response: missing required fields')
|
||||
return null
|
||||
}
|
||||
|
||||
ctx.info('successfully fetched oEmbed data', { type: data.type })
|
||||
return data
|
||||
} catch (error) {
|
||||
// Don't throw - oEmbed failure should fall back to OG parsing
|
||||
ctx.warn('failed to fetch oEmbed data', {
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
// return null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function convertOEmbedToPreview (
|
||||
oembed: OEmbedResponse,
|
||||
baseUrl: string,
|
||||
hostname: string,
|
||||
host: string
|
||||
): LinkPreviewDetails {
|
||||
const descriptionParts: string[] = []
|
||||
if (oembed.provider_name !== undefined && oembed.provider_name.length > 0) {
|
||||
descriptionParts.push(oembed.provider_name)
|
||||
}
|
||||
if (oembed.author_name !== undefined && oembed.author_name.length > 0) {
|
||||
descriptionParts.push(oembed.author_name)
|
||||
}
|
||||
const description = descriptionParts.length > 0 ? descriptionParts.join(' - ') : undefined
|
||||
|
||||
let image: string | undefined
|
||||
if (oembed.thumbnail_url !== undefined && oembed.thumbnail_url.length > 0) {
|
||||
image = oembed.thumbnail_url
|
||||
} else if (oembed.type === 'photo' && oembed.url !== undefined && oembed.url.length > 0) {
|
||||
image = oembed.url
|
||||
}
|
||||
|
||||
let imageWidth: number | undefined
|
||||
if (oembed.thumbnail_width !== undefined) {
|
||||
imageWidth = oembed.thumbnail_width
|
||||
} else if (oembed.type === 'photo' && oembed.width !== undefined) {
|
||||
imageWidth = oembed.width
|
||||
}
|
||||
|
||||
let imageHeight: number | undefined
|
||||
if (oembed.thumbnail_height !== undefined) {
|
||||
imageHeight = oembed.thumbnail_height
|
||||
} else if (oembed.type === 'photo' && oembed.height !== undefined) {
|
||||
imageHeight = oembed.height
|
||||
}
|
||||
|
||||
return {
|
||||
title: oembed.title,
|
||||
description,
|
||||
url: baseUrl,
|
||||
hostname,
|
||||
host,
|
||||
image,
|
||||
imageWidth,
|
||||
imageHeight
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Charset Parsing
|
||||
// ============================================================================
|
||||
|
||||
function parseCharset (headers: Headers, $: ReturnType<typeof cheerio.load>): string {
|
||||
// Try Content-Type header first
|
||||
const headerContentType = headers.get('content-type')
|
||||
const headerCharset = extractCharsetFromContentType(headerContentType)
|
||||
if (isNonEmptyString(headerCharset)) return headerCharset
|
||||
|
||||
// Try <meta charset="...">
|
||||
const metaCharset = $('meta[charset]').attr('charset')
|
||||
if (isNonEmptyString(metaCharset)) {
|
||||
return normalizeCharset(metaCharset)
|
||||
}
|
||||
|
||||
// Try <meta http-equiv="Content-Type" content="...">
|
||||
const metaContentType = $('meta[http-equiv="Content-Type" i]').attr('content')
|
||||
const bodyCharset = extractCharsetFromContentType(metaContentType)
|
||||
if (isNonEmptyString(bodyCharset)) return bodyCharset
|
||||
|
||||
return 'utf-8'
|
||||
}
|
||||
|
||||
function extractCharsetFromContentType (contentType: string | null | undefined): string | null {
|
||||
if (contentType === null || contentType === undefined) return null
|
||||
if (!contentType.includes('charset=')) return null
|
||||
|
||||
const parts = contentType.split('charset=')
|
||||
if (parts.length < 2) return null
|
||||
|
||||
const charsetPart = parts[1]
|
||||
if (!isNonEmptyString(charsetPart)) return null
|
||||
|
||||
// Handle quoted values and trim; also handle trailing parameters
|
||||
const cleanedCharset = charsetPart.split(';')[0].split(',')[0]
|
||||
if (!isNonEmptyString(cleanedCharset)) return null
|
||||
|
||||
return normalizeCharset(cleanedCharset)
|
||||
}
|
||||
|
||||
function normalizeCharset (charset: string): string {
|
||||
return charset.toLowerCase().trim().replace(/^["']/, '').replace(/["']$/, '')
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Description Parsing
|
||||
// ============================================================================
|
||||
|
||||
function parseDescription ($: ReturnType<typeof cheerio.load>, config: Config): string | undefined {
|
||||
const paragraphs = $('p')
|
||||
if (paragraphs.length === 0) return undefined
|
||||
|
||||
const sentences: string[] = []
|
||||
const maxSentences = config.DescriptionMaxSentences
|
||||
|
||||
for (const p of paragraphs.toArray()) {
|
||||
const rawText = $(p).text()
|
||||
if (rawText === undefined || rawText === null) continue
|
||||
|
||||
const text = rawText.trim()
|
||||
if (text.length === 0) continue
|
||||
|
||||
// Split on sentence-ending punctuation followed by space or end
|
||||
const extracted = extractSentences(text)
|
||||
sentences.push(...extracted)
|
||||
|
||||
if (sentences.length >= maxSentences) break
|
||||
}
|
||||
|
||||
if (sentences.length === 0) return undefined
|
||||
|
||||
let result = sentences.slice(0, maxSentences).join(' ')
|
||||
|
||||
const maxLength = config.DescriptionMaxLength
|
||||
if (result.length > maxLength && maxLength >= 3) {
|
||||
result = result.substring(0, maxLength - 3) + '...'
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function extractSentences (text: string): string[] {
|
||||
// Match sentences ending with . ? or !
|
||||
const sentenceRegex = /[^.!?]*[.!?]+/g
|
||||
const matches = text.match(sentenceRegex)
|
||||
|
||||
if (matches === null || matches.length === 0) {
|
||||
// No sentence-ending punctuation, return whole text as one "sentence"
|
||||
const trimmed = text.trim()
|
||||
return trimmed.length > 0 ? [trimmed] : []
|
||||
}
|
||||
|
||||
return matches.map((s) => s.trim()).filter((s) => s.length > 0)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Image Size Loading
|
||||
// ============================================================================
|
||||
|
||||
async function loadImageSize (ctx: MeasureContext, url: string, config: Config): Promise<ImageDimensions | undefined> {
|
||||
const timeoutMs = config.TimeoutMs ?? DEFAULT_TIMEOUT_MS
|
||||
const maxBytes = config.MaxImageBytes ?? DEFAULT_MAX_IMAGE_BYTES
|
||||
|
||||
try {
|
||||
// Validate the image URL too
|
||||
validateUrl(url)
|
||||
|
||||
const response = await fetchWithTimeout(url, { headers: { 'User-Agent': config.UserAgent } }, timeoutMs)
|
||||
|
||||
if (!response.ok) {
|
||||
ctx.warn('failed to fetch image', { status: response.status, url })
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Check content length before downloading
|
||||
const contentLengthHeader = response.headers.get('content-length')
|
||||
if (isNonEmptyString(contentLengthHeader)) {
|
||||
const contentLength = parseInt(contentLengthHeader, 10)
|
||||
if (!Number.isNaN(contentLength) && contentLength > maxBytes) {
|
||||
ctx.warn('image too large', { contentLength, maxBytes, url })
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
// Verify it's actually an image
|
||||
const contentType = response.headers.get('content-type')
|
||||
if (contentType === null || !contentType.startsWith('image/')) {
|
||||
ctx.warn('URL is not an image', { contentType: contentType ?? 'null', url })
|
||||
return undefined
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer()
|
||||
|
||||
// Double-check size after download (in case Content-Length was missing/wrong)
|
||||
if (arrayBuffer.byteLength > maxBytes) {
|
||||
ctx.warn('downloaded image too large', { size: arrayBuffer.byteLength, maxBytes, url })
|
||||
return undefined
|
||||
}
|
||||
|
||||
const uint8Array = new Uint8Array(arrayBuffer)
|
||||
const size = imageSize(uint8Array)
|
||||
|
||||
if (typeof size.width === 'number' && typeof size.height === 'number') {
|
||||
return { width: size.width, height: size.height }
|
||||
}
|
||||
|
||||
return undefined
|
||||
} catch (error) {
|
||||
// Don't propagate image size errors - they're non-critical
|
||||
ctx.warn('failed to get image size', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
url
|
||||
})
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Open Graph Parsing
|
||||
// ============================================================================
|
||||
|
||||
function parseOpenGraphData (
|
||||
$: ReturnType<typeof cheerio.load>,
|
||||
config: Config,
|
||||
parsedUrl: URL,
|
||||
finalUrl: string
|
||||
): Omit<LinkPreviewDetails, 'imageWidth' | 'imageHeight'> {
|
||||
const host = `${parsedUrl.protocol}//${parsedUrl.host}`
|
||||
|
||||
const ogSiteName = $('meta[property="og:site_name"]').attr('content')
|
||||
const hostname = isNonEmptyString(ogSiteName) ? ogSiteName : parsedUrl.hostname
|
||||
|
||||
// Title: OG > Twitter > title tag
|
||||
const ogTitle = $('meta[property="og:title"]').attr('content')
|
||||
const twitterTitle = $('meta[name="twitter:title"]').attr('content')
|
||||
const htmlTitle = $('title').text().trim()
|
||||
|
||||
let title: string | undefined
|
||||
if (isNonEmptyString(ogTitle)) {
|
||||
title = ogTitle
|
||||
} else if (isNonEmptyString(twitterTitle)) {
|
||||
title = twitterTitle
|
||||
} else if (htmlTitle.length > 0) {
|
||||
title = htmlTitle
|
||||
}
|
||||
|
||||
// Description: OG > Twitter > meta description > paragraph extraction
|
||||
const ogDescription = $('meta[property="og:description"]').attr('content')
|
||||
const twitterDescription = $('meta[name="twitter:description"]').attr('content')
|
||||
const metaDescription = $('meta[name="description"]').attr('content')
|
||||
|
||||
let description: string | undefined
|
||||
if (isNonEmptyString(ogDescription)) {
|
||||
description = ogDescription
|
||||
} else if (isNonEmptyString(twitterDescription)) {
|
||||
description = twitterDescription
|
||||
} else if (isNonEmptyString(metaDescription)) {
|
||||
description = metaDescription
|
||||
} else {
|
||||
description = parseDescription($, config)
|
||||
}
|
||||
|
||||
// URL: canonical > OG URL > original
|
||||
const canonicalHref = $('link[rel="canonical"]').attr('href')
|
||||
const ogUrl = $('meta[property="og:url"]').attr('content')
|
||||
|
||||
let url = finalUrl
|
||||
if (isNonEmptyString(canonicalHref)) {
|
||||
url = resolveUrl(canonicalHref, finalUrl)
|
||||
} else if (isNonEmptyString(ogUrl)) {
|
||||
url = resolveUrl(ogUrl, finalUrl)
|
||||
}
|
||||
|
||||
// Icon: shortcut icon > icon > apple-touch-icon
|
||||
const shortcutIcon = $('link[rel="shortcut icon"]').attr('href')
|
||||
const iconLink = $('link[rel="icon"]').attr('href')
|
||||
const appleTouchIcon = $('link[rel="apple-touch-icon"]').attr('href')
|
||||
|
||||
let icon: string | undefined
|
||||
if (isNonEmptyString(shortcutIcon)) {
|
||||
icon = shortcutIcon
|
||||
} else if (isNonEmptyString(iconLink)) {
|
||||
icon = iconLink
|
||||
} else if (isNonEmptyString(appleTouchIcon)) {
|
||||
icon = appleTouchIcon
|
||||
}
|
||||
|
||||
if (icon !== undefined && !icon.startsWith('http')) {
|
||||
icon = new URL(icon, finalUrl).href
|
||||
}
|
||||
|
||||
// Image: OG > Twitter
|
||||
const ogImage = $('meta[property="og:image"]').attr('content')
|
||||
const ogImageUrl = $('meta[property="og:image:url"]').attr('content')
|
||||
const twitterImage = $('meta[name="twitter:image"]').attr('content')
|
||||
const twitterImageSrc = $('meta[name="twitter:image:src"]').attr('content')
|
||||
|
||||
let image: string | undefined
|
||||
if (isNonEmptyString(ogImage)) {
|
||||
image = resolveUrl(ogImage, finalUrl)
|
||||
} else if (isNonEmptyString(ogImageUrl)) {
|
||||
image = resolveUrl(ogImageUrl, finalUrl)
|
||||
} else if (isNonEmptyString(twitterImage)) {
|
||||
image = resolveUrl(twitterImage, finalUrl)
|
||||
} else if (isNonEmptyString(twitterImageSrc)) {
|
||||
image = resolveUrl(twitterImageSrc, finalUrl)
|
||||
}
|
||||
|
||||
return { title, description, url, hostname, host, icon, image }
|
||||
}
|
||||
|
||||
function resolveUrl (url: string, base: string): string {
|
||||
if (url.startsWith('https://') || url.startsWith('http://')) {
|
||||
return url
|
||||
}
|
||||
try {
|
||||
return new URL(url, base).href
|
||||
} catch {
|
||||
return base
|
||||
}
|
||||
}
|
||||
|
||||
function isNonEmptyString (value: string | undefined | null): value is string {
|
||||
return value !== undefined && value !== null && value.length > 0
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main Export
|
||||
// ============================================================================
|
||||
|
||||
export async function parseLinkPreviewDetails (
|
||||
ctx: MeasureContext,
|
||||
config: Config,
|
||||
query: string
|
||||
): Promise<LinkPreviewDetails> {
|
||||
const timeoutMs = config.TimeoutMs ?? DEFAULT_TIMEOUT_MS
|
||||
|
||||
// Validate URL first (SSRF protection)
|
||||
const parsedUrl = validateUrl(query)
|
||||
|
||||
// Fetch the page
|
||||
const response = await fetchWithTimeout(
|
||||
query,
|
||||
{
|
||||
redirect: 'follow',
|
||||
headers: {
|
||||
Accept: 'text/html,application/xhtml+xml',
|
||||
'User-Agent': config.UserAgent
|
||||
}
|
||||
},
|
||||
timeoutMs
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new LinkPreviewError(`Failed to fetch URL: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
// Check if response is an image
|
||||
const contentType = response.headers.get('content-type') ?? ''
|
||||
if (contentType.startsWith('image/')) {
|
||||
const size = await loadImageSize(ctx, query, config)
|
||||
return {
|
||||
url: query,
|
||||
image: query,
|
||||
host: `${parsedUrl.protocol}//${parsedUrl.host}`,
|
||||
hostname: parsedUrl.hostname,
|
||||
imageWidth: size?.width,
|
||||
imageHeight: size?.height
|
||||
}
|
||||
}
|
||||
|
||||
// Parse HTML
|
||||
const rawBody = await response.arrayBuffer()
|
||||
const utf8Body = new TextDecoder('utf-8').decode(rawBody)
|
||||
const $utf8 = cheerio.load(utf8Body)
|
||||
|
||||
const detectedCharset = parseCharset(response.headers, $utf8)
|
||||
const $ = detectedCharset === 'utf-8' ? $utf8 : cheerio.load(new TextDecoder(detectedCharset).decode(rawBody))
|
||||
|
||||
const host = `${parsedUrl.protocol}//${parsedUrl.host}`
|
||||
|
||||
// Try oEmbed first
|
||||
const oembedData = await fetchOEmbedData(ctx, $, query, timeoutMs)
|
||||
if (oembedData !== null) {
|
||||
const ogSiteName = $('meta[property="og:site_name"]').attr('content')
|
||||
const hostname = isNonEmptyString(ogSiteName) ? ogSiteName : parsedUrl.hostname
|
||||
ctx.info('using oEmbed data', { url: query })
|
||||
return convertOEmbedToPreview(oembedData, query, hostname, host)
|
||||
}
|
||||
|
||||
// Fall back to Open Graph / meta tag parsing
|
||||
ctx.info('using Open Graph data', { url: query })
|
||||
const preview = parseOpenGraphData($, config, parsedUrl, query)
|
||||
|
||||
// Get image dimensions if we have an image but no dimensions
|
||||
let imageWidth: number | undefined
|
||||
let imageHeight: number | undefined
|
||||
|
||||
const ogWidth = $('meta[property="og:image:width"]').attr('content')
|
||||
const ogHeight = $('meta[property="og:image:height"]').attr('content')
|
||||
|
||||
if (isNonEmptyString(ogWidth) && isNonEmptyString(ogHeight)) {
|
||||
const width = parseInt(ogWidth, 10)
|
||||
const height = parseInt(ogHeight, 10)
|
||||
if (!Number.isNaN(width) && !Number.isNaN(height) && width > 0 && height > 0) {
|
||||
imageWidth = width
|
||||
imageHeight = height
|
||||
}
|
||||
}
|
||||
|
||||
if (preview.image !== undefined && imageWidth === undefined) {
|
||||
const size = await loadImageSize(ctx, preview.image, config)
|
||||
if (size !== undefined) {
|
||||
imageWidth = size.width
|
||||
imageHeight = size.height
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...preview,
|
||||
charset: detectedCharset,
|
||||
imageWidth,
|
||||
imageHeight
|
||||
}
|
||||
}
|
||||
|
||||
function oembedSchemeToRegex (scheme: string): RegExp {
|
||||
// Escape special regex characters except *
|
||||
const regex = scheme
|
||||
.replace(/[.+?^${}()|[\]\\]/g, '\\$&')
|
||||
// ** matches anything (including slashes)
|
||||
.replace(/\*\*/g, '.*')
|
||||
// * matches anything except slashes
|
||||
.replace(/\*/g, '[^/]*')
|
||||
|
||||
return new RegExp(`^${regex}$`)
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
//
|
||||
// Copyright © 2026 Hardcore Engineering Inc.
|
||||
//
|
||||
// Licensed under the Eclipse Public License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License. You may
|
||||
// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
import { Analytics } from '@hcengineering/analytics'
|
||||
import { MeasureContext, metricsAggregate } from '@hcengineering/core'
|
||||
import { getCPUInfo, getMemoryInfo } from '@hcengineering/server-core'
|
||||
import { decodeToken, TokenError } from '@hcengineering/server-token'
|
||||
|
||||
import cors from 'cors'
|
||||
import express, { type Express, type NextFunction, type Response } from 'express'
|
||||
import { type Server } from 'http'
|
||||
import morgan from 'morgan'
|
||||
|
||||
import { type Config } from './config'
|
||||
import { type RequestWithAuth, errorHandler, keepAlive, withAuthorization } from './middleware'
|
||||
import { parseLinkPreviewDetails } from './parse'
|
||||
|
||||
const KEEP_ALIVE_TIMEOUT = 5 // seconds
|
||||
const KEEP_ALIVE_MAX = 1000
|
||||
|
||||
const cacheControlNoCache = 'public, no-store, no-cache, must-revalidate, max-age=0'
|
||||
|
||||
type AsyncRequestHandler = (ctx: MeasureContext, req: RequestWithAuth, res: Response) => Promise<void>
|
||||
|
||||
const handleRequest = async (
|
||||
ctx: MeasureContext,
|
||||
name: string,
|
||||
fn: AsyncRequestHandler,
|
||||
req: RequestWithAuth,
|
||||
res: Response,
|
||||
next: NextFunction
|
||||
): Promise<void> => {
|
||||
try {
|
||||
const source = req.token?.extra?.service ?? '🤦♂️user'
|
||||
await ctx.with(name, { source }, (ctx) => {
|
||||
return fn(ctx, req, res)
|
||||
})
|
||||
} catch (err: unknown) {
|
||||
next(err)
|
||||
}
|
||||
}
|
||||
|
||||
export async function createServer (ctx: MeasureContext, config: Config): Promise<{ app: Express, close: () => void }> {
|
||||
const app = express()
|
||||
|
||||
app.use(
|
||||
cors({
|
||||
maxAge: 86400
|
||||
})
|
||||
)
|
||||
app.use(express.json({ limit: '1mb' }))
|
||||
app.use(keepAlive({ timeout: KEEP_ALIVE_TIMEOUT, max: KEEP_ALIVE_MAX }))
|
||||
|
||||
const childLogger = ctx.logger.childLogger?.('requests', { enableConsole: 'true' })
|
||||
const requests = ctx.newChild('requests', {}, { logger: childLogger, span: false })
|
||||
class LogStream {
|
||||
write (text: string): void {
|
||||
requests.info(text)
|
||||
}
|
||||
}
|
||||
|
||||
const wrapRequest =
|
||||
(ctx: MeasureContext, name: string, fn: AsyncRequestHandler) =>
|
||||
(req: RequestWithAuth, res: Response, next: NextFunction) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
||||
handleRequest(ctx, name, fn, req, res, next)
|
||||
}
|
||||
|
||||
app.use(morgan('short', { stream: new LogStream() }))
|
||||
|
||||
app.get(
|
||||
'/details',
|
||||
withAuthorization,
|
||||
wrapRequest(ctx, 'getLinkPreviewDetails', async (ctx, req, res) => {
|
||||
if (req.query === undefined) {
|
||||
res.status(400).json({ message: 'Bad Request' })
|
||||
return
|
||||
}
|
||||
if (req.query.q === undefined) {
|
||||
res.status(400).json({ message: 'Bad Request' })
|
||||
return
|
||||
}
|
||||
const url = req.query.q as string
|
||||
try {
|
||||
const result = await parseLinkPreviewDetails(ctx, config, url)
|
||||
res.status(200).json(result)
|
||||
} catch (err) {
|
||||
console.error({ message: 'failed to parse link preview details', url, err })
|
||||
res.status(422).json({ message: 'Unprocessable Entity' })
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
app.get('/api/v1/statistics', (req, res) => {
|
||||
try {
|
||||
const token = req.query.token as string
|
||||
const payload = decodeToken(token)
|
||||
const admin = payload.extra?.admin === 'true'
|
||||
res.setHeader('Content-Type', 'application/json')
|
||||
res.setHeader('Cache-Control', cacheControlNoCache)
|
||||
|
||||
const json = JSON.stringify({
|
||||
metrics: metricsAggregate((ctx as any).metrics),
|
||||
statistics: {
|
||||
cpu: getCPUInfo(),
|
||||
memory: getMemoryInfo()
|
||||
},
|
||||
admin
|
||||
})
|
||||
res.status(200).send(json)
|
||||
} catch (err: any) {
|
||||
if (err instanceof TokenError) {
|
||||
res.status(401).send()
|
||||
return
|
||||
}
|
||||
ctx.error('statistics error', { err })
|
||||
Analytics.handleError(err)
|
||||
res.status(404).send()
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/', (_req, res) => {
|
||||
res.send(`
|
||||
Huly® Link Preview Service™ <a href="https://huly.io">https://huly.io</a>
|
||||
© 2026 <a href="https://hulylabs.com">Huly Labs</a>
|
||||
`)
|
||||
})
|
||||
|
||||
app.use((req, res) => {
|
||||
res.status(404).json({ message: 'Not Found' })
|
||||
})
|
||||
|
||||
app.use(errorHandler({ ctx }))
|
||||
|
||||
return {
|
||||
app,
|
||||
close: () => {
|
||||
// No cleanup needed for this service
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function listen (e: Express, port: number, host?: string): Server {
|
||||
const cb = (): void => {
|
||||
console.log(`Service started at ${host ?? '*'}:${port}`)
|
||||
}
|
||||
|
||||
const server = host !== undefined ? e.listen(port, host, cb) : e.listen(port, cb)
|
||||
server.keepAliveTimeout = KEEP_ALIVE_TIMEOUT * 1000 + 1000
|
||||
server.headersTimeout = KEEP_ALIVE_TIMEOUT * 1000 + 2000
|
||||
|
||||
return server
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"extends": "./node_modules/@hcengineering/platform-rig/profiles/node/tsconfig.json",
|
||||
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./lib",
|
||||
"declarationDir": "./types",
|
||||
"tsBuildInfoFile": ".build/build.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "lib", "dist", "types", "bundle"]
|
||||
}
|
||||
Reference in New Issue
Block a user