diff --git a/.github/issue_template.md b/.github/issue_template.md index acd31ec849..c030c415fc 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -2,7 +2,7 @@ A clear and concise description of the issue. ### Your environment -* Version of huly +* Version of Huly * Browser (and version) * Your operating system (and version) diff --git a/.vscode/launch.json b/.vscode/launch.json index c1f2a92066..120bfe5e16 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -95,9 +95,11 @@ "ACCOUNT_PORT": "3000", "FRONT_URL": "http://localhost:8080", "SES_URL": "", + // "WS_LIVENESS_DAYS": "1", "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", - "MINIO_ENDPOINT": "localhost" + "MINIO_ENDPOINT": "localhost", + // "DISABLE_SIGNUP": "true", // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml", // "INIT_WORKSPACE": "onboarding", }, diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index b9784be4ce..4dbab23640 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -83,8 +83,10 @@ services: - RESERVED_DB_NAMES=telegram,gmail,github - MODEL_ENABLED=* - LAST_NAME_FIRST=true + # - WS_LIVENESS_DAYS=1 - ACCOUNTS_URL=http://host.docker.internal:3000 - BRANDING_PATH=/var/cfg/branding.json + # - DISABLE_SIGNUP=true # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped @@ -197,6 +199,7 @@ services: - DESKTOP_UPDATES_URL=https://dist.huly.io - DESKTOP_UPDATES_CHANNEL=dev - BRANDING_URL=http://host.docker.internal:8087/branding.json + # - DISABLE_SIGNUP=true restart: unless-stopped transactor: image: hardcoreeng/transactor diff --git a/dev/tool/src/db.ts b/dev/tool/src/db.ts index 34f376ae3d..1d88ac4d26 100644 --- a/dev/tool/src/db.ts +++ b/dev/tool/src/db.ts @@ -54,6 +54,10 @@ export async function moveFromMongoToPG ( client.close() } +function escapeBackticks (str: string): string { + return str.replaceAll("'", "''") +} + async function moveWorkspace ( accountDb: AccountDB, mongo: MongoClient, @@ -77,32 +81,38 @@ async function moveWorkspace ( for (const collection of collections) { const cursor = collection.find() const domain = translateDomain(collection.collectionName) + const current = await pgClient.query(`SELECT _id FROM ${domain} WHERE "workspaceId" = $1`, [ws.workspace]) + const currentIds = new Set(current.rows.map((r) => r._id)) console.log('move domain', domain) + const docs: Doc[] = [] while (true) { - const doc = (await cursor.next()) as Doc | null - if (doc === null) break - try { - const converted = convertDoc(doc, ws.workspaceName ?? ws.workspace) - await retryTxn(pgClient, async (client) => { - await client.query( - `INSERT INTO ${domain} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, - [ - converted._id, - converted.workspaceId, - converted._class, - converted.createdBy ?? converted.modifiedBy, - converted.modifiedBy, - converted.modifiedOn, - converted.createdOn ?? converted.modifiedOn, - converted.space, - converted.attachedTo, - converted.data - ] - ) - }) - } catch (err) { - console.log('error when move doc', doc._id, doc._class, err) - continue + while (docs.length < 50000) { + const doc = (await cursor.next()) as Doc | null + if (doc === null) break + if (currentIds.has(doc._id)) continue + docs.push(doc) + } + if (docs.length === 0) break + while (docs.length > 0) { + const part = docs.splice(0, 500) + const vals = part + .map((doc) => { + const d = convertDoc(doc, ws.workspace) + return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${ + d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL' + }, '${escapeBackticks(JSON.stringify(d.data))}')` + }) + .join(', ') + try { + await retryTxn(pgClient, async (client) => { + await client.query( + `INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}` + ) + }) + } catch (err) { + console.log('error when move doc to', domain, err) + continue + } } } } diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 3e2dbe5711..fab8aa4cc2 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -1598,7 +1598,7 @@ export function devTool ( throw new Error('mongodbUri is not set') } - await withDatabase(dbUrl, async (db) => { + await withDatabase(mongodbUri, async (db) => { const workspaces = await listWorkspacesRaw(db) workspaces.sort((a, b) => b.lastVisit - a.lastVisit) await moveFromMongoToPG( @@ -1617,7 +1617,7 @@ export function devTool ( throw new Error('mongodbUri is not set') } - await withDatabase(dbUrl, async (db) => { + await withDatabase(mongodbUri, async (db) => { const workspaceInfo = await getWorkspaceById(db, workspace) if (workspaceInfo === null) { throw new Error(`workspace ${workspace} not found`) diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index 1d67df97c1..03a33c3bd2 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -112,7 +112,7 @@ export async function recreateElastic ( const db = getWorkspaceMongoDB(_client, workspaceId) await db .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {} } }) + .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {}, needIndex: true } }) await connection.sendForceClose() } finally { client.close() diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index 1109e5549e..e7772d1147 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -955,7 +955,8 @@ export function createModel (builder: Builder): void { ) // Allow to use fuzzy search for mixins - builder.mixin(contact.class.Contact, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: contact.class.Contact, fullTextSummary: true }) diff --git a/models/controlled-documents/src/index.ts b/models/controlled-documents/src/index.ts index 64145c2ebf..528dfa2fcd 100644 --- a/models/controlled-documents/src/index.ts +++ b/models/controlled-documents/src/index.ts @@ -549,7 +549,8 @@ export function createModel (builder: Builder): void { func: documents.function.GetAllDocumentStates }) - builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: documents.class.Document, fullTextSummary: true, childProcessingAllowed: true }) @@ -886,11 +887,13 @@ export function defineNotifications (builder: Builder): void { } export function defineSearch (builder: Builder): void { - builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: documents.class.Document, parentPropagate: true }) - builder.mixin(documents.class.DocumentMeta, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: documents.class.DocumentMeta, fullTextSummary: true, childProcessingAllowed: true, propagate: [] diff --git a/models/core/src/core.ts b/models/core/src/core.ts index b7e6651995..3942868da8 100644 --- a/models/core/src/core.ts +++ b/models/core/src/core.ts @@ -361,8 +361,10 @@ export class TDocIndexState extends TDoc implements DocIndexState { generationId?: string } -@MMixin(core.mixin.FullTextSearchContext, core.class.Class) -export class TFullTextSearchContext extends TClass implements FullTextSearchContext {} +@Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL) +export class TFullTextSearchContext extends TDoc implements FullTextSearchContext { + toClass!: Ref>> +} @MMixin(core.mixin.ConfigurationElement, core.class.Class) export class TConfigurationElement extends TClass implements ConfigurationElement { diff --git a/models/core/src/index.ts b/models/core/src/index.ts index b80246b6cf..0ef71c7ae9 100644 --- a/models/core/src/index.ts +++ b/models/core/src/index.ts @@ -309,7 +309,8 @@ export function createModel (builder: Builder): void { ] }) - builder.mixin(core.class.Space, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: core.class.Space, childProcessingAllowed: false }) diff --git a/models/core/src/migration.ts b/models/core/src/migration.ts index 844d1e96f0..5645eec853 100644 --- a/models/core/src/migration.ts +++ b/models/core/src/migration.ts @@ -262,7 +262,11 @@ export const coreOperation: MigrateOperation = { async migrate (client: MigrationClient): Promise { // We need to delete all documents in doc index state for missing classes const allClasses = client.hierarchy.getDescendants(core.class.Doc) - const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it)) + const contexts = new Map( + client.model.findAllSync(core.class.FullTextSearchContext, {}).map((it) => [it.toClass, it]) + ) + + const allIndexed = allClasses.filter((it) => isClassIndexable(client.hierarchy, it, contexts)) // Next remove all non indexed classes and missing classes as well. await client.update( diff --git a/models/gmail/src/index.ts b/models/gmail/src/index.ts index 6e02c0d38c..c7853dcdb4 100644 --- a/models/gmail/src/index.ts +++ b/models/gmail/src/index.ts @@ -214,7 +214,8 @@ export function createModel (builder: Builder): void { gmail.action.WriteEmail ) - builder.mixin(gmail.class.Message, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: gmail.class.Message, parentPropagate: false }) diff --git a/models/recruit/src/index.ts b/models/recruit/src/index.ts index b1f588e624..5a337a6351 100644 --- a/models/recruit/src/index.ts +++ b/models/recruit/src/index.ts @@ -1438,13 +1438,15 @@ export function createModel (builder: Builder): void { ) // Allow to use fuzzy search for mixins - builder.mixin(recruit.class.Vacancy, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: recruit.class.Vacancy, fullTextSummary: true, childProcessingAllowed: true, propagate: [] }) - builder.mixin(recruit.mixin.Candidate, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: recruit.mixin.Candidate, fullTextSummary: true, propagate: [recruit.class.Applicant], childProcessingAllowed: true, @@ -1457,7 +1459,8 @@ export function createModel (builder: Builder): void { }) // Allow to use fuzzy search for mixins - builder.mixin(recruit.class.Applicant, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: recruit.class.Applicant, fullTextSummary: true, forceIndex: true, childProcessingAllowed: true, diff --git a/models/telegram/src/index.ts b/models/telegram/src/index.ts index c950f28c55..80e8a139c4 100644 --- a/models/telegram/src/index.ts +++ b/models/telegram/src/index.ts @@ -179,7 +179,8 @@ export function createModel (builder: Builder): void { telegram.ids.TelegramMessageSharedActivityViewlet ) - builder.mixin(telegram.class.Message, core.class.Class, core.mixin.FullTextSearchContext, { + builder.createDoc(core.class.FullTextSearchContext, core.space.Model, { + toClass: telegram.class.Message, parentPropagate: false, childProcessingAllowed: true }) diff --git a/models/text-editor/src/index.ts b/models/text-editor/src/index.ts index ac7802ced6..cbf19db02e 100644 --- a/models/text-editor/src/index.ts +++ b/models/text-editor/src/index.ts @@ -331,6 +331,15 @@ export function createModel (builder: Builder): void { index: 10 }) + builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { + kind: 'image', + action: textEditor.function.DownloadImage, + icon: textEditor.icon.Download, + label: textEditor.string.Download, + category: 90, + index: 15 + }) + builder.createDoc(textEditor.class.TextEditorAction, core.space.Model, { kind: 'image', action: textEditor.function.MoreImageActions, diff --git a/models/text-editor/src/plugin.ts b/models/text-editor/src/plugin.ts index 19439fe536..ad954bb19f 100644 --- a/models/text-editor/src/plugin.ts +++ b/models/text-editor/src/plugin.ts @@ -29,6 +29,7 @@ export default mergeIds(textEditorId, textEditor, { ExpandImage: '' as Resource, MoreImageActions: '' as Resource, ConfigureNote: '' as Resource, + DownloadImage: '' as Resource, IsEditableTableActive: '' as Resource, IsEditableNote: '' as Resource, diff --git a/packages/core/src/classes.ts b/packages/core/src/classes.ts index 0ce2d198f6..064205df7c 100644 --- a/packages/core/src/classes.ts +++ b/packages/core/src/classes.ts @@ -578,7 +578,8 @@ export interface BlobLookup extends Blob { * * If defined for class, this class will be enabled for embedding search like openai. */ -export interface FullTextSearchContext extends Class { +export interface FullTextSearchContext extends Doc { + toClass: Ref> fullTextSummary?: boolean forceIndex?: boolean diff --git a/packages/core/src/component.ts b/packages/core/src/component.ts index 08343ec7f3..5349a3a8c2 100644 --- a/packages/core/src/component.ts +++ b/packages/core/src/component.ts @@ -142,10 +142,10 @@ export default plugin(coreId, { StatusCategory: '' as Ref>, MigrationState: '' as Ref>, - BenchmarkDoc: '' as Ref> + BenchmarkDoc: '' as Ref>, + FullTextSearchContext: '' as Ref> }, mixin: { - FullTextSearchContext: '' as Ref>, ConfigurationElement: '' as Ref>, IndexConfiguration: '' as Ref>>, SpacesTypeData: '' as Ref> diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index f27ed14624..4987f6d617 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -46,9 +46,9 @@ import core from './component' import { Hierarchy } from './hierarchy' import { TxOperations } from './operations' import { isPredicate } from './predicate' +import { Branding, BrandingMap } from './server' import { DocumentQuery, FindResult } from './storage' import { DOMAIN_TX } from './tx' -import { Branding, BrandingMap } from './server' function toHex (value: number, chars: number): string { const result = value.toString(16) @@ -686,39 +686,48 @@ export function getFullTextIndexableAttributes ( return result } +const ctxKey = 'indexer_ftc' /** * @public */ export function getFullTextContext ( hierarchy: Hierarchy, - objectClass: Ref> + objectClass: Ref>, + contexts: Map>, FullTextSearchContext> ): Omit> { - let objClass = hierarchy.getClass(objectClass) - - while (true) { - if (hierarchy.hasMixin(objClass, core.mixin.FullTextSearchContext)) { - const ctx = hierarchy.as, FullTextSearchContext>(objClass, core.mixin.FullTextSearchContext) + let ctx: Omit> | undefined = hierarchy.getClassifierProp(objectClass, ctxKey) + if (ctx !== undefined) { + return ctx + } + if (typeof ctx !== 'string') { + const anc = hierarchy.getAncestors(objectClass) + for (const oc of anc) { + const ctx = contexts.get(oc) if (ctx !== undefined) { + hierarchy.setClassifierProp(objectClass, ctxKey, ctx) return ctx } } - if (objClass.extends === undefined) { - break - } - objClass = hierarchy.getClass(objClass.extends) } - return { + ctx = { + toClass: objectClass, fullTextSummary: false, forceIndex: false, propagate: [], childProcessingAllowed: true } + hierarchy.setClassifierProp(objectClass, ctxKey, ctx) + return ctx } /** * @public */ -export function isClassIndexable (hierarchy: Hierarchy, c: Ref>): boolean { +export function isClassIndexable ( + hierarchy: Hierarchy, + c: Ref>, + contexts: Map>, FullTextSearchContext> +): boolean { const indexed = hierarchy.getClassifierProp(c, 'class_indexed') if (indexed !== undefined) { return indexed as boolean @@ -756,13 +765,13 @@ export function isClassIndexable (hierarchy: Hierarchy, c: Ref>): boo let result = true - if (attrs.length === 0 && !(getFullTextContext(hierarchy, c)?.forceIndex ?? false)) { + if (attrs.length === 0 && !(getFullTextContext(hierarchy, c, contexts)?.forceIndex ?? false)) { result = false // We need check if document has collections with indexable fields. const attrs = hierarchy.getAllAttributes(c).values() for (const attr of attrs) { if (attr.type._class === core.class.Collection) { - if (isClassIndexable(hierarchy, (attr.type as Collection).of)) { + if (isClassIndexable(hierarchy, (attr.type as Collection).of, contexts)) { result = true break } diff --git a/packages/presentation/src/sound.ts b/packages/presentation/src/sound.ts index 1be62940a7..eff939be10 100644 --- a/packages/presentation/src/sound.ts +++ b/packages/presentation/src/sound.ts @@ -3,10 +3,10 @@ import { type Asset, getMetadata, getResource } from '@hcengineering/platform' import { getClient } from '.' import notification from '@hcengineering/notification' -const sounds = new Map() +const sounds = new Map() const context = new AudioContext() -export async function prepareSound (key: string, _class?: Ref>, loop = false, play = false): Promise { +export async function prepareSound (key: string, _class?: Ref>): Promise { if (_class === undefined) return const client = getClient() @@ -23,39 +23,45 @@ export async function prepareSound (key: string, _class?: Ref>, loop try { const soundUrl = getMetadata(key as Asset) as string - const audioBuffer = await fetch(soundUrl) - .then(async (res) => await res.arrayBuffer()) - .then(async (ArrayBuffer) => await context.decodeAudioData(ArrayBuffer)) + const rawAudio = await fetch(soundUrl) + const rawBuffer = await rawAudio.arrayBuffer() + const decodedBuffer = await context.decodeAudioData(rawBuffer) + + sounds.set(key as Asset, decodedBuffer) + } catch (err) { + console.error('Sound not found', key) + } +} + +export async function playSound ( + soundKey: string, + _class?: Ref>, + loop = false +): Promise<(() => void) | null> { + const soundAssetKey = soundKey as Asset + if (!sounds.has(soundAssetKey)) { + await prepareSound(soundKey, _class) + } + + const sound = sounds.get(soundKey as Asset) + if (sound === undefined) { + console.error('Cannot prepare audio buffer', soundKey) + return null + } + + try { const audio = context.createBufferSource() - audio.buffer = audioBuffer + audio.buffer = sound audio.loop = loop - sounds.set(key as Asset, audio) - if (play) { - playSound(key) + audio.connect(context.destination) + audio.start() + + return (): void => { + audio.stop() + audio.disconnect(context.destination) } } catch (err) { - console.error('sound not found', key) - } -} - -export function playSound (soundKey: string, _class?: Ref>, loop = false): void { - const sound = sounds.get(soundKey as Asset) - if (sound !== undefined) { - try { - sound.connect(context.destination) - sound.start() - } catch (err) { - console.error('error happened during sound play', soundKey, err) - } - } else { - void prepareSound(soundKey, _class, loop, true) - } -} - -export function stopSound (soundKey: string): void { - const sound = sounds.get(soundKey as Asset) - if (sound !== undefined && sound?.context.state === 'running') { - sound.stop() - sound.disconnect(context.destination) + console.error('Error when playing sound back', soundKey, err) + return null } } diff --git a/packages/text/src/ydoc.ts b/packages/text/src/ydoc.ts index 7debe06300..0661f2fc55 100644 --- a/packages/text/src/ydoc.ts +++ b/packages/text/src/ydoc.ts @@ -13,7 +13,7 @@ // limitations under the License. // -import { Markup } from '@hcengineering/core' +import { generateId, Markup } from '@hcengineering/core' import { Extensions, getSchema } from '@tiptap/core' import { Node, Schema } from '@tiptap/pm/model' import { prosemirrorJSONToYDoc, prosemirrorToYDoc, yDocToProsemirrorJSON } from 'y-prosemirror' @@ -50,7 +50,7 @@ export function jsonToYDocNoSchema (json: MarkupNode, field: string): YDoc { const nodes = json.type === 'doc' ? json.content ?? [] : [json] const content = nodes.map(nodeToYXmlElement) - const ydoc = new YDoc() + const ydoc = new YDoc({ guid: generateId() }) const fragment = ydoc.getXmlFragment(field) fragment.push(content) @@ -113,7 +113,10 @@ export function yDocContentToNodes (content: ArrayBuffer, schema?: Schema, exten const nodes: Node[] = [] try { - const ydoc = new YDoc() + const ydoc = new YDoc({ + gc: false, + guid: generateId() + }) const uint8arr = new Uint8Array(content) applyUpdate(ydoc, uint8arr) @@ -144,8 +147,8 @@ export function updateYDocContent ( schema ??= extensions === undefined ? defaultSchema : getSchema(extensions ?? defaultExtensions) try { - const ydoc = new YDoc({ gc: false }) - const res = new YDoc({ gc: false }) + const ydoc = new YDoc({ guid: generateId(), gc: false }) + const res = new YDoc({ guid: generateId(), gc: false }) const uint8arr = new Uint8Array(content) applyUpdate(ydoc, uint8arr) diff --git a/packages/ui/src/components/NavItem.svelte b/packages/ui/src/components/NavItem.svelte index e2e0290585..f511ab7bc0 100644 --- a/packages/ui/src/components/NavItem.svelte +++ b/packages/ui/src/components/NavItem.svelte @@ -44,7 +44,6 @@ export let disabled: boolean = false export let isFold: boolean = false export let isOpen: boolean = false - export let isSecondary: boolean = false export let withBackground: boolean = false export let showMenu: boolean = false export let shouldTooltip: boolean = false @@ -78,7 +77,7 @@