diff --git a/.gitignore b/.gitignore index c7d04a649c..a175778200 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ _api-extractor-temp/ temp/ .idea pods/workspace/init/ +pods/workspace/init-scripts/ # Logs *.log diff --git a/.vscode/launch.json b/.vscode/launch.json index 3f5bed63ac..8efbb30f73 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -276,12 +276,14 @@ "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost", - "MODEL_VERSION": "v0.7.1", + "MODEL_VERSION": "v0.7.75", "WS_OPERATION": "all+backup", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", - "BACKUP_BUCKET": "dev-backups" - // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", - // "INIT_WORKSPACE": "staging-dev" + "BACKUP_BUCKET": "dev-backups", + "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", + "INIT_WORKSPACE": "staging-dev", + "QUEUE_CONFIG": "huly.local:19092", + "QUEUE_REGION": "" }, "runtimeVersion": "20", "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], @@ -309,10 +311,11 @@ "MINIO_ACCESS_KEY": "minioadmin", "MINIO_SECRET_KEY": "minioadmin", "MINIO_ENDPOINT": "localhost", - "MODEL_VERSION": "0.7.1", + "MODEL_VERSION": "0.7.75", "WS_OPERATION": "all+backup", "BACKUP_STORAGE": "minio|minio?accessKey=minioadmin&secretKey=minioadmin", - "BACKUP_BUCKET": "dev-backups" + "BACKUP_BUCKET": "dev-backups", + "QUEUE_CONFIG": "huly.local:19092" // "INIT_REPO_DIR": "${workspaceRoot}/pods/workspace/init", // "INIT_WORKSPACE": "staging-dev" }, diff --git a/dev/docker-compose.min.yaml b/dev/docker-compose.min.yaml index 09ef0e1462..57a9dec657 100644 --- a/dev/docker-compose.min.yaml +++ b/dev/docker-compose.min.yaml @@ -53,6 +53,10 @@ services: - 9001:9001 volumes: - files:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + retries: 10 restart: unless-stopped elastic: image: 'elasticsearch:7.14.2' @@ -143,7 +147,7 @@ services: - BACKUP_STORAGE=${BACKUP_STORAGE_CONFIG} - BACKUP_BUCKET=${BACKUP_BUCKET_NAME} - MAIL_URL= - # - INIT_WORKSPACE=staging-dev + - INIT_WORKSPACE=staging-dev restart: unless-stopped collaborator: image: hardcoreeng/collaborator @@ -255,6 +259,30 @@ services: - STATS_URL=http://huly.local:4900 - REKONI_URL=http://huly.local:4004 - ACCOUNTS_URL=http://huly.local:3000 + datalake: + image: hardcoreeng/datalake + extra_hosts: + - 'huly.local:host-gateway' + depends_on: + minio: + condition: service_healthy + cockroach: + condition: service_started + stats: + condition: service_started + account: + condition: service_started + ports: + - 4030:4030 + environment: + - PORT=4030 + - SECRET=secret + - ACCOUNTS_URL=http://huly.local:3000 + - STATS_URL=http://huly.local:4900 + - STREAM_URL=http://huly.local:1080/recording + - DB_URL=${DB_CR_URL} + - BUCKETS=blobs,eu|http://minio:9000?accessKey=minioadmin&secretKey=minioadmin + restart: unless-stopped volumes: db: dbpg: diff --git a/models/card/src/migration.ts b/models/card/src/migration.ts index 94043fcdf0..3d064bd113 100644 --- a/models/card/src/migration.ts +++ b/models/card/src/migration.ts @@ -96,10 +96,11 @@ async function setParentInfo (client: MigrationClient): Promise { ) } -function extractObjectProps (doc: T): Data { +function extractObjectData (doc: T): Data { + const dataKeys = ['_id', 'space', 'modifiedOn', 'modifiedBy', 'createdBy', 'createdOn'] const data: any = {} for (const key in doc) { - if (key === '_id') { + if (dataKeys.includes(key)) { continue } data[key] = doc[key] @@ -114,8 +115,8 @@ async function migrateViewlets (client: Client): Promise { const currentViewlets = await client.findAll(view.class.Viewlet, { attachTo: { $in: masterTags.map((p) => p._id) } }) for (const masterTag of masterTags) { for (const viewlet of viewlets) { - const base = extractObjectProps(viewlet) - const resConfig = base.config + const base = extractObjectData(viewlet) + const resConfig = [...base.config] let index = -1 if (viewlet.descriptor === view.viewlet.List) { index = viewlet.config.findIndex((p) => typeof p !== 'string' && p.displayProps?.grow === true) diff --git a/models/contact/package.json b/models/contact/package.json index 9e797e7cf0..a8474684be 100644 --- a/models/contact/package.json +++ b/models/contact/package.json @@ -43,6 +43,7 @@ "@hcengineering/model-presentation": "^0.6.0", "@hcengineering/model-view": "^0.6.0", "@hcengineering/model-workbench": "^0.6.1", + "@hcengineering/model-card": "^0.6.0", "@hcengineering/notification": "^0.6.23", "@hcengineering/platform": "^0.6.11", "@hcengineering/setting": "^0.6.17", diff --git a/models/contact/src/index.ts b/models/contact/src/index.ts index c012135dee..4c4fad48ac 100644 --- a/models/contact/src/index.ts +++ b/models/contact/src/index.ts @@ -15,7 +15,7 @@ // import activity from '@hcengineering/activity' -import card, { type Role, type Card } from '@hcengineering/card' +import { type Role, type Card } from '@hcengineering/card' import { AvatarType, type UserRole, @@ -36,7 +36,6 @@ import { } from '@hcengineering/contact' import { AccountRole, - ClassifierKind, DOMAIN_MODEL, DateRangeMode, IndexKind, @@ -52,6 +51,7 @@ import { type SocialIdType, type Timestamp } from '@hcengineering/core' +import { createSystemType } from '@hcengineering/model-card' import { Collection as CollectionType, Hidden, @@ -280,55 +280,6 @@ export class TUserRole extends TDoc implements UserRole { role!: Ref } -function createUserProfileTag (builder: Builder): void { - builder.createDoc( - card.class.MasterTag, - core.space.Model, - { - extends: card.class.Card, - label: contact.string.UserProfile, - kind: ClassifierKind.CLASS, - icon: contact.icon.Person - }, - contact.class.UserProfile - ) - - builder.createDoc(core.class.Attribute, core.space.Model, { - attributeOf: contact.class.UserProfile, - name: 'person', - label: contact.string.Person, - icon: contact.icon.Person, - type: TypeRef(contact.class.Person), - readonly: true - }) - - builder.mixin(contact.class.UserProfile, core.class.Mixin, setting.mixin.Editable, { - value: false - }) - builder.mixin(contact.class.UserProfile, core.class.Mixin, setting.mixin.UserMixin, {}) -} - -function createUserProfileViewlet (builder: Builder): void { - builder.createDoc( - view.class.Viewlet, - core.space.Model, - { - attachTo: contact.class.UserProfile, - descriptor: view.viewlet.Table, - config: [ - '', - { key: 'modifiedOn', displayProps: { key: 'modified', fixed: 'right' } }, - { key: 'createdBy', displayProps: { fixed: 'left', key: 'app' } } - ], - configOptions: { - hiddenKeys: ['name'], - sortable: true - } - }, - contact.viewlet.TableUserProfile - ) -} - export function createModel (builder: Builder): void { builder.createModel( TAvatarProvider, @@ -1319,6 +1270,14 @@ export function createModel (builder: Builder): void { }) createAttributePresenter(builder, contact.component.SpaceMembersEditor, core.class.Space, 'members', 'array') - createUserProfileTag(builder) - createUserProfileViewlet(builder) + createSystemType(builder, contact.class.UserProfile, contact.icon.Person, contact.string.UserProfile) + builder.createDoc(core.class.Attribute, core.space.Model, { + attributeOf: contact.class.UserProfile, + name: 'person', + label: contact.string.Person, + icon: contact.icon.Person, + type: TypeRef(contact.class.Person), + isCustom: true, + readonly: true + }) } diff --git a/packages/importer/src/huly/cards.ts b/packages/importer/src/huly/cards.ts index dcdb8105c7..dde21f3393 100644 --- a/packages/importer/src/huly/cards.ts +++ b/packages/importer/src/huly/cards.ts @@ -32,11 +32,10 @@ import * as yaml from 'js-yaml' import { contentType } from 'mime-types' import * as path from 'path' import { IntlString } from '../../../platform/types' -import { Props, UnifiedDoc, UnifiedUpdate, UnifiedFile, UnifiedMixin } from '../types' -import { MetadataRegistry, AssociationMetadata } from './registry' -import { readMarkdownContent, readYamlHeader } from './parsing' import { Logger } from '../importer/logger' -import { validateSchema } from './validation' +import { Props, UnifiedDoc, UnifiedFile, UnifiedMixin, UnifiedUpdate } from '../types' +import { UnifiedFormatParser } from './parser' +import { AssociationMetadata, MetadataRegistry } from './registry' import { AssociationSchema, BaseFieldType, @@ -51,6 +50,7 @@ import { StringFieldType, TagSchema } from './schema' +import { validateSchema } from './validation' export interface UnifiedDocProcessResult { docs: Map>> @@ -62,6 +62,7 @@ export interface UnifiedDocProcessResult { export class CardsProcessor { constructor ( private readonly metadataRegistry: MetadataRegistry, + private readonly parser: UnifiedFormatParser, private readonly logger: Logger ) {} @@ -199,7 +200,7 @@ export class CardsProcessor { for (const entry of entries) { if (entry.isFile() && entry.name.endsWith('.md')) { const cardPath = path.join(currentPath, entry.name) - const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) + const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath) if (masterTagId !== undefined) { await this.processCard(result, cardPath, cardProps, masterTagId, masterTagAssociaions, masterTagAttributes) @@ -231,7 +232,7 @@ export class CardsProcessor { for (const entry of entries) { if (entry.isFile() && entry.name.endsWith('.md')) { const cardPath = path.join(currentDir, entry.name) - const { class: cardType, ...cardProps } = await readYamlHeader(cardPath) + const { class: cardType, ...cardProps } = this.parser.readYamlHeader(cardPath) if (cardType !== undefined && cardType.startsWith('card:types:') === false) { throw new Error('Unsupported card type: ' + cardType + ' in ' + cardPath) @@ -310,7 +311,7 @@ export class CardsProcessor { for (const entry of entries) { const childCardPath = path.join(cardDir, entry.name) - const { class: cardClass, ...cardProps } = await readYamlHeader(childCardPath) + const { class: cardClass, ...cardProps } = this.parser.readYamlHeader(childCardPath) await this.processCard( result, childCardPath, @@ -594,7 +595,7 @@ export class CardsProcessor { { _class: masterTagId, collabField: 'content', - contentProvider: () => readMarkdownContent(cardPath), + contentProvider: () => Promise.resolve(this.parser.readMarkdownContent(cardPath)), props: cardProps as Props }, ...relations diff --git a/packages/importer/src/huly/huly.ts b/packages/importer/src/huly/huly.ts index 58e06bc182..a940364adf 100644 --- a/packages/importer/src/huly/huly.ts +++ b/packages/importer/src/huly/huly.ts @@ -15,27 +15,16 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { type Attachment } from '@hcengineering/attachment' import card from '@hcengineering/card' -import contact, { Employee, type Person, SocialIdentity } from '@hcengineering/contact' +import contact, { Employee, type Person } from '@hcengineering/contact' import documents, { ControlledDocument, DocumentCategory, DocumentMeta, DocumentState } from '@hcengineering/controlled-documents' -import { - AccountUuid, - type Class, - type Doc, - generateId, - PersonId, - type Ref, - SocialIdType, - type Space, - type TxOperations -} from '@hcengineering/core' +import { AccountUuid, generateId, PersonId, type Ref, SocialIdType, type TxOperations } from '@hcengineering/core' import document, { type Document } from '@hcengineering/document' import core from '@hcengineering/model-core' -import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text' import tracker, { type Issue, Project } from '@hcengineering/tracker' import * as fs from 'fs' import sizeOf from 'image-size' @@ -59,11 +48,11 @@ import { WorkspaceImporter } from '../importer/importer' import { type Logger } from '../importer/logger' -import { BaseMarkdownPreprocessor } from '../importer/preprocessor' import { type FileUploader } from '../importer/uploader' import { CardsProcessor } from './cards' -import { MetadataRegistry, MentionMetadata } from './registry' -import { readMarkdownContent, readYamlHeader } from './parsing' +import { UnifiedFormatParser } from './parser' +import { HulyMarkdownPreprocessor, type AttachmentMetadata } from './preprocessor' +import { MetadataRegistry } from './registry' export interface HulyComment { author: string text: string @@ -165,175 +154,10 @@ export interface HulyOrgSpaceSettings extends HulySpaceSettings { qara?: string } -class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { - constructor ( - private readonly urlProvider: (id: string) => string, - private readonly logger: Logger, - private readonly metadataRegistry: MetadataRegistry, - private readonly attachMetaByPath: Map, - personsByName: Map> - ) { - super(personsByName) - } - - process (json: MarkupNode, id: Ref, spaceId: Ref): MarkupNode { - traverseNode(json, (node) => { - if (node.type === MarkupNodeType.image) { - this.processImageNode(node, id, spaceId) - } else { - this.processLinkMarks(node, id, spaceId) - this.processMentions(node) - } - return true - }) - return json - } - - private processImageNode (node: MarkupNode, id: Ref, spaceId: Ref): void { - const src = node.attrs?.src - if (src === undefined) return - - const sourcePath = this.getSourcePath(id) - if (sourcePath == null) return - - const href = decodeURI(src as string) - const fullPath = path.resolve(path.dirname(sourcePath), href) - const attachmentMeta = this.attachMetaByPath.get(fullPath) - - if (attachmentMeta === undefined) { - this.logger.error(`Attachment image not found for ${fullPath}`) - return - } - - if (!this.metadataRegistry.hasRefMetadata(sourcePath)) { - this.logger.error(`Source metadata not found for ${sourcePath}`) - return - } - - const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath) - this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta) - this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name) - } - - private processLinkMarks (node: MarkupNode, id: Ref, spaceId: Ref): void { - traverseNodeMarks(node, (mark) => { - if (mark.type !== MarkupMarkType.link) return - - const sourcePath = this.getSourcePath(id) - if (sourcePath == null) return - - const href = decodeURI(mark.attrs?.href ?? '') - const fullPath = path.resolve(path.dirname(sourcePath), href) - - if (this.metadataRegistry.hasRefMetadata(fullPath)) { - const targetDocMeta = this.metadataRegistry.getRefMetadata(fullPath) - this.alterMentionNode(node, targetDocMeta) - } else if (this.attachMetaByPath.has(fullPath)) { - const attachmentMeta = this.attachMetaByPath.get(fullPath) - if (attachmentMeta !== undefined) { - this.alterAttachmentLinkNode(node, attachmentMeta) - if (this.metadataRegistry.hasRefMetadata(sourcePath)) { - const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath) - this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta) - } - } - } else { - this.logger.log('Unknown link type, leave it as is: ' + href) - } - }) - } - - private alterImageNode (node: MarkupNode, id: string, name: string): void { - node.type = MarkupNodeType.image - if (node.attrs !== undefined) { - node.attrs = { - 'file-id': id, - src: this.urlProvider(id), - width: node.attrs.width ?? null, - height: node.attrs.height ?? null, - align: node.attrs.align ?? null, - alt: name, - title: name - } - const mimeType = this.getContentType(name) - if (mimeType !== undefined) { - node.attrs['data-file-type'] = mimeType - } - } - } - - private alterMentionNode (node: MarkupNode, targetMeta: MentionMetadata): void { - node.type = MarkupNodeType.reference - node.attrs = { - id: targetMeta.id, - label: targetMeta.refTitle, - objectclass: targetMeta.class, - text: '', - content: '' - } - } - - private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void { - const stats = fs.statSync(targetMeta.path) - node.type = MarkupNodeType.file - node.attrs = { - 'file-id': targetMeta.id, - 'data-file-name': targetMeta.name, - 'data-file-size': stats.size, - 'data-file-href': targetMeta.path - } - const mimeType = this.getContentType(targetMeta.name) - if (mimeType !== undefined) { - node.attrs['data-file-type'] = mimeType - } - } - - private getContentType (fileName: string): string | undefined { - const mimeType = contentType(fileName) - return mimeType !== false ? mimeType : undefined - } - - private getSourcePath (id: Ref): string | null { - const sourcePath = this.metadataRegistry.getPath(id) - if (sourcePath === undefined) { - this.logger.error(`Source file path not found for ${id}`) - return null - } - return sourcePath - } - - private updateAttachmentMetadata ( - fullPath: string, - attachmentMeta: AttachmentMetadata, - id: Ref, - spaceId: Ref, - sourceMeta: MentionMetadata - ): void { - this.attachMetaByPath.set(fullPath, { - ...attachmentMeta, - spaceId, - parentId: id, - parentClass: sourceMeta.class as Ref>> - }) - } -} - -interface AttachmentMetadata { - id: Ref - name: string - path: string - parentId?: Ref - parentClass?: Ref>> - spaceId?: Ref -} - export class HulyFormatImporter { - private readonly importerEmailPlaceholder = 'newuser@huly.io' - private readonly importerNamePlaceholder = 'New User' - - private personsByName = new Map>() + private readonly personsByName = new Map>() private employeesByName = new Map>() - private accountsByEmail = new Map() + private readonly accountsByName = new Map() private readonly personIdByEmail = new Map() private controlledDocumentCategories = new Map>() @@ -341,20 +165,20 @@ export class HulyFormatImporter { private readonly metadataRegistry = new MetadataRegistry() private readonly cardsProcessor: CardsProcessor + private readonly parser: UnifiedFormatParser constructor ( private readonly client: TxOperations, private readonly fileUploader: FileUploader, private readonly logger: Logger, - private readonly importerSocialId?: PersonId, - private readonly importerPerson?: Ref + variables?: Record ) { - this.cardsProcessor = new CardsProcessor(this.metadataRegistry, this.logger) + this.parser = new UnifiedFormatParser(variables ?? {}) + this.cardsProcessor = new CardsProcessor(this.metadataRegistry, this.parser, this.logger) } private async initCaches (): Promise { await this.cachePersonsByNames() - await this.cacheAccountsByEmails() await this.cacheEmployeesByName() await this.cacheControlledDocumentCategories() } @@ -485,7 +309,7 @@ export class HulyFormatImporter { try { this.logger.log(`Processing ${spaceName}...`) - const spaceConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as HulySpaceSettings + const spaceConfig = this.parser.readYaml(yamlPath) as HulySpaceSettings if (spaceConfig?.class === undefined) { this.logger.error(`Skipping ${spaceName}: not a space - no class specified`) @@ -560,7 +384,7 @@ export class HulyFormatImporter { for (const issueFile of issueFiles) { const issuePath = path.join(currentPath, issueFile) - const issueHeader = (await readYamlHeader(issuePath)) as HulyIssueHeader + const issueHeader = this.parser.readYamlHeader(issuePath) as HulyIssueHeader if (issueHeader.class === undefined) { this.logger.error(`Skipping ${issueFile}: not an issue`) @@ -578,7 +402,7 @@ export class HulyFormatImporter { class: tracker.class.Issue, title: issueHeader.title, number: parseInt(issueNumber ?? 'NaN'), - descrProvider: async () => await readMarkdownContent(issuePath), + descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(issuePath)), status: { name: issueHeader.status }, priority: issueHeader.priority, estimation: issueHeader.estimation, @@ -606,9 +430,6 @@ export class HulyFormatImporter { return undefined } - if (name === this.importerNamePlaceholder && this.importerPerson != null) { - return this.importerPerson - } const person = this.personsByName.get(name) if (person === undefined) { throw new Error(`Person not found: ${name}`) @@ -617,10 +438,6 @@ export class HulyFormatImporter { } private async getPersonIdByEmail (email: string): Promise { - if (email === this.importerEmailPlaceholder && this.importerSocialId != null) { - return this.importerSocialId - } - const personId = this.personIdByEmail.get(email) if (personId !== undefined) { return personId @@ -640,10 +457,10 @@ export class HulyFormatImporter { return socialId._id } - private findAccountByEmail (email: string): AccountUuid { - const account = this.accountsByEmail.get(email) + private findAccountByName (name: string): AccountUuid { + const account = this.accountsByName.get(name) if (account === undefined) { - throw new Error(`Account not found: ${email}`) + throw new Error(`Account not found: ${name}`) } return account } @@ -666,7 +483,7 @@ export class HulyFormatImporter { for (const docFile of docFiles) { const docPath = path.join(currentPath, docFile) - const docHeader = (await readYamlHeader(docPath)) as HulyDocumentHeader + const docHeader = this.parser.readYamlHeader(docPath) as HulyDocumentHeader if (docHeader.class === undefined) { this.logger.error(`Skipping ${docFile}: not a document`) @@ -680,7 +497,7 @@ export class HulyFormatImporter { id: this.metadataRegistry.getRef(docPath) as Ref, class: document.class.Document, title: docHeader.title, - descrProvider: async () => await readMarkdownContent(docPath), + descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)), subdocs: [] // Will be added via builder } @@ -707,7 +524,7 @@ export class HulyFormatImporter { for (const docFile of docFiles) { const docPath = path.join(currentPath, docFile) - const docHeader = (await readYamlHeader(docPath)) as HulyControlledDocumentHeader | HulyDocumentTemplateHeader + const docHeader = this.parser.readYamlHeader(docPath) as HulyControlledDocumentHeader | HulyDocumentTemplateHeader if (docHeader.class === undefined) { this.logger.error(`Skipping ${docFile}: not a document`) @@ -788,55 +605,51 @@ export class HulyFormatImporter { ) } - private async processProject (projectHeader: HulyProjectSettings): Promise { + private async processProject (data: HulyProjectSettings): Promise { return { class: tracker.class.Project, - id: projectHeader.id as Ref, - title: projectHeader.title, - identifier: projectHeader.identifier, - private: projectHeader.private ?? false, - autoJoin: projectHeader.autoJoin ?? true, - archived: projectHeader.archived ?? false, - description: projectHeader.description, - emoji: projectHeader.emoji, - defaultIssueStatus: - projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined, - owners: - projectHeader.owners !== undefined ? projectHeader.owners.map((email) => this.findAccountByEmail(email)) : [], - members: - projectHeader.members !== undefined ? projectHeader.members.map((email) => this.findAccountByEmail(email)) : [], + id: data.id as Ref, + title: data.title, + identifier: data.identifier, + private: data.private ?? false, + autoJoin: data.autoJoin ?? true, + archived: data.archived ?? false, + description: data.description, + emoji: data.emoji, + defaultIssueStatus: data.defaultIssueStatus !== undefined ? { name: data.defaultIssueStatus } : undefined, + owners: data.owners !== undefined ? data.owners.map((name) => this.findAccountByName(name)) : [], + members: data.members !== undefined ? data.members.map((name) => this.findAccountByName(name)) : [], docs: [] } } - private async processTeamspace (spaceHeader: HulyTeamspaceSettings): Promise { + private async processTeamspace (data: HulyTeamspaceSettings): Promise { return { class: document.class.Teamspace, - title: spaceHeader.title, - private: spaceHeader.private ?? false, - autoJoin: spaceHeader.autoJoin ?? true, - archived: spaceHeader.archived ?? false, - description: spaceHeader.description, - emoji: spaceHeader.emoji, - owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [], - members: - spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [], + title: data.title, + private: data.private ?? false, + autoJoin: data.autoJoin ?? true, + archived: data.archived ?? false, + description: data.description, + emoji: data.emoji, + owners: data.owners !== undefined ? data.owners.map((name) => this.findAccountByName(name)) : [], + members: data.members !== undefined ? data.members.map((name) => this.findAccountByName(name)) : [], docs: [] } } - private async processOrgSpace (spaceHeader: HulyOrgSpaceSettings): Promise { + private async processOrgSpace (data: HulyOrgSpaceSettings): Promise { return { class: documents.class.OrgSpace, - title: spaceHeader.title, - private: spaceHeader.private ?? false, - archived: spaceHeader.archived ?? false, - description: spaceHeader.description, - owners: spaceHeader.owners?.map((email) => this.findAccountByEmail(email)) ?? [], - members: spaceHeader.members?.map((email) => this.findAccountByEmail(email)) ?? [], - qualified: spaceHeader.qualified !== undefined ? this.findAccountByEmail(spaceHeader.qualified) : undefined, - manager: spaceHeader.manager !== undefined ? this.findAccountByEmail(spaceHeader.manager) : undefined, - qara: spaceHeader.qara !== undefined ? this.findAccountByEmail(spaceHeader.qara) : undefined, + title: data.title, + private: data.private ?? false, + archived: data.archived ?? false, + description: data.description, + owners: data.owners?.map((name) => this.findAccountByName(name)) ?? [], + members: data.members?.map((name) => this.findAccountByName(name)) ?? [], + qualified: data.qualified !== undefined ? this.findAccountByName(data.qualified) : undefined, + manager: data.manager !== undefined ? this.findAccountByName(data.manager) : undefined, + qara: data.qara !== undefined ? this.findAccountByName(data.qara) : undefined, docs: [] } } @@ -876,10 +689,10 @@ export class HulyFormatImporter { author, owner, abstract: header.abstract, - reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], - approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], - coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], - descrProvider: async () => await readMarkdownContent(docPath), + reviewers: header.reviewers?.map((name) => this.findEmployeeByName(name)) ?? [], + approvers: header.approvers?.map((name) => this.findEmployeeByName(name)) ?? [], + coAuthors: header.coAuthors?.map((name) => this.findEmployeeByName(name)) ?? [], + descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)), ccReason: header.changeControl?.reason, ccImpact: header.changeControl?.impact, ccDescription: header.changeControl?.description, @@ -919,10 +732,10 @@ export class HulyFormatImporter { author, owner, abstract: header.abstract, - reviewers: header.reviewers?.map((email) => this.findEmployeeByName(email)) ?? [], - approvers: header.approvers?.map((email) => this.findEmployeeByName(email)) ?? [], - coAuthors: header.coAuthors?.map((email) => this.findEmployeeByName(email)) ?? [], - descrProvider: async () => await readMarkdownContent(docPath), + reviewers: header.reviewers?.map((name) => this.findEmployeeByName(name)) ?? [], + approvers: header.approvers?.map((name) => this.findEmployeeByName(name)) ?? [], + coAuthors: header.coAuthors?.map((name) => this.findEmployeeByName(name)) ?? [], + descrProvider: () => Promise.resolve(this.parser.readMarkdownContent(docPath)), ccReason: header.changeControl?.reason, ccImpact: header.changeControl?.impact, ccDescription: header.changeControl?.description, @@ -930,54 +743,14 @@ export class HulyFormatImporter { } } - private async cacheAccountsByEmails (): Promise { - const employees = await this.client.findAll( - contact.mixin.Employee, - { active: true }, - { lookup: { _id: { socialIds: contact.class.SocialIdentity } } } - ) - - this.accountsByEmail = employees.reduce((map, employee) => { - employee.$lookup?.socialIds?.forEach((socialId) => { - if ((socialId as SocialIdentity).type === SocialIdType.EMAIL) { - map.set((socialId as SocialIdentity).value, employee.personUuid) - } - }) - - return map - }, new Map()) - } - - private async cachePersonIdsByEmails (): Promise { - const employees = await this.client.findAll( - contact.mixin.Employee, - { active: true }, - { lookup: { _id: { socialIds: contact.class.SocialIdentity } } } - ) - - this.accountsByEmail = employees.reduce((map, employee) => { - employee.$lookup?.socialIds?.forEach((socialId) => { - if ((socialId as SocialIdentity).type === SocialIdType.EMAIL) { - map.set((socialId as SocialIdentity).value, employee.personUuid) - } - }) - - return map - }, new Map()) - } - private async cachePersonsByNames (): Promise { - this.personsByName = (await this.client.findAll(contact.class.Person, {})) - .map((person) => { - return { - _id: person._id, - name: person.name.split(',').reverse().join(' ') - } - }) - .reduce((refByName, person) => { - refByName.set(person.name, person._id) - return refByName - }, new Map()) + ;(await this.client.findAll(contact.class.Person, {})).forEach((person) => { + const name = person.name.split(',').reverse().join(' ') + this.personsByName.set(name, person._id) + if (person.personUuid !== undefined) { + this.accountsByName.set(name, person.personUuid as AccountUuid) + } + }) } private async cacheEmployeesByName (): Promise { diff --git a/packages/importer/src/huly/parser.ts b/packages/importer/src/huly/parser.ts new file mode 100644 index 0000000000..682474f7bc --- /dev/null +++ b/packages/importer/src/huly/parser.ts @@ -0,0 +1,79 @@ +// +// Copyright © 2025 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import * as fs from 'fs' +import * as yaml from 'js-yaml' + +const VARIABLE_REGEX = /\${\S+?}/ +const YAML_HEADER_REGEX = /^---\n([\s\S]*?)\n---/ +const MARKDOWN_CONTENT_REGEX = /^---\n[\s\S]*?\n---\n(.*)$/s + +export class UnifiedFormatParser { + constructor (private readonly variables: Record) {} + + public readYaml (filePath: string): Record { + const content = fs.readFileSync(filePath, 'utf8') + const data = yaml.load(content) as Record + return this.resolveProps(data) + } + + public readYamlHeader (filePath: string): Record { + const content = fs.readFileSync(filePath, 'utf8') + const match = content.match(YAML_HEADER_REGEX) + if (match != null) { + const header = yaml.load(match[1]) + return this.resolveProps(header as Record) + } + return {} + } + + public readMarkdownContent (filePath: string): string { + const content = fs.readFileSync(filePath, 'utf8') + const match = content.match(MARKDOWN_CONTENT_REGEX) + return match != null ? match[1] : content + } + + public resolveProps (data: Record): Record { + for (const key in data) { + const value = (data as any)[key] + ;(data as any)[key] = this.resolveValue(value) + } + return data + } + + private resolveValue (value: any): any { + if (typeof value === 'object') { + if (Array.isArray(value)) { + return value.map((v) => this.resolveValue(v)) + } else { + return this.resolveProps(value) + } + } else if (typeof value === 'string') { + while (true) { + const matched = VARIABLE_REGEX.exec(value) + if (matched === null) break + const result = this.variables[matched[0]] + if (result === undefined) { + throw new Error(`Variable ${matched[0]} not found`) + } else { + value = value.replaceAll(matched[0], result) + VARIABLE_REGEX.lastIndex = 0 + } + } + return value + } + return value + } +} diff --git a/packages/importer/src/huly/parsing.ts b/packages/importer/src/huly/parsing.ts deleted file mode 100644 index 3846a1ceed..0000000000 --- a/packages/importer/src/huly/parsing.ts +++ /dev/null @@ -1,32 +0,0 @@ -// -// Copyright © 2025 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import * as fs from 'fs' -import * as yaml from 'js-yaml' - -export async function readYamlHeader (filePath: string): Promise { - const content = fs.readFileSync(filePath, 'utf8') - const match = content.match(/^---\n([\s\S]*?)\n---/) - if (match != null) { - return yaml.load(match[1]) - } - return {} -} - -export async function readMarkdownContent (filePath: string): Promise { - const content = fs.readFileSync(filePath, 'utf8') - const match = content.match(/^---\n[\s\S]*?\n---\n(.*)$/s) - return match != null ? match[1] : content -} diff --git a/packages/importer/src/huly/preprocessor.ts b/packages/importer/src/huly/preprocessor.ts new file mode 100644 index 0000000000..87c12726df --- /dev/null +++ b/packages/importer/src/huly/preprocessor.ts @@ -0,0 +1,186 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. + +import { Attachment } from '@hcengineering/attachment' +import { type Person } from '@hcengineering/contact' +import { type Class, type Doc, type Ref, type Space } from '@hcengineering/core' +import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text' +import * as fs from 'fs' +import { contentType } from 'mime-types' +import * as path from 'path' +import { type Logger } from '../importer/logger' +import { BaseMarkdownPreprocessor } from '../importer/preprocessor' +import { MentionMetadata, MetadataRegistry } from './registry' + +export interface AttachmentMetadata { + id: Ref + name: string + path: string + parentId?: Ref + parentClass?: Ref>> + spaceId?: Ref +} + +export class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { + constructor ( + private readonly urlProvider: (id: string) => string, + private readonly logger: Logger, + private readonly metadataRegistry: MetadataRegistry, + private readonly attachMetaByPath: Map, + personsByName: Map> + ) { + super(personsByName) + } + + process (json: MarkupNode, id: Ref, spaceId: Ref): MarkupNode { + traverseNode(json, (node) => { + if (node.type === MarkupNodeType.image) { + this.processImageNode(node, id, spaceId) + } else { + this.processLinkMarks(node, id, spaceId) + this.processMentions(node) + } + return true + }) + return json + } + + private processImageNode (node: MarkupNode, id: Ref, spaceId: Ref): void { + const src = node.attrs?.src + if (src === undefined) return + + const sourcePath = this.getSourcePath(id) + if (sourcePath == null) return + + const href = decodeURI(src as string) + const fullPath = path.resolve(path.dirname(sourcePath), href) + const attachmentMeta = this.attachMetaByPath.get(fullPath) + + if (attachmentMeta === undefined) { + this.logger.error(`Attachment image not found for ${fullPath}`) + return + } + + if (!this.metadataRegistry.hasRefMetadata(sourcePath)) { + this.logger.error(`Source metadata not found for ${sourcePath}`) + return + } + + const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath) + this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta) + this.alterImageNode(node, attachmentMeta.id, attachmentMeta.name) + } + + private processLinkMarks (node: MarkupNode, id: Ref, spaceId: Ref): void { + traverseNodeMarks(node, (mark) => { + if (mark.type !== MarkupMarkType.link) return + + const sourcePath = this.getSourcePath(id) + if (sourcePath == null) return + + const href = decodeURI(mark.attrs?.href ?? '') + const fullPath = path.resolve(path.dirname(sourcePath), href) + + if (this.metadataRegistry.hasRefMetadata(fullPath)) { + const targetDocMeta = this.metadataRegistry.getRefMetadata(fullPath) + this.alterMentionNode(node, targetDocMeta) + } else if (this.attachMetaByPath.has(fullPath)) { + const attachmentMeta = this.attachMetaByPath.get(fullPath) + if (attachmentMeta !== undefined) { + this.alterAttachmentLinkNode(node, attachmentMeta) + if (this.metadataRegistry.hasRefMetadata(sourcePath)) { + const sourceMeta = this.metadataRegistry.getRefMetadata(sourcePath) + this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta) + } + } + } else { + this.logger.log('Unknown link type, leave it as is: ' + href) + } + }) + } + + private alterImageNode (node: MarkupNode, id: string, name: string): void { + node.type = MarkupNodeType.image + if (node.attrs !== undefined) { + node.attrs = { + 'file-id': id, + src: this.urlProvider(id), + width: node.attrs.width ?? null, + height: node.attrs.height ?? null, + align: node.attrs.align ?? null, + alt: name, + title: name + } + const mimeType = this.getContentType(name) + if (mimeType !== undefined) { + node.attrs['data-file-type'] = mimeType + } + } + } + + private alterMentionNode (node: MarkupNode, targetMeta: MentionMetadata): void { + node.type = MarkupNodeType.reference + node.attrs = { + id: targetMeta.id, + label: targetMeta.refTitle, + objectclass: targetMeta.class, + text: '', + content: '' + } + } + + private alterAttachmentLinkNode (node: MarkupNode, targetMeta: AttachmentMetadata): void { + const stats = fs.statSync(targetMeta.path) + node.type = MarkupNodeType.file + node.attrs = { + 'file-id': targetMeta.id, + 'data-file-name': targetMeta.name, + 'data-file-size': stats.size, + 'data-file-href': targetMeta.path + } + const mimeType = this.getContentType(targetMeta.name) + if (mimeType !== undefined) { + node.attrs['data-file-type'] = mimeType + } + } + + private getContentType (fileName: string): string | undefined { + const mimeType = contentType(fileName) + return mimeType !== false ? mimeType : undefined + } + + private getSourcePath (id: Ref): string | null { + const sourcePath = this.metadataRegistry.getPath(id) + if (sourcePath === undefined) { + this.logger.error(`Source file path not found for ${id}`) + return null + } + return sourcePath + } + + private updateAttachmentMetadata ( + fullPath: string, + attachmentMeta: AttachmentMetadata, + id: Ref, + spaceId: Ref, + sourceMeta: MentionMetadata + ): void { + this.attachMetaByPath.set(fullPath, { + ...attachmentMeta, + spaceId, + parentId: id, + parentClass: sourceMeta.class as Ref>> + }) + } +} diff --git a/server-plugins/card-resources/src/index.ts b/server-plugins/card-resources/src/index.ts index 48f1deb794..5d06bff350 100644 --- a/server-plugins/card-resources/src/index.ts +++ b/server-plugins/card-resources/src/index.ts @@ -47,37 +47,40 @@ async function OnAttribute (ctx: TxCreateDoc[], control: TriggerCo for (const des of desc) { const viewlets = control.modelDb.findAllSync(view.class.Viewlet, { attachTo: des, variant: { $exists: false } }) for (const viewlet of viewlets) { + const updatedConfig = [...viewlet.config] // let push it after grow for the list if (viewlet.descriptor === view.viewlet.List) { const index = viewlet.config.findIndex((p) => typeof p !== 'string' && p.displayProps?.grow === true) if (index !== -1) { - viewlet.config.splice(index + 1, 0, attr.name) + updatedConfig.splice(index + 1, 0, attr.name) } else { - viewlet.config.push(attr.name) + updatedConfig.push(attr.name) } } else { - viewlet.config.push(attr.name) + updatedConfig.push(attr.name) } res.push( control.txFactory.createTxUpdateDoc(viewlet._class, viewlet.space, viewlet._id, { - config: viewlet.config + config: updatedConfig }) ) + const prefs = await control.findAll(control.ctx, view.class.ViewletPreference, { attachedTo: viewlet._id }) for (const pref of prefs) { + const updatedPrefConfig = [...pref.config] if (viewlet.descriptor === view.viewlet.List) { - const index = viewlet.config.findIndex((p) => typeof p !== 'string' && p.displayProps?.grow === true) + const index = updatedPrefConfig.findIndex((p) => typeof p !== 'string' && p.displayProps?.grow === true) if (index !== -1) { - viewlet.config.splice(index + 1, 0, attr.name) + updatedPrefConfig.splice(index + 1, 0, attr.name) } else { - viewlet.config.push(attr.name) + updatedPrefConfig.push(attr.name) } } else { - viewlet.config.push(attr.name) + updatedPrefConfig.push(attr.name) } res.push( control.txFactory.createTxUpdateDoc(pref._class, pref.space, pref._id, { - config: pref.config + config: updatedPrefConfig }) ) } @@ -97,18 +100,16 @@ async function OnAttributeRemove (ctx: TxRemoveDoc[], control: Tri for (const des of desc) { const viewlets = control.modelDb.findAllSync(view.class.Viewlet, { attachTo: des }) for (const viewlet of viewlets) { - viewlet.config = viewlet.config.filter((p) => p !== attr.name) res.push( control.txFactory.createTxUpdateDoc(viewlet._class, viewlet.space, viewlet._id, { - config: viewlet.config + config: viewlet.config.filter((p) => p !== attr.name) }) ) const prefs = await control.findAll(control.ctx, view.class.ViewletPreference, { attachedTo: viewlet._id }) for (const pref of prefs) { - pref.config = pref.config.filter((p) => p !== attr.name) res.push( control.txFactory.createTxUpdateDoc(pref._class, pref.space, pref._id, { - config: pref.config + config: pref.config.filter((p) => p !== attr.name) }) ) } @@ -200,10 +201,11 @@ async function removeTagRelations (control: TriggerControl, tag: Ref (doc: T): Data { +function extractObjectData (doc: T): Data { + const dataKeys = ['_id', 'space', 'modifiedOn', 'modifiedBy', 'createdBy', 'createdOn'] const data: any = {} for (const key in doc) { - if (key === '_id') { + if (dataKeys.includes(key)) { continue } data[key] = doc[key] @@ -229,7 +231,7 @@ async function OnMasterTagCreate (ctx: TxCreateDoc[], control: variant: { $exists: false } }) for (const viewlet of viewlets) { - const base = extractObjectProps(viewlet) + const base = extractObjectData(viewlet) res.push( control.txFactory.createTxCreateDoc(view.class.Viewlet, core.space.Model, { ...base, diff --git a/server-plugins/contact-resources/src/index.ts b/server-plugins/contact-resources/src/index.ts index 7c2f6d204a..48cb8ff1cd 100644 --- a/server-plugins/contact-resources/src/index.ts +++ b/server-plugins/contact-resources/src/index.ts @@ -167,7 +167,7 @@ export async function OnPersonCreate (_txes: Tx[], control: TriggerControl): Pro const lastOne = ( await control.findAll(control.ctx, card.class.Card, {}, { sort: { rank: SortingOrder.Descending }, limit: 1 }) )[0] - const userProfileTx = control.txFactory.createTxCreateDoc(contact.class.UserProfile, ctx.space, { + const userProfileTx = control.txFactory.createTxCreateDoc(contact.class.UserProfile, ctx.objectSpace, { person: ctx.objectId, title: formatName(ctx.attributes.name), rank: makeRank(lastOne?.rank, undefined), @@ -178,7 +178,7 @@ export async function OnPersonCreate (_txes: Tx[], control: TriggerControl): Pro result.push(userProfileTx) result.push( - control.txFactory.createTxUpdateDoc(ctx.objectClass, ctx.space, ctx.objectId, { + control.txFactory.createTxUpdateDoc(ctx.objectClass, ctx.objectSpace, ctx.objectId, { profile: userProfileTx.objectId }) ) diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index afe0dc6cca..f29903b57b 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -208,9 +208,8 @@ export async function initializeWorkspace ( progress: (value: number) => Promise ): Promise { const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) - const initRepoDir = getMetadata(toolPlugin.metadata.InitRepoDir) + const initRepoDir = getMetadata(toolPlugin.metadata.InitRepoDir) ?? '' ctx.info('Init script details', { initWS, initRepoDir }) - if (initWS === undefined || initRepoDir === undefined) return const initScriptFile = path.resolve(initRepoDir, 'script.yaml') if (!fs.existsSync(initScriptFile)) { diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index a8a5613a03..aafb080e1c 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -205,9 +205,7 @@ export class WorkspaceInitializer { try { const uploader = new StorageFileUploader(this.ctx, this.storageAdapter, this.wsIds) const initPath = path.resolve(this.initRepoDir, step.path) - // eslint-disable-next-line no-template-curly-in-string - const initPerson = vars[`\${${this.creatorPersonVar}}`] - const importer = new HulyFormatImporter(this.client, uploader, logger, this.socialId, initPerson) + const importer = new HulyFormatImporter(this.client, uploader, logger, vars) await importer.importFolder(initPath) } catch (error) { logger.error('Import failed', error)