WEBDIENSTE/huly-platform
1
0
Fork 0
mirror of https://github.com/hcengineering/platform.git synced 2026-09-23 18:15:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

EZQMS-1145: Fixes doc import tool (#6204)

Browse Source 
* EZQMS-1145: Fixes doc import tool
Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>
This commit is contained in:
Alexey Zinoviev
2024-08-01 12:05:07 +07:00
committed by GitHub
parent 0bb4bb04ac
commit c55e6f00b0
15 changed files with 346 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/doc-import-tool/src/__start.ts
+1 -3
View File
@@ -14,6 +14,4 @@
//
import { docImportTool } from '.'
const productId = process.env.PRODUCT_ID ?? 'ezqms'
docImportTool(productId)
docImportTool()
dev/doc-import-tool/src/commands.ts
+12 -3
View File
@@ -1,5 +1,8 @@
import docx4js from 'docx4js'
import { AnyNode } from 'domhandler'
import extract from './extract/extract'
import { read } from './extract/types'
import { MetadataContainer, read } from './extract/types'
import importExtractedFile from './import'
import convert from './convert/convert'
import { Config } from './config'
@@ -10,9 +13,15 @@ export async function importDoc (config: Config): Promise<void> {
const spec = await read(specFile)
console.log(`Spec: ${JSON.stringify(spec, undefined, 2)}`)
let headerRoot: AnyNode | undefined
if (spec.metadata.in === MetadataContainer.PageHeaderTableRow) {
const headerIdx = spec.metadata.headerIdx ?? 1
const docx = await docx4js.load(config.doc)
headerRoot = docx.getObjectPart(`word/header${headerIdx}.xml`).root()[0]
}
const contents = await convert(doc, backend)
const extractedFile = await extract(contents, spec)
// console.log(`Extracted data: ${JSON.stringify(extractedFile, undefined, 2)}`)
const extractedFile = await extract(contents, spec, headerRoot)
await importExtractedFile(config, extractedFile)
}
dev/doc-import-tool/src/extract/extract.ts
+6 -6
View File
@@ -1,5 +1,5 @@
import { parseDocument } from 'htmlparser2'
import { Document } from 'domhandler'
import { AnyNode, Document } from 'domhandler'
import { FileSpec, FileSpecType, TocFileSpec } from './types'
import { createMetadataExtractor } from './meta'
@@ -28,10 +28,10 @@ class TocContentExtractor implements ContentExtractor {
readonly type = FileSpecType.TOC
) {}
extract (doc: Document): ExtractedFile {
extract (doc: Document, headerRoot?: AnyNode): ExtractedFile {
const metadataExtractor = createMetadataExtractor(this.spec.metadata)
const title = metadataExtractor.extractName(doc)
const oldId = metadataExtractor.extractId(doc)
const title = metadataExtractor.extractName(doc, headerRoot)
const oldId = metadataExtractor.extractId(doc, headerRoot)
const docSpec = this.spec.spec
@@ -59,10 +59,10 @@ class TocContentExtractor implements ContentExtractor {
* @public
* Extracts HTML file contents
*/
export async function extract (contents: string, spec: FileSpec): Promise<ExtractedFile> {
export async function extract (contents: string, spec: FileSpec, headerRoot?: AnyNode): Promise<ExtractedFile> {
const extractor = new TocContentExtractor(spec)
const doc = parseDocument(contents)
return extractor.extract(doc)
return extractor.extract(doc, headerRoot)
}
export default extract
dev/doc-import-tool/src/extract/meta.ts
+42 -3
View File
@@ -1,8 +1,15 @@
import { Document, Element } from 'domhandler'
import { AnyNode, Document, Element, Text } from 'domhandler'
import { find } from 'domutils'
import { ElementType } from 'htmlparser2'
import { DocMetadataSpec, MetadataContainer, DocTableRowMetadata, DocMetaTagsMetadata } from './types'
import {
DocMetadataSpec,
MetadataContainer,
DocTableRowMetadata,
DocMetaTagsMetadata,
PageHeaderTableRowMetadata,
MetadataTableCell
} from './types'
import { ELEMENT_LIMIT } from './common'
import { TableNodeExtractor } from './nodes'
import { TableContainer } from './container'
@@ -73,7 +80,37 @@ export class TableRowDocMetadataExtractor implements DocMetadataExtractor {
}
}
type AnyDocMetadataExtractor = MetaTagsDocMetadataExtractor | TableRowDocMetadataExtractor
const maxElems = 10000
export class PageHeaderTableRowDocMetadataExtractor implements DocMetadataExtractor {
constructor (readonly tableMetadata: PageHeaderTableRowMetadata) {}
private getCellText (meta: MetadataTableCell, headerRoot?: AnyNode): string {
if (headerRoot === undefined) {
return ''
}
const rows = find((n) => n.type === ElementType.Tag && n.name === 'w:tr', [headerRoot], true, maxElems)
const { row, col, slice } = meta.extract
const cell = find((n) => n.type === ElementType.Tag && n.name === 'w:tc', [rows[row]], true, maxElems)[col]
const textNodes = find((n) => n.type === ElementType.Text, [cell], true, maxElems) as Text[]
const text = textNodes.map((n) => n.data).join('')
return slice === undefined ? text : text.slice(slice.start, slice.end)
}
extractName (doc: Document, headerRoot?: AnyNode): string {
return this.getCellText(this.tableMetadata.docName, headerRoot)
}
extractId (doc: Document, headerRoot?: AnyNode): string {
return this.getCellText(this.tableMetadata.docId, headerRoot)
}
}
type AnyDocMetadataExtractor =
| MetaTagsDocMetadataExtractor
| TableRowDocMetadataExtractor
| PageHeaderTableRowDocMetadataExtractor
export function createMetadataExtractor (metadata: DocMetadataSpec): AnyDocMetadataExtractor {
switch (metadata.in) {
@@ -81,5 +118,7 @@ export function createMetadataExtractor (metadata: DocMetadataSpec): AnyDocMetad
return new MetaTagsDocMetadataExtractor(metadata)
case MetadataContainer.TableRow:
return new TableRowDocMetadataExtractor(metadata)
case MetadataContainer.PageHeaderTableRow:
return new PageHeaderTableRowDocMetadataExtractor(metadata)
}
}
dev/doc-import-tool/src/extract/nodes.ts
+18 -4
View File
@@ -103,7 +103,7 @@ export class TableNodeExtractor implements NodeExtractor {
private parseRows (table: AnyDomNode): AnyNode[][] {
const header = findOne((n) => n.tagName === 'thead', [table])
const body = findOne((n) => n.tagName === 'tbody', [table])
const bodyRows =
let bodyRows =
body != null
? getChildren(body).filter((n) => clean(innerText(n)) !== '')
: findAll((n) => n.tagName === 'tr' && clean(innerText(n)) !== '', [table])
@@ -111,14 +111,28 @@ export class TableNodeExtractor implements NodeExtractor {
if (header != null) {
const firstRow = findOne((n) => n.tagName === 'tr', [header])
if (bodyRows.length > 0) {
if (getChildren(bodyRows[0]).find((n) => n.type === ElementType.Tag && n.tagName === 'th') != null) {
bodyRows = bodyRows.slice(1)
}
}
return [
findAll((n) => n.tagName === 'th', firstRow != null ? [firstRow] : []),
...bodyRows.map((r) => getChildren(r).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'))
...bodyRows.map((r) =>
getChildren(r).filter((n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th'))
)
]
} else if (bodyRows.length > 0) {
return [
getChildren(bodyRows[0]).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'),
...bodyRows.slice(1).map((r) => getChildren(r).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'))
getChildren(bodyRows[0]).filter(
(n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th')
),
...bodyRows
.slice(1)
.map((r) =>
getChildren(r).filter((n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th'))
)
]
}
dev/doc-import-tool/src/extract/types.ts
+19 -3
View File
@@ -114,7 +114,8 @@ export type TocSectionSpec = z.infer<typeof tocSection>
export enum MetadataContainer {
MetaTags = 'meta-tags',
TableRow = 'table-row'
TableRow = 'table-row',
PageHeaderTableRow = 'page-header-table-row'
}
const metaTagsMetadata = z.object({
@@ -126,9 +127,16 @@ const metaTagsMetadata = z.object({
const metadataTableCell = z.object({
extract: z.object({
row: z.number().min(0),
col: z.number().min(0)
col: z.number().min(0),
slice: z
.object({
start: z.number().min(0).optional(),
end: z.number().min(0).optional()
})
.optional()
})
})
export type MetadataTableCell = z.infer<typeof metadataTableCell>
const tableRowMetadata = z.object({
in: z.literal(MetadataContainer.TableRow),
@@ -137,11 +145,19 @@ const tableRowMetadata = z.object({
docId: metadataTableCell
})
const docMetadata = z.union([metaTagsMetadata, tableRowMetadata])
const pageHeaderTableRowMetadata = z.object({
in: z.literal(MetadataContainer.PageHeaderTableRow),
headerIdx: z.number().min(1).optional(),
docName: metadataTableCell,
docId: metadataTableCell
})
const docMetadata = z.union([metaTagsMetadata, tableRowMetadata, pageHeaderTableRowMetadata])
export type DocMetadataSpec = z.infer<typeof docMetadata>
export type DocMetaTagsMetadata = z.infer<typeof metaTagsMetadata>
export type DocTableRowMetadata = z.infer<typeof tableRowMetadata>
export type PageHeaderTableRowMetadata = z.infer<typeof pageHeaderTableRowMetadata>
// #endregion
dev/doc-import-tool/src/import.ts
+18 -22
View File
@@ -41,18 +41,13 @@ import { compareStrExact, uploadFile } from './helpers'
export default async function importExtractedFile (config: Config, extractedFile: ExtractedFile): Promise<void> {
const { workspaceId } = config
const token = generateToken(systemAccountEmail, workspaceId)
const transactorUrl = await getTransactorEndpoint(token)
const transactorUrl = await getTransactorEndpoint(token, 'external')
console.log(`Connecting to transactor: ${transactorUrl} (ws: '${workspaceId.name}')`)
const connection = (await createClient(transactorUrl, token)) as CoreClient & BackupClient
try {
console.log(`Connected to ${transactorUrl}`)
const txops = new TxOperations(connection, core.account.System)
try {
@@ -73,17 +68,18 @@ async function createDocument (
config: Config
): Promise<Ref<Document>> {
const { owner, space } = config
console.log('Creating document from extracted data')
const templateId = await createTemplateIfNotExist(txops, extractedFile.prefix, config)
const { title, prefix } = extractedFile
const { title, prefix, oldId } = extractedFile
const docId: Ref<ControlledDocument> = generateId()
const ccRecordId = generateId<ChangeControl>()
const data: AttachedData<ControlledDocument> = {
title,
prefix,
code: '',
code: oldId,
seqNumber: 0,
major: 0,
minor: 1,
@@ -95,7 +91,7 @@ async function createDocument (
reviewers: [],
approvers: [],
coAuthors: [],
changeControl: '' as Ref<ChangeControl>,
changeControl: ccRecordId,
author: owner,
owner,
category: '' as Ref<DocumentCategory>,
@@ -103,13 +99,13 @@ async function createDocument (
effectiveDate: 0,
reviewInterval: DEFAULT_PERIODIC_REVIEW_INTERVAL,
content: getCollaborativeDoc(generateId()),
snapshots: 0
snapshots: 0,
plannedEffectiveDate: 0
}
const ccRecordId = generateId<ChangeControl>()
const ccRecord: Data<ChangeControl> = {
description: '',
reason: '', // TODO: move to config
reason: 'Imported document', // TODO: move to config
impact: '',
impactedDocuments: []
}
@@ -135,29 +131,28 @@ async function createTemplateIfNotExist (
): Promise<Ref<DocumentTemplate>> {
const { owner, space } = config
console.log(`Getting template ${prefix}`)
console.log(`Getting template with doc ${prefix}`)
const template = await txops.findOne(documents.mixin.DocumentTemplate, { prefix })
const template = await txops.findOne(documents.mixin.DocumentTemplate, { docPrefix: prefix })
if (template != null) {
return template._id
}
console.log(`Creating template with prefix: ${prefix}`)
console.log(`Creating template with doc prefix: ${prefix}`)
const ccRecordId = generateId<ChangeControl>()
const ccRecord: Data<ChangeControl> = {
description: '',
reason: '', // TODO: move to config
reason: 'Imported template', // TODO: move to config
impact: '',
impactedDocuments: []
}
const templateId: Ref<ControlledDocument> = generateId()
const category = '' as Ref<DocumentCategory> // TODO: move to config
const data: AttachedData<ControlledDocument> = {
prefix: 'IMP',
const data = {
title: 'Import template',
code: templateId,
code: '',
seqNumber: 0,
sections: 0,
category,
@@ -172,12 +167,13 @@ async function createTemplateIfNotExist (
coAuthors: [],
changeControl: ccRecordId,
content: getCollaborativeDoc(generateId()),
snapshots: 0
snapshots: 0,
plannedEffectiveDate: 0
}
const { success } = await createDocumentTemplate(
txops,
documents.class.Document,
documents.class.ControlledDocument,
space,
documents.mixin.DocumentTemplate,
documents.ids.NoProject,
dev/doc-import-tool/src/index.ts
+8 -8
View File
@@ -28,16 +28,16 @@ import { getBackend } from './convert/convert'
/**
* @public
*/
export function docImportTool (productId: string): void {
export function docImportTool (): void {
const serverSecret = process.env.SERVER_SECRET
if (serverSecret === undefined) {
console.error('please provide server secret')
process.exit(1)
}
const accountUrl = process.env.ACCOUNT_URL
const accountUrl = process.env.ACCOUNTS_URL
if (accountUrl === undefined) {
console.error('please provide transactor url')
console.error('please provide account url')
process.exit(1)
}
@@ -64,7 +64,7 @@ export function docImportTool (productId: string): void {
program
.command('import <doc> <workspace> <owner>')
.description('import doc into workspace')
.option('-s|--spec <specFile>', 'Specification file')
.option('-s|--spec <spec>', 'Specification file')
.option('-b|--backend <backend>', 'Conversion backend', 'pandoc')
.option('--space <space>', 'Doc space ID', documents.space.QualityDocuments)
.action(
@@ -72,22 +72,22 @@ export function docImportTool (productId: string): void {
doc: string,
workspace: string,
owner: Ref<Employee>,
cmd: { backend: string, space: Ref<DocumentSpace>, specFile?: string }
cmd: { backend: string, space: Ref<DocumentSpace>, spec?: string }
) => {
console.log(
`Importing document '${doc}' into workspace '${workspace}', owner: ${JSON.stringify(owner)}, spec: ${
cmd.specFile
cmd.spec
}, space: ${cmd.space}, backend: ${cmd.backend}`
)
try {
const workspaceId = getWorkspaceId(workspace, productId)
const workspaceId = getWorkspaceId(workspace)
const config: Config = {
doc,
workspaceId,
owner,
backend: getBackend(cmd.backend),
specFile: cmd.specFile,
specFile: cmd.spec,
space: cmd.space,
uploadURL: uploadUrl,
collaboratorApiURL: collaboratorApiUrl,
dev/doc-import-tool/src/type/docx4js.d.ts
Vendored
+3
View File
@@ -0,0 +1,3 @@
declare module 'docx4js' {
export = any
}