EZQMS-1145: Fixes doc import tool (#6204)

* EZQMS-1145: Fixes doc import tool
Signed-off-by: Alexey Zinoviev <alexey.zinoviev@xored.com>

dev/doc-import-tool/src/extract/extract.ts

@@ -1,5 +1,5 @@
 import { parseDocument } from 'htmlparser2'
-import { Document } from 'domhandler'
+import { AnyNode, Document } from 'domhandler'
 
 import { FileSpec, FileSpecType, TocFileSpec } from './types'
 import { createMetadataExtractor } from './meta'
@@ -28,10 +28,10 @@ class TocContentExtractor implements ContentExtractor {
     readonly type = FileSpecType.TOC
   ) {}
 
-  extract (doc: Document): ExtractedFile {
+  extract (doc: Document, headerRoot?: AnyNode): ExtractedFile {
     const metadataExtractor = createMetadataExtractor(this.spec.metadata)
-    const title = metadataExtractor.extractName(doc)
-    const oldId = metadataExtractor.extractId(doc)
+    const title = metadataExtractor.extractName(doc, headerRoot)
+    const oldId = metadataExtractor.extractId(doc, headerRoot)
 
     const docSpec = this.spec.spec
 
@@ -59,10 +59,10 @@ class TocContentExtractor implements ContentExtractor {
  * @public
  * Extracts HTML file contents
  */
-export async function extract (contents: string, spec: FileSpec): Promise<ExtractedFile> {
+export async function extract (contents: string, spec: FileSpec, headerRoot?: AnyNode): Promise<ExtractedFile> {
   const extractor = new TocContentExtractor(spec)
   const doc = parseDocument(contents)
-  return extractor.extract(doc)
+  return extractor.extract(doc, headerRoot)
 }
 
 export default extract

dev/doc-import-tool/src/extract/meta.ts

@@ -1,8 +1,15 @@
-import { Document, Element } from 'domhandler'
+import { AnyNode, Document, Element, Text } from 'domhandler'
 import { find } from 'domutils'
 import { ElementType } from 'htmlparser2'
 
-import { DocMetadataSpec, MetadataContainer, DocTableRowMetadata, DocMetaTagsMetadata } from './types'
+import {
+  DocMetadataSpec,
+  MetadataContainer,
+  DocTableRowMetadata,
+  DocMetaTagsMetadata,
+  PageHeaderTableRowMetadata,
+  MetadataTableCell
+} from './types'
 import { ELEMENT_LIMIT } from './common'
 import { TableNodeExtractor } from './nodes'
 import { TableContainer } from './container'
@@ -73,7 +80,37 @@ export class TableRowDocMetadataExtractor implements DocMetadataExtractor {
   }
 }
 
-type AnyDocMetadataExtractor = MetaTagsDocMetadataExtractor | TableRowDocMetadataExtractor
+const maxElems = 10000
+export class PageHeaderTableRowDocMetadataExtractor implements DocMetadataExtractor {
+  constructor (readonly tableMetadata: PageHeaderTableRowMetadata) {}
+
+  private getCellText (meta: MetadataTableCell, headerRoot?: AnyNode): string {
+    if (headerRoot === undefined) {
+      return ''
+    }
+
+    const rows = find((n) => n.type === ElementType.Tag && n.name === 'w:tr', [headerRoot], true, maxElems)
+    const { row, col, slice } = meta.extract
+    const cell = find((n) => n.type === ElementType.Tag && n.name === 'w:tc', [rows[row]], true, maxElems)[col]
+    const textNodes = find((n) => n.type === ElementType.Text, [cell], true, maxElems) as Text[]
+    const text = textNodes.map((n) => n.data).join('')
+
+    return slice === undefined ? text : text.slice(slice.start, slice.end)
+  }
+
+  extractName (doc: Document, headerRoot?: AnyNode): string {
+    return this.getCellText(this.tableMetadata.docName, headerRoot)
+  }
+
+  extractId (doc: Document, headerRoot?: AnyNode): string {
+    return this.getCellText(this.tableMetadata.docId, headerRoot)
+  }
+}
+
+type AnyDocMetadataExtractor =
+  | MetaTagsDocMetadataExtractor
+  | TableRowDocMetadataExtractor
+  | PageHeaderTableRowDocMetadataExtractor
 
 export function createMetadataExtractor (metadata: DocMetadataSpec): AnyDocMetadataExtractor {
   switch (metadata.in) {
@@ -81,5 +118,7 @@ export function createMetadataExtractor (metadata: DocMetadataSpec): AnyDocMetad
       return new MetaTagsDocMetadataExtractor(metadata)
     case MetadataContainer.TableRow:
       return new TableRowDocMetadataExtractor(metadata)
+    case MetadataContainer.PageHeaderTableRow:
+      return new PageHeaderTableRowDocMetadataExtractor(metadata)
   }
 }

dev/doc-import-tool/src/extract/nodes.ts

@@ -103,7 +103,7 @@ export class TableNodeExtractor implements NodeExtractor {
   private parseRows (table: AnyDomNode): AnyNode[][] {
     const header = findOne((n) => n.tagName === 'thead', [table])
     const body = findOne((n) => n.tagName === 'tbody', [table])
-    const bodyRows =
+    let bodyRows =
       body != null
         ? getChildren(body).filter((n) => clean(innerText(n)) !== '')
         : findAll((n) => n.tagName === 'tr' && clean(innerText(n)) !== '', [table])
@@ -111,14 +111,28 @@ export class TableNodeExtractor implements NodeExtractor {
     if (header != null) {
       const firstRow = findOne((n) => n.tagName === 'tr', [header])
 
+      if (bodyRows.length > 0) {
+        if (getChildren(bodyRows[0]).find((n) => n.type === ElementType.Tag && n.tagName === 'th') != null) {
+          bodyRows = bodyRows.slice(1)
+        }
+      }
+
       return [
         findAll((n) => n.tagName === 'th', firstRow != null ? [firstRow] : []),
-        ...bodyRows.map((r) => getChildren(r).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'))
+        ...bodyRows.map((r) =>
+          getChildren(r).filter((n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th'))
+        )
       ]
     } else if (bodyRows.length > 0) {
       return [
-        getChildren(bodyRows[0]).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'),
-        ...bodyRows.slice(1).map((r) => getChildren(r).filter((n) => n.type === ElementType.Tag && n.tagName === 'td'))
+        getChildren(bodyRows[0]).filter(
+          (n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th')
+        ),
+        ...bodyRows
+          .slice(1)
+          .map((r) =>
+            getChildren(r).filter((n) => n.type === ElementType.Tag && (n.tagName === 'td' || n.tagName === 'th'))
+          )
       ]
     }
 

dev/doc-import-tool/src/extract/types.ts

@@ -114,7 +114,8 @@ export type TocSectionSpec = z.infer<typeof tocSection>
 
 export enum MetadataContainer {
   MetaTags = 'meta-tags',
-  TableRow = 'table-row'
+  TableRow = 'table-row',
+  PageHeaderTableRow = 'page-header-table-row'
 }
 
 const metaTagsMetadata = z.object({
@@ -126,9 +127,16 @@ const metaTagsMetadata = z.object({
 const metadataTableCell = z.object({
   extract: z.object({
     row: z.number().min(0),
-    col: z.number().min(0)
+    col: z.number().min(0),
+    slice: z
+      .object({
+        start: z.number().min(0).optional(),
+        end: z.number().min(0).optional()
+      })
+      .optional()
   })
 })
+export type MetadataTableCell = z.infer<typeof metadataTableCell>
 
 const tableRowMetadata = z.object({
   in: z.literal(MetadataContainer.TableRow),
@@ -137,11 +145,19 @@ const tableRowMetadata = z.object({
   docId: metadataTableCell
 })
 
-const docMetadata = z.union([metaTagsMetadata, tableRowMetadata])
+const pageHeaderTableRowMetadata = z.object({
+  in: z.literal(MetadataContainer.PageHeaderTableRow),
+  headerIdx: z.number().min(1).optional(),
+  docName: metadataTableCell,
+  docId: metadataTableCell
+})
+
+const docMetadata = z.union([metaTagsMetadata, tableRowMetadata, pageHeaderTableRowMetadata])
 
 export type DocMetadataSpec = z.infer<typeof docMetadata>
 export type DocMetaTagsMetadata = z.infer<typeof metaTagsMetadata>
 export type DocTableRowMetadata = z.infer<typeof tableRowMetadata>
+export type PageHeaderTableRowMetadata = z.infer<typeof pageHeaderTableRowMetadata>
 
 // #endregion