Fake data generator (#594)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>

dev/generator/src/attachments.ts

@@ -0,0 +1,46 @@
+import attachment, { Attachment } from '@anticrm/attachment'
+import { Class, Doc, generateId, Ref, Space, TxOperations } from '@anticrm/core'
+import faker from 'faker'
+import { Client } from 'minio'
+import PDFDocument from 'pdfkit'
+
+export interface AttachmentOptions {
+  min: number
+  max: number
+  deleteFactor: number // 0-100 value, will delete just added attachment, below min with rate
+}
+
+export async function addAttachments<T extends Doc> (options: AttachmentOptions, client: TxOperations, minio: Client, dbName: string, space: Ref<Space>, objectId: Ref<T>, _class: Ref<Class<T>>, collection: string): Promise<void> {
+  const attachmentCount = options.min + faker.datatype.number(options.max)
+  for (let i = 0; i < attachmentCount; i++) {
+    const attachmentId = `candidate-attachment-${generateId()}-${i}` as Ref<Attachment>
+
+    const needDelete = i >= options.min && (faker.datatype.number(100) > options.deleteFactor)
+
+    let bufLen = 0
+    if (!needDelete) {
+      const doc = new PDFDocument()
+      doc
+        .fontSize(16)
+        .text(faker.lorem.paragraph(faker.datatype.number(50)))
+
+      doc.end()
+
+      const buf = doc.read()
+      bufLen = buf.length
+      await minio.putObject(dbName, attachmentId, buf, bufLen, { 'Content-Type': 'application/pdf' })
+    }
+
+    await client.addCollection(attachment.class.Attachment, space, objectId, _class, 'attachments', {
+      name: faker.system.commonFileName('pdf'),
+      file: attachmentId,
+      type: 'application/pdf',
+      size: bufLen,
+      lastModified: faker.date.past().getTime()
+    }, attachmentId)
+
+    if (needDelete) {
+      await client.removeCollection(attachment.class.Attachment, space, attachmentId, objectId, _class, 'attachments')
+    }
+  }
+}

dev/generator/src/comments.ts

@@ -0,0 +1,29 @@
+import chunter, { Comment } from '@anticrm/chunter'
+import { AttachedData, Class, Doc, generateId, Ref, Space, TxOperations } from '@anticrm/core'
+import faker from 'faker'
+
+export interface CommentOptions {
+  min: number
+  max: number
+  paragraphMin: number
+  paragraphMax: number
+  updateFactor: number // 0-100 value, will generate random value and if value is less updateFactor it will be updated.
+}
+
+export async function addComments<T extends Doc> (options: CommentOptions, client: TxOperations, space: Ref<Space>, objectId: Ref<T>, _class: Ref<Class<T>>, collection: string): Promise<void> {
+  const commentsCount = options.min + faker.datatype.number(options.max)
+  for (let i = 0; i < commentsCount; i++) {
+    const commentId = `candidate-comment-${generateId()}-${i}` as Ref<Comment>
+
+    const commentData: AttachedData<Comment> = {
+      message: faker.lorem.paragraphs(options.paragraphMin + faker.datatype.number(options.paragraphMax))
+    }
+    await client.addCollection(chunter.class.Comment, space, objectId, _class, collection, commentData, commentId)
+
+    if (faker.datatype.number(100) > options.updateFactor) {
+      const updateMsg = faker.lorem.paragraphs(options.paragraphMin + faker.datatype.number(options.paragraphMax))
+
+      await client.updateCollection(chunter.class.Comment, space, commentId, objectId, _class, collection, { message: updateMsg })
+    }
+  }
+}

dev/generator/src/connect.ts

@@ -0,0 +1,18 @@
+
+import client from '@anticrm/client'
+import clientResources from '@anticrm/client-resources'
+import { Client } from '@anticrm/core'
+import { setMetadata } from '@anticrm/platform'
+import { encode } from 'jwt-simple'
+
+// eslint-disable-next-line
+const WebSocket = require('ws')
+
+export async function connect (transactorUrl: string, workspace: string): Promise<Client> {
+  console.log('connecting to transactor...')
+  const token = encode({ email: 'anticrm@hc.engineering', workspace }, 'secret')
+
+  // We need to override default factory with 'ws' one.
+  setMetadata(client.metadata.ClientSocketFactory, (url) => new WebSocket(url))
+  return await (await clientResources()).function.GetClient(token, transactorUrl)
+}

dev/generator/src/index.ts

@@ -0,0 +1,72 @@
+//
+// Copyright © 2020, 2021 Anticrm Platform Contributors.
+// Copyright © 2021 Hardcore Engineering Inc.
+//
+// Licensed under the Eclipse Public License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License. You may
+// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+import { program } from 'commander'
+import { Client } from 'minio'
+import { generateContacts } from './recruit'
+
+const transactorUrl = process.env.TRANSACTOR_URL
+if (transactorUrl === undefined) {
+  console.error('please provide transactor url.')
+  process.exit(1)
+}
+
+const minioEndpoint = process.env.MINIO_ENDPOINT
+if (minioEndpoint === undefined) {
+  console.error('please provide minio endpoint')
+  process.exit(1)
+}
+
+const minioAccessKey = process.env.MINIO_ACCESS_KEY
+if (minioAccessKey === undefined) {
+  console.error('please provide minio access key')
+  process.exit(1)
+}
+
+const minioSecretKey = process.env.MINIO_SECRET_KEY
+if (minioSecretKey === undefined) {
+  console.error('please provide minio secret key')
+  process.exit(1)
+}
+
+const minio = new Client({
+  endPoint: minioEndpoint,
+  port: 9000,
+  useSSL: false,
+  accessKey: minioAccessKey,
+  secretKey: minioSecretKey
+})
+
+program.version('0.0.1')
+
+program
+  .command('gen-recruit <workspace> <count>')
+  .description('generate a bunch of random candidates with attachemnts and comments.')
+  .option('-r, --random', 'generate random ids. So every call will add count <count> more candidates.', false)
+  .action(async (workspace: string, count: number, cmd) => {
+    return await generateContacts(transactorUrl, workspace, {
+      contacts: count,
+      random: (cmd.random as boolean),
+      comments: { min: 1, max: 10, paragraphMin: 1, paragraphMax: 20, updateFactor: 30 },
+      attachments: {
+        min: 1, max: 3, deleteFactor: 20
+      },
+      vacancy: 3,
+      applicantUpdateFactor: 70
+    }, minio)
+  })
+
+program.parse(process.argv)

dev/generator/src/kanban.ts

@@ -0,0 +1,36 @@
+import core, { Ref, SpaceWithStates, State, TxOperations } from '@anticrm/core'
+import { findOrUpdate } from './utils'
+import view, { Kanban } from '@anticrm/view'
+
+export async function createUpdateSpaceKanban (spaceId: Ref<SpaceWithStates>, client: TxOperations): Promise<Ref<State>[]> {
+  const states = [
+    { color: '#7C6FCD', name: 'Initial' },
+    { color: '#6F7BC5', name: 'Intermidiate' },
+    { color: '#77C07B', name: 'OverIntermidiate' },
+    { color: '#A5D179', name: 'Done' },
+    { color: '#F28469', name: 'Invalid' }
+  ]
+  const ids: Array<Ref<State>> = []
+  for (const st of states) {
+    const sid = ('generated-' + spaceId + '.state.' + st.name.toLowerCase().replace(' ', '_')) as Ref<State>
+    await findOrUpdate(client, spaceId, core.class.State,
+      sid,
+      {
+        title: st.name,
+        color: st.color
+      }
+    )
+    ids.push(sid)
+  }
+
+  await findOrUpdate(client, spaceId,
+    view.class.Kanban,
+    ('generated-' + spaceId + '.kanban') as Ref<Kanban>,
+    {
+      attachedTo: spaceId,
+      states: ids,
+      order: []
+    }
+  )
+  return ids
+}

dev/generator/src/recruit.ts

@@ -0,0 +1,146 @@
+import contact from '@anticrm/contact'
+import core, { AttachedData, Data, generateId, Ref, TxOperations } from '@anticrm/core'
+import recruit from '@anticrm/model-recruit'
+import { Applicant, Candidate, Vacancy } from '@anticrm/recruit'
+import faker from 'faker'
+import jpeg, { BufferRet } from 'jpeg-js'
+import { Client } from 'minio'
+import { addAttachments, AttachmentOptions } from './attachments'
+import { addComments, CommentOptions } from './comments'
+import { connect } from './connect'
+import { createUpdateSpaceKanban } from './kanban'
+import { findOrUpdate, findOrUpdateAttached } from './utils'
+
+export interface RecruitOptions {
+  random: boolean // random id prefix.
+  contacts: number // how many contacts to add
+  vacancy: number // Will add number of vacancies with applications.
+  // Comment generation control
+  comments: CommentOptions
+  // Attachment generation control
+  attachments: AttachmentOptions
+
+  applicantUpdateFactor: number
+}
+
+export async function generateContacts (transactorUrl: string, dbName: string, options: RecruitOptions, minio: Client): Promise<void> {
+  const connection = await connect(transactorUrl, dbName)
+
+  const accounts = await connection.findAll(contact.class.EmployeeAccount, {})
+  const accountIds = accounts.map(a => a._id)
+  const emoloyeeIds = accounts.map(a => a.employee)
+
+  const account = faker.random.arrayElement(accounts)
+
+  const client = new TxOperations(connection, account._id)
+
+  const candidates: Ref<Candidate>[] = []
+
+  for (let i = 0; i < options.contacts; i++) {
+    const fName = faker.name.firstName()
+    const lName = faker.name.lastName()
+
+    const { imgId, jpegImageData } = generateAvatar(i)
+    await minio.putObject(dbName, imgId, jpegImageData.data, jpegImageData.data.length, { 'Content-Type': 'image/jpeg' })
+    const candidate: Data<Candidate> = {
+      name: fName + ',' + lName,
+      city: faker.address.city(),
+      title: faker.name.title(),
+      channels: [
+        { provider: contact.channelProvider.Email, value: faker.internet.email(fName, lName) }
+      ],
+      onsite: faker.datatype.boolean(),
+      remote: faker.datatype.boolean(),
+      avatar: imgId,
+      source: faker.lorem.lines(1)
+    }
+    const candidateId = (options.random ? `candidate-${generateId()}-${i}` : `candidate-genid-${i}`) as Ref<Candidate>
+    candidates.push(candidateId)
+
+    // Update or create candidate
+    await findOrUpdate(client, recruit.space.CandidatesPublic, recruit.class.Candidate, candidateId, candidate)
+
+    await addComments(options.comments, client, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'comments')
+
+    await addAttachments(options.attachments, client, minio, dbName, recruit.space.CandidatesPublic, candidateId, recruit.class.Candidate, 'attachments')
+
+    console.log('Candidate', fName, lName, ' generated')
+  }
+  // Work on Vacancy/Applications.
+  for (let i = 0; i < options.vacancy; i++) {
+    const vacancy: Data<Vacancy> = {
+      name: faker.company.companyName(),
+      description: faker.lorem.sentences(2),
+      fullDescription: faker.lorem.sentences(10),
+      location: faker.address.city(),
+      company: faker.company.companyName(),
+      members: accountIds,
+      private: false
+    }
+    const vacancyId = (options.random ? `vacancy-${generateId()}-${i}` : `vacancy-genid-${i}`) as Ref<Vacancy>
+
+    console.log('Creating vacandy', vacancy.name)
+    // Update or create candidate
+    await findOrUpdate(client, core.space.Model, recruit.class.Vacancy, vacancyId, vacancy)
+
+    console.log('Vacandy generated', vacancy.name)
+
+    await addAttachments(options.attachments, client, minio, dbName, vacancyId, vacancyId, recruit.class.Vacancy, 'attachments')
+
+    console.log('Vacandy attachments generated', vacancy.name)
+
+    const states = await createUpdateSpaceKanban(vacancyId, client)
+
+    console.log('States generated', vacancy.name)
+
+    for (const candidateId of candidates) {
+      const applicantId = `vacancy-${vacancyId}-${candidateId}` as Ref<Applicant>
+
+      const applicant: AttachedData<Applicant> = {
+        number: faker.datatype.number(),
+        employee: faker.random.arrayElement(emoloyeeIds),
+        state: faker.random.arrayElement(states)
+      }
+
+      // Update or create candidate
+      await findOrUpdateAttached(client, vacancyId, recruit.class.Applicant, applicantId, applicant, { attachedTo: candidateId, attachedClass: recruit.class.Candidate, collection: 'applications' })
+
+      await addComments(options.comments, client, vacancyId, applicantId, recruit.class.Vacancy, 'comments')
+
+      await addAttachments(options.attachments, client, minio, dbName, vacancyId, applicantId, recruit.class.Applicant, 'attachments')
+
+      if (faker.datatype.number(100) > options.applicantUpdateFactor) {
+        await client.updateCollection(recruit.class.Applicant, vacancyId, applicantId, candidateId, recruit.class.Applicant, 'applications', {
+          state: faker.random.arrayElement(states)
+        })
+      }
+    }
+  }
+
+  await connection.close()
+}
+function generateAvatar (pos: number): {imgId: string, jpegImageData: BufferRet } {
+  const imgId = generateId()
+  const width = 128
+  const height = 128
+  const frameData = Buffer.alloc(width * height * 4)
+  let i = 0
+
+  const baseR = faker.datatype.number(255)
+  const baseG = faker.datatype.number(255)
+  const baseB = faker.datatype.number(255)
+
+  while (i < frameData.length) {
+    frameData[i++] = (baseR + faker.datatype.number(100)) % 255 // red
+    frameData[i++] = (baseG + faker.datatype.number(100)) % 255 // green
+    frameData[i++] = (baseB + faker.datatype.number(100)) % 255 // blue
+    frameData[i++] = 0xff // alpha - ignored in JPEGs
+  }
+  const rawImageData = {
+    data: frameData,
+    width: width,
+    height: height
+  }
+  const jpegImageData = jpeg.encode(rawImageData, 50)
+  return { imgId, jpegImageData }
+}

dev/generator/src/utils.ts

@@ -0,0 +1,18 @@
+import { AttachedData, AttachedDoc, Class, Data, Doc, DocumentUpdate, Ref, Space, TxOperations } from '@anticrm/core'
+
+export async function findOrUpdate<T extends Doc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: Data<T>): Promise<void> {
+  const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
+  if (existingObj !== undefined) {
+    await client.updateDoc(_class, space, objectId, data)
+  } else {
+    await client.createDoc(_class, space, data, objectId)
+  }
+}
+export async function findOrUpdateAttached<T extends AttachedDoc> (client: TxOperations, space: Ref<Space>, _class: Ref<Class<T>>, objectId: Ref<T>, data: AttachedData<T>, attached: {attachedTo: Ref<Doc>, attachedClass: Ref<Class<Doc>>, collection: string}): Promise<void> {
+  const existingObj = await client.findOne<Doc>(_class, { _id: objectId, space })
+  if (existingObj !== undefined) {
+    await client.updateCollection(_class, space, objectId, attached.attachedTo, attached.attachedClass, attached.collection, data as unknown as DocumentUpdate<T>)
+  } else {
+    await client.addCollection(_class, space, attached.attachedTo, attached.attachedClass, attached.collection, data, objectId)
+  }
+}