UBERF-9764: Adjust gmail for new accounts (#8681)

Signed-off-by: Artem Savchenko <armisav@gmail.com>
2026-09-13 21:27:46 +02:00 · 2025-04-28 11:37:30 +05:00
parent 9efefe3693
commit ac249c8273
59 changed files with 4326 additions and 966 deletions
@@ -103,4 +103,13 @@ export function getAccountDBUrl (): string {
  return url
 }

+export function getKvsUrl (): string {
+  const url = process.env.KVS_URL
+  if (url === undefined) {
+    console.error('please provide KVS_URL')
+    process.exit(1)
+  }
+  return url
+}
+
 devTool(prepareTools)
@@ -0,0 +1,311 @@
+import { getAccountClient } from '@hcengineering/server-client'
+import { getClient as getKvsClient } from '@hcengineering/kvs-client'
+import { generateToken } from '@hcengineering/server-token'
+import { MongoMemoryServer } from 'mongodb-memory-server'
+import { MongoClient, type Db } from 'mongodb'
+import { performGmailAccountMigrations } from '../gmail'
+
+// Mock dependencies
+jest.mock('@hcengineering/server-client')
+jest.mock('@hcengineering/server-token')
+jest.mock('@hcengineering/kvs-client')
+
+describe('Gmail Migrations', () => {
+  // Setup MongoDB in-memory server
+  let mongoServer: MongoMemoryServer
+  let mongoClient: MongoClient
+  let db: Db
+
+  // Mock implementations
+  const mockAccountClient = {
+    listWorkspaces: jest.fn(),
+    findFullSocialIdBySocialKey: jest.fn(),
+    getIntegration: jest.fn(),
+    createIntegration: jest.fn(),
+    getIntegrationSecret: jest.fn(),
+    addIntegrationSecret: jest.fn(),
+    updateIntegrationSecret: jest.fn()
+  }
+
+  const mockKvsClient = {
+    getValue: jest.fn(),
+    setValue: jest.fn(),
+    deleteKey: jest.fn(),
+    listKeys: jest.fn()
+  }
+
+  beforeAll(async () => {
+    mongoServer = await MongoMemoryServer.create()
+    const uri = mongoServer.getUri()
+    mongoClient = await MongoClient.connect(uri)
+    db = mongoClient.db('test-db')
+
+    // Setup mocks
+    ;(getAccountClient as jest.Mock).mockReturnValue(mockAccountClient)
+    ;(getKvsClient as jest.Mock).mockReturnValue(mockKvsClient)
+    ;(generateToken as jest.Mock).mockReturnValue('mock-token')
+  })
+
+  afterAll(async () => {
+    await mongoClient.close()
+    await mongoServer.stop()
+  })
+
+  beforeEach(async () => {
+    // Reset mocks
+    jest.clearAllMocks()
+
+    // Setup collections
+    await db.collection('tokens').deleteMany({})
+    await db.collection('histories').deleteMany({})
+
+    // Reset mock implementations
+    mockAccountClient.listWorkspaces.mockReset()
+    mockAccountClient.findFullSocialIdBySocialKey.mockReset()
+    mockAccountClient.getIntegration.mockReset()
+    mockAccountClient.createIntegration.mockReset()
+    mockAccountClient.getIntegrationSecret.mockReset()
+    mockAccountClient.addIntegrationSecret.mockReset()
+    mockAccountClient.updateIntegrationSecret.mockReset()
+
+    mockKvsClient.getValue.mockReset()
+    mockKvsClient.setValue.mockReset()
+  }, 10000)
+
+  it('should migrate tokens to new integration format', async () => {
+    // Setup test data
+    const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
+    const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' }
+
+    // Mock workspace list
+    mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2])
+
+    // Setup tokens in DB
+    await db.collection('tokens').insertMany([
+      {
+        userId: 'user1@example.com',
+        workspace: 'ws1',
+        token: 'token1',
+        refresh_token: 'refresh1',
+        access_token: 'access1'
+      },
+      {
+        userId: 'user2@example.com',
+        workspace: 'oldWs2',
+        token: 'token2',
+        refresh_token: 'refresh2',
+        access_token: 'access2'
+      }
+    ])
+
+    // Mock social ID lookup
+    mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
+      if (key === 'email:user1@example.com') {
+        return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
+      } else if (key === 'email:user2@example.com') {
+        return Promise.resolve({ _id: 'social2', personUuid: 'person2' })
+      }
+      return Promise.resolve(undefined)
+    })
+
+    // Mock integration checks
+    mockAccountClient.getIntegration.mockResolvedValue(null)
+    mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
+
+    // Run migration
+    await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
+
+    // Verify tokens were migrated
+    expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2)
+    expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2)
+
+    // Check that tokens were migrated with correct data
+    const calls = mockAccountClient.addIntegrationSecret.mock.calls
+    expect(calls).toContainEqual([
+      expect.objectContaining({
+        socialId: 'social1',
+        workspaceUuid: 'ws1',
+        secret: expect.stringContaining('user1@example.com')
+      })
+    ])
+    expect(calls).toContainEqual([
+      expect.objectContaining({
+        socialId: 'social2',
+        workspaceUuid: 'ws2',
+        secret: expect.stringContaining('user2@example.com')
+      })
+    ])
+  })
+
+  it('should migrate with oids and github ids', async () => {
+    // Setup test data
+    const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
+    const workspace2 = { uuid: 'ws2', dataId: 'oldWs2', name: 'Workspace 2' }
+
+    // Mock workspace list
+    mockAccountClient.listWorkspaces.mockResolvedValue([workspace1, workspace2])
+
+    // Setup tokens in DB
+    await db.collection('tokens').insertMany([
+      {
+        userId: 'github:user1',
+        workspace: 'ws1',
+        token: 'token1',
+        refresh_token: 'refresh1',
+        access_token: 'access1'
+      },
+      {
+        userId: 'openid:user2',
+        workspace: 'oldWs2',
+        token: 'token2',
+        refresh_token: 'refresh2',
+        access_token: 'access2'
+      }
+    ])
+
+    // Mock social ID lookup
+    mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
+      if (key === 'github:user1') {
+        return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
+      } else if (key === 'oidc:user2') {
+        return Promise.resolve({ _id: 'social2', personUuid: 'person2' })
+      }
+      return Promise.resolve(undefined)
+    })
+
+    // Mock integration checks
+    mockAccountClient.getIntegration.mockResolvedValue(null)
+    mockAccountClient.getIntegrationSecret.mockResolvedValue(null)
+
+    // Run migration
+    await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
+
+    // Verify tokens were migrated
+    expect(mockAccountClient.createIntegration).toHaveBeenCalledTimes(2)
+    expect(mockAccountClient.addIntegrationSecret).toHaveBeenCalledTimes(2)
+
+    // Check that tokens were migrated with correct data
+    const calls = mockAccountClient.addIntegrationSecret.mock.calls
+    expect(calls).toContainEqual([
+      expect.objectContaining({
+        socialId: 'social1',
+        workspaceUuid: 'ws1',
+        secret: expect.stringContaining('github:user1')
+      })
+    ])
+    expect(calls).toContainEqual([
+      expect.objectContaining({
+        socialId: 'social2',
+        workspaceUuid: 'ws2',
+        secret: expect.stringContaining('openid:user2')
+      })
+    ])
+  })
+
+  it('should migrate history records to KVS', async () => {
+    // Setup test data
+    const workspace1 = { uuid: 'ws1', name: 'Workspace 1' }
+
+    // Mock workspace list
+    mockAccountClient.listWorkspaces.mockResolvedValue([workspace1])
+
+    // Setup histories in DB
+    await db.collection('histories').insertMany([
+      {
+        userId: 'user1@example.com',
+        workspace: 'ws1',
+        token: 'token1',
+        historyId: 'history1'
+      }
+    ])
+
+    // Mock social ID lookup
+    mockAccountClient.findFullSocialIdBySocialKey.mockImplementation((key) => {
+      if (key === 'email:user1@example.com') {
+        return Promise.resolve({ _id: 'social1', personUuid: 'person1' })
+      }
+      return Promise.resolve(undefined)
+    })
+
+    // Mock KVS client responses
+    mockKvsClient.getValue.mockResolvedValue(null)
+
+    // Run migration
+    await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
+
+    // Verify KVS calls
+    expect(mockKvsClient.setValue).toHaveBeenCalledWith(
+      'history:ws1:social1',
+      expect.objectContaining({
+        historyId: 'history1',
+        email: 'user1@example.com',
+        userId: 'person1',
+        workspace: 'ws1'
+      })
+    )
+  })
+
+  it('should handle errors gracefully', async () => {
+    // Mock console.error to capture errors
+    const originalConsoleError = console.error
+    console.error = jest.fn()
+
+    // Setup test data that will cause errors
+    mockAccountClient.listWorkspaces.mockResolvedValue([])
+    mockAccountClient.findFullSocialIdBySocialKey.mockRejectedValue(new Error('Network error'))
+
+    // Insert some data
+    await db.collection('tokens').insertOne({
+      userId: 'error@example.com',
+      workspace: 'non-existent',
+      token: 'token-error'
+    })
+
+    // Run migration
+    await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
+
+    // Should not throw but log errors
+    expect(console.error).toHaveBeenCalled()
+
+    // Restore console.error
+    console.error = originalConsoleError
+  })
+
+  it('should update existing integration secrets', async () => {
+    // Setup test data
+    const workspace = { uuid: 'ws1', name: 'Workspace 1' }
+
+    // Mock workspace list
+    mockAccountClient.listWorkspaces.mockResolvedValue([workspace])
+
+    // Setup token in DB
+    await db.collection('tokens').insertOne({
+      userId: 'user1@example.com',
+      workspace: 'ws1',
+      token: 'token1',
+      refresh_token: 'refresh1'
+    })
+
+    // Mock social ID lookup
+    mockAccountClient.findFullSocialIdBySocialKey.mockResolvedValue({ _id: 'social1', personUuid: 'person1' })
+
+    // Mock existing integration and token
+    mockAccountClient.getIntegration.mockResolvedValue({ _id: 'integration1' })
+    mockAccountClient.getIntegrationSecret.mockResolvedValue({
+      scope: 'old-scope',
+      token_type: 'Bearer'
+    })
+
+    // Run migration
+    await performGmailAccountMigrations(db, 'test-region', 'http://kvs-url')
+
+    // Verify update was called instead of add
+    expect(mockAccountClient.createIntegration).not.toHaveBeenCalled()
+    expect(mockAccountClient.addIntegrationSecret).not.toHaveBeenCalled()
+    expect(mockAccountClient.updateIntegrationSecret).toHaveBeenCalledWith(
+      expect.objectContaining({
+        secret: expect.stringContaining('refresh1')
+      })
+    )
+  })
+})
@@ -0,0 +1,225 @@
+import {
+  type AccountUuid,
+  type PersonId,
+  type SocialId,
+  type WorkspaceInfoWithStatus,
+  type WorkspaceUuid,
+  buildSocialIdString,
+  systemAccountUuid
+} from '@hcengineering/core'
+import { getAccountClient } from '@hcengineering/server-client'
+import { getClient as getKvsClient } from '@hcengineering/kvs-client'
+import { generateToken } from '@hcengineering/server-token'
+import { getSocialKeyByOldEmail } from '@hcengineering/model-core'
+import type { Db } from 'mongodb'
+
+// Old token and history types
+interface Credentials {
+  refresh_token?: string | null
+  expiry_date?: number | null
+  access_token?: string | null
+  token_type?: string | null
+  id_token?: string | null
+  scope?: string
+}
+
+interface User {
+  userId: string
+  workspace: WorkspaceUuid
+  token: string
+}
+
+type History = User & {
+  historyId: string
+}
+
+// Updated token and history types
+interface UserV2 {
+  userId: AccountUuid
+  email?: string
+  socialId: SocialId
+  workspace: WorkspaceUuid
+  token: string
+}
+
+export type TokenV2 = UserV2 & Credentials
+
+export type HistoryV2 = UserV2 & {
+  historyId: string
+}
+
+type Token = User & Credentials
+
+interface WorkspaceInfoProvider {
+  getWorkspaceInfo: (workspaceUuid: WorkspaceUuid) => Promise<WorkspaceInfoWithStatus | undefined>
+}
+
+const GMAIL_INTEGRATION = 'gmail'
+const TOKEN_TYPE = 'token'
+
+export async function performGmailAccountMigrations (db: Db, region: string | null, kvsUrl: string): Promise<void> {
+  console.log('Start Gmail migrations')
+  const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' })
+  const accountClient = getAccountClient(token)
+
+  const allWorkpaces = await accountClient.listWorkspaces(region)
+  const byId = new Map(allWorkpaces.map((it) => [it.uuid, it]))
+  const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it]))
+  const workspaceProvider: WorkspaceInfoProvider = {
+    getWorkspaceInfo: async (workspaceUuid: WorkspaceUuid) => {
+      const ws = oldNewIds.get(workspaceUuid as any) ?? byId.get(workspaceUuid as any)
+      if (ws == null) {
+        console.error('No workspace found for token', workspaceUuid)
+        return undefined
+      }
+      return ws
+    }
+  }
+
+  await migrateGmailIntegrations(db, token, workspaceProvider)
+
+  await migrateGmailHistory(db, token, kvsUrl, workspaceProvider)
+  console.log('Finished Gmail migrations')
+}
+
+async function migrateGmailIntegrations (
+  db: Db,
+  token: string,
+  workspaceProvider: WorkspaceInfoProvider
+): Promise<void> {
+  try {
+    console.log('Start Gmail account migrations')
+    const gmailToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'gmail' })
+    const accountClient = getAccountClient(token)
+
+    const gmailAccountClient = getAccountClient(gmailToken)
+
+    const tokens = db.collection<Token>('tokens')
+
+    const allTokens = await tokens.find({}).toArray()
+
+    for (const token of allTokens) {
+      try {
+        const ws = await workspaceProvider.getWorkspaceInfo(token.workspace as any)
+        if (ws == null) {
+          continue
+        }
+        token.workspace = ws.uuid
+
+        const socialKey = buildSocialIdString(getSocialKeyByOldEmail(token.userId))
+        const socialId =
+          socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined
+        if (socialId == null) {
+          console.error('No socialId found for token', token)
+          continue
+        }
+        // Check/create integration in account
+        const existing = await gmailAccountClient.getIntegration({
+          kind: GMAIL_INTEGRATION,
+          workspaceUuid: ws?.uuid,
+          socialId: socialId._id
+        })
+
+        if (existing == null) {
+          await gmailAccountClient.createIntegration({
+            kind: GMAIL_INTEGRATION,
+            workspaceUuid: ws?.uuid,
+            socialId: socialId._id
+          })
+        }
+
+        const existingToken = await gmailAccountClient.getIntegrationSecret({
+          key: TOKEN_TYPE,
+          kind: GMAIL_INTEGRATION,
+          socialId: socialId._id,
+          workspaceUuid: ws?.uuid
+        })
+        const newToken: TokenV2 = {
+          ...token,
+          workspace: ws?.uuid,
+          email: token.userId,
+          userId: socialId.personUuid as AccountUuid,
+          socialId
+        }
+        if (existingToken == null) {
+          await gmailAccountClient.addIntegrationSecret({
+            key: TOKEN_TYPE,
+            kind: GMAIL_INTEGRATION,
+            socialId: socialId._id,
+            secret: JSON.stringify(newToken),
+            workspaceUuid: token.workspace
+          })
+        } else {
+          const updatedToken = {
+            ...existingToken,
+            ...newToken
+          }
+          await gmailAccountClient.updateIntegrationSecret({
+            key: TOKEN_TYPE,
+            kind: GMAIL_INTEGRATION,
+            socialId: socialId._id,
+            secret: JSON.stringify(updatedToken),
+            workspaceUuid: token.workspace
+          })
+        }
+      } catch (e) {
+        console.error('Error migrating token', token, e)
+      }
+    }
+    console.log('Gmail integrations migrations done, integration count:', allTokens.length)
+  } catch (e) {
+    console.error('Error migrating tokens', e)
+  }
+}
+
+async function migrateGmailHistory (
+  db: Db,
+  token: string,
+  kvsUrl: string,
+  workspaceProvider: WorkspaceInfoProvider
+): Promise<void> {
+  try {
+    console.log('Start Gmail history migrations')
+    const accountClient = getAccountClient(token)
+    const history = db.collection<History>('histories')
+    const allHistories = await history.find({}).toArray()
+
+    const kvsClient = getKvsClient(kvsUrl, token)
+
+    for (const history of allHistories) {
+      try {
+        const socialKey = buildSocialIdString(getSocialKeyByOldEmail(history.userId))
+        const socialId =
+          socialKey !== undefined ? await accountClient.findFullSocialIdBySocialKey(socialKey) : undefined
+        if (socialId == null) {
+          console.error('No socialId found for history', history)
+          continue
+        }
+        // Update/create history in KVS
+        const ws = await workspaceProvider.getWorkspaceInfo(history.workspace as any)
+        if (ws == null) {
+          continue
+        }
+        const historyKey = getHistoryKey(ws.uuid, socialId._id)
+        const existingHistory = await kvsClient.getValue<HistoryV2>(historyKey)
+        const updatedHistory: HistoryV2 = {
+          ...(existingHistory ?? history),
+          email: history.userId,
+          workspace: ws.uuid,
+          userId: socialId.personUuid as AccountUuid,
+          socialId
+        }
+        await kvsClient.setValue(historyKey, updatedHistory)
+      } catch (e) {
+        console.error('Error migrating history', history, e)
+      }
+    }
+    console.log('Finished migrating gmail history, count:', allHistories.length)
+  } catch (e) {
+    console.error('Error migrating gmail history', e)
+  }
+}
+
+function getHistoryKey (workspace: WorkspaceUuid, userId: PersonId): string {
+  return `history:${workspace}:${userId}`
+}
@@ -89,13 +89,14 @@ import {
  type QueueWorkspaceMessage,
  type StorageAdapter
 } from '@hcengineering/server-core'
-import { getAccountDBUrl, getMongoDBUrl } from './__start'
+import { getAccountDBUrl, getKvsUrl, getMongoDBUrl } from './__start'
 // import { fillGithubUsers, fixAccountEmails, renameAccount } from './account'
 import { changeConfiguration } from './configuration'

 import { performGithubAccountMigrations } from './github'
 import { migrateCreatedModifiedBy, ensureGlobalPersonsForLocalAccounts, moveAccountDbFromMongoToPG } from './db'
 import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
+import { performGmailAccountMigrations } from './gmail'

 const colorConstants = {
  colorRed: '\u001b[31m',
@@ -2429,6 +2430,21 @@ export function devTool (
      await queue.createTopics(parseInt(cmd.tx ?? '1'))
    })

+  program
+    .command('migrate-gmail-account')
+    .option('--db <db>', 'DB name', 'gmail-service')
+    .option('--region <region>', 'DB region')
+    .action(async (cmd: { db: string, region?: string }) => {
+      const mongodbUri = getMongoDBUrl()
+      const client = getMongoClient(mongodbUri)
+      const _client = await client.getClient()
+
+      const kvsUrl = getKvsUrl()
+      await performGmailAccountMigrations(_client.db(cmd.db), cmd.region ?? null, kvsUrl)
+      await _client.close()
+      client.close()
+    })
+
  extendProgram?.(program)

  program.parse(process.argv)