UBERF-9137: Fix Support for suspended installations (#7667)

Signed-off-by: Andrey Sobolev <haiodo@gmail.com>
This commit is contained in:
Andrey Sobolev
2025-01-15 23:23:23 +07:00
committed by GitHub
parent 0c6feb646e
commit c8dde57123
24 changed files with 435 additions and 336 deletions
+2 -2
View File
@@ -28,8 +28,8 @@ import { fillConfiguration, pluginFilterTx } from '../utils'
import { connect } from './connection'
import { genMinModel } from './minmodel'
function filterPlugin (plugin: Plugin): (txes: Tx[]) => Promise<Tx[]> {
return async (txes) => {
function filterPlugin (plugin: Plugin): (txes: Tx[]) => Tx[] {
return (txes) => {
const configs = new Map<Ref<PluginConfiguration>, PluginConfiguration>()
fillConfiguration(txes, configs)
+21 -12
View File
@@ -215,7 +215,7 @@ export interface TxPersistenceStore {
store: (model: LoadModelResponse) => Promise<void>
}
export type ModelFilter = (tx: Tx[]) => Promise<Tx[]>
export type ModelFilter = (tx: Tx[]) => Tx[]
/**
* @public
@@ -256,17 +256,21 @@ export async function createClient (
}
const conn = await ctx.with('connect', {}, () => connect(txHandler))
const { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence))
let { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence))
switch (mode) {
case 'same':
case 'upgrade':
await ctx.with('build-model', {}, (ctx) => buildModel(ctx, current, modelFilter, hierarchy, model))
ctx.withSync('build-model', {}, (ctx) => {
buildModel(ctx, current, modelFilter, hierarchy, model)
})
break
case 'addition':
await ctx.with('build-model', {}, (ctx) =>
ctx.withSync('build-model', {}, (ctx) => {
buildModel(ctx, current.concat(addition), modelFilter, hierarchy, model)
)
})
}
current = []
addition = []
txBuffer = txBuffer.filter((tx) => tx.space !== core.space.Model)
@@ -287,7 +291,7 @@ export async function createClient (
return
}
// Find all new transactions and apply
const { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence))
let { mode, current, addition } = await ctx.with('load-model', {}, (ctx) => loadModel(ctx, conn, txPersistence))
switch (mode) {
case 'upgrade':
@@ -296,16 +300,21 @@ export async function createClient (
model = new ModelDb(hierarchy)
;(client as ClientImpl).setModel(hierarchy, model)
await ctx.with('build-model', {}, (ctx) => buildModel(ctx, current, modelFilter, hierarchy, model))
ctx.withSync('build-model', {}, (ctx) => {
buildModel(ctx, current, modelFilter, hierarchy, model)
})
current = []
await oldOnConnect?.(ClientConnectEvent.Upgraded, _lastTx, data)
// No need to fetch more stuff since upgrade was happened.
break
case 'addition':
await ctx.with('build-model', {}, (ctx) =>
ctx.withSync('build-model', {}, (ctx) => {
buildModel(ctx, current.concat(addition), modelFilter, hierarchy, model)
)
})
break
}
current = []
addition = []
if (lastTx === undefined) {
// No need to do anything here since we connected.
@@ -391,13 +400,13 @@ async function loadModel (
return { mode: 'addition', current: current.transactions, addition: result.transactions }
}
async function buildModel (
function buildModel (
ctx: MeasureContext,
transactions: Tx[],
modelFilter: ModelFilter | undefined,
hierarchy: Hierarchy,
model: ModelDb
): Promise<void> {
): void {
const systemTx: Tx[] = []
const userTx: Tx[] = []
@@ -416,7 +425,7 @@ async function buildModel (
let txes = systemTx.concat(userTx)
if (modelFilter !== undefined) {
txes = await modelFilter(txes)
txes = modelFilter(txes)
}
ctx.withSync('build hierarchy', {}, () => {
+72 -33
View File
@@ -87,6 +87,8 @@ class RequestPromise {
chunks?: { index: number, data: FindResult<any> }[]
}
const globalRPCHandler: RPCHandler = new RPCHandler()
class Connection implements ClientConnection {
private websocket: ClientSocket | null = null
binaryMode = false
@@ -115,7 +117,7 @@ class Connection implements ClientConnection {
onConnect?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise<void>
rpcHandler = new RPCHandler()
rpcHandler: RPCHandler
lastHash?: string
@@ -145,6 +147,7 @@ class Connection implements ClientConnection {
} else {
this.sessionId = generateId()
}
this.rpcHandler = opt?.useGlobalRPCHandler === true ? globalRPCHandler : new RPCHandler()
this.onConnect = opt?.onConnect
@@ -187,6 +190,8 @@ class Connection implements ClientConnection {
this.pingResponse = Date.now()
}
}
}).catch((err) => {
this.ctx.error('failed to send msg', { err })
})
} else {
clearInterval(this.interval)
@@ -336,7 +341,9 @@ class Connection implements ClientConnection {
helloResp.reconnect === true ? ClientConnectEvent.Reconnected : ClientConnectEvent.Connected,
helloResp.lastTx,
this.sessionId
)
)?.catch((err) => {
this.ctx.error('failed to call onConnect', { err })
})
this.schedulePing(socketId)
return
} else {
@@ -345,7 +352,9 @@ class Connection implements ClientConnection {
return
}
if (resp.result === pingConst) {
void this.sendRequest({ method: pingConst, params: [] })
void this.sendRequest({ method: pingConst, params: [] }).catch((err) => {
this.ctx.error('failed to send ping', { err })
})
return
}
if (resp.id !== undefined) {
@@ -416,14 +425,21 @@ class Connection implements ClientConnection {
promise.reject(new PlatformError(resp.error))
} else {
if (request?.handleResult !== undefined) {
void request.handleResult(resp.result).then(() => {
promise.resolve(resp.result)
})
void request
.handleResult(resp.result)
.then(() => {
promise.resolve(resp.result)
})
.catch((err) => {
this.ctx.error('failed to handleResult', { err })
})
} else {
promise.resolve(resp.result)
}
}
void broadcastEvent(client.event.NetworkRequests, this.requests.size)
void broadcastEvent(client.event.NetworkRequests, this.requests.size).catch((err) => {
this.ctx.error('failed to broadcast', { err })
})
} else {
const txArr = Array.isArray(resp.result) ? (resp.result as Tx[]) : [resp.result as Tx]
@@ -437,10 +453,14 @@ class Connection implements ClientConnection {
this.handler(...txArr)
clearTimeout(this.incomingTimer)
void broadcastEvent(client.event.NetworkRequests, this.requests.size + 1)
void broadcastEvent(client.event.NetworkRequests, this.requests.size + 1).catch((err) => {
this.ctx.error('failed to broadcast', { err })
})
this.incomingTimer = setTimeout(() => {
void broadcastEvent(client.event.NetworkRequests, this.requests.size)
void broadcastEvent(client.event.NetworkRequests, this.requests.size).catch((err) => {
this.ctx.error('failed to broadcast', { err })
})
}, 500)
}
}
@@ -476,7 +496,9 @@ class Connection implements ClientConnection {
this.dialTimer = setTimeout(() => {
this.dialTimer = null
if (!opened && !this.closed) {
void this.opt?.onDialTimeout?.()
void this.opt?.onDialTimeout?.()?.catch((err) => {
this.ctx.error('failed to handle dial timeout', { err })
})
this.scheduleOpen(this.ctx, true)
}
}, dialTimeout)
@@ -494,7 +516,9 @@ class Connection implements ClientConnection {
return
}
if (event.data === pingConst) {
void this.sendRequest({ method: pingConst, params: [] })
void this.sendRequest({ method: pingConst, params: [] }).catch((err) => {
this.ctx.error('failed to send ping', { err })
})
return
}
if (
@@ -503,7 +527,9 @@ class Connection implements ClientConnection {
) {
const text = new TextDecoder().decode(event.data)
if (text === pingConst) {
void this.sendRequest({ method: pingConst, params: [] })
void this.sendRequest({ method: pingConst, params: [] }).catch((err) => {
this.ctx.error('failed to send ping', { err })
})
}
if (text === pongConst) {
this.pingResponse = Date.now()
@@ -511,27 +537,32 @@ class Connection implements ClientConnection {
return
}
if (event.data instanceof Blob) {
void event.data.arrayBuffer().then((data) => {
if (this.compressionMode && this.helloReceived) {
void event.data
.arrayBuffer()
.then((data) => {
if (this.compressionMode && this.helloReceived) {
try {
data = uncompress(data)
} catch (err: any) {
// Ignore
console.error(err)
}
}
try {
data = uncompress(data)
const resp = this.rpcHandler.readResponse<any>(data, this.binaryMode)
this.handleMsg(socketId, resp)
} catch (err: any) {
// Ignore
console.error(err)
if (!this.helloReceived) {
// Just error and ignore for now.
console.error(err)
} else {
throw err
}
}
}
try {
const resp = this.rpcHandler.readResponse<any>(data, this.binaryMode)
this.handleMsg(socketId, resp)
} catch (err: any) {
if (!this.helloReceived) {
// Just error and ignore for now.
console.error(err)
} else {
throw err
}
}
})
})
.catch((err) => {
this.ctx.error('failed to decode array buffer', { err })
})
} else {
let data = event.data
if (this.compressionMode && this.helloReceived) {
@@ -561,7 +592,9 @@ class Connection implements ClientConnection {
return
}
// console.log('client websocket closed', socketId, ev?.reason)
void broadcastEvent(client.event.NetworkRequests, -1)
void broadcastEvent(client.event.NetworkRequests, -1).catch((err) => {
this.ctx.error('failed broadcast', { err })
})
this.scheduleOpen(this.ctx, true)
}
wsocket.onopen = () => {
@@ -591,7 +624,9 @@ class Connection implements ClientConnection {
if (opened) {
console.error('client websocket error:', socketId, this.url, this.workspace, this.email)
}
void broadcastEvent(client.event.NetworkRequests, -1)
void broadcastEvent(client.event.NetworkRequests, -1).catch((err) => {
this.ctx.error('failed to broadcast', { err })
})
}
}
@@ -669,7 +704,11 @@ class Connection implements ClientConnection {
ctx.withSync('send-data', {}, () => {
sendData()
})
void ctx.with('broadcast-event', {}, () => broadcastEvent(client.event.NetworkRequests, this.requests.size))
void ctx
.with('broadcast-event', {}, () => broadcastEvent(client.event.NetworkRequests, this.requests.size))
.catch((err) => {
this.ctx.error('failed to broadcast', { err })
})
if (data.method !== pingConst) {
return await promise.promise
}
+13 -2
View File
@@ -133,7 +133,7 @@ export default async () => {
return await Promise.resolve(clientConnection)
}
const modelFilter: ModelFilter = async (txes) => {
const modelFilter: ModelFilter = (txes) => {
if (filterModel === 'client') {
return returnClientTxes(txes)
}
@@ -177,7 +177,18 @@ function returnClientTxes (txes: Tx[]): Tx[] {
'templates:class:TemplateField' as Ref<Class<Doc>>,
'activity:class:DocUpdateMessageViewlet' as Ref<Class<Doc>>,
'core:class:PluginConfiguration' as Ref<Class<Doc>>,
'core:class:DomainIndexConfiguration' as Ref<Class<Doc>>
'core:class:DomainIndexConfiguration' as Ref<Class<Doc>>,
'view:class:ViewletDescriptor' as Ref<Class<Doc>>,
'presentation:class:ComponentPointExtension' as Ref<Class<Doc>>,
'activity:class:ActivityMessagesFilter' as Ref<Class<Doc>>,
'view:class:ActionCategory' as Ref<Class<Doc>>,
'activity:class:ActivityExtension' as Ref<Class<Doc>>,
'chunter:class:ChatMessageViewlet' as Ref<Class<Doc>>,
'activity:class:ActivityMessageControl' as Ref<Class<Doc>>,
'notification:class:ActivityNotificationViewlet' as Ref<Class<Doc>>,
'setting:class:SettingsCategory' as Ref<Class<Doc>>,
'setting:class:WorkspaceSettingCategory' as Ref<Class<Doc>>,
'notification:class:NotificationProvider' as Ref<Class<Doc>>
])
const result = pluginFilterTx(excludedPlugins, configs, txes).filter((tx) => {
+2
View File
@@ -65,6 +65,8 @@ export interface ClientFactoryOptions {
onConnect?: (event: ClientConnectEvent, lastTx: string | undefined, data: any) => Promise<void>
ctx?: MeasureContext
onDialTimeout?: () => void | Promise<void>
useGlobalRPCHandler?: boolean
}
/**
@@ -50,7 +50,7 @@
object._class,
{ _id: object._id },
(res) => {
relationsA = res[0].$associations ?? {}
relationsA = res?.[0]?.$associations ?? {}
},
{ associations: associationsA.map((a) => [a._id, 1]) }
)
@@ -60,7 +60,7 @@
object._class,
{ _id: object._id },
(res) => {
relationsB = res[0].$associations ?? {}
relationsB = res?.[0]?.$associations ?? {}
},
{ associations: associationsB.map((a) => [a._id, -1]) }
)
+2 -1
View File
@@ -88,6 +88,7 @@
"AuthenticationRevokedGithub": "Re-authorization for {login} is required for the proper functioning of the GitHub App",
"UnlinkInstallationTitle": "Uninstall Github App",
"UnlinkInstallation": "Are you sure you want to uninstall the GitHub App? Synchronization will be disabled.",
"RemoveInstallation": "Uninstall"
"RemoveInstallation": "Uninstall",
"Suspended": "Suspended"
}
}
+2 -1
View File
@@ -88,6 +88,7 @@
"AuthenticationRevokedGithub": "É necessária uma nova autorização para {login} para o funcionamento adequado do aplicativo GitHub",
"UnlinkInstallationTitle": "Desinstalar aplicativo Github",
"UnlinkInstallation": "Tem certeza de que deseja desinstalar o aplicativo GitHub? A sincronização será desabilitada.",
"RemoveInstallation": "Desinstalar"
"RemoveInstallation": "Desinstalar",
"Suspended": "Suspended"
}
}
+2 -1
View File
@@ -88,6 +88,7 @@
"AuthenticationRevokedGithub": "Требуется повторная авторизация для {login} для правильной работы приложения GitHub",
"UnlinkInstallationTitle": "Удалить приложение Github",
"UnlinkInstallation": "Вы уверены, что хотите удалить приложение GitHub? Синхронизация будет отключена.",
"RemoveInstallation": "Удалить"
"RemoveInstallation": "Удалить",
"Suspended": "Suspended"
}
}
+2 -1
View File
@@ -88,6 +88,7 @@
"AuthenticationRevokedGithub": "Se requiere una nueva autorización para {login} para el correcto funcionamiento de la aplicación de GitHub",
"UnlinkInstallationTitle": "Desinstalar aplicación de Github",
"UnlinkInstallation": "¿Estás seguro de que deseas desinstalar la aplicación de GitHub? La sincronización se deshabilitará.",
"RemoveInstallation": "Desinstalar"
"RemoveInstallation": "Desinstalar",
"Suspended": "Suspended"
}
}
@@ -26,6 +26,7 @@
import ConnectProject from './ConnectProject.svelte'
import { githubLanguageColors } from './languageColors'
import { sendGHServiceRequest } from './utils'
import { BackgroundColor } from '@hcengineering/text'
export let integration: WithLookup<GithubIntegration>
export let projects: Project[] = []
@@ -141,145 +142,151 @@
}
</script>
<Expandable expanded={true}>
<svelte:fragment slot="title">
<span class="fs-title flex-row-center flex-between flex-grow flex">
<div class="ml-2 mr-2">
<img class="svg-large" src={integration.name.replace('github.com', 'avatars.githubusercontent.com')} />
</div>
{#if integration.name.length > 0}
{integration.name}
{#if (integration.type ?? '') !== ''}
({integration.type})
{/if}
{:else}
<Label label={github.string.ConnectionPending} />
{/if}
{#if !integration.alive}
<Label label={github.string.Closed} />
{/if}
</span>
</svelte:fragment>
<svelte:fragment slot="tools">
<Button
kind={'dangerous'}
label={github.string.RemoveInstallation}
on:click={() => {
showPopup(MessageBox, {
label: github.string.UnlinkInstallationTitle,
message: github.string.UnlinkInstallation,
params: {},
richMessage: true,
action: async () => {
await sendGHServiceRequest('installation-remove', {
installationId: integration.installationId,
token: getMetadata(presentation.metadata.Token) ?? ''
})
}
})
}}
/>
</svelte:fragment>
<div class="flex flex-row-center flex-between m-0-5">
<SearchEdit bind:value={search} width={'100%'} />
</div>
{#each repos.slice(0, limit) as repository}
{@const prj = giProjects.find((it) => it.repositories.includes(repository._id))}
<div
class="repository-card flex-col m-0-5"
class:selected={prj !== undefined}
class:disabled={prj !== undefined && !repository.enabled}
>
<div class="flex flex-row-center flex-between">
<div class="flex-row-center">
<NavLink href={repository.htmlURL}>{repository.name}</NavLink>
<div class="ml-2 visibility">
{repository.visibility}
</div>
{#if !repository.enabled && prj !== undefined}
<div class="ml-2 visibility">
<Label label={github.string.Disabled} />
</div>
<div
style:background-color={!integration.alive ? 'var(--primary-button-disabled)' : undefined}
style:border-radius="0.5rem"
class="p-1"
>
<Expandable expanded={true}>
<svelte:fragment slot="title">
<span class="fs-title flex-row-center flex-between flex-grow flex">
<div class="ml-2 mr-2">
<img class="svg-large" src={integration.name.replace('github.com', 'avatars.githubusercontent.com')} />
</div>
{#if integration.name.length > 0}
{integration.name}
{#if (integration.type ?? '') !== ''}
({integration.type})
{/if}
</div>
<div class="flex flex-row-center">
{#if prj !== undefined}
<div class="mr-2">
<Label label={github.string.LinkedWith} />
</div>
<ObjectPresenter _class={prj._class} objectId={prj._id} value={prj} />
<div class="ml-2">
<Button
kind={'dangerous'}
label={github.string.UnlinkFromProject}
size={'medium'}
on:click={(evt) => {
void onDisconnect(evt, prj, repository)
}}
/>
</div>
<div class="ml-2">
<Button
icon={IconMoreV}
size={'small'}
on:click={(evt) => {
void showMenu(evt, prj, repository)
}}
/>
</div>
{:else}
<ConnectProject {integration} {repository} {projects} />
{/if}
</div>
</div>
<div class="mt-2">
{repository.description ?? ''}
</div>
<div class="flex flex-row-center mt-4">
{#if repository.language != null}
<div class="flex-row-center mr-4">
<div class="lcolor-pin mr-1" style:background-color={githubLanguageColors[repository.language] ?? ''} />
{repository.language}
</div>
{:else}
<Label label={github.string.ConnectionPending} />
{/if}
<div class="flex-row-center">
<Icon icon={IconColStar} fill={'none'} size={'small'} />
<span class="ml-1">{repository.stargazers ?? 0}</span>
</div>
<div class="flex-row-center ml-4">
<Icon icon={github.icon.Forks} size={'small'} />
<span class="ml-1">{repository.forks ?? 0}</span>
</div>
<div class="flex-row-center ml-4">
<Icon icon={tracker.icon.Issue} size={'small'} />
<span class="ml-1">{repository.openIssues ?? 0}</span>
</div>
{#if repository.updatedAt !== undefined}
<div class="flex-row-center ml-4">
<Label label={github.string.Updated} />
<span class="ml-2">
<TimeSince value={repository.updatedAt} />
</span>
</div>
{#if !integration.alive}
<Label label={github.string.Suspended} />
{/if}
</div>
</span>
</svelte:fragment>
<svelte:fragment slot="tools">
<Button
kind={'dangerous'}
label={github.string.RemoveInstallation}
on:click={() => {
showPopup(MessageBox, {
label: github.string.UnlinkInstallationTitle,
message: github.string.UnlinkInstallation,
params: {},
richMessage: true,
action: async () => {
await sendGHServiceRequest('installation-remove', {
installationId: integration.installationId,
token: getMetadata(presentation.metadata.Token) ?? ''
})
}
})
}}
/>
</svelte:fragment>
<div class="flex flex-row-center flex-between m-0-5">
<SearchEdit bind:value={search} width={'100%'} />
</div>
{/each}
{#if repos.length > limit}
<Button
label={ui.string.ShowMore}
on:click={() => {
limit = limit + 10
}}
/>
{/if}
</Expandable>
{#each repos.slice(0, limit) as repository}
{@const prj = giProjects.find((it) => it.repositories.includes(repository._id))}
<div
class="repository-card flex-col m-0-5"
class:selected={prj !== undefined}
class:disabled={prj !== undefined && !repository.enabled}
>
<div class="flex flex-row-center flex-between">
<div class="flex-row-center">
<NavLink href={repository.htmlURL}>{repository.name}</NavLink>
<div class="ml-2 visibility">
{repository.visibility}
</div>
{#if !repository.enabled && prj !== undefined}
<div class="ml-2 visibility">
<Label label={github.string.Disabled} />
</div>
{/if}
</div>
<div class="flex flex-row-center">
{#if prj !== undefined}
<div class="mr-2">
<Label label={github.string.LinkedWith} />
</div>
<ObjectPresenter _class={prj._class} objectId={prj._id} value={prj} />
<div class="ml-2">
<Button
kind={'dangerous'}
label={github.string.UnlinkFromProject}
size={'medium'}
on:click={(evt) => {
void onDisconnect(evt, prj, repository)
}}
/>
</div>
<div class="ml-2">
<Button
icon={IconMoreV}
size={'small'}
on:click={(evt) => {
void showMenu(evt, prj, repository)
}}
/>
</div>
{:else}
<ConnectProject {integration} {repository} {projects} />
{/if}
</div>
</div>
<div class="mt-2">
{repository.description ?? ''}
</div>
<div class="flex flex-row-center mt-4">
{#if repository.language != null}
<div class="flex-row-center mr-4">
<div class="lcolor-pin mr-1" style:background-color={githubLanguageColors[repository.language] ?? ''} />
{repository.language}
</div>
{/if}
<div class="flex-row-center">
<Icon icon={IconColStar} fill={'none'} size={'small'} />
<span class="ml-1">{repository.stargazers ?? 0}</span>
</div>
<div class="flex-row-center ml-4">
<Icon icon={github.icon.Forks} size={'small'} />
<span class="ml-1">{repository.forks ?? 0}</span>
</div>
<div class="flex-row-center ml-4">
<Icon icon={tracker.icon.Issue} size={'small'} />
<span class="ml-1">{repository.openIssues ?? 0}</span>
</div>
{#if repository.updatedAt !== undefined}
<div class="flex-row-center ml-4">
<Label label={github.string.Updated} />
<span class="ml-2">
<TimeSince value={repository.updatedAt} />
</span>
</div>
{/if}
</div>
</div>
{/each}
{#if repos.length > limit}
<Button
label={ui.string.ShowMore}
on:click={() => {
limit = limit + 10
}}
/>
{/if}
</Expandable>
</div>
<style lang="scss">
.bordered {
+2 -1
View File
@@ -600,6 +600,7 @@ export default plugin(githubId, {
AuthenticatedWithGithub: '' as IntlString,
AuthenticationRevokedGithub: '' as IntlString,
AuthenticatedWithGithubEmployee: '' as IntlString,
AuthenticatedWithGithubRequired: '' as IntlString
AuthenticatedWithGithubRequired: '' as IntlString,
Suspended: '' as IntlString
}
})
+1 -1
View File
@@ -10,4 +10,4 @@ export MINIO_SECRET_KEY=minioadmin
export MINIO_ENDPOINT=localhost
export MONGO_URL=mongodb://localhost:27017
rush bundle --to @hcengineering/pod-github
node $@ bundle/bundle.js
node $@ bundle/bundle.js $@
+2 -1
View File
@@ -44,7 +44,8 @@ export async function createPlatformClient (
const connection = await (
await clientResources()
).function.GetClient(token, endpoint, {
onConnect: reconnect
onConnect: reconnect,
useGlobalRPCHandler: true
})
return { client: connection, endpoint }
+14 -6
View File
@@ -31,15 +31,23 @@ Analytics.setTag('application', 'github-service')
let doOnClose: () => Promise<void> = async () => {}
void start(metricsContext, loadBrandingMap(config.BrandingPath)).then((r) => {
doOnClose = r
})
void start(metricsContext, loadBrandingMap(config.BrandingPath))
.then((r) => {
doOnClose = r
})
.catch((err) => {
metricsContext.error('Error', { error: err })
})
const onClose = (): void => {
metricsContext.info('Closed')
void doOnClose().then((r) => {
process.exit(0)
})
void doOnClose()
.then((r) => {
process.exit(0)
})
.catch((err) => {
metricsContext.error('Error', { error: err })
})
}
process.on('uncaughtException', (e) => {
+35 -19
View File
@@ -52,6 +52,7 @@ export interface InstallationRecord {
repositories?: InstallationCreatedEvent['repositories'] | InstallationUnsuspendEvent['repositories']
type: 'Bot' | 'User' | 'Organization'
octokit: Octokit
suspended: boolean
}
export class PlatformWorker {
@@ -593,7 +594,8 @@ export class PlatformWorker {
login: tinst.account.login,
loginNodeId: tinst.account.node_id,
type: tinst.account?.type ?? 'User',
installationName: `${tinst.account?.html_url ?? ''}`
installationName: `${tinst.account?.html_url ?? ''}`,
suspended: install.data.suspended_at != null
}
this.updateInstallationRecord(installationId, val)
}
@@ -609,6 +611,7 @@ export class PlatformWorker {
current.loginNodeId = val.loginNodeId
current.type = val.type
current.installationName = val.installationName
current.suspended = val.suspended
if (val.repositories !== undefined) {
current.repositories = val.repositories
}
@@ -625,7 +628,8 @@ export class PlatformWorker {
login: tinst.account.login,
loginNodeId: tinst.account.node_id,
type: tinst.account?.type ?? 'User',
installationName: `${tinst.account?.html_url ?? ''}`
installationName: `${tinst.account?.html_url ?? ''}`,
suspended: install.installation.suspended_at != null
}
this.updateInstallationRecord(install.installation.id, val)
ctx.info('Found installation', {
@@ -650,11 +654,17 @@ export class PlatformWorker {
type: install.account?.type ?? 'User',
loginNodeId: install.account.node_id,
installationName: iName,
repositories
repositories,
suspended: !enabled
})
const worker = this.getWorker(install.id)
if (worker !== undefined) {
const integeration = worker.integrations.get(install.id)
if (integeration !== undefined) {
integeration.enabled = enabled
}
await worker.syncUserData(this.ctx, await this.getUsers(worker.workspace.name))
await worker.reloadRepositories(install.id)
@@ -662,13 +672,6 @@ export class PlatformWorker {
worker.triggerSync()
}
// Need to inform workspace
const integeration = this.integrations.find((it) => it.installationId === install.id)
if (integeration !== undefined) {
const worker = this.clients.get(integeration.workspace) as GithubWorker
worker?.triggerUpdate()
}
// Check if no workspace was available
this.triggerCheckWorkspaces()
}
@@ -810,17 +813,28 @@ export class PlatformWorker {
this.storageAdapter,
(workspace, event) => {
if (event === ClientConnectEvent.Refresh || event === ClientConnectEvent.Upgraded) {
void this.clients.get(workspace)?.refreshClient(event === ClientConnectEvent.Upgraded)
void this.clients
.get(workspace)
?.refreshClient(event === ClientConnectEvent.Upgraded)
?.catch((err) => {
workerCtx.error('Failed to refresh', { error: err })
})
}
if (initialized) {
// We need to check if workspace is inactive
void this.checkWorkspaceIsActive(token, workspace).then((res) => {
if (res === undefined) {
this.ctx.warn('Workspace is inactive, removing from clients list.', { workspace })
this.clients.delete(workspace)
void worker?.close()
}
})
void this.checkWorkspaceIsActive(token, workspace)
.then((res) => {
if (res === undefined) {
this.ctx.warn('Workspace is inactive, removing from clients list.', { workspace })
this.clients.delete(workspace)
void worker?.close().catch((err) => {
this.ctx.error('Failed to close workspace', { workspace, error: err })
})
}
})
.catch((err) => {
this.ctx.error('Failed to check workspace is active', { workspace, error: err })
})
}
}
)
@@ -879,7 +893,9 @@ export class PlatformWorker {
try {
this.ctx.info('workspace removed from tracking list', { workspace: deleted })
this.clients.delete(deleted)
void ws.close()
void ws.close().catch((err) => {
this.ctx.error('Error', { error: err })
})
} catch (err: any) {
Analytics.handleError(err)
errors++
@@ -207,9 +207,9 @@ export abstract class IssueSyncManagerBase {
})
if (syncData !== undefined) {
const milestone = (
await this.provider.liveQuery.queryFind<GithubMilestone>(github.mixin.GithubMilestone, {})
).find((it) => it.projectNodeId === projectId)
const milestone = await this.client.findOne<GithubMilestone>(github.mixin.GithubMilestone, {
projectNodeId: projectId
})
const target: IssueSyncTarget | undefined =
milestone !== undefined
@@ -264,11 +264,6 @@ export abstract class IssueSyncManagerBase {
let structure = integration.projectStructure.get(target.target._id)
const repositories = await this.provider.liveQuery.queryFind<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{}
)
for (const f of target.prjData.fieldValues?.nodes ?? []) {
if (!('id' in f)) {
continue
@@ -281,8 +276,13 @@ export abstract class IssueSyncManagerBase {
needProjectRefresh = true
}
}
if (needProjectRefresh) {
const repo = repositories.find((it) => it._id === syncData.repository)
if (needProjectRefresh && syncData.repository != null) {
const repo = await this.provider.liveQuery.findOne<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{
_id: syncData.repository
}
)
if (repo !== undefined) {
await this.provider.handleEvent(github.class.GithubIntegration, integration.installationId, repo, {})
@@ -1153,9 +1153,9 @@ export abstract class IssueSyncManagerBase {
if (existingIssue !== undefined) {
// Select a milestone project
if (existingIssue.milestone != null) {
const milestone = (
await this.provider.liveQuery.queryFind<GithubMilestone>(github.mixin.GithubMilestone, {})
).find((it) => it._id === existingIssue.milestone)
const milestone = await this.provider.liveQuery.findOne<GithubMilestone>(github.mixin.GithubMilestone, {
_id: existingIssue.milestone as Ref<GithubMilestone>
})
if (milestone === undefined) {
return
}
+8 -14
View File
@@ -95,13 +95,15 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
if (projectV2Event) {
const projectV2Event = event as ProjectsV2ItemEvent
const githubProjects = await this.provider.liveQuery.queryFind(github.mixin.GithubProject, {})
const githubProjects = await this.provider.liveQuery.findAll(github.mixin.GithubProject, {
archived: false
})
let prj = githubProjects.find((it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id)
if (prj === undefined) {
// Checking for milestones
const m = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).find(
(it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id
)
const m = await this.provider.liveQuery.findOne(github.mixin.GithubMilestone, {
projectNodeId: projectV2Event.projects_v2_item.project_node_id
})
if (m !== undefined) {
prj = githubProjects.find((it) => it._id === m.space)
}
@@ -353,13 +355,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
}
if (info.repository == null) {
// No need to sync if component it not yet set
const repos = (await this.provider.getProjectRepositories(container.project._id))
.map((it) => it.name)
.join(', ')
this.ctx.error('Not syncing repository === null', {
url: info.url,
identifier: (existing as Issue).identifier,
repos
identifier: (existing as Issue).identifier
})
return { needSync: githubSyncVersion }
}
@@ -372,13 +370,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
if (info.external === undefined && existing !== undefined) {
const repository = await this.provider.getRepositoryById(info.repository)
if (repository === undefined) {
const repos = (await this.provider.getProjectRepositories(container.project._id))
.map((it) => it.name)
.join(', ')
this.ctx.error('Not syncing repository === undefined', {
url: info.url,
identifier: (existing as Issue).identifier,
repos
identifier: (existing as Issue).identifier
})
return { needSync: githubSyncVersion }
}
@@ -446,9 +446,9 @@ export class ProjectsSyncManager implements DocSyncManager {
if (syncConfig.SupportMilestones && integration.type === 'Organization') {
// Check project milestones and sync their structure as well.
const milestones = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).filter(
(it) => it.space === prj._id
)
const milestones = await this.provider.liveQuery.findAll(github.mixin.GithubMilestone, {
space: prj._id
})
for (const m of milestones) {
if (this.provider.isClosing()) {
break
@@ -100,13 +100,15 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
if (projectV2Event) {
const projectV2Event = _event as ProjectsV2ItemEvent
const githubProjects = await this.provider.liveQuery.queryFind(github.mixin.GithubProject, {})
const githubProjects = await this.provider.liveQuery.findAll(github.mixin.GithubProject, {
archived: false
})
let prj = githubProjects.find((it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id)
if (prj === undefined) {
// Checking for milestones
const m = (await this.provider.liveQuery.queryFind(github.mixin.GithubMilestone, {})).find(
(it) => it.projectNodeId === projectV2Event.projects_v2_item.project_node_id
)
const m = await this.provider.liveQuery.findOne(github.mixin.GithubMilestone, {
projectNodeId: projectV2Event.projects_v2_item.project_node_id
})
if (m !== undefined) {
prj = githubProjects.find((it) => it._id === m.space)
}
@@ -48,9 +48,9 @@ export class RepositorySyncMapper implements DocSyncManager {
if (repositories !== undefined) {
// We have a list of repositories, so we could create them if they are missing.
// Need to find all repositories, not only active, so passed repositories are not work.
const allRepositories = (
await this.provider.liveQuery.queryFind(github.class.GithubIntegrationRepository, {})
).filter((it) => it.attachedTo === integration.integration._id)
const allRepositories = await this.provider.liveQuery.findAll(github.class.GithubIntegrationRepository, {
attachedTo: integration.integration._id
})
const allRepos: GithubIntegrationRepository[] = [...allRepositories]
for (const repository of repositories) {
@@ -259,9 +259,9 @@ export class RepositorySyncMapper implements DocSyncManager {
const iterable = this.app.eachRepository.iterator({ installationId: integration.installationId })
// Need to find all repositories, not only active, so passed repositories are not work.
const allRepositories = (
await this.provider.liveQuery.queryFind(github.class.GithubIntegrationRepository, {})
).filter((it) => it.attachedTo === integration.integration._id)
const allRepositories = await this.provider.liveQuery.findAll(github.class.GithubIntegrationRepository, {
attachedTo: integration.integration._id
})
let allRepos: GithubIntegrationRepository[] = [...allRepositories]
-2
View File
@@ -126,8 +126,6 @@ export interface IntegrationManager {
isPlatformUser: (account: Ref<PersonAccount>) => Promise<boolean>
getProjectRepositories: (space: Ref<Space>) => Promise<GithubIntegrationRepository[]>
getRepositoryById: (ref?: Ref<GithubIntegrationRepository> | null) => Promise<GithubIntegrationRepository | undefined>
isClosing: () => boolean
+77 -69
View File
@@ -44,7 +44,6 @@ import github, {
GithubIntegration,
GithubIntegrationRepository,
GithubIssue,
GithubMilestone,
GithubProject,
GithubUserInfo,
githubId
@@ -198,11 +197,9 @@ export class GithubWorker implements IntegrationManager {
}
async getContainer (space: Ref<Space>): Promise<ContainerFocus | undefined> {
const project = (
await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, {
_id: space as Ref<GithubProject>
})
).shift()
const project = await this.liveQuery.findOne<GithubProject>(github.mixin.GithubProject, {
_id: space as Ref<GithubProject>
})
if (project !== undefined) {
for (const v of this.integrations.values()) {
if (v.octokit === undefined) {
@@ -219,21 +216,13 @@ export class GithubWorker implements IntegrationManager {
}
}
async getProjectRepositories (space: Ref<Space>): Promise<GithubIntegrationRepository[]> {
const repositories = await this.liveQuery.queryFind<GithubIntegrationRepository>(
github.class.GithubIntegrationRepository,
{}
)
return repositories.filter((it) => it.githubProject === space)
}
async getRepositoryById (
_id?: Ref<GithubIntegrationRepository> | null
): Promise<GithubIntegrationRepository | undefined> {
if (_id != null) {
return (
await this.liveQuery.queryFind<GithubIntegrationRepository>(github.class.GithubIntegrationRepository, { _id })
).shift()
return await this.liveQuery.findOne<GithubIntegrationRepository>(github.class.GithubIntegrationRepository, {
_id
})
}
}
@@ -289,7 +278,9 @@ export class GithubWorker implements IntegrationManager {
})
}
const account = await this.liveQuery.findOne(contact.class.PersonAccount, { email: `github:${userInfo.login}` })
const account = await this.client
.getModel()
.findOne(contact.class.PersonAccount, { email: `github:${userInfo.login}` })
if (account !== undefined) {
const person = await this.liveQuery.findOne(contact.class.Person, { _id: account.person })
// We need to be sure employee are exists.
@@ -330,7 +321,7 @@ export class GithubWorker implements IntegrationManager {
person,
role: AccountRole.User
})
const acc = await this.liveQuery.findOne(contact.class.PersonAccount, { _id: id })
const acc = await this.client.getModel().findOne(contact.class.PersonAccount, { _id: id })
return acc
}
}
@@ -420,7 +411,7 @@ export class GithubWorker implements IntegrationManager {
let person: Ref<Person> | undefined
// try to find by account.
if (userInfo.email != null && userInfo.email.trim().length > 0) {
const personAccount = await this.liveQuery.findOne(contact.class.PersonAccount, { email: userInfo.email })
const personAccount = await this.client.getModel().findOne(contact.class.PersonAccount, { email: userInfo.email })
person = personAccount?.person
}
@@ -472,7 +463,7 @@ export class GithubWorker implements IntegrationManager {
}
async getGithubLogin (container: IntegrationContainer, person: Ref<Person>): Promise<UserInfo | undefined> {
const accounts = await this.liveQuery.queryFind(contact.class.PersonAccount, {})
const accounts = this.client.getModel().findAllSync(contact.class.PersonAccount, {})
const acc = accounts.find((it) => it.person === person && it.email.startsWith('github:'))
if (acc === undefined) {
return // Nobody, will use system account.
@@ -527,7 +518,11 @@ export class GithubWorker implements IntegrationManager {
const ops = new TxOperations(this.client, accountRef)
await syncUser(ctx, record, userAuth, ops, accountRef)
} catch (err: any) {
await this.platform.revokeUserAuth(record)
try {
await this.platform.revokeUserAuth(record)
} catch (err: any) {
ctx.error(`Failed to revoke user ${record._id}`, err)
}
if (err.response?.data?.message !== 'Bad credentials') {
ctx.error(`Failed to sync user ${record._id}`, err)
Analytics.handleError(err)
@@ -551,7 +546,7 @@ export class GithubWorker implements IntegrationManager {
let record = await this.platform.getAccountByRef(this.workspace.name, account)
// const accountRef = this.accounts.find((it) => it._id === account)
const [accountRef] = await this.liveQuery.queryFind(contact.class.PersonAccount, { _id: account })
const [accountRef] = this.client.getModel().findAllSync(contact.class.PersonAccount, { _id: account })
if (record === undefined) {
if (accountRef !== undefined) {
const accounts = this._client.getModel().getAccountByPersonId(accountRef.person)
@@ -651,7 +646,7 @@ export class GithubWorker implements IntegrationManager {
async getProjectStatuses (type: Ref<ProjectType> | undefined): Promise<Status[]> {
if (type === undefined) return []
const statuses = await this.liveQuery.queryFind(core.class.Status, {})
const statuses = this.client.getModel().findAllSync(core.class.Status, {})
const projectType = await this.getProjectType(type)
@@ -663,7 +658,7 @@ export class GithubWorker implements IntegrationManager {
if (type === undefined) return []
const taskType = await this.getTaskType(type)
const statuses = await this.liveQuery.queryFind(core.class.Status, {})
const statuses = this.client.getModel().findAllSync(core.class.Status, {})
const allowedTypes = new Set(taskType?.statuses ?? [])
return statuses.filter((it) => allowedTypes.has(it._id))
@@ -743,37 +738,27 @@ export class GithubWorker implements IntegrationManager {
}
projects: GithubProject[] = []
milestones: GithubMilestone[] = []
async queryProjects (): Promise<void> {
await new Promise<void>((resolve) => {
this.liveQuery.query(github.mixin.GithubProject, {}, (res) => {
let needRefresh = false
if (!equalExceptKeys(this.projects, res, ['sequence', 'modifiedOn', 'modifiedBy'])) {
needRefresh = true
this.liveQuery.query(
github.mixin.GithubProject,
{
archived: false
},
(res) => {
let needRefresh = false
if (!equalExceptKeys(this.projects, res, ['sequence', 'modifiedOn', 'modifiedBy'])) {
needRefresh = true
}
this.projects = res
resolve()
if (needRefresh || this.projects.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
}
this.projects = res
resolve()
if (needRefresh || this.projects.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
})
})
await new Promise<void>((resolve) => {
this.liveQuery.query(github.mixin.GithubMilestone, {}, (res) => {
let needRefresh = false
if (!equalExceptKeys(this.milestones, res, ['modifiedOn', 'modifiedBy'])) {
needRefresh = true
}
this.milestones = res
resolve()
if (needRefresh || this.milestones.length !== res.length) {
// Do not trigger update if only sequence is changed.
this.triggerUpdate()
}
})
)
})
}
@@ -796,7 +781,7 @@ export class GithubWorker implements IntegrationManager {
loginNodeId: inst.loginNodeId ?? '',
type: inst.type ?? 'User',
installationName: inst?.installationName ?? '',
enabled: true,
enabled: !inst.suspended,
synchronized: new Set(),
projectStructure: new Map(),
syncLock: new Map()
@@ -866,7 +851,7 @@ export class GithubWorker implements IntegrationManager {
private async queryAccounts (): Promise<void> {
const updateAccounts = async (accounts: PersonAccount[]): Promise<void> => {
const persons = await this.liveQuery.queryFind(contact.class.Person, {
const persons = await this.liveQuery.findAll(contact.class.Person, {
_id: { $in: accounts.map((it) => it.person) }
})
const h = this.client.getHierarchy()
@@ -1112,7 +1097,9 @@ export class GithubWorker implements IntegrationManager {
if (this.updateRequests > 0) {
this.updateRequests = 0 // Just in case
await this.updateIntegrations()
void this.performFullSync()
void this.performFullSync().catch((err) => {
this.ctx.error('Failed to perform full sync', { error: err })
})
}
const { projects, repositories } = await this.collectActiveProjects()
@@ -1150,7 +1137,10 @@ export class GithubWorker implements IntegrationManager {
}
}
private async performSync (projects: GithubProject[], repositories: GithubIntegrationRepository[]): Promise<boolean> {
private async performSync (
projects: GithubProject[],
repositories: Pick<GithubIntegrationRepository, '_id'>[]
): Promise<boolean> {
const _projects = toIdMap(projects)
const _repositories = repositories.map((it) => it._id)
@@ -1207,12 +1197,21 @@ export class GithubWorker implements IntegrationManager {
const projects: GithubProject[] = []
const repositories: GithubIntegrationRepository[] = []
const allProjects = await this.liveQuery.queryFind<GithubProject>(github.mixin.GithubProject, { archived: false })
const allRepositories = await this.liveQuery.queryFind(github.class.GithubIntegrationRepository, { enabled: true })
const allProjects = await this.liveQuery.findAll<GithubProject>(github.mixin.GithubProject, {
archived: false
})
const allRepositories = (await this.liveQuery.findAll(github.class.GithubIntegrationRepository, {})).filter(
(it) => it.enabled
)
for (const it of Array.from(this.integrations.values())) {
if (it.enabled) {
const _projects = allProjects.filter((p) => !syncConfig.MainProject || it.projectStructure.has(p._id))
const _projects = []
for (const p of allProjects) {
if (p.integration === it.integration._id && (!syncConfig.MainProject || it.projectStructure.has(p._id))) {
_projects.push(p)
}
}
const prjIds = new Set(_projects.map((it) => it._id))
@@ -1237,13 +1236,13 @@ export class GithubWorker implements IntegrationManager {
const integration = await this._client.findOne(github.class.GithubIntegration, {
installationId: intgr.installationId
})
if (integration === undefined && this.installations.has(intgr.installationId)) {
const installation = this.installations.get(intgr.installationId) as InstallationRecord
const installation = this.installations.get(intgr.installationId) as InstallationRecord
if (integration === undefined && installation !== undefined) {
await this._client.createDoc(
github.class.GithubIntegration,
core.space.Configuration,
{
alive: true,
alive: !installation.suspended,
installationId: intgr.installationId,
clientId: config.ClientID,
name: installation.installationName,
@@ -1257,7 +1256,7 @@ export class GithubWorker implements IntegrationManager {
this.triggerUpdate()
} else if (integration !== undefined) {
await this._client.diffUpdate(integration, {
alive: true
alive: !installation.suspended
})
}
}
@@ -1455,9 +1454,7 @@ export class GithubWorker implements IntegrationManager {
'external sync',
{ installation: integration.installationName, workspace: this.workspace.name },
async () => {
if (!integration.enabled || integration.octokit === undefined) {
return
}
const enabled = integration.enabled && integration.octokit !== undefined
const upd: DocumentUpdate<GithubIntegration> = {}
if (integration.integration.byUser !== integration.login) {
@@ -1470,14 +1467,19 @@ export class GithubWorker implements IntegrationManager {
upd.clientId = config.ClientID
}
if (integration.integration.name !== integration.installationName || !integration.integration.alive) {
if (integration.integration.name !== integration.installationName) {
upd.name = integration.installationName
upd.alive = true
}
if (integration.integration.alive !== enabled) {
upd.alive = enabled
}
if (Object.keys(upd).length > 0) {
await this._client.diffUpdate(integration.integration, upd, Date.now(), integration.integration.createdBy)
this.triggerUpdate()
}
if (!enabled) {
return
}
const derivedClient = new TxOperations(this.client, core.account.System, true)
const { projects, repositories } = await this.collectActiveProjects()
@@ -1591,7 +1593,13 @@ export class GithubWorker implements IntegrationManager {
branding
)
ctx.info('Init worker', { workspace: workspace.workspaceUrl, workspaceId: workspace.workspaceName })
void worker.init()
void worker.init().catch((err) => {
ctx.error('Failed to init worker', {
workspace: workspace.workspaceUrl,
workspaceId: workspace.workspaceName,
error: err
})
})
return worker
} catch (err: any) {
ctx.error('timeout during to connect', { workspace, error: err })
@@ -1,11 +1,11 @@
import { expect, test } from '@playwright/test'
import { generateId, PlatformSetting, PlatformURI } from '../utils'
import { NavigationMenuPage } from '../model/recruiting/navigation-menu-page'
import { ApplicationsPage } from '../model/recruiting/applications-page'
import { test } from '@playwright/test'
import { ApplicationsDetailsPage } from '../model/recruiting/applications-details-page'
import { VacancyDetailsPage } from '../model/recruiting/vacancy-details-page'
import { VacanciesPage } from '../model/recruiting/vacancies-page'
import { ApplicationsPage } from '../model/recruiting/applications-page'
import { NavigationMenuPage } from '../model/recruiting/navigation-menu-page'
import { TalentsPage } from '../model/recruiting/talents-page'
import { VacanciesPage } from '../model/recruiting/vacancies-page'
import { VacancyDetailsPage } from '../model/recruiting/vacancy-details-page'
import { generateId, PlatformSetting, PlatformURI } from '../utils'
test.use({
storageState: PlatformSetting
@@ -106,7 +106,6 @@ test.describe('Application tests', () => {
await applicationsPage.openApplicationByTalentName(talentName)
const applicationId = await applicationsDetailsPage.getApplicationId()
await applicationsDetailsPage.deleteEntity()
expect(page.url()).toContain(applicationId)
await navigationMenuPage.clickButtonApplications()
await applicationsPage.checkApplicationNotExist(applicationId)
})