Fix translate service (#10118)

dev/docker-compose.yaml

@@ -577,6 +577,7 @@ services:
 #      - HULYLAKE_URL=http://huly.local:8096
 #      - QUEUE_CONFIG=${QUEUE_CONFIG}
 #      - QUEUE_REGION=cockroach
+#      - OPENAI_API_KEY=
 #    restart: unless-stopped
   backup-cockroach:
     image: hardcoreeng/backup

plugins/communication-assets/lang/cs.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Anonymní test",
     "Direct": "Přímý",
     "Directs": "Přímé",
-    "Members": "Členové"
+    "Members": "Členové",
+    "Translate": "Přeložit"
   }
 }

plugins/communication-assets/lang/de.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Anonyme quiz",
     "Direct": "Direkt",
     "Directs": "Direkte",
-    "Members": "Mitglieder"
+    "Members": "Mitglieder",
+    "Translate": "Übersetzen"
   }
 }

plugins/communication-assets/lang/en.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Anonymous quiz",
     "Direct": "Direct",
     "Directs": "Directs",
-    "Members": "Members"
+    "Members": "Members",
+    "Translate": "Translate"
   }
 }

plugins/communication-assets/lang/es.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Quiz anónimo",
     "Direct": "Directo",
     "Directs": "Directos",
-    "Members": "Miembros"
+    "Members": "Miembros",
+    "Translate": "Traducir"
   }
 }

plugins/communication-assets/lang/fr.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Quiz anonyme",
     "Direct": "Direct",
     "Directs": "Directs",
-    "Members": "Membres"
+    "Members": "Membres",
+    "Translate": "Traduire"
   }
 }

plugins/communication-assets/lang/it.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Quiz anonimo",
     "Direct": "Diretto",
     "Directs": "Diretti",
-    "Members": "Membri"
+    "Members": "Membri",
+    "Translate": "Traduci"
   }
 }

plugins/communication-assets/lang/ja.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "匿名クイズ",
     "Direct": "ダイレクト",
     "Directs": "ダイレクト",
-    "Members": "メンバー"
+    "Members": "メンバー",
+    "Translate": "翻訳"
   }
 }

plugins/communication-assets/lang/pt.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Quiz anônimo",
     "Direct": "Direto",
     "Directs": "Diretos",
-    "Members": "Membros"
+    "Members": "Membros",
+    "Translate": "Traduzir"
   }
 }

plugins/communication-assets/lang/ru.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Анонимная викторина",
     "Direct": "Личные сообщения",
     "Directs": "Личные сообщения",
-    "Members": "Участники"
+    "Members": "Участники",
+    "Translate": "Перевести"
   }
 }

plugins/communication-assets/lang/tr.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "Anonim quiz",
     "Direct": "Direkt",
     "Directs": "Direktler",
-    "Members": "Üyeler"
+    "Members": "Üyeler",
+    "Translate": "Çevir"
   }
 }

plugins/communication-assets/lang/zh.json

@@ -89,6 +89,7 @@
     "AnonymousQuiz": "匿名测验",
     "Direct": "私信",
     "Directs": "私信",
-    "Members": "成员"
+    "Members": "成员",
+    "Translate": "翻译"
   }
 }

plugins/communication-resources/src/actions.ts

@@ -47,7 +47,8 @@ import {
   messageEditingStore,
   showOriginalMessagesStore,
   threadCreateMessageStore,
-  translateMessagesStore
+  translateMessagesStore,
+  translateToStore
 } from './stores'
 
 export const addReaction: MessageActionFunction = async (message, card: Card, evt, onOpen, onClose) => {
@@ -155,15 +156,16 @@ export const canReplyInThread: MessageActionVisibilityTester = (message: Message
 }
 
 export const translateMessage: MessageActionFunction = async (message: Message): Promise<void> => {
+  const language = get(translateToStore) ?? get(languageStore)
+
   if (isMessageManualTranslating(message.cardId, message.id)) return
   const result = get(translateMessagesStore).find((it) => it.cardId === message.cardId && it.messageId === message.id)
 
-  if (result != null) {
-    showOriginalMessagesStore.update((store) =>
-      store.filter(([cId, mId]) => cId !== message.cardId || mId !== message.id)
-    )
-    return
-  }
+  showOriginalMessagesStore.update((store) =>
+    store.filter(([cId, mId]) => cId !== message.cardId || mId !== message.id)
+  )
+
+  if (result != null) return
 
   translateMessagesStore.update((store) => {
     store.push({ inProgress: true, messageId: message.id, cardId: message.cardId })
@@ -171,9 +173,8 @@ export const translateMessage: MessageActionFunction = async (message: Message):
   })
 
   const markup = toMarkup(message.content)
-  const lang = get(languageStore)
-  const currentTranslate = message?.translates?.[lang] ?? ''
-  const response = currentTranslate !== '' ? toMarkup(currentTranslate) : (await aiTranslate(markup, lang))?.text
+  const currentTranslate = message?.translates?.[language] ?? ''
+  const response = currentTranslate !== '' ? toMarkup(currentTranslate) : (await aiTranslate(markup, language))?.text
 
   if (response !== undefined) {
     translateMessagesStore.update((store) => {

plugins/communication-resources/src/components/message/MessageBody.svelte

@@ -31,9 +31,10 @@
     showOriginalMessagesStore,
     isMessageTranslated,
     translateToStore,
-    dontTranslateStore
+    dontTranslateStore,
+    isMessageOriginalShown
   } from '../../stores'
-  import { showOriginalMessage } from '../../actions'
+  import { showOriginalMessage, translateMessage } from '../../actions'
 
   export let card: Card
   export let author: Person | undefined
@@ -68,6 +69,13 @@
     $translateMessagesStore,
     $showOriginalMessagesStore
   )
+  $: isOriginalShown = isMessageOriginalShown(
+    message,
+    $translateToStore,
+    $dontTranslateStore,
+    $translateMessagesStore,
+    $showOriginalMessagesStore
+  )
 </script>
 
 {#if compact || hideHeader}
@@ -132,11 +140,14 @@
           <div class="message__translating">
             <Label label={communication.string.Translating} />
           </div>
-        {/if}
-        {#if isTranslated}
+        {:else if isTranslated}
           <div class="message__show-original" on:click={() => showOriginalMessage(message, card)}>
             <Label label={communication.string.ShowOriginal} />
           </div>
+        {:else if isOriginalShown}
+          <div class="message__translate" on:click={() => translateMessage(message, card)}>
+            <Label label={communication.string.Translate} />
+          </div>
         {/if}
       </div>
       {#if !isEditing}
@@ -224,7 +235,8 @@
     font-weight: 400;
   }
 
-  .message__show-original {
+  .message__show-original,
+  .message__translate {
     font-size: 0.75rem;
     color: var(--global-tertiary-TextColor);
     cursor: pointer;

plugins/communication-resources/src/components/message/MessageContentViewer.svelte

@@ -51,6 +51,7 @@
     $translateMessagesStore,
     $showOriginalMessagesStore
   )
+
   $: displayMarkup = translatedMarkup ?? toMarkup(message.content)
 
   $: if (prevContent !== message.content) {

plugins/communication-resources/src/plugin.ts

@@ -78,6 +78,7 @@ export default mergeIds(communicationId, communication, {
     LeftThe: '' as IntlString,
     Translating: '' as IntlString,
     ShowOriginal: '' as IntlString,
+    Translate: '' as IntlString,
     AddReaction: '' as IntlString,
     ReplyInThread: '' as IntlString,
     TranslateMessage: '' as IntlString,

plugins/communication-resources/src/stores.ts

@@ -13,7 +13,14 @@
 
 import { get, writable, derived } from 'svelte/store'
 import { createLabelsQuery, createQuery, onClient, onCommunicationClient } from '@hcengineering/presentation'
-import { type CardID, type Label, type LabelID, type Message, type MessageID } from '@hcengineering/communication-types'
+import {
+  MessageType,
+  type CardID,
+  type Label,
+  type LabelID,
+  type Message,
+  type MessageID
+} from '@hcengineering/communication-types'
 import core, { getCurrentAccount, type Markup, type Ref } from '@hcengineering/core'
 import { languageStore } from '@hcengineering/ui'
 import cardPlugin, { type Card } from '@hcengineering/card'
@@ -64,16 +71,13 @@ export function isShownManualTranslatedMessage (cardId: CardID, messageId: Messa
   return !showOriginal && result?.result != null
 }
 
-export function isMessageTranslated (
+export function hasTranslate (
   message: Message,
   translateTo: string | undefined,
   dontTranslate: string[],
-  translatedMessages: TranslateMessagesStatus[],
-  showOriginalMessage: Array<[CardID, MessageID]>
+  translatedMessages: TranslateMessagesStatus[]
 ): boolean {
-  const showOriginal = showOriginalMessage.some(([cId, mId]) => cId === message.cardId && mId === message.id)
-  if (showOriginal) return false
-
+  if (message.type !== MessageType.Text) return false
   const manualTranslate = translatedMessages.find((it) => it.cardId === message.cardId && it.messageId === message.id)
   if (manualTranslate?.result != null) return true
 
@@ -87,6 +91,32 @@ export function isMessageTranslated (
   return res !== ''
 }
 
+export function isMessageTranslated (
+  message: Message,
+  translateTo: string | undefined,
+  dontTranslate: string[],
+  translatedMessages: TranslateMessagesStatus[],
+  showOriginalMessage: Array<[CardID, MessageID]>
+): boolean {
+  const showOriginal = showOriginalMessage.some(([cId, mId]) => cId === message.cardId && mId === message.id)
+  if (showOriginal) return false
+
+  return hasTranslate(message, translateTo, dontTranslate, translatedMessages)
+}
+
+export function isMessageOriginalShown (
+  message: Message,
+  translateTo: string | undefined,
+  dontTranslate: string[],
+  translatedMessages: TranslateMessagesStatus[],
+  showOriginalMessage: Array<[CardID, MessageID]>
+): boolean {
+  const showOriginal = showOriginalMessage.some(([cId, mId]) => cId === message.cardId && mId === message.id)
+  if (!showOriginal) return false
+
+  return hasTranslate(message, translateTo, dontTranslate, translatedMessages)
+}
+
 export function getMessageTranslation (
   message: Message,
   translateTo: string | undefined,
@@ -94,7 +124,9 @@ export function getMessageTranslation (
   translatedMessages: TranslateMessagesStatus[],
   showOriginalMessage: Array<[CardID, MessageID]>
 ): Markup | undefined {
+  if (message.type !== MessageType.Text) return undefined
   const showOriginal = showOriginalMessage.some(([cId, mId]) => cId === message.cardId && mId === message.id)
+
   if (showOriginal) return undefined
 
   const manualTranslate = translatedMessages.find((it) => it.cardId === message.cardId && it.messageId === message.id)

services/translate/src/conroller.ts

@@ -33,7 +33,6 @@ import { BlobID, CardID, Markdown, Message, MessageID } from '@hcengineering/com
 import { withRetry } from '@hcengineering/retry'
 import { Analytics } from '@hcengineering/analytics'
 import OpenAI from 'openai'
-import { PlatformQueue } from '@hcengineering/server-core'
 import { MessageEventType, TranslateMessageEvent, UpdatePatchEvent } from '@hcengineering/communication-sdk-types'
 
 import { Storage } from './storage'
@@ -46,8 +45,7 @@ export class Controller {
 
   constructor (
     private readonly ctx: MeasureContext,
-    private readonly openai: OpenAI,
-    private readonly queue: PlatformQueue
+    private readonly openai: OpenAI
   ) {
     this.storage = new Storage(this.ctx)
   }
@@ -139,6 +137,7 @@ export class Controller {
     blobId: BlobID
   ): Promise<void> {
     const initialLanguage = message.language
+
     const translateTo = await this.getLanguages(workspace)
     if (translateTo.length === 0) return
 
@@ -183,6 +182,7 @@ export class Controller {
     }
 
     if (originalLanguage != null && originalLanguage !== '' && originalLanguage !== initialLanguage) {
+      ctx.info('update original language', { originalLanguage, id: message.id, content: message.content.slice(0, 50) })
       txes.unshift(this.getUpdateLanguageTx(message.cardId, message.id, originalLanguage))
     }
 
@@ -281,24 +281,29 @@ export class Controller {
     markdown: string,
     lang: string
   ): Promise<{ original_language?: string, translation?: string } | undefined> {
-    const systemPropmpt = `You are a translation model. 
-You are a translation model. Your only task is to translate the given markdown text into the ${lang} language.
+    const systemPrompt = `
+You are a translation model.
+Your only task is to translate the given Markdown text into the ${lang} language.
 Detect the original language of the input.
-If the input markdown is already written in ${lang}, do not translate. Preserve names and terms if they have no clear equivalent.
- Be as literal and accurate as possible while keeping the meaning natural. Keep markdown structire.
+If the input Markdown is already written in ${lang}, do not translate.
+Preserve names and terms if they have no clear equivalent.
+Be as literal and accurate as possible while keeping the meaning natural.
+Keep the Markdown structure.
 Output the result strictly as a JSON object in the following format:
 {
   "original_language": "<detected language in iso 639-1>",
   "translation": "<translated markdown or empty string if no translation was needed>"
 }
-Do not add any explanations, comments, or extra text outside the JSON.`
+Do not add any explanations, comments, or extra text outside the JSON.
+`.trim()
 
     const response = await this.openai.chat.completions.create({
       model: config.OpenAIModel,
+      response_format: { type: 'json_object' },
       messages: [
         {
           role: 'system',
-          content: systemPropmpt
+          content: systemPrompt
         },
         {
           role: 'user',
@@ -307,17 +312,17 @@ Do not add any explanations, comments, or extra text outside the JSON.`
       ]
     })
 
-    const res = response.choices[0].message.content ?? ''
+    const res = response.choices[0]?.message.content ?? ''
 
     try {
       const parsed = JSON.parse(res)
       if (typeof parsed !== 'object' || parsed === null) {
-        console.log('Failed to parse translation response', { response: res })
+        console.error('Failed to parse translation response', { response: res })
         return undefined
       }
       return parsed
     } catch (e) {
-      console.log('Failed to parse translation response', { response: res, error: e })
+      console.error('Failed to parse translation response', { response: res, error: e })
       return undefined
     }
   }

services/translate/src/index.ts

@@ -55,7 +55,7 @@ async function main (): Promise<void> {
   })
 
   const queue = getPlatformQueue(config.ServiceId, config.QueueRegion)
-  const controller = new Controller(ctx, openai, queue)
+  const controller = new Controller(ctx, openai)
 
   const consumer = queue.createConsumer<Tx>(ctx, QueueTopic.Tx, queue.getClientId(), async (ctx, queueMessage) => {
     const ws = queueMessage.workspace