From b4b91f1afd5c50627e135a022787fecafe07fa5a Mon Sep 17 00:00:00 2001 From: Mike Watson Date: Tue, 11 Feb 2025 11:05:19 -0500 Subject: [PATCH 1/7] Jetpack AI: Adding translation support using Chrome's Gemini AI mini * Adds optional support for translations using Chrome's built in AI * Will fall back to our standard OpenAI translations if Gemini can't be found or can't translate * Puts the entire feature behind a beta feature flag * API key currently read from WPCOM, changes TBD there --- .../ai-client/src/chrome-ai/factory.ts | 103 +++++++++++++ .../ai-client/src/chrome-ai/index.ts | 2 + .../ai-client/src/chrome-ai/suggestions.ts | 139 ++++++++++++++++++ .../src/hooks/use-ai-suggestions/index.ts | 10 +- projects/js-packages/ai-client/src/index.ts | 5 + .../blocks/ai-assistant/ai-assistant.php | 66 +++++++++ .../components/toolbar-controls/index.js | 1 + .../plugins/jetpack/extensions/index.json | 3 +- 8 files changed, 327 insertions(+), 2 deletions(-) create mode 100644 projects/js-packages/ai-client/src/chrome-ai/factory.ts create mode 100644 projects/js-packages/ai-client/src/chrome-ai/index.ts create mode 100644 projects/js-packages/ai-client/src/chrome-ai/suggestions.ts diff --git a/projects/js-packages/ai-client/src/chrome-ai/factory.ts b/projects/js-packages/ai-client/src/chrome-ai/factory.ts new file mode 100644 index 0000000000000..5d5e0a8e00d6d --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/factory.ts @@ -0,0 +1,103 @@ +import { getJetpackExtensionAvailability } from '@automattic/jetpack-shared-extension-utils'; +import { + PROMPT_TYPE_CHANGE_LANGUAGE, + //PROMPT_TYPE_SUMMARIZE, +} from '../constants.js'; +import { PromptProp } from '../types.js'; +import ChromeAISuggestionsEventSource from './suggestions.js'; + +/** + * Check for the feature flag. + * + * @return boolean + */ +function shouldUseChromeAI() { + return getJetpackExtensionAvailability( 'ai-use-chrome-ai-sometimes' ).available === true; +} + +/** + * This will return an instance of ChromeAISuggestionsEventSource or false. + * + * @param promptArg - The messages array of the prompt. + * @return ChromeAISuggestionsEventSource | bool + */ +export default async function ChromeAIFactory( promptArg: PromptProp ) { + if ( ! shouldUseChromeAI() ) { + return false; + } + + let context; + let promptType = ''; + if ( Array.isArray( promptArg ) ) { + context = promptArg[ promptArg.length - 1 ].context; + promptType = context.type; + } + + if ( promptType.startsWith( 'ai-assistant-change-language' ) ) { + const [ language ] = context.language.split( ' ' ); + + if ( + ! ( 'translation' in self ) || + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ! ( self.translation as any ).createTranslator || + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ! ( self.translation as any ).canTranslate + ) { + return false; + } + + const languageOpts = { + sourceLanguage: 'en', + targetLanguage: language, + }; + + // see if we can detect the source language + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ( 'ai' in self && ( self.ai as any ).languageDetector ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const detector = await ( self.ai as any ).languageDetector.create(); + const confidences = await detector.detect( context.content ); + + for ( const confidence of confidences ) { + // 75% confidence is just a value that was picked. Generally + // 80% of higher is pretty safe, but the source language is + // required for the translator to work at all, which is also + // why en is the default language. + if ( confidence.confidence > 0.75 ) { + languageOpts.sourceLanguage = confidence.detectedLanguage; + break; + } + } + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const canTranslate = await ( self.translation as any ).canTranslate( languageOpts ); + + if ( canTranslate === 'no' ) { + return false; + } + + const chromeAI = new ChromeAISuggestionsEventSource( { + content: context.content, + promptType: PROMPT_TYPE_CHANGE_LANGUAGE, + options: languageOpts, + } ); + + return chromeAI; + } + + // TODO + if ( promptType.startsWith( 'ai-assistant-summarize' ) ) { + /* + return new ChromeAISuggestionsEventSource({ + content: "", + promptType: PROMPT_TYPE_SUMMARIZE, + options: {}, + } ); + */ + + return false; + } + + return false; +} diff --git a/projects/js-packages/ai-client/src/chrome-ai/index.ts b/projects/js-packages/ai-client/src/chrome-ai/index.ts new file mode 100644 index 0000000000000..13e1bdc070d5c --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/index.ts @@ -0,0 +1,2 @@ +export { default as ChromeAIFactory } from './factory.js'; +export { default as ChromeAISuggestionsEventSource } from './suggestions.js'; diff --git a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts new file mode 100644 index 0000000000000..b6b3c3f095ee7 --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts @@ -0,0 +1,139 @@ +import { EventSourceMessage } from '@microsoft/fetch-event-source'; +import { PROMPT_TYPE_CHANGE_LANGUAGE, PROMPT_TYPE_SUMMARIZE } from '../constants.js'; +import { getErrorData } from '../hooks/use-ai-suggestions/index.js'; +import { AiModelTypeProp, ERROR_RESPONSE, ERROR_NETWORK } from '../types.js'; + +type ChromeAISuggestionsEventSourceConstructorArgs = { + content: string; + promptType: string; + options?: { + postId?: number | string; + feature?: 'ai-assistant-experimental' | string | undefined; + + // translation + sourceLanguage?: string; + targetLanguage?: string; + + // not sure if we need these + functions?: Array< object >; + model?: AiModelTypeProp; + }; +}; + +type ChromeAIEvent = { + type: string; + message: string; + complete?: boolean; +}; + +type FunctionCallProps = { + name?: string; + arguments?: string; +}; + +export default class ChromeAISuggestionsEventSource extends EventTarget { + fullMessage: string; + fullFunctionCall: FunctionCallProps; + isPromptClear: boolean; + controller: AbortController; + + errorUnclearPromptTriggered: boolean; + + constructor( data: ChromeAISuggestionsEventSourceConstructorArgs ) { + super(); + this.fullMessage = ''; + this.fullFunctionCall = { + name: '', + arguments: '', + }; + this.isPromptClear = false; + + this.controller = new AbortController(); + + this.initSource( data ); + } + + initSource( { + content, + promptType, + options = {}, + }: ChromeAISuggestionsEventSourceConstructorArgs ) { + if ( promptType === PROMPT_TYPE_CHANGE_LANGUAGE ) { + this.translate( content, options.targetLanguage, options.sourceLanguage ); + } + + if ( promptType === PROMPT_TYPE_SUMMARIZE ) { + this.summarize( content ); + } + } + + async initEventSource() {} + + close() {} + + checkForUnclearPrompt() {} + + processEvent( e: EventSourceMessage ) { + let data: ChromeAIEvent; + try { + data = JSON.parse( e.data ); + } catch ( err ) { + this.processErrorEvent( err ); + return; + } + + if ( e.event === 'translation' ) { + this.dispatchEvent( new CustomEvent( 'suggestion', { detail: data.message } ) ); + } + + if ( data.complete ) { + this.dispatchEvent( new CustomEvent( 'done', { detail: data.message } ) ); + } + } + + processErrorEvent( e ) { + // Dispatch a generic network error event + this.dispatchEvent( new CustomEvent( ERROR_NETWORK, { detail: e } ) ); + this.dispatchEvent( + new CustomEvent( ERROR_RESPONSE, { + detail: getErrorData( ERROR_NETWORK ), + } ) + ); + } + + // use the Chrome AI translator + async translate( text: string, target: string, source: string = '' ) { + if ( ! ( 'translation' in self ) ) { + return; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const translator = await ( self.translation as any ).createTranslator( { + sourceLanguage: source, + targetLanguage: target, + } ); + + if ( ! translator ) { + return; + } + + try { + const translation = await translator.translate( text ); + this.processEvent( { + id: '', + event: 'translation', + data: JSON.stringify( { + message: translation, + complete: true, + } ), + } ); + } catch ( error ) { + this.processErrorEvent( error ); + } + } + + // TODO + async summarize( text: string ) { + return text; + } +} diff --git a/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts b/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts index d90a3d9feef40..60bfb2b06c33f 100644 --- a/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts +++ b/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts @@ -7,6 +7,7 @@ import { __ } from '@wordpress/i18n'; * Internal dependencies */ import askQuestion from '../../ask-question/index.js'; +import ChromeAIFactory from '../../chrome-ai/factory.js'; import { ERROR_CONTEXT_TOO_LARGE, ERROR_MODERATION, @@ -314,7 +315,14 @@ export default function useAiSuggestions( { // Set the request status. setRequestingState( 'requesting' ); - eventSourceRef.current = await askQuestion( promptArg, options ); + // check if we can (or should) use Chrome AI + const chromeAI = await ChromeAIFactory( promptArg ); + + if ( chromeAI !== false ) { + eventSourceRef.current = chromeAI; + } else { + eventSourceRef.current = await askQuestion( promptArg, options ); + } if ( ! eventSourceRef?.current ) { return; diff --git a/projects/js-packages/ai-client/src/index.ts b/projects/js-packages/ai-client/src/index.ts index 8342cf0899312..dfeb888d88fed 100644 --- a/projects/js-packages/ai-client/src/index.ts +++ b/projects/js-packages/ai-client/src/index.ts @@ -55,3 +55,8 @@ export * from './constants.js'; * Logo Generator */ export * from './logo-generator/index.js'; + +/** + * Chrome AI + */ +export * from './chrome-ai/index.js'; diff --git a/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php b/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php index d3e39253a5758..a7777798360df 100644 --- a/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php +++ b/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php @@ -10,8 +10,10 @@ namespace Automattic\Jetpack\Extensions\AIAssistant; use Automattic\Jetpack\Blocks; +use Automattic\Jetpack\Connection\Client; use Automattic\Jetpack\Status; use Automattic\Jetpack\Status\Host; +use Automattic\Jetpack\Status\Visitor; use Jetpack_Gutenberg; /** @@ -54,6 +56,54 @@ function load_assets( $attr, $content ) { ); } +/** + * Retrieve the Chrome trial AI token for use with the Chrome AI feature. + * This ultimately sets an Origin-Trial header with the token. + */ +function add_chrome_ai_token_header() { + $token_transient_name = 'jetpack-ai-chrome-ai-token'; + + $cached_token = get_transient( $token_transient_name ); + + if ( ! $cached_token ) { + $blog_id = \Jetpack_Options::get_option( 'id' ); + + // get the token from wpcom + $wpcom_request = Client::wpcom_json_api_request_as_user( + sprintf( '/sites/%d/jetpack-ai/ai-assistant-feature', $blog_id ), + 'v2', + array( + 'method' => 'GET', + 'headers' => array( + 'X-Forwarded-For' => ( new Visitor() )->get_ip( true ), + ), + 'timeout' => 30, + ), + null, + 'wpcom' + ); + + $response_code = wp_remote_retrieve_response_code( $wpcom_request ); + if ( 200 === $response_code ) { + $ai_assistant_feature_data = json_decode( wp_remote_retrieve_body( $wpcom_request ), true ); + + if ( ! empty( $ai_assistant_feature_data['chrome-ai-token'] ) ) { + set_transient( + $token_transient_name, + $ai_assistant_feature_data['chrome-ai-token'], + 3600 // cache for an hour, but this can probably be longer + ); + + $cached_token = $ai_assistant_feature_data['chrome-ai-token']; + } + } + } + + if ( $cached_token ) { + header( "Origin-Trial: {$cached_token}" ); + } +} + /** * Register extensions. */ @@ -120,3 +170,19 @@ function () { } } ); + +/** + * Register the `ai-use-chrome-ai-sometimes` extension. + */ +add_action( + 'jetpack_register_gutenberg_extensions', + function () { + if ( apply_filters( 'jetpack_ai_enabled', true ) && + apply_filters( 'ai_chrome_ai_enabled', false ) + ) { + \Jetpack_Gutenberg::set_extension_available( 'ai-use-chrome-ai-sometimes' ); + + add_chrome_ai_token_header(); + } + } +); diff --git a/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js b/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js index eede407e2145f..1acb040c7f587 100644 --- a/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js +++ b/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js @@ -83,6 +83,7 @@ const ToolbarControls = ( { type: 'suggestion', suggestion: PROMPT_TYPE_CHANGE_LANGUAGE, } ); + getSuggestionFromOpenAI( PROMPT_TYPE_CHANGE_LANGUAGE, { language, contentType: contentIsLoaded ? 'generated' : null, diff --git a/projects/plugins/jetpack/extensions/index.json b/projects/plugins/jetpack/extensions/index.json index a2fe1a4a7e689..e72c8c74afa61 100644 --- a/projects/plugins/jetpack/extensions/index.json +++ b/projects/plugins/jetpack/extensions/index.json @@ -81,7 +81,8 @@ "videopress/video-chapters", "ai-assistant-backend-prompts", "ai-list-to-table-transform", - "ai-seo-assistant" + "ai-seo-assistant", + "ai-use-chrome-ai-sometimes" ], "experimental": [], "no-post-editor": [ From 6aac5e08298d04785425626de5b6b0d7d40f7345 Mon Sep 17 00:00:00 2001 From: Mike Watson Date: Tue, 11 Feb 2025 11:10:08 -0500 Subject: [PATCH 2/7] changelog --- .../changelog/change-add-chrome-ai-translate-support | 4 ++++ .../jetpack/changelog/change-add-chrome-ai-translate-support | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support create mode 100644 projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support diff --git a/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support b/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support new file mode 100644 index 0000000000000..b17b8e888f7de --- /dev/null +++ b/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support @@ -0,0 +1,4 @@ +Significance: patch +Type: added + +Jetpack AI: Adding translation support using Chrome's Gemini AI mini diff --git a/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support b/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support new file mode 100644 index 0000000000000..dea253ea9fe33 --- /dev/null +++ b/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support @@ -0,0 +1,4 @@ +Significance: patch +Type: other + +Jetpack AI: Adding translation support using Chrome's Gemini AI mini From cd9466f210bee0d80b133e1baffa1c7c38f21268 Mon Sep 17 00:00:00 2001 From: Douglas Date: Tue, 11 Feb 2025 18:51:36 -0300 Subject: [PATCH 3/7] fix types --- projects/js-packages/ai-client/global.d.ts | 28 +++++++++++++++++++ .../ai-client/src/chrome-ai/factory.ts | 15 ++++------ .../ai-client/src/chrome-ai/suggestions.ts | 3 +- projects/js-packages/ai-client/tsconfig.json | 1 + 4 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 projects/js-packages/ai-client/global.d.ts diff --git a/projects/js-packages/ai-client/global.d.ts b/projects/js-packages/ai-client/global.d.ts new file mode 100644 index 0000000000000..d48d1b682e623 --- /dev/null +++ b/projects/js-packages/ai-client/global.d.ts @@ -0,0 +1,28 @@ +export declare global { + interface Window { + translation: { + canTranslate: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< 'no' | 'yes' | string >; + createTranslator: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< { + translate: ( text: string ) => Promise< string >; + } >; + }; + ai: { + languageDetector: { + create: () => Promise< { + detect: ( text: string ) => Promise< + { + detectedLanguage: string; + confidence: number; + }[] + >; + } >; + }; + }; + } +} diff --git a/projects/js-packages/ai-client/src/chrome-ai/factory.ts b/projects/js-packages/ai-client/src/chrome-ai/factory.ts index 5d5e0a8e00d6d..d5ef508b341df 100644 --- a/projects/js-packages/ai-client/src/chrome-ai/factory.ts +++ b/projects/js-packages/ai-client/src/chrome-ai/factory.ts @@ -38,10 +38,8 @@ export default async function ChromeAIFactory( promptArg: PromptProp ) { if ( ! ( 'translation' in self ) || - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ! ( self.translation as any ).createTranslator || - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ! ( self.translation as any ).canTranslate + ! self.translation.createTranslator || + ! self.translation.canTranslate ) { return false; } @@ -52,10 +50,8 @@ export default async function ChromeAIFactory( promptArg: PromptProp ) { }; // see if we can detect the source language - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if ( 'ai' in self && ( self.ai as any ).languageDetector ) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const detector = await ( self.ai as any ).languageDetector.create(); + if ( 'ai' in self && self.ai.languageDetector ) { + const detector = await self.ai.languageDetector.create(); const confidences = await detector.detect( context.content ); for ( const confidence of confidences ) { @@ -70,8 +66,7 @@ export default async function ChromeAIFactory( promptArg: PromptProp ) { } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const canTranslate = await ( self.translation as any ).canTranslate( languageOpts ); + const canTranslate = await self.translation.canTranslate( languageOpts ); if ( canTranslate === 'no' ) { return false; diff --git a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts index b6b3c3f095ee7..a004fd9c352ae 100644 --- a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts +++ b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts @@ -107,8 +107,7 @@ export default class ChromeAISuggestionsEventSource extends EventTarget { return; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const translator = await ( self.translation as any ).createTranslator( { + const translator = await self.translation.createTranslator( { sourceLanguage: source, targetLanguage: target, } ); diff --git a/projects/js-packages/ai-client/tsconfig.json b/projects/js-packages/ai-client/tsconfig.json index cf2f897aca63b..cda961512b73f 100644 --- a/projects/js-packages/ai-client/tsconfig.json +++ b/projects/js-packages/ai-client/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "jetpack-js-tools/tsconfig.tsc.json", "include": [ "./src/**/*" ], + "files": [ "./global.d.ts" ], "compilerOptions": { "typeRoots": [ "./node_modules/@types/", "src/*" ], "sourceMap": false, From a5dfba625de28a5e73ae91b8b388e0a11e1960bf Mon Sep 17 00:00:00 2001 From: Douglas Date: Tue, 11 Feb 2025 19:04:23 -0300 Subject: [PATCH 4/7] changing tsconfig entry --- projects/js-packages/ai-client/global.d.ts | 4 ++-- projects/js-packages/ai-client/tsconfig.json | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/js-packages/ai-client/global.d.ts b/projects/js-packages/ai-client/global.d.ts index d48d1b682e623..a1606ecb16bad 100644 --- a/projects/js-packages/ai-client/global.d.ts +++ b/projects/js-packages/ai-client/global.d.ts @@ -1,6 +1,6 @@ export declare global { interface Window { - translation: { + translation?: { canTranslate: ( options: { sourceLanguage: string; targetLanguage: string; @@ -12,7 +12,7 @@ export declare global { translate: ( text: string ) => Promise< string >; } >; }; - ai: { + ai?: { languageDetector: { create: () => Promise< { detect: ( text: string ) => Promise< diff --git a/projects/js-packages/ai-client/tsconfig.json b/projects/js-packages/ai-client/tsconfig.json index cda961512b73f..c342b19ef262c 100644 --- a/projects/js-packages/ai-client/tsconfig.json +++ b/projects/js-packages/ai-client/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "jetpack-js-tools/tsconfig.tsc.json", - "include": [ "./src/**/*" ], - "files": [ "./global.d.ts" ], + "include": [ "./src/**/*", "./global.d.ts" ], "compilerOptions": { "typeRoots": [ "./node_modules/@types/", "src/*" ], "sourceMap": false, From 77c9d6190f64d03d2d1b2cd41c67978af6035093 Mon Sep 17 00:00:00 2001 From: Douglas Date: Wed, 12 Feb 2025 12:37:37 -0300 Subject: [PATCH 5/7] move declaration to regular types file --- projects/js-packages/ai-client/global.d.ts | 28 ------------------- projects/js-packages/ai-client/src/types.ts | 29 ++++++++++++++++++++ projects/js-packages/ai-client/tsconfig.json | 2 +- 3 files changed, 30 insertions(+), 29 deletions(-) delete mode 100644 projects/js-packages/ai-client/global.d.ts diff --git a/projects/js-packages/ai-client/global.d.ts b/projects/js-packages/ai-client/global.d.ts deleted file mode 100644 index a1606ecb16bad..0000000000000 --- a/projects/js-packages/ai-client/global.d.ts +++ /dev/null @@ -1,28 +0,0 @@ -export declare global { - interface Window { - translation?: { - canTranslate: ( options: { - sourceLanguage: string; - targetLanguage: string; - } ) => Promise< 'no' | 'yes' | string >; - createTranslator: ( options: { - sourceLanguage: string; - targetLanguage: string; - } ) => Promise< { - translate: ( text: string ) => Promise< string >; - } >; - }; - ai?: { - languageDetector: { - create: () => Promise< { - detect: ( text: string ) => Promise< - { - detectedLanguage: string; - confidence: number; - }[] - >; - } >; - }; - }; - } -} diff --git a/projects/js-packages/ai-client/src/types.ts b/projects/js-packages/ai-client/src/types.ts index 0fc6479d5ec8c..4fd220fa83620 100644 --- a/projects/js-packages/ai-client/src/types.ts +++ b/projects/js-packages/ai-client/src/types.ts @@ -132,3 +132,32 @@ export interface BlockEditorStore { [ key in keyof typeof BlockEditorSelectors ]: ( typeof BlockEditorSelectors )[ key ]; }; } + +declare global { + interface Window { + translation?: { + canTranslate: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< 'no' | 'yes' | string >; + createTranslator: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< { + translate: ( text: string ) => Promise< string >; + } >; + }; + ai?: { + languageDetector: { + create: () => Promise< { + detect: ( text: string ) => Promise< + { + detectedLanguage: string; + confidence: number; + }[] + >; + } >; + }; + }; + } +} diff --git a/projects/js-packages/ai-client/tsconfig.json b/projects/js-packages/ai-client/tsconfig.json index c342b19ef262c..cf2f897aca63b 100644 --- a/projects/js-packages/ai-client/tsconfig.json +++ b/projects/js-packages/ai-client/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "jetpack-js-tools/tsconfig.tsc.json", - "include": [ "./src/**/*", "./global.d.ts" ], + "include": [ "./src/**/*" ], "compilerOptions": { "typeRoots": [ "./node_modules/@types/", "src/*" ], "sourceMap": false, From 6ac9102ab03b0b2f194e12f3b71e0843206792d9 Mon Sep 17 00:00:00 2001 From: Mike Watson Date: Wed, 12 Feb 2025 15:56:09 -0500 Subject: [PATCH 6/7] Better handling of PrompType payloads and convert markdown to HTML (and then back) so translations work better. This fixes the issue where a full AI generated article could not be translated to a different language. --- .../ai-client/src/chrome-ai/factory.ts | 23 ++++++++++++++++--- .../ai-client/src/chrome-ai/suggestions.ts | 5 ++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/projects/js-packages/ai-client/src/chrome-ai/factory.ts b/projects/js-packages/ai-client/src/chrome-ai/factory.ts index d5ef508b341df..115b138e20286 100644 --- a/projects/js-packages/ai-client/src/chrome-ai/factory.ts +++ b/projects/js-packages/ai-client/src/chrome-ai/factory.ts @@ -26,11 +26,28 @@ export default async function ChromeAIFactory( promptArg: PromptProp ) { return false; } - let context; + const context = {}; let promptType = ''; if ( Array.isArray( promptArg ) ) { - context = promptArg[ promptArg.length - 1 ].context; - promptType = context.type; + for ( const prompt of promptArg ) { + if ( prompt.content ) { + context.content = prompt.content; + } + + if ( prompt.context ) { + if ( prompt.context.type ) { + promptType = prompt.context.type; + } + + if ( prompt.context.language ) { + context.language = prompt.context.language; + } + + if ( prompt.context.content ) { + context.content = prompt.context.content; + } + } + } } if ( promptType.startsWith( 'ai-assistant-change-language' ) ) { diff --git a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts index a004fd9c352ae..1c3a26deca750 100644 --- a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts +++ b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts @@ -1,6 +1,7 @@ import { EventSourceMessage } from '@microsoft/fetch-event-source'; import { PROMPT_TYPE_CHANGE_LANGUAGE, PROMPT_TYPE_SUMMARIZE } from '../constants.js'; import { getErrorData } from '../hooks/use-ai-suggestions/index.js'; +import { renderHTMLFromMarkdown, renderMarkdownFromHTML } from '../libs/markdown/index.js'; import { AiModelTypeProp, ERROR_RESPONSE, ERROR_NETWORK } from '../types.js'; type ChromeAISuggestionsEventSourceConstructorArgs = { @@ -117,12 +118,12 @@ export default class ChromeAISuggestionsEventSource extends EventTarget { } try { - const translation = await translator.translate( text ); + const translation = await translator.translate( renderHTMLFromMarkdown( { content: text } ) ); this.processEvent( { id: '', event: 'translation', data: JSON.stringify( { - message: translation, + message: renderMarkdownFromHTML( { content: translation } ), complete: true, } ), } ); From 5b22871e8a38ccbdfb6d3143fa228fae5bee56ec Mon Sep 17 00:00:00 2001 From: Mike Watson Date: Wed, 12 Feb 2025 16:49:43 -0500 Subject: [PATCH 7/7] Fix for some build errors that were accidentally committed --- .../ai-client/src/chrome-ai/factory.ts | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/projects/js-packages/ai-client/src/chrome-ai/factory.ts b/projects/js-packages/ai-client/src/chrome-ai/factory.ts index 115b138e20286..87eeffb73474e 100644 --- a/projects/js-packages/ai-client/src/chrome-ai/factory.ts +++ b/projects/js-packages/ai-client/src/chrome-ai/factory.ts @@ -3,7 +3,7 @@ import { PROMPT_TYPE_CHANGE_LANGUAGE, //PROMPT_TYPE_SUMMARIZE, } from '../constants.js'; -import { PromptProp } from '../types.js'; +import { PromptProp, PromptItemProps } from '../types.js'; import ChromeAISuggestionsEventSource from './suggestions.js'; /** @@ -15,6 +15,12 @@ function shouldUseChromeAI() { return getJetpackExtensionAvailability( 'ai-use-chrome-ai-sometimes' ).available === true; } +interface PromptContext { + type?: string; + content?: string; + language?: string; +} + /** * This will return an instance of ChromeAISuggestionsEventSource or false. * @@ -26,26 +32,34 @@ export default async function ChromeAIFactory( promptArg: PromptProp ) { return false; } - const context = {}; + const context = { + content: '', + language: '', + }; let promptType = ''; if ( Array.isArray( promptArg ) ) { - for ( const prompt of promptArg ) { + for ( let i = 0; i < promptArg.length; i++ ) { + const prompt: PromptItemProps = promptArg[ i ]; if ( prompt.content ) { context.content = prompt.content; } - if ( prompt.context ) { - if ( prompt.context.type ) { - promptType = prompt.context.type; - } + if ( ! ( 'context' in prompt ) ) { + continue; + } - if ( prompt.context.language ) { - context.language = prompt.context.language; - } + const promptContext: PromptContext = prompt.context; - if ( prompt.context.content ) { - context.content = prompt.context.content; - } + if ( promptContext.type ) { + promptType = promptContext.type; + } + + if ( promptContext.language ) { + context.language = promptContext.language; + } + + if ( promptContext.content ) { + context.content = promptContext.content; } } }