diff --git a/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support b/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support new file mode 100644 index 0000000000000..b17b8e888f7de --- /dev/null +++ b/projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support @@ -0,0 +1,4 @@ +Significance: patch +Type: added + +Jetpack AI: Adding translation support using Chrome's Gemini AI mini diff --git a/projects/js-packages/ai-client/src/chrome-ai/factory.ts b/projects/js-packages/ai-client/src/chrome-ai/factory.ts new file mode 100644 index 0000000000000..87eeffb73474e --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/factory.ts @@ -0,0 +1,129 @@ +import { getJetpackExtensionAvailability } from '@automattic/jetpack-shared-extension-utils'; +import { + PROMPT_TYPE_CHANGE_LANGUAGE, + //PROMPT_TYPE_SUMMARIZE, +} from '../constants.js'; +import { PromptProp, PromptItemProps } from '../types.js'; +import ChromeAISuggestionsEventSource from './suggestions.js'; + +/** + * Check for the feature flag. + * + * @return boolean + */ +function shouldUseChromeAI() { + return getJetpackExtensionAvailability( 'ai-use-chrome-ai-sometimes' ).available === true; +} + +interface PromptContext { + type?: string; + content?: string; + language?: string; +} + +/** + * This will return an instance of ChromeAISuggestionsEventSource or false. + * + * @param promptArg - The messages array of the prompt. + * @return ChromeAISuggestionsEventSource | bool + */ +export default async function ChromeAIFactory( promptArg: PromptProp ) { + if ( ! shouldUseChromeAI() ) { + return false; + } + + const context = { + content: '', + language: '', + }; + let promptType = ''; + if ( Array.isArray( promptArg ) ) { + for ( let i = 0; i < promptArg.length; i++ ) { + const prompt: PromptItemProps = promptArg[ i ]; + if ( prompt.content ) { + context.content = prompt.content; + } + + if ( ! ( 'context' in prompt ) ) { + continue; + } + + const promptContext: PromptContext = prompt.context; + + if ( promptContext.type ) { + promptType = promptContext.type; + } + + if ( promptContext.language ) { + context.language = promptContext.language; + } + + if ( promptContext.content ) { + context.content = promptContext.content; + } + } + } + + if ( promptType.startsWith( 'ai-assistant-change-language' ) ) { + const [ language ] = context.language.split( ' ' ); + + if ( + ! ( 'translation' in self ) || + ! self.translation.createTranslator || + ! self.translation.canTranslate + ) { + return false; + } + + const languageOpts = { + sourceLanguage: 'en', + targetLanguage: language, + }; + + // see if we can detect the source language + if ( 'ai' in self && self.ai.languageDetector ) { + const detector = await self.ai.languageDetector.create(); + const confidences = await detector.detect( context.content ); + + for ( const confidence of confidences ) { + // 75% confidence is just a value that was picked. Generally + // 80% of higher is pretty safe, but the source language is + // required for the translator to work at all, which is also + // why en is the default language. + if ( confidence.confidence > 0.75 ) { + languageOpts.sourceLanguage = confidence.detectedLanguage; + break; + } + } + } + + const canTranslate = await self.translation.canTranslate( languageOpts ); + + if ( canTranslate === 'no' ) { + return false; + } + + const chromeAI = new ChromeAISuggestionsEventSource( { + content: context.content, + promptType: PROMPT_TYPE_CHANGE_LANGUAGE, + options: languageOpts, + } ); + + return chromeAI; + } + + // TODO + if ( promptType.startsWith( 'ai-assistant-summarize' ) ) { + /* + return new ChromeAISuggestionsEventSource({ + content: "", + promptType: PROMPT_TYPE_SUMMARIZE, + options: {}, + } ); + */ + + return false; + } + + return false; +} diff --git a/projects/js-packages/ai-client/src/chrome-ai/index.ts b/projects/js-packages/ai-client/src/chrome-ai/index.ts new file mode 100644 index 0000000000000..13e1bdc070d5c --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/index.ts @@ -0,0 +1,2 @@ +export { default as ChromeAIFactory } from './factory.js'; +export { default as ChromeAISuggestionsEventSource } from './suggestions.js'; diff --git a/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts new file mode 100644 index 0000000000000..1c3a26deca750 --- /dev/null +++ b/projects/js-packages/ai-client/src/chrome-ai/suggestions.ts @@ -0,0 +1,139 @@ +import { EventSourceMessage } from '@microsoft/fetch-event-source'; +import { PROMPT_TYPE_CHANGE_LANGUAGE, PROMPT_TYPE_SUMMARIZE } from '../constants.js'; +import { getErrorData } from '../hooks/use-ai-suggestions/index.js'; +import { renderHTMLFromMarkdown, renderMarkdownFromHTML } from '../libs/markdown/index.js'; +import { AiModelTypeProp, ERROR_RESPONSE, ERROR_NETWORK } from '../types.js'; + +type ChromeAISuggestionsEventSourceConstructorArgs = { + content: string; + promptType: string; + options?: { + postId?: number | string; + feature?: 'ai-assistant-experimental' | string | undefined; + + // translation + sourceLanguage?: string; + targetLanguage?: string; + + // not sure if we need these + functions?: Array< object >; + model?: AiModelTypeProp; + }; +}; + +type ChromeAIEvent = { + type: string; + message: string; + complete?: boolean; +}; + +type FunctionCallProps = { + name?: string; + arguments?: string; +}; + +export default class ChromeAISuggestionsEventSource extends EventTarget { + fullMessage: string; + fullFunctionCall: FunctionCallProps; + isPromptClear: boolean; + controller: AbortController; + + errorUnclearPromptTriggered: boolean; + + constructor( data: ChromeAISuggestionsEventSourceConstructorArgs ) { + super(); + this.fullMessage = ''; + this.fullFunctionCall = { + name: '', + arguments: '', + }; + this.isPromptClear = false; + + this.controller = new AbortController(); + + this.initSource( data ); + } + + initSource( { + content, + promptType, + options = {}, + }: ChromeAISuggestionsEventSourceConstructorArgs ) { + if ( promptType === PROMPT_TYPE_CHANGE_LANGUAGE ) { + this.translate( content, options.targetLanguage, options.sourceLanguage ); + } + + if ( promptType === PROMPT_TYPE_SUMMARIZE ) { + this.summarize( content ); + } + } + + async initEventSource() {} + + close() {} + + checkForUnclearPrompt() {} + + processEvent( e: EventSourceMessage ) { + let data: ChromeAIEvent; + try { + data = JSON.parse( e.data ); + } catch ( err ) { + this.processErrorEvent( err ); + return; + } + + if ( e.event === 'translation' ) { + this.dispatchEvent( new CustomEvent( 'suggestion', { detail: data.message } ) ); + } + + if ( data.complete ) { + this.dispatchEvent( new CustomEvent( 'done', { detail: data.message } ) ); + } + } + + processErrorEvent( e ) { + // Dispatch a generic network error event + this.dispatchEvent( new CustomEvent( ERROR_NETWORK, { detail: e } ) ); + this.dispatchEvent( + new CustomEvent( ERROR_RESPONSE, { + detail: getErrorData( ERROR_NETWORK ), + } ) + ); + } + + // use the Chrome AI translator + async translate( text: string, target: string, source: string = '' ) { + if ( ! ( 'translation' in self ) ) { + return; + } + + const translator = await self.translation.createTranslator( { + sourceLanguage: source, + targetLanguage: target, + } ); + + if ( ! translator ) { + return; + } + + try { + const translation = await translator.translate( renderHTMLFromMarkdown( { content: text } ) ); + this.processEvent( { + id: '', + event: 'translation', + data: JSON.stringify( { + message: renderMarkdownFromHTML( { content: translation } ), + complete: true, + } ), + } ); + } catch ( error ) { + this.processErrorEvent( error ); + } + } + + // TODO + async summarize( text: string ) { + return text; + } +} diff --git a/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts b/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts index d90a3d9feef40..60bfb2b06c33f 100644 --- a/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts +++ b/projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts @@ -7,6 +7,7 @@ import { __ } from '@wordpress/i18n'; * Internal dependencies */ import askQuestion from '../../ask-question/index.js'; +import ChromeAIFactory from '../../chrome-ai/factory.js'; import { ERROR_CONTEXT_TOO_LARGE, ERROR_MODERATION, @@ -314,7 +315,14 @@ export default function useAiSuggestions( { // Set the request status. setRequestingState( 'requesting' ); - eventSourceRef.current = await askQuestion( promptArg, options ); + // check if we can (or should) use Chrome AI + const chromeAI = await ChromeAIFactory( promptArg ); + + if ( chromeAI !== false ) { + eventSourceRef.current = chromeAI; + } else { + eventSourceRef.current = await askQuestion( promptArg, options ); + } if ( ! eventSourceRef?.current ) { return; diff --git a/projects/js-packages/ai-client/src/index.ts b/projects/js-packages/ai-client/src/index.ts index 8342cf0899312..dfeb888d88fed 100644 --- a/projects/js-packages/ai-client/src/index.ts +++ b/projects/js-packages/ai-client/src/index.ts @@ -55,3 +55,8 @@ export * from './constants.js'; * Logo Generator */ export * from './logo-generator/index.js'; + +/** + * Chrome AI + */ +export * from './chrome-ai/index.js'; diff --git a/projects/js-packages/ai-client/src/types.ts b/projects/js-packages/ai-client/src/types.ts index 0fc6479d5ec8c..4fd220fa83620 100644 --- a/projects/js-packages/ai-client/src/types.ts +++ b/projects/js-packages/ai-client/src/types.ts @@ -132,3 +132,32 @@ export interface BlockEditorStore { [ key in keyof typeof BlockEditorSelectors ]: ( typeof BlockEditorSelectors )[ key ]; }; } + +declare global { + interface Window { + translation?: { + canTranslate: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< 'no' | 'yes' | string >; + createTranslator: ( options: { + sourceLanguage: string; + targetLanguage: string; + } ) => Promise< { + translate: ( text: string ) => Promise< string >; + } >; + }; + ai?: { + languageDetector: { + create: () => Promise< { + detect: ( text: string ) => Promise< + { + detectedLanguage: string; + confidence: number; + }[] + >; + } >; + }; + }; + } +} diff --git a/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support b/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support new file mode 100644 index 0000000000000..dea253ea9fe33 --- /dev/null +++ b/projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support @@ -0,0 +1,4 @@ +Significance: patch +Type: other + +Jetpack AI: Adding translation support using Chrome's Gemini AI mini diff --git a/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php b/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php index d3e39253a5758..a7777798360df 100644 --- a/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php +++ b/projects/plugins/jetpack/extensions/blocks/ai-assistant/ai-assistant.php @@ -10,8 +10,10 @@ namespace Automattic\Jetpack\Extensions\AIAssistant; use Automattic\Jetpack\Blocks; +use Automattic\Jetpack\Connection\Client; use Automattic\Jetpack\Status; use Automattic\Jetpack\Status\Host; +use Automattic\Jetpack\Status\Visitor; use Jetpack_Gutenberg; /** @@ -54,6 +56,54 @@ function load_assets( $attr, $content ) { ); } +/** + * Retrieve the Chrome trial AI token for use with the Chrome AI feature. + * This ultimately sets an Origin-Trial header with the token. + */ +function add_chrome_ai_token_header() { + $token_transient_name = 'jetpack-ai-chrome-ai-token'; + + $cached_token = get_transient( $token_transient_name ); + + if ( ! $cached_token ) { + $blog_id = \Jetpack_Options::get_option( 'id' ); + + // get the token from wpcom + $wpcom_request = Client::wpcom_json_api_request_as_user( + sprintf( '/sites/%d/jetpack-ai/ai-assistant-feature', $blog_id ), + 'v2', + array( + 'method' => 'GET', + 'headers' => array( + 'X-Forwarded-For' => ( new Visitor() )->get_ip( true ), + ), + 'timeout' => 30, + ), + null, + 'wpcom' + ); + + $response_code = wp_remote_retrieve_response_code( $wpcom_request ); + if ( 200 === $response_code ) { + $ai_assistant_feature_data = json_decode( wp_remote_retrieve_body( $wpcom_request ), true ); + + if ( ! empty( $ai_assistant_feature_data['chrome-ai-token'] ) ) { + set_transient( + $token_transient_name, + $ai_assistant_feature_data['chrome-ai-token'], + 3600 // cache for an hour, but this can probably be longer + ); + + $cached_token = $ai_assistant_feature_data['chrome-ai-token']; + } + } + } + + if ( $cached_token ) { + header( "Origin-Trial: {$cached_token}" ); + } +} + /** * Register extensions. */ @@ -120,3 +170,19 @@ function () { } } ); + +/** + * Register the `ai-use-chrome-ai-sometimes` extension. + */ +add_action( + 'jetpack_register_gutenberg_extensions', + function () { + if ( apply_filters( 'jetpack_ai_enabled', true ) && + apply_filters( 'ai_chrome_ai_enabled', false ) + ) { + \Jetpack_Gutenberg::set_extension_available( 'ai-use-chrome-ai-sometimes' ); + + add_chrome_ai_token_header(); + } + } +); diff --git a/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js b/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js index eede407e2145f..1acb040c7f587 100644 --- a/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js +++ b/projects/plugins/jetpack/extensions/blocks/ai-assistant/components/toolbar-controls/index.js @@ -83,6 +83,7 @@ const ToolbarControls = ( { type: 'suggestion', suggestion: PROMPT_TYPE_CHANGE_LANGUAGE, } ); + getSuggestionFromOpenAI( PROMPT_TYPE_CHANGE_LANGUAGE, { language, contentType: contentIsLoaded ? 'generated' : null, diff --git a/projects/plugins/jetpack/extensions/index.json b/projects/plugins/jetpack/extensions/index.json index a2fe1a4a7e689..e72c8c74afa61 100644 --- a/projects/plugins/jetpack/extensions/index.json +++ b/projects/plugins/jetpack/extensions/index.json @@ -81,7 +81,8 @@ "videopress/video-chapters", "ai-assistant-backend-prompts", "ai-list-to-table-transform", - "ai-seo-assistant" + "ai-seo-assistant", + "ai-use-chrome-ai-sometimes" ], "experimental": [], "no-post-editor": [