Jetpack AI: Adding translation support using Chrome's Gemini AI mini (#…

…41724)

* Jetpack AI: Adding translation support using Chrome's Gemini AI mini
* Adds optional support for translations using Chrome's built in AI
* Will fall back to our standard OpenAI translations if Gemini can't be found or can't translate
* Puts the entire feature behind a beta feature flag
* API key currently read from WPCOM, changes TBD there

* changelog

* fix types

* changing tsconfig entry

* move declaration to regular types file

* Better handling of PrompType payloads and convert markdown to HTML (and then back) so translations work better. This fixes the issue where a full AI generated article could not be translated to a different language.

* Fix for some build errors that were accidentally committed

---------

Co-authored-by: Douglas <[email protected]>

projects/js-packages/ai-client/changelog/change-add-chrome-ai-translate-support

@@ -0,0 +1,4 @@
+Significance: patch
+Type: added
+
+Jetpack AI: Adding translation support using Chrome's Gemini AI mini

projects/js-packages/ai-client/src/chrome-ai/factory.ts

@@ -0,0 +1,129 @@
+import { getJetpackExtensionAvailability } from '@automattic/jetpack-shared-extension-utils';
+import {
+	PROMPT_TYPE_CHANGE_LANGUAGE,
+	//PROMPT_TYPE_SUMMARIZE,
+} from '../constants.js';
+import { PromptProp, PromptItemProps } from '../types.js';
+import ChromeAISuggestionsEventSource from './suggestions.js';
+
+/**
+ * Check for the feature flag.
+ *
+ * @return boolean
+ */
+function shouldUseChromeAI() {
+	return getJetpackExtensionAvailability( 'ai-use-chrome-ai-sometimes' ).available === true;
+}
+
+interface PromptContext {
+	type?: string;
+	content?: string;
+	language?: string;
+}
+
+/**
+ * This will return an instance of ChromeAISuggestionsEventSource or false.
+ *
+ * @param promptArg - The messages array of the prompt.
+ * @return ChromeAISuggestionsEventSource | bool
+ */
+export default async function ChromeAIFactory( promptArg: PromptProp ) {
+	if ( ! shouldUseChromeAI() ) {
+		return false;
+	}
+
+	const context = {
+		content: '',
+		language: '',
+	};
+	let promptType = '';
+	if ( Array.isArray( promptArg ) ) {
+		for ( let i = 0; i < promptArg.length; i++ ) {
+			const prompt: PromptItemProps = promptArg[ i ];
+			if ( prompt.content ) {
+				context.content = prompt.content;
+			}
+
+			if ( ! ( 'context' in prompt ) ) {
+				continue;
+			}
+
+			const promptContext: PromptContext = prompt.context;
+
+			if ( promptContext.type ) {
+				promptType = promptContext.type;
+			}
+
+			if ( promptContext.language ) {
+				context.language = promptContext.language;
+			}
+
+			if ( promptContext.content ) {
+				context.content = promptContext.content;
+			}
+		}
+	}
+
+	if ( promptType.startsWith( 'ai-assistant-change-language' ) ) {
+		const [ language ] = context.language.split( ' ' );
+
+		if (
+			! ( 'translation' in self ) ||
+			! self.translation.createTranslator ||
+			! self.translation.canTranslate
+		) {
+			return false;
+		}
+
+		const languageOpts = {
+			sourceLanguage: 'en',
+			targetLanguage: language,
+		};
+
+		// see if we can detect the source language
+		if ( 'ai' in self && self.ai.languageDetector ) {
+			const detector = await self.ai.languageDetector.create();
+			const confidences = await detector.detect( context.content );
+
+			for ( const confidence of confidences ) {
+				// 75% confidence is just a value that was picked. Generally
+				// 80% of higher is pretty safe, but the source language is
+				// required for the translator to work at all, which is also
+				// why en is the default language.
+				if ( confidence.confidence > 0.75 ) {
+					languageOpts.sourceLanguage = confidence.detectedLanguage;
+					break;
+				}
+			}
+		}
+
+		const canTranslate = await self.translation.canTranslate( languageOpts );
+
+		if ( canTranslate === 'no' ) {
+			return false;
+		}
+
+		const chromeAI = new ChromeAISuggestionsEventSource( {
+			content: context.content,
+			promptType: PROMPT_TYPE_CHANGE_LANGUAGE,
+			options: languageOpts,
+		} );
+
+		return chromeAI;
+	}
+
+	// TODO
+	if ( promptType.startsWith( 'ai-assistant-summarize' ) ) {
+		/*
+		return new ChromeAISuggestionsEventSource({
+			content: "",
+			promptType: PROMPT_TYPE_SUMMARIZE,
+			options: {},
+		} );
+		*/
+
+		return false;
+	}
+
+	return false;
+}

projects/js-packages/ai-client/src/chrome-ai/index.ts

@@ -0,0 +1,2 @@
+export { default as ChromeAIFactory } from './factory.js';
+export { default as ChromeAISuggestionsEventSource } from './suggestions.js';

projects/js-packages/ai-client/src/chrome-ai/suggestions.ts

@@ -0,0 +1,139 @@
+import { EventSourceMessage } from '@microsoft/fetch-event-source';
+import { PROMPT_TYPE_CHANGE_LANGUAGE, PROMPT_TYPE_SUMMARIZE } from '../constants.js';
+import { getErrorData } from '../hooks/use-ai-suggestions/index.js';
+import { renderHTMLFromMarkdown, renderMarkdownFromHTML } from '../libs/markdown/index.js';
+import { AiModelTypeProp, ERROR_RESPONSE, ERROR_NETWORK } from '../types.js';
+
+type ChromeAISuggestionsEventSourceConstructorArgs = {
+	content: string;
+	promptType: string;
+	options?: {
+		postId?: number | string;
+		feature?: 'ai-assistant-experimental' | string | undefined;
+
+		// translation
+		sourceLanguage?: string;
+		targetLanguage?: string;
+
+		// not sure if we need these
+		functions?: Array< object >;
+		model?: AiModelTypeProp;
+	};
+};
+
+type ChromeAIEvent = {
+	type: string;
+	message: string;
+	complete?: boolean;
+};
+
+type FunctionCallProps = {
+	name?: string;
+	arguments?: string;
+};
+
+export default class ChromeAISuggestionsEventSource extends EventTarget {
+	fullMessage: string;
+	fullFunctionCall: FunctionCallProps;
+	isPromptClear: boolean;
+	controller: AbortController;
+
+	errorUnclearPromptTriggered: boolean;
+
+	constructor( data: ChromeAISuggestionsEventSourceConstructorArgs ) {
+		super();
+		this.fullMessage = '';
+		this.fullFunctionCall = {
+			name: '',
+			arguments: '',
+		};
+		this.isPromptClear = false;
+
+		this.controller = new AbortController();
+
+		this.initSource( data );
+	}
+
+	initSource( {
+		content,
+		promptType,
+		options = {},
+	}: ChromeAISuggestionsEventSourceConstructorArgs ) {
+		if ( promptType === PROMPT_TYPE_CHANGE_LANGUAGE ) {
+			this.translate( content, options.targetLanguage, options.sourceLanguage );
+		}
+
+		if ( promptType === PROMPT_TYPE_SUMMARIZE ) {
+			this.summarize( content );
+		}
+	}
+
+	async initEventSource() {}
+
+	close() {}
+
+	checkForUnclearPrompt() {}
+
+	processEvent( e: EventSourceMessage ) {
+		let data: ChromeAIEvent;
+		try {
+			data = JSON.parse( e.data );
+		} catch ( err ) {
+			this.processErrorEvent( err );
+			return;
+		}
+
+		if ( e.event === 'translation' ) {
+			this.dispatchEvent( new CustomEvent( 'suggestion', { detail: data.message } ) );
+		}
+
+		if ( data.complete ) {
+			this.dispatchEvent( new CustomEvent( 'done', { detail: data.message } ) );
+		}
+	}
+
+	processErrorEvent( e ) {
+		// Dispatch a generic network error event
+		this.dispatchEvent( new CustomEvent( ERROR_NETWORK, { detail: e } ) );
+		this.dispatchEvent(
+			new CustomEvent( ERROR_RESPONSE, {
+				detail: getErrorData( ERROR_NETWORK ),
+			} )
+		);
+	}
+
+	// use the Chrome AI translator
+	async translate( text: string, target: string, source: string = '' ) {
+		if ( ! ( 'translation' in self ) ) {
+			return;
+		}
+
+		const translator = await self.translation.createTranslator( {
+			sourceLanguage: source,
+			targetLanguage: target,
+		} );
+
+		if ( ! translator ) {
+			return;
+		}
+
+		try {
+			const translation = await translator.translate( renderHTMLFromMarkdown( { content: text } ) );
+			this.processEvent( {
+				id: '',
+				event: 'translation',
+				data: JSON.stringify( {
+					message: renderMarkdownFromHTML( { content: translation } ),
+					complete: true,
+				} ),
+			} );
+		} catch ( error ) {
+			this.processErrorEvent( error );
+		}
+	}
+
+	// TODO
+	async summarize( text: string ) {
+		return text;
+	}
+}

projects/js-packages/ai-client/src/hooks/use-ai-suggestions/index.ts

@@ -7,6 +7,7 @@ import { __ } from '@wordpress/i18n';
  * Internal dependencies
  */
 import askQuestion from '../../ask-question/index.js';
+import ChromeAIFactory from '../../chrome-ai/factory.js';
 import {
 	ERROR_CONTEXT_TOO_LARGE,
 	ERROR_MODERATION,
@@ -314,7 +315,14 @@ export default function useAiSuggestions( {
 			// Set the request status.
 			setRequestingState( 'requesting' );
 
-			eventSourceRef.current = await askQuestion( promptArg, options );
+			// check if we can (or should) use Chrome AI
+			const chromeAI = await ChromeAIFactory( promptArg );
+
+			if ( chromeAI !== false ) {
+				eventSourceRef.current = chromeAI;
+			} else {
+				eventSourceRef.current = await askQuestion( promptArg, options );
+			}
 
 			if ( ! eventSourceRef?.current ) {
 				return;

projects/js-packages/ai-client/src/index.ts

@@ -55,3 +55,8 @@ export * from './constants.js';
  * Logo Generator
  */
 export * from './logo-generator/index.js';
+
+/**
+ * Chrome AI
+ */
+export * from './chrome-ai/index.js';

projects/js-packages/ai-client/src/types.ts

@@ -132,3 +132,32 @@ export interface BlockEditorStore {
 		[ key in keyof typeof BlockEditorSelectors ]: ( typeof BlockEditorSelectors )[ key ];
 	};
 }
+
+declare global {
+	interface Window {
+		translation?: {
+			canTranslate: ( options: {
+				sourceLanguage: string;
+				targetLanguage: string;
+			} ) => Promise< 'no' | 'yes' | string >;
+			createTranslator: ( options: {
+				sourceLanguage: string;
+				targetLanguage: string;
+			} ) => Promise< {
+				translate: ( text: string ) => Promise< string >;
+			} >;
+		};
+		ai?: {
+			languageDetector: {
+				create: () => Promise< {
+					detect: ( text: string ) => Promise<
+						{
+							detectedLanguage: string;
+							confidence: number;
+						}[]
+					>;
+				} >;
+			};
+		};
+	}
+}

projects/plugins/jetpack/changelog/change-add-chrome-ai-translate-support

@@ -0,0 +1,4 @@
+Significance: patch
+Type: other
+
+Jetpack AI: Adding translation support using Chrome's Gemini AI mini