Skip to content

Commit

Permalink
refactor: Office document parsing will merge sub-documents into a sin…
Browse files Browse the repository at this point in the history 
…gle document

refactor: No more AI naming of topics for empty conversations
refactor: Optimize question output experience
fix: Fixed the issue that cloudflare page does not support fetch cache
  • Loading branch information
@Amery2010
Amery2010 committed Feb 9, 2025
1 parent 97993cb commit 726b39d
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 31 deletions.
4 changes: 1 addition & 3 deletions app/api/models/route.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@ export async function GET(req: NextRequest) {

try {
const apiBaseUrl = geminiApiBaseUrl || 'https://generativelanguage.googleapis.com'
const response = await fetch(`${apiBaseUrl}/v1beta/models?key=${geminiApiKey}`, {
cache: 'no-store',
})
const response = await fetch(`${apiBaseUrl}/v1beta/models?key=${geminiApiKey}`)
const result = await response.json()
return NextResponse.json(result)
} catch (error) {
Expand Down
10 changes: 5 additions & 5 deletions app/page.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,6 @@ export default function Home() {
onMessage: (content) => {
text += content
setMessage(text)
// scrollToBottom()
},
onStatement: (statement) => {
if (talkMode === 'voice') {
Expand All @@ -370,7 +369,6 @@ export default function Home() {
addMessage(message)
setMessage('')
setThinkingMessage('')
// scrollToBottom()
setIsThinking(false)
stopGeneratingRef.current = false
setExecutingPlugins([])
Expand Down Expand Up @@ -657,6 +655,7 @@ export default function Home() {
setContent('')
clearAttachment()
setTextareaHeight(TEXTAREA_DEFAULT_HEIGHT)
scrollToBottom()
await fetchAnswer({
messages,
model,
Expand All @@ -665,7 +664,7 @@ export default function Home() {
onError: handleError,
})
},
[isOldVisionModel, fetchAnswer, handleResponse, handleFunctionCall, handleError, checkAccessStatus],
[isOldVisionModel, fetchAnswer, handleResponse, handleFunctionCall, handleError, checkAccessStatus, scrollToBottom],
)

const handleResubmit = useCallback(
Expand All @@ -684,6 +683,7 @@ export default function Home() {
}
}
}
scrollToBottom()
await fetchAnswer({
messages: [...messagesRef.current],
model,
Expand All @@ -692,7 +692,7 @@ export default function Home() {
onError: handleError,
})
},
[fetchAnswer, handleResponse, handleFunctionCall, handleError, checkAccessStatus],
[fetchAnswer, handleResponse, handleFunctionCall, handleError, checkAccessStatus, scrollToBottom],
)

const handleCleanMessage = useCallback(() => {
Expand Down Expand Up @@ -935,7 +935,7 @@ export default function Home() {
}, [])

return (
<main className="mx-auto flex h-screen w-full max-w-screen-md flex-col justify-between overflow-hidden">
<main className="mx-auto flex h-screen max-h-[-webkit-fill-available] w-full max-w-screen-md flex-col justify-between overflow-hidden">
<div className="flex justify-between px-4 pb-2 pr-2 pt-10 max-md:pt-4 max-sm:pr-2 max-sm:pt-4">
<div className="flex flex-row text-xl leading-8 text-red-400 max-sm:text-base">
<MessageCircleHeart className="h-10 w-10 max-sm:h-8 max-sm:w-8" />
Expand Down
4 changes: 4 additions & 0 deletions components/AppSidebar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,17 @@ function ConversationItem(props: Props) {
const { lang, apiKey, apiProxy, password } = useSettingStore.getState()
const { currentId, query, addOrUpdate } = useConversationStore.getState()
const { messages, systemInstruction, setTitle } = useMessageStore.getState()

const conversation = query(id)
const config: RequestProps = {
apiKey,
lang,
messages: id === currentId ? messages : conversation.messages,
systemRole: id === currentId ? systemInstruction : conversation.systemInstruction,
}

if (config.messages.length === 0) return false

if (apiKey !== '') {
config.baseUrl = apiProxy || GEMINI_API_BASE_URL
} else {
Expand Down
8 changes: 2 additions & 6 deletions components/FileUploader.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,8 @@ function FileUploader({ beforeUpload, afterUpload }: Props) {
const file = files[i]
if (mimeType.includes(file.type)) {
if (isOfficeFile(file.type)) {
const data = await parseOffice(file, { type: 'file' })
if (Array.isArray(data)) {
data.forEach((item) => {
fileList.push(item)
})
}
const newFile = await parseOffice(file, { type: 'file' })
if (newFile instanceof File) fileList.push(newFile)
} else {
fileList.push(file)
}
Expand Down
4 changes: 2 additions & 2 deletions components/PluginList.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ function PluginList() {
<PopoverContent className="max-h-[330px] w-48 overflow-y-auto">
<div>
<h3 className="p-2 text-sm text-slate-400">{t('officialPlugins')}</h3>
<div className="flex rounded-sm px-2 py-1 hover:bg-gray-100 dark:hover:bg-gray-900">
{/* <div className="flex rounded-sm px-2 py-1 hover:bg-gray-100 dark:hover:bg-gray-900">
<Label
className="inline-flex flex-1 cursor-pointer overflow-hidden leading-6 text-slate-500"
htmlFor={OFFICAL_PLUGINS.IMAGEN}
Expand All @@ -96,7 +96,7 @@ function PluginList() {
defaultChecked={enabledTools.includes(OFFICAL_PLUGINS.IMAGEN)}
onCheckedChange={(checkedState) => handleUsePlugin(OFFICAL_PLUGINS.IMAGEN, checkedState === true)}
/>
</div>
</div> */}
<div className="flex rounded-sm px-2 py-1 hover:bg-gray-100 dark:hover:bg-gray-900">
<Label
className="inline-flex flex-1 cursor-pointer overflow-hidden leading-6 text-slate-500"
Expand Down
68 changes: 53 additions & 15 deletions utils/officeParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,51 @@ function parseXMLString(xml: string) {
return parser.parseFromString(xml, 'text/xml')
}

function createMergedXmlDocument(): Document {
return document.implementation.createDocument(null, 'root') // 将 "root" 替换为您想要的根元素名称
}

function mergeXmlDocuments(mergedDoc: Document, xmlDocs: Document[]): void {
const mergedRoot = mergedDoc.documentElement
xmlDocs.forEach((xmlDoc) => {
const root = xmlDoc.documentElement
if (root) {
Array.from(root.childNodes).forEach((node) => {
mergedRoot.appendChild(mergedDoc.importNode(node, true))
})
}
})
}

function serializeXmlDocument(xmlDoc: Document): string {
const serializer = new XMLSerializer()
return serializer.serializeToString(xmlDoc)
}

function stringToBlob(xmlString: string): Blob {
return new Blob([xmlString], { type: 'text/xml' })
}

function blobToFile(blob: Blob, fileName: string): File {
return new File([blob], fileName, { type: blob.type })
}

async function mergeXmlBlobs(blobs: Blob[]): Promise<Blob> {
const xmlStrings = await Promise.all(blobs.map((blob) => blob.text()))
const xmlDocs = xmlStrings.map((xmlString) => parseXMLString(xmlString))

const mergedDoc = createMergedXmlDocument()
mergeXmlDocuments(mergedDoc, xmlDocs)

const mergedXmlString = serializeXmlDocument(mergedDoc)
return stringToBlob(mergedXmlString)
}

function extractFiles(zipInput: File, filterFn: (filename: string) => boolean): Promise<ExtractedFiles[]> {
return new Promise(async (resolve, reject) => {
const extractedFiles: ExtractedFiles[] = []
const processZipfile = async (entry: Entry) => {
if (filterFn(entry.filename)) {
console.log(entry)
if (entry.getData) {
const data = await entry.getData(new BlobWriter())
extractedFiles.push({
Expand All @@ -99,7 +138,7 @@ function extractFiles(zipInput: File, filterFn: (filename: string) => boolean):
})
}

export function parseWord(file: File, config: Partial<OfficeParserConfig>): Promise<string | File[]> {
export function parseWord(file: File, config: Partial<OfficeParserConfig>): Promise<string | File> {
/** The target content xml file for the docx file. */
const mainContentFileRegex = /word\/document[\d+]?.xml/g
const footnotesFileRegex = /word\/footnotes[\d+]?.xml/g
Expand Down Expand Up @@ -128,9 +167,8 @@ export function parseWord(file: File, config: Partial<OfficeParserConfig>): Prom
// ******************************************************************************************************
.then(async (files: ExtractedFiles[]) => {
if (config.type === 'file') {
return resolve(
files.map((item) => new File([item.data], `${file.name}(${item.filename})`, { type: 'text/xml' })),
)
const mergedBlob = await mergeXmlBlobs(files.map((item) => item.data))
return resolve(blobToFile(mergedBlob, file.name))
}

/** Store all the text content to respond. */
Expand All @@ -140,6 +178,7 @@ export function parseWord(file: File, config: Partial<OfficeParserConfig>): Prom
for await (const file of files) {
xmlContentArray.push(await file.data.text())
}

xmlContentArray.forEach((xmlContent) => {
/** Find text nodes with w:p tags */
const xmlParagraphNodesList = parseXMLString(xmlContent).getElementsByTagName('w:p')
Expand Down Expand Up @@ -168,7 +207,7 @@ export function parseWord(file: File, config: Partial<OfficeParserConfig>): Prom
})
}

export function parsePowerPoint(file: File, config: Partial<OfficeParserConfig>): Promise<string | File[]> {
export function parsePowerPoint(file: File, config: Partial<OfficeParserConfig>): Promise<string | File> {
// Files regex that hold our content of interest
const allFilesRegex = /ppt\/(notesSlides|slides)\/(notesSlide|slide)\d+.xml/g
const slidesRegex = /ppt\/slides\/slide\d+.xml/g
Expand Down Expand Up @@ -209,7 +248,8 @@ export function parsePowerPoint(file: File, config: Partial<OfficeParserConfig>)
// ******************************************************************************************************
.then(async (files: ExtractedFiles[]) => {
if (config.type === 'file') {
resolve(files.map((item) => new File([item.data], `${file.name}(${item.filename})`, { type: 'text/xml' })))
const mergedBlob = await mergeXmlBlobs(files.map((item) => item.data))
return resolve(blobToFile(mergedBlob, file.name))
}

/** Store all the text content to respond */
Expand Down Expand Up @@ -246,7 +286,7 @@ export function parsePowerPoint(file: File, config: Partial<OfficeParserConfig>)
})
}

export function parseExcel(file: File, config: Partial<OfficeParserConfig>): Promise<string | File[]> {
export function parseExcel(file: File, config: Partial<OfficeParserConfig>): Promise<string | File> {
// Files regex that hold our content of interest
const sheetsRegex = /xl\/worksheets\/sheet\d+.xml/g
const drawingsRegex = /xl\/drawings\/drawing\d+.xml/g
Expand Down Expand Up @@ -291,9 +331,8 @@ export function parseExcel(file: File, config: Partial<OfficeParserConfig>): Pro
files.push(fileOrFiles)
}
}
return resolve(
files.map((item) => new File([item.data], `${file.name}(${item.filename})`, { type: 'text/xml' })),
)
const mergedBlob = await mergeXmlBlobs(files.map((item) => item.data))
return resolve(blobToFile(mergedBlob, file.name))
}

/** Store all the text content to respond */
Expand Down Expand Up @@ -413,7 +452,7 @@ export function parseExcel(file: File, config: Partial<OfficeParserConfig>): Pro
})
}

export function parseOpenOffice(file: File, config: Partial<OfficeParserConfig>): Promise<string | File[]> {
export function parseOpenOffice(file: File, config: Partial<OfficeParserConfig>): Promise<string | File> {
/** The target content xml file for the openoffice file. */
const mainContentFilePath = 'content.xml'
const objectContentFilesRegex = /Object \d+\/content.xml/g
Expand Down Expand Up @@ -446,9 +485,8 @@ export function parseOpenOffice(file: File, config: Partial<OfficeParserConfig>)
files.push(fileOrFiles)
}
}
return resolve(
files.map((item) => new File([item.data], `${file.name}(${item.filename})`, { type: 'text/xml' })),
)
const mergedBlob = await mergeXmlBlobs(files.map((item) => item.data))
return resolve(blobToFile(mergedBlob, file.name))
}

/** Store all the notes text content to respond */
Expand Down

0 comments on commit 726b39d

Please sign in to comment.