Fix: Auto detect updated file encoding and read file with encoding[IN…

…S-5017] (#8428)

* Add a read file method in process with file encoding detection
* Add a file encoding picker
* Fix the unnecessary try-catch

packages/insomnia/package.json

@@ -49,6 +49,7 @@
     "aws4": "^1.12.0",
     "chai": "^4.3.4",
     "chai-json-schema": "1.5.1",
+    "chardet": "^2.0.0",
     "clone": "^2.1.2",
     "color": "^4.2.3",
     "content-disposition": "^0.5.4",

packages/insomnia/src/main/ipc/electron.ts

@@ -28,6 +28,7 @@ export type HandleChannels =
   | 'webSocket.open'
   | 'webSocket.readyState'
   | 'writeFile'
+  | 'readFile'
   | 'extractJsonFileFromPostmanDataDumpArchive'
   | 'secretStorage.setSecret'
   | 'secretStorage.getSecret'

packages/insomnia/src/main/ipc/main.ts

@@ -1,7 +1,9 @@
 import * as Sentry from '@sentry/electron/main';
+import chardet from 'chardet';
 import type { MarkerRange } from 'codemirror';
 import { app, BrowserWindow, type IpcRendererEvent, type MenuItemConstructorOptions, shell } from 'electron';
 import fs from 'fs';
+import iconv from 'iconv-lite';
 
 import { APP_START_TIME, LandingPage, SentryMetrics } from '../../common/sentry';
 import type { HiddenBrowserWindowBridgeAPI } from '../../hidden-window';
@@ -32,6 +34,7 @@ export interface RendererToMainBridgeAPI {
   setMenuBarVisibility: (visible: boolean) => void;
   installPlugin: typeof installPlugin;
   writeFile: (options: { path: string; content: string }) => Promise<string>;
+  readFile: (options: { path: string; encoding?: string }) => Promise<{ content: string; encoding: string }>;
   cancelCurlRequest: typeof cancelCurlRequest;
   curlRequest: typeof curlRequest;
   on: (channel: RendererOnChannels, listener: (event: IpcRendererEvent, ...args: any[]) => void) => () => void;
@@ -103,6 +106,33 @@ export function registerMainHandlers() {
     }
   });
 
+  ipcMainHandle('readFile', async (_, options: { path: string; encoding?: string }) => {
+    const defaultEncoding = 'utf8';
+    const contentBuffer = await fs.promises.readFile(options.path);
+    const { encoding } = options;
+    if (encoding) {
+      if (iconv.encodingExists(encoding)) {
+        const content = iconv.decode(contentBuffer, encoding);
+        return { content, encoding };
+      };
+      throw new Error(`Unsupported encoding: ${encoding} to read file`);
+    }
+    // using chardet to detect encoding
+    const detecedEncoding = chardet.detect(contentBuffer);
+    if (detecedEncoding) {
+      if (iconv.encodingExists(detecedEncoding)) {
+        const content = iconv.decode(contentBuffer, detecedEncoding);
+        return { content, encoding: detecedEncoding };
+      };
+      throw new Error(`Unsupported encoding: ${detecedEncoding} to read file`);
+    }
+    // failed to detect encoding, use default utf-8 as fallback
+    return {
+      content: iconv.decode(contentBuffer, defaultEncoding),
+      encoding: defaultEncoding,
+    };
+  });
+
   ipcMainHandle('curlRequest', (_, options: Parameters<typeof curlRequest>[0]) => {
     return curlRequest(options);
   });

packages/insomnia/src/preload.ts

@@ -108,6 +108,7 @@ const main: Window['main'] = {
   curlRequest: options => ipcRenderer.invoke('curlRequest', options),
   cancelCurlRequest: options => ipcRenderer.send('cancelCurlRequest', options),
   writeFile: options => ipcRenderer.invoke('writeFile', options),
+  readFile: options => ipcRenderer.invoke('readFile', options),
   on: (channel, listener) => {
     ipcRenderer.on(channel, listener);
     return () => ipcRenderer.removeListener(channel, listener);

packages/insomnia/src/ui/components/encoding-picker.tsx

@@ -0,0 +1,107 @@
+import React from 'react';
+import { Button, ComboBox, Group, Input, ListBox, ListBoxItem, Popover } from 'react-aria-components';
+
+import { fuzzyMatch } from '../../common/misc';
+import { Icon } from './icon';
+
+const BUILT_IN_ENCODINGS = [
+  { key: 'UTF-8', label: 'UTF-8' },
+  { key: 'UTF-16LE', label: 'UTF-16 LE' },
+  { key: 'UTF-16BE', label: 'UTF-16 BE' },
+  { key: 'UTF-32LE', label: 'UTF-32 LE' },
+  { key: 'UTF-32BE', label: 'UTF-32 BE' },
+  { key: 'ASCII', label: 'ASCII' },
+  { key: 'ISO-8859-1', label: 'Western European (Latin-1)' },
+  { key: 'ISO-8859-2', label: 'Central European (Latin-2)' },
+  { key: 'ISO-8859-3', label: 'South European (Latin-3)' },
+  { key: 'ISO-8859-4', label: 'North European (Latin-4)' },
+  { key: 'ISO-8859-5', label: 'Cyrillic' },
+  { key: 'ISO-8859-6', label: 'Arabic' },
+  { key: 'ISO-8859-7', label: 'Greek' },
+  { key: 'ISO-8859-8', label: 'Hebrew' },
+  { key: 'ISO-8859-9', label: 'Turkish (Latin-5)' },
+  { key: 'ISO-8859-10', label: 'Nordic (Latin-6)' },
+  { key: 'ISO-8859-11', label: 'Thai' },
+  { key: 'ISO-8859-12', label: 'Ethiopic' },
+  { key: 'ISO-8859-13', label: 'Baltic (Latin-7)' },
+  { key: 'ISO-8859-14', label: 'Celtic (Latin-8)' },
+  { key: 'ISO-8859-15', label: 'Western European (Latin-9)' },
+  { key: 'ISO-8859-16', label: 'Southeastern European (Latin-10)' },
+  { key: 'windows-1250', label: 'Windows-1250' },
+  { key: 'windows-1251', label: 'Windows-1251' },
+  { key: 'windows-1252', label: 'Windows-1252' },
+  { key: 'windows-1253', label: 'Windows-1253' },
+  { key: 'windows-1254', label: 'Windows-1254' },
+  { key: 'windows-1255', label: 'Windows-1255' },
+  { key: 'windows-1256', label: 'Windows-1256' },
+  { key: 'windows-1257', label: 'Windows-1257' },
+  { key: 'windows-1258', label: 'Windows-1258' },
+  { key: 'GB18030', label: 'GB 18030' },
+  { key: 'EUC-JP', label: 'EUC-JP' },
+  { key: 'EUC-KR', label: 'EUC-KR' },
+  { key: 'EUC-CN', label: 'EUC-CN' },
+  { key: 'Big5', label: 'Big5' },
+  { key: 'Shift_JIS', label: 'Shift_JIS' },
+  { key: 'KOI8-R', label: 'KOI8-R' },
+  { key: 'KOI8-U', label: 'KOI8-U' },
+  { key: 'KOI8-RU', label: 'KOI8-RU' },
+  { key: 'KOI8-T', label: 'KOI8-T' },
+];
+
+export const EncodingPicker = ({ encoding, onChange }: { encoding: string; onChange: (value: string) => void }) => {
+  return (
+    <ComboBox
+      aria-label='Encoding Selector'
+      className='inline-block'
+      selectedKey={encoding}
+      onSelectionChange={key => {
+        if (key) {
+          onChange(key as string);
+        }
+      }}
+      defaultFilter={(textValue, filter) => {
+        const encodingKey = BUILT_IN_ENCODINGS.find(e => e.label === textValue)?.key || '';
+        return Boolean(fuzzyMatch(
+          filter,
+          encodingKey,
+          { splitSpace: false, loose: true }
+        )?.indexes) || textValue.toLowerCase().includes(filter.toLowerCase());
+      }}
+    >
+      <Group className='flex border-solid border border-[--hl-sm] w-full pr-2 min-w-64'>
+        <Input className='flex-1 py-1 px-2'/>
+        <Button className="flex items-center transition-all bg-transparent">
+         <Icon icon="caret-down" />
+        </Button>
+      </Group>
+      <Popover className="overflow-y-hidden flex flex-col">
+        <ListBox
+          className="border select-none text-sm max-h-80 border-solid border-[--hl-sm] shadow-lg bg-[--color-bg] py-1 rounded-md overflow-y-auto focus:outline-none"
+          items={BUILT_IN_ENCODINGS}
+          aria-label="Encoding List"
+          autoFocus
+        >
+          {item => (
+            <ListBoxItem
+              aria-label={item.label}
+              textValue={item.label}
+              className="aria-disabled:opacity-30 rounded aria-disabled:cursor-not-allowed flex gap-2 px-[--padding-md] aria-selected:font-bold items-center text-[--color-font] h-[--line-height-xs] w-full text-md whitespace-nowrap bg-transparent hover:bg-[--hl-sm] disabled:cursor-not-allowed focus:bg-[--hl-xs] data-[focused]:bg-[--hl-xs] focus:outline-none transition-colors"
+            >
+              {({ isSelected }) => (
+                <>
+                  <span>{item.label}</span>
+                  {isSelected && (
+                    <Icon
+                      icon="check"
+                      className="ml-1 text-[--color-success] justify-self-end"
+                    />
+                  )}
+                </>
+              )}
+            </ListBoxItem>
+          )}
+        </ListBox>
+      </Popover>
+    </ComboBox>
+  );
+};

packages/insomnia/src/ui/components/modals/upload-runner-data-modal.tsx

@@ -14,6 +14,7 @@ import {
   TableHeader,
 } from 'react-aria-components';
 
+import { EncodingPicker } from '../encoding-picker';
 import { Icon } from '../icon';
 
 export type UploadDataType = Record<string, any>;
@@ -25,6 +26,10 @@ export interface UploadDataModalProps {
 
 const rowHeaderStyle = 'sticky normal-case top-[-8px] p-2 z-10 border-b border-[--hl-sm] bg-[--hl-xs] text-left text-xs font-semibold backdrop-blur backdrop-filter focus:outline-none';
 const rowCellStyle = 'whitespace-nowrap text-sm font-medium border-b border-solid border-[--hl-sm] group-last-of-type:border-none focus:outline-none';
+const supportedFileTypes = [
+  'application/json',
+  'text/csv',
+];
 
 export const genPreviewTableData = (uploadData: UploadDataType[]) => {
   // generate header and body data for preview table from upload data
@@ -45,21 +50,12 @@ export const UploadDataModal = ({ onUploadFile, onClose, userUploadData }: Uploa
   const [file, setUploadFile] = useState<File | null>(null);
   const [uploadDataHeaders, setUploadDataHeaders] = useState<string[]>([]);
   const [uploadData, setUploadData] = useState<UploadDataType[]>([]);
+  const [fileEncoding, setFileEncoding] = useState('');
   const [invalidFileReason, setInvalidFileReason] = useState('');
 
-  const handleFileSelect = (fileList: FileList | null) => {
-    setInvalidFileReason('');
-    setUploadData([]);
-    if (!fileList) {
-      return;
-    };
-    const files = Array.from(fileList);
-    const file = files[0];
-    setUploadFile(file);
-    const reader = new FileReader();
-    reader.onload = (e: ProgressEvent<FileReader>) => {
-      const content = e.target?.result as string;
-      if (file.type === 'application/json') {
+  const parseFileContent = (content: string, fileType: string) => {
+    try {
+      if (fileType === 'application/json') {
         try {
           const jsonDataContent = JSON.parse(content);
           if (Array.isArray(jsonDataContent)) {
@@ -76,7 +72,7 @@ export const UploadDataModal = ({ onUploadFile, onClose, userUploadData }: Uploa
         } catch (error) {
           setInvalidFileReason('Upload JSON file can not be parsed');
         }
-      } else if (file.type === 'text/csv') {
+      } else if (fileType === 'text/csv') {
         // Replace CRLF (Windows line break) and CR (Mac link break) with \n, then split into csv arrays
         const csvRows = content.replace(/\r\n|\r/g, '\n').split('\n').map(row => row.split(','));
         // at least 2 rows required for csv
@@ -93,13 +89,51 @@ export const UploadDataModal = ({ onUploadFile, onClose, userUploadData }: Uploa
           setInvalidFileReason('CSV file must contain at least two rows with first row as variable names');
         }
       } else {
-        setInvalidFileReason(`Uploaded file is unsupported ${file.type}`);
+
       }
+    } catch (error) {
+      setInvalidFileReason(`Failed to read file ${error?.message}`);
+    }
+  };
+
+  const handleFileSelect = async (fileList: FileList | null) => {
+    setInvalidFileReason('');
+    setUploadData([]);
+    if (!fileList) {
+      return;
     };
-    reader.onerror = () => {
-      setInvalidFileReason(`Failed to read file ${reader.error?.message}`);
+    const files = Array.from(fileList);
+    const file = files[0];
+    const fileType = file.type;
+    if (!supportedFileTypes.includes(fileType)) {
+      setInvalidFileReason(`Uploaded file is unsupported ${file.type}`);
+      return;
     };
-    reader.readAsText(file);
+    const filePath = window.webUtils.getPathForFile(file);
+    try {
+      const { content, encoding } = await window.main.readFile({ path: filePath });
+      setFileEncoding(encoding);
+      parseFileContent(content, fileType);
+    } catch (error) {
+      setInvalidFileReason(`Failed to read file ${error?.message}`);
+      return;
+    }
+    setUploadFile(file);
+  };
+
+  const handleEncodingChange = async (newEncoding: string) => {
+    setFileEncoding(newEncoding);
+    setInvalidFileReason('');
+    if (file) {
+      const filePath = window.webUtils.getPathForFile(file);
+      const fileType = file.type;
+      try {
+        const { content } = await window.main.readFile({ path: filePath, encoding: newEncoding });
+        parseFileContent(content, fileType);
+      } catch (error) {
+        setInvalidFileReason(`Failed to read file ${error?.message}`);
+      }
+    }
   };
 
   const handleUploadData = () => {
@@ -157,12 +191,21 @@ export const UploadDataModal = ({ onUploadFile, onClose, userUploadData }: Uploa
                   onSelect={handleFileSelect}
                   acceptedFileTypes={['.csv', '.json']}
                 >
-                  <Button className="flex flex-1 flex-shrink-0 border-solid border border-[--hl-`sm] py-1 gap-2 items-center justify-center px-2 aria-pressed:bg-[--hl-sm] aria-selected:bg-[--hl-sm] rounded-sm text-[--color-font] hover:bg-[--hl-xs] focus:ring-inset ring-1 ring-transparent transition-all text-base">
+                  <Button className="flex flex-1 flex-shrink-0 border-solid border border-[--hl-sm] py-1 gap-2 items-center justify-center px-2 aria-pressed:bg-[--hl-sm] aria-selected:bg-[--hl-sm] rounded-sm text-[--color-font] hover:bg-[--hl-xs] focus:ring-inset ring-1 ring-transparent transition-all text-base">
                     <Icon icon="upload" />
                     <span>{uploadData.length > 0 ? 'Change Data File' : 'Select Data File'}</span>
                   </Button>
                 </FileTrigger>
               </div>
+              {file && uploadData.length > 0 &&
+                <div>
+                  <span className='mr-4'>File Encoding</span>
+                  <EncodingPicker
+                    encoding={fileEncoding}
+                    onChange={handleEncodingChange}
+                  />
+                </div>
+              }
               {invalidFileReason !== '' &&
                 <div className="notice error margin-top-sm">
                   <p>{invalidFileReason}</p>