feat: TS extractors are back (#129)

package.json

@@ -37,37 +37,46 @@
     "glob": "^11.0.0",
     "json5": "^2.2.3",
     "jsonschema": "^1.4.1",
-    "openapi-fetch": "^0.10.6",
+    "openapi-fetch": "^0.13.0",
     "unescape-js": "^1.1.4",
     "vscode-oniguruma": "^2.0.1",
     "vscode-textmate": "^9.1.0",
-    "yauzl": "^3.1.3"
+    "yauzl": "^3.2.0"
   },
   "devDependencies": {
-    "@eslint/js": "^9.8.0",
+    "@eslint/js": "^9.15.0",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/git": "^10.0.1",
     "@tsconfig/node18": "^18.2.4",
-    "@tsconfig/recommended": "^1.0.7",
+    "@tsconfig/recommended": "^1.0.8",
     "@types/eslint__js": "^8.42.3",
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^22.1.0",
+    "@types/node": "^22.9.0",
     "@types/yauzl": "^2.10.3",
     "cross-env": "^7.0.3",
-    "eslint": "^9.8.0",
+    "eslint": "~9.14.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
+    "jiti": "^2.4.0",
     "js-yaml": "^4.1.0",
-    "json-schema-to-typescript": "^15.0.0",
-    "openapi-typescript": "^7.3.0",
+    "json-schema-to-typescript": "^15.0.3",
+    "openapi-typescript": "^7.4.3",
     "premove": "^4.0.0",
     "prettier": "^3.3.3",
-    "semantic-release": "^24.0.0",
+    "semantic-release": "^24.2.0",
     "tree-cli": "^0.6.7",
-    "tsx": "^4.17.0",
-    "typescript": "^5.5.4",
-    "typescript-eslint": "^8.0.1",
-    "vitest": "^2.0.5"
+    "tsx": "^4.19.2",
+    "typescript": "^5.6.3",
+    "typescript-eslint": "^8.14.0",
+    "vitest": "^2.1.5"
+  },
+  "peerDependencies": {
+    "jiti": ">= 2"
+  },
+  "peerDependenciesMeta": {
+    "jiti": {
+      "optional": true
+    }
   },
   "engines": {
     "node": ">= 18"

src/extractor/runner.ts

@@ -23,16 +23,27 @@ function parseVerbose(v: VerboseOption[] | boolean | undefined) {
 
 export async function extractKeysFromFile(
   file: string,
-  parserType: ParserType,
+  parserType: ParserType | undefined,
   options: ExtractOptions,
-  extractor?: string
+  extractor: string | undefined
 ) {
-  return callWorker({
-    extractor: extractor,
-    parserType,
-    file: file,
-    options,
-  });
+  if (typeof extractor !== 'undefined') {
+    return callWorker({
+      extractor,
+      file,
+      options,
+    });
+  } else if (typeof parserType !== 'undefined') {
+    return callWorker({
+      parserType,
+      file,
+      options,
+    });
+  }
+
+  throw new Error(
+    'Internal error: neither the parser type nor a custom extractors have been defined! Please report this.'
+  );
 }
 
 export function findPossibleFrameworks(fileNames: string[]) {
@@ -101,8 +112,7 @@ export async function extractKeysOfFiles(opts: Opts) {
   }
 
   let parserType = opts.parser;
-
-  if (!parserType) {
+  if (!parserType && !opts.extractor) {
     parserType = detectParserType(files);
   }
 

src/extractor/worker.ts

@@ -1,49 +1,52 @@
 import type {
-  ExtractOptions,
   ExtractionResult,
+  ExtractOptions,
   Extractor,
   ParserType,
 } from './index.js';
 import { fileURLToPath } from 'url';
-import { resolve, extname } from 'path';
-import { Worker, isMainThread, parentPort } from 'worker_threads';
-import { readFile } from 'fs/promises';
+import { extname, resolve } from 'path';
+import { isMainThread, parentPort, SHARE_ENV, Worker } from 'worker_threads';
+import { readFileSync } from 'fs';
 
 import internalExtractor from './extractor.js';
 import { loadModule } from '../utils/moduleLoader.js';
-import { type Deferred, createDeferred } from '../utils/deferred.js';
+import { createDeferred, type Deferred } from '../utils/deferred.js';
 
 const FILE_TIME_LIMIT = 60 * 1000; // one minute
 
-export type WorkerParams = {
-  extractor?: string;
-  file: string;
-  parserType: ParserType;
-  options: ExtractOptions;
-};
+export type WorkerParams =
+  | {
+      file: string;
+      parserType: ParserType;
+      options: ExtractOptions;
+    }
+  | {
+      extractor: string;
+      file: string;
+      options: ExtractOptions;
+    };
 
-const IS_TS_NODE = extname(import.meta.url) === '.ts';
+const IS_TSX = extname(import.meta.url) === '.ts';
 
 // --- Worker functions
 
-let loadedExtractor: string | undefined | symbol = Symbol('unloaded');
 let extractor: Extractor;
 
 async function handleJob(args: WorkerParams): Promise<ExtractionResult> {
   const file = resolve(args.file);
-  const code = await readFile(file, 'utf8');
-  if (args.extractor) {
-    if (args.extractor !== loadedExtractor) {
-      loadedExtractor = args.extractor;
+  const code = readFileSync(file, 'utf8');
+  if ('extractor' in args) {
+    if (!extractor) {
       extractor = await loadModule(args.extractor).then((mdl) => mdl.default);
     }
     return extractor(code, file, args.options);
-  } else {
-    return internalExtractor(code, file, args.parserType, args.options);
   }
+
+  return internalExtractor(code, file, args.parserType, args.options);
 }
 
-async function workerInit() {
+function workerInit() {
   parentPort!.on('message', (params) => {
     handleJob(params)
       .then((res) => parentPort!.postMessage({ data: res }))
@@ -59,15 +62,20 @@ let worker: Worker;
 const jobQueue: Array<[WorkerParams, Deferred]> = [];
 
 function createWorker() {
-  const worker = IS_TS_NODE
-    ? new Worker(
-        fileURLToPath(new URL(import.meta.url)).replace('.ts', '.js'),
-        {
-          // ts-node workaround
-          execArgv: ['--require', 'ts-node/register'],
-        }
-      )
-    : new Worker(fileURLToPath(new URL(import.meta.url)));
+  let worker: Worker;
+  if (IS_TSX) {
+    worker = new Worker(
+      `import('tsx/esm/api').then(({ register }) => { register(); import('${fileURLToPath(new URL(import.meta.url))}') })`,
+      {
+        env: SHARE_ENV,
+        eval: true,
+      }
+    );
+  } else {
+    worker = new Worker(fileURLToPath(new URL(import.meta.url)), {
+      env: SHARE_ENV,
+    });
+  }
 
   let timeout: NodeJS.Timeout;
   let currentDeferred: Deferred;

src/utils/moduleLoader.ts

@@ -1,33 +1,26 @@
-import { extname } from 'path';
-
-let tsService: any;
-
-async function registerTsNode() {
-  if (!tsService) {
-    // try {
-    //   const tsNode = await import('ts-node');
-    //   tsService = tsNode.register({ compilerOptions: { module: 'CommonJS' } });
-    // } catch (e: any) {
-    //   if (e.code === 'ERR_MODULE_NOT_FOUND') {
-    //     throw new Error('ts-node is required to load TypeScript files.');
-    //   }
-    //   throw e;
-    // }
-  }
-}
+import type { Jiti } from 'jiti';
+
+let jiti: Jiti;
 
+// https://github.com/eslint/eslint/blob/6f37b0747a14dfa9a9e3bdebc5caed1f39b6b0e2/lib/config/config-loader.js#L164-L197
 async function importTypeScript(file: string) {
-  if (extname(import.meta.url) === '.ts') {
+  // @ts-ignore
+  if (!!globalThis.Bun || !!globalThis.Deno) {
+    // We're in an env that natively supports TS
     return import(file);
   }
 
-  await registerTsNode();
+  if (!jiti) {
+    const { createJiti } = await import('jiti').catch(() => {
+      throw new Error(
+        "The 'jiti' library is required for loading TypeScript extractors. Make sure to install it."
+      );
+    });
 
-  tsService.enabled(true);
-  const mdl = await import(file);
-  tsService.enabled(false);
+    jiti = createJiti(import.meta.url);
+  }
 
-  return mdl;
+  return jiti.import(file);
 }
 
 export async function loadModule(module: string) {

test/__fixtures__/customExtractors/extract-js.js

@@ -0,0 +1,14 @@
+export default function (code) {
+  // Very simple, trivial extractor for the purposes of testing
+  const keys = [];
+  const lines = code.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    for (const [str] of lines[i].matchAll(/STR_[A-Z_]+/g)) {
+      keys.push({ keyName: str, line: i + 1 });
+    }
+  }
+
+  return {
+    keys,
+  };
+}

test/__fixtures__/customExtractors/extract-ts.ts

@@ -0,0 +1,16 @@
+import type { ExtractionResult, ExtractedKey } from '#cli/extractor/index.js';
+
+export default function (code: string): ExtractionResult {
+  // Very simple, trivial extractor for the purposes of testing
+  const keys: ExtractedKey[] = [];
+  const lines = code.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    for (const [str] of lines[i].matchAll(/STR_[A-Z_]+/g)) {
+      keys.push({ keyName: str, line: i + 1 });
+    }
+  }
+
+  return {
+    keys,
+  };
+}

test/__fixtures__/customExtractors/testfile.txt

@@ -0,0 +1,7 @@
+Hey this is a very STR_CRUDE test
+to see if the custom extractors are STR_WORKING
+properly.
+
+Hopefully, this will catch any STR_NEW
+issue that may happen before it makes its
+way to STR_PRODUCTION!

test/e2e/extractCustom.test.ts

@@ -0,0 +1,40 @@
+import { fileURLToPathSlash } from './utils/toFilePath.js';
+import { run } from './utils/run.js';
+
+const FIXTURES_PATH = new URL('../__fixtures__/', import.meta.url);
+const FAKE_PROJECT = new URL('./customExtractors/', FIXTURES_PATH);
+const TEST_FILE = fileURLToPathSlash(new URL('./testfile.txt', FAKE_PROJECT));
+const JS_EXTRACTOR = fileURLToPathSlash(
+  new URL('./extract-js.js', FAKE_PROJECT)
+);
+const TS_EXTRACTOR = fileURLToPathSlash(
+  new URL('./extract-ts.ts', FAKE_PROJECT)
+);
+
+it('successfully uses a custom extractor written in JS', async () => {
+  const out = await run(
+    ['extract', 'print', '--extractor', JS_EXTRACTOR, '--patterns', TEST_FILE],
+    undefined,
+    50e3
+  );
+
+  expect(out.code).toBe(0);
+  expect(out.stdout).toContain('STR_CRUDE');
+  expect(out.stdout).toContain('STR_WORKING');
+  expect(out.stdout).toContain('STR_NEW');
+  expect(out.stdout).toContain('STR_PRODUCTION');
+}, 60e3);
+
+it('successfully uses a custom extractor written in TS', async () => {
+  const out = await run(
+    ['extract', 'print', '--extractor', TS_EXTRACTOR, '--patterns', TEST_FILE],
+    undefined,
+    50e3
+  );
+
+  expect(out.code).toBe(0);
+  expect(out.stdout).toContain('STR_CRUDE');
+  expect(out.stdout).toContain('STR_WORKING');
+  expect(out.stdout).toContain('STR_NEW');
+  expect(out.stdout).toContain('STR_PRODUCTION');
+}, 60e3);