diff --git a/package.json b/package.json index 683c7e538..972a1a480 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,11 @@ "dist/**/*.js", "dist/**/*.lua", "dist/**/*.ts", - "dist/lualib/**/*.json" + "dist/lualib/**/*.json", + "src/lualib/**/*.ts", + "src/lualib/**/*.json", + "src/lualib-build/**/*.ts", + "language-extensions/index.d.ts" ], "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/src/CompilerOptions.ts b/src/CompilerOptions.ts index 7f5562c07..3dcb8824e 100644 --- a/src/CompilerOptions.ts +++ b/src/CompilerOptions.ts @@ -20,12 +20,15 @@ export interface TransformerImport { export interface LuaPluginImport { name: string; import?: string; + skipRecompileLuaLib?: boolean; [option: string]: any; } export interface InMemoryLuaPlugin { plugin: Plugin | ((options: Record) => Plugin); + skipRecompileLuaLib?: boolean; + [option: string]: any; } @@ -46,6 +49,7 @@ export interface TypeScriptToLuaOptions { tstlVerbose?: boolean; lua51AllowTryCatchInAsyncAwait?: boolean; measurePerformance?: boolean; + recompileLuaLib?: boolean; } export type CompilerOptions = OmitIndexSignature & diff --git a/src/LuaLib.ts b/src/LuaLib.ts index 6320bc99c..35781a9d2 100644 --- a/src/LuaLib.ts +++ b/src/LuaLib.ts @@ -1,8 +1,11 @@ import * as path from "path"; -import { EmitHost } from "./transpilation"; +import { EmitHost, transpileProject } from "./transpilation"; import * as lua from "./LuaAST"; -import { LuaTarget } from "./CompilerOptions"; +import { LuaTarget, type CompilerOptions } from "./CompilerOptions"; import { getOrUpdate } from "./utils"; +import { createEmitOutputCollector, type TranspiledFile } from "./transpilation/output-collector"; +import { parseConfigFileWithSystem } from "./cli/tsconfig"; +import { createDiagnosticReporter } from "./cli/report"; export enum LuaLibFeature { ArrayAt = "ArrayAt", @@ -139,12 +142,16 @@ export function resolveLuaLibDir(luaTarget: LuaTarget) { export const luaLibModulesInfoFileName = "lualib_module_info.json"; const luaLibModulesInfo = new Map(); -export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost): LuaLibModulesInfo { - if (!luaLibModulesInfo.has(luaTarget)) { +export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost, useCache = true): LuaLibModulesInfo { + if (!useCache || !luaLibModulesInfo.has(luaTarget)) { const lualibPath = path.join(resolveLuaLibDir(luaTarget), luaLibModulesInfoFileName); const result = emitHost.readFile(lualibPath); if (result !== undefined) { - luaLibModulesInfo.set(luaTarget, JSON.parse(result) as LuaLibModulesInfo); + const info = JSON.parse(result) as LuaLibModulesInfo; + if (!useCache) { + return info; + } + luaLibModulesInfo.set(luaTarget, info); } else { throw new Error(`Could not load lualib dependencies from '${lualibPath}'`); } @@ -175,7 +182,21 @@ export function getLuaLibExportToFeatureMap( const lualibFeatureCache = new Map>(); -export function readLuaLibFeature(feature: LuaLibFeature, luaTarget: LuaTarget, emitHost: EmitHost): string { +export function readLuaLibFeature( + feature: LuaLibFeature, + luaTarget: LuaTarget, + emitHost: EmitHost, + useCache = true +): string { + if (!useCache) { + const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`); + const luaLibFeature = emitHost.readFile(featurePath); + if (luaLibFeature === undefined) { + throw new Error(`Could not load lualib feature from '${featurePath}'`); + } + return luaLibFeature; + } + const featureMap = getOrUpdate(lualibFeatureCache, luaTarget, () => new Map()); if (!featureMap.has(feature)) { const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`); @@ -257,9 +278,69 @@ export function loadImportedLualibFeatures( return statements; } +const recompileLualibCache = new WeakMap(); + +function recompileLuaLibFiles(sourceOptions: CompilerOptions, emitHost: EmitHost): TranspiledFile[] { + let transpiledFiles = recompileLualibCache.get(emitHost); + if (!transpiledFiles) { + const tsconfigPath = + sourceOptions.luaTarget === LuaTarget.Lua50 + ? path.join(__dirname, "../src/lualib/tsconfig.lua50.json") + : path.join(__dirname, "../src/lualib/tsconfig.json"); + const config = parseConfigFileWithSystem(tsconfigPath); + const options = config.options; + const sourcePlugins = (sourceOptions.luaPlugins ?? []).filter(p => !p.skipRecompileLuaLib); + options.luaPlugins = [...(options.luaPlugins ?? []), ...sourcePlugins]; + + const collector = createEmitOutputCollector(options.extension); + const reportDiagnostic = createDiagnosticReporter(false); + + const { diagnostics } = transpileProject(tsconfigPath, options, collector.writeFile); + diagnostics.forEach(reportDiagnostic); + + transpiledFiles = collector.files; + recompileLualibCache.set(emitHost, transpiledFiles); + } + + return transpiledFiles; +} + +function recompileLuaLibBundle(sourceOptions: CompilerOptions, emitHost: EmitHost): string | undefined { + const transpiledFiles = recompileLuaLibFiles(sourceOptions, emitHost); + const lualibBundle = transpiledFiles.find(f => f.outPath.endsWith("lualib_bundle.lua")); + return lualibBundle?.lua; +} + +export function recompileInlineLualibFeatures( + features: Iterable, + options: CompilerOptions, + emitHost: EmitHost +): string { + const luaTarget = options.luaTarget ?? LuaTarget.Universal; + const transpiledFiles = recompileLuaLibFiles(options, emitHost); + emitHost = { + readFile(filePath: string) { + const file = transpiledFiles.find(f => f.outPath === filePath); + return file ? file.text : undefined; + }, + } as any as EmitHost; + const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, false); + return resolveRecursiveLualibFeatures(features, luaTarget, emitHost, moduleInfo) + .map(feature => readLuaLibFeature(feature, luaTarget, emitHost, false)) + .join("\n"); +} + const luaLibBundleContent = new Map(); -export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost): string { +export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost, options: CompilerOptions): string { + if (options.recompileLuaLib) { + const result = recompileLuaLibBundle(options, emitHost); + if (!result) { + throw new Error(`Failed to recompile lualib bundle`); + } + return result; + } + const lualibPath = path.join(resolveLuaLibDir(luaTarget), "lualib_bundle.lua"); if (!luaLibBundleContent.has(lualibPath)) { const result = emitHost.readFile(lualibPath); @@ -279,13 +360,26 @@ export function getLualibBundleReturn(exportedValues: string[]): string { export function buildMinimalLualibBundle( features: Iterable, - luaTarget: LuaTarget, + options: CompilerOptions, emitHost: EmitHost ): string { - const code = loadInlineLualibFeatures(features, luaTarget, emitHost); - const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost); - const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports); + const luaTarget = options.luaTarget ?? LuaTarget.Universal; + let code; + if (options.recompileLuaLib) { + code = recompileInlineLualibFeatures(features, options, emitHost); + const transpiledFiles = recompileLuaLibFiles(options, emitHost); + emitHost = { + readFile(filePath: string) { + const file = transpiledFiles.find(f => f.outPath === filePath); + return file ? file.text : undefined; + }, + } as any as EmitHost; + } else { + code = loadInlineLualibFeatures(features, luaTarget, emitHost); + } + const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, !options.recompileLuaLib); + const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports); return code + getLualibBundleReturn(exports); } diff --git a/src/LuaPrinter.ts b/src/LuaPrinter.ts index b0a02dfaf..527fcb3c4 100644 --- a/src/LuaPrinter.ts +++ b/src/LuaPrinter.ts @@ -3,7 +3,12 @@ import { Mapping, SourceMapGenerator, SourceNode } from "source-map"; import * as ts from "typescript"; import { CompilerOptions, isBundleEnabled, LuaLibImportKind, LuaTarget } from "./CompilerOptions"; import * as lua from "./LuaAST"; -import { loadImportedLualibFeatures, loadInlineLualibFeatures, LuaLibFeature } from "./LuaLib"; +import { + loadImportedLualibFeatures, + loadInlineLualibFeatures, + LuaLibFeature, + recompileInlineLualibFeatures, +} from "./LuaLib"; import { isValidLuaIdentifier, shouldAllowUnicode } from "./transformation/utils/safe-names"; import { EmitHost, getEmitPath } from "./transpilation"; import { intersperse, normalizeSlashes } from "./utils"; @@ -246,7 +251,11 @@ export class LuaPrinter { } else if (luaLibImport === LuaLibImportKind.Inline && file.luaLibFeatures.size > 0) { // Inline lualib features sourceChunks.push("-- Lua Library inline imports\n"); - sourceChunks.push(loadInlineLualibFeatures(file.luaLibFeatures, luaTarget, this.emitHost)); + if (this.options.recompileLuaLib) { + sourceChunks.push(recompileInlineLualibFeatures(file.luaLibFeatures, this.options, this.emitHost)); + } else { + sourceChunks.push(loadInlineLualibFeatures(file.luaLibFeatures, luaTarget, this.emitHost)); + } sourceChunks.push("-- End of Lua Library inline imports\n"); } diff --git a/src/cli/parse.ts b/src/cli/parse.ts index ca4f039ad..c45ee9254 100644 --- a/src/cli/parse.ts +++ b/src/cli/parse.ts @@ -104,6 +104,11 @@ export const optionDeclarations: CommandLineOption[] = [ description: "Measure performance of the tstl compiler.", type: "boolean", }, + { + name: "recompileLuaLib", + description: "Recompile the Lua standard library with custom plugins.", + type: "boolean", + }, ]; export function updateParsedConfigFile(parsedConfigFile: ts.ParsedCommandLine): ParsedCommandLine { diff --git a/src/lualib-build/plugin.ts b/src/lualib-build/plugin.ts index 79233f3df..66137e614 100644 --- a/src/lualib-build/plugin.ts +++ b/src/lualib-build/plugin.ts @@ -18,7 +18,8 @@ import { isImport, isRequire, } from "./util"; -import { createDiagnosticFactoryWithCode } from "../utils"; +import { createDiagnosticFactoryWithCode, normalizeSlashes } from "../utils"; +import type { CompilerOptions } from ".."; export const lualibDiagnostic = createDiagnosticFactoryWithCode(200000, (message: string, file?: ts.SourceFile) => ({ messageText: message, @@ -168,6 +169,27 @@ class LuaLibPlugin implements tstl.Plugin { } return { result: result as LuaLibModulesInfo, diagnostics }; } + + public moduleResolution( + moduleIdentifier: string, + requiringFile: string, + _options: CompilerOptions, + emitHost: EmitHost + ): string | undefined { + const tstlRoot = path.resolve(__dirname, "../.."); + const relativeRequiringFile = normalizeSlashes(path.relative(tstlRoot, requiringFile)); + if (!relativeRequiringFile.startsWith("src/lualib/")) { + return; + } + + const tryingPaths = ["./", "./universal/"]; + for (const tryingPath of tryingPaths) { + const possiblePath = path.join(tstlRoot, "src", "lualib", tryingPath, `${moduleIdentifier}.ts`); + if (emitHost.fileExists(possiblePath)) { + return possiblePath; + } + } + } } class LuaLibPrinter extends tstl.LuaPrinter { diff --git a/src/transpilation/output-collector.ts b/src/transpilation/output-collector.ts index 1458b2403..10e5552f6 100644 --- a/src/transpilation/output-collector.ts +++ b/src/transpilation/output-collector.ts @@ -12,6 +12,8 @@ export interface TranspiledFile { js?: string; /** @internal */ jsSourceMap?: string; + /** @internal */ + text?: string; } export function createEmitOutputCollector(luaExtension = ".lua") { @@ -38,6 +40,7 @@ export function createEmitOutputCollector(luaExtension = ".lua") { } else if (fileName.endsWith(".d.ts.map")) { file.declarationMap = data; } + file.text = data; }; return { writeFile, files }; diff --git a/src/transpilation/transpile.ts b/src/transpilation/transpile.ts index 6adfa5fc4..a52b5172c 100644 --- a/src/transpilation/transpile.ts +++ b/src/transpilation/transpile.ts @@ -145,9 +145,16 @@ export function getProgramTranspileResult( transpiledFiles = []; } + const proxyEmitHost = + writeFileResult !== emitHost.writeFile + ? new Proxy(emitHost, { + get: (target, prop) => (prop === "writeFile" ? writeFileResult : target[prop as keyof EmitHost]), + }) + : emitHost; + for (const plugin of plugins) { if (plugin.afterPrint) { - const pluginDiagnostics = plugin.afterPrint(program, options, emitHost, transpiledFiles) ?? []; + const pluginDiagnostics = plugin.afterPrint(program, options, proxyEmitHost, transpiledFiles) ?? []; diagnostics.push(...pluginDiagnostics); } } diff --git a/src/transpilation/transpiler.ts b/src/transpilation/transpiler.ts index 9bac1f5bf..5dc5f66b2 100644 --- a/src/transpilation/transpiler.ts +++ b/src/transpilation/transpiler.ts @@ -162,9 +162,9 @@ export class Transpiler { this.emitHost, resolvedFiles.map(f => f.code) ); - return buildMinimalLualibBundle(usedFeatures, luaTarget, this.emitHost); + return buildMinimalLualibBundle(usedFeatures, options, this.emitHost); } else { - return getLuaLibBundle(luaTarget, this.emitHost); + return getLuaLibBundle(luaTarget, this.emitHost, options); } } } diff --git a/test/transpile/lualib.spec.ts b/test/transpile/lualib.spec.ts index 20288bd8a..4b38cc274 100644 --- a/test/transpile/lualib.spec.ts +++ b/test/transpile/lualib.spec.ts @@ -1,7 +1,8 @@ import * as ts from "typescript"; -import { LuaLibFeature, LuaTarget } from "../../src"; +import { LuaLibFeature, LuaLibImportKind, LuaTarget } from "../../src"; import { readLuaLibFeature } from "../../src/LuaLib"; import * as util from "../util"; +import path = require("path"); test.each(Object.entries(LuaLibFeature))("Lualib does not use ____exports (%p)", (_, feature) => { const lualibCode = readLuaLibFeature(feature, LuaTarget.Lua54, ts.sys); @@ -29,3 +30,58 @@ test("Lualib bundle does not assign globals", () => { .withLanguageExtensions() .expectNoExecutionError(); }); + +test("Lualib recompile with import kind require", () => { + const { transpiledFiles } = util.testExpression`Array.isArray({})` + .setOptions({ + recompileLuaLib: true, + luaPlugins: [{ name: path.join(__dirname, "./plugins/beforeEmit.ts") }], + }) + .expectToHaveNoDiagnostics() + .getLuaResult(); + const lualibBundle = transpiledFiles.find(f => f.outPath === "lualib_bundle.lua"); + expect(lualibBundle).toBeDefined(); + + // plugin apply twice: main compilation and lualib recompile + expect(lualibBundle?.lua).toContain( + "-- Comment added by beforeEmit plugin\n-- Comment added by beforeEmit plugin\n" + ); +}); + +test("Lualib recompile with import kind require minimal", () => { + const builder = util.testExpression`Array.isArray(123)` + .setOptions({ + luaLibImport: LuaLibImportKind.RequireMinimal, + recompileLuaLib: true, + luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }], + }) + .expectToHaveNoDiagnostics() + .expectToEqual(true); + const transpiledFiles = builder.getLuaResult().transpiledFiles; + const lualibBundle = transpiledFiles.find(f => f.outPath === "lualib_bundle.lua"); + expect(lualibBundle).toBeDefined(); +}); + +test("Lualib recompile with import kind inline", () => { + util.testExpression`Array.isArray(123)` + .setOptions({ + luaLibImport: LuaLibImportKind.Inline, + recompileLuaLib: true, + luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }], + }) + .expectToHaveNoDiagnostics() + .expectToEqual(true); +}); + +test("Lualib recompile with bundling", () => { + util.testExpression`Array.isArray(123)` + .setOptions({ + luaBundle: "bundle.lua", + luaBundleEntry: "main.ts", + luaLibImport: LuaLibImportKind.RequireMinimal, + recompileLuaLib: true, + luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }], + }) + .expectToHaveNoDiagnostics() + .expectToEqual(true); +}); diff --git a/tsconfig-schema.json b/tsconfig-schema.json index 87366ea76..eb3d96ac2 100644 --- a/tsconfig-schema.json +++ b/tsconfig-schema.json @@ -102,6 +102,10 @@ "measurePerformance": { "description": "Measure and report performance of the tstl compiler.", "type": "boolean" + }, + "recompileLuaLib": { + "description": "Recompile the Lua standard library with custom plugins.", + "type": "boolean" } }, "dependencies": {