[gjs/esm/dynamic-imports: 2/3] Implement dynamic imports

[Philip Chimento <pchimento src gnome org>]

commit 97c2fd83488d567c1d83481478ac7bee2335505b
Author: Evan Welsh <contact evanwelsh com>
Date:   Sat Nov 14 14:36:27 2020 -0600

    Implement dynamic imports
    
    (Changes from Philip folded in: tests, moving file operations into
    internal.cpp, some added comments)

 gjs/context.cpp                          |   1 +
 gjs/global.cpp                           |   2 +
 gjs/internal.cpp                         | 150 +++++++++++++++++++++++++++++++
 gjs/module.cpp                           | 138 ++++++++++++++++++++++++++++
 gjs/module.h                             |   6 ++
 installed-tests/js/.eslintrc.yml         |   1 +
 installed-tests/js/jsunit.gresources.xml |   1 +
 installed-tests/js/modules/say.js        |  10 +++
 installed-tests/js/testESModules.js      |  44 +++++++++
 modules/internal/.eslintrc.yml           |   1 +
 modules/internal/loader.js               |  80 +++++++++++++++++
 11 files changed, 434 insertions(+)
---
diff --git a/gjs/context.cpp b/gjs/context.cpp
index 4d8e7144..64ef9605 100644
--- a/gjs/context.cpp
+++ b/gjs/context.cpp
@@ -553,6 +553,7 @@ GjsContextPrivate::GjsContextPrivate(JSContext* cx, GjsContext* public_context)
     }
 
     JS::SetModuleResolveHook(rt, gjs_module_resolve);
+    JS::SetModuleDynamicImportHook(rt, gjs_dynamic_module_resolve);
     JS::SetModuleMetadataHook(rt, gjs_populate_module_meta);
 
     if (!JS_DefineProperty(m_cx, internal_global, "moduleGlobalThis", global,
diff --git a/gjs/global.cpp b/gjs/global.cpp
index 6e0dac3b..95a39399 100644
--- a/gjs/global.cpp
+++ b/gjs/global.cpp
@@ -270,6 +270,8 @@ class GjsInternalGlobal : GjsBaseGlobal {
               0),
         JS_FN("getRegistry", gjs_internal_get_registry, 1, 0),
         JS_FN("loadResourceOrFile", gjs_internal_load_resource_or_file, 1, 0),
+        JS_FN("loadResourceOrFileAsync",
+              gjs_internal_load_resource_or_file_async, 1, 0),
         JS_FN("parseURI", gjs_internal_parse_uri, 1, 0),
         JS_FN("resolveRelativeResourceOrFile",
               gjs_internal_resolve_relative_resource_or_file, 2, 0),
diff --git a/gjs/internal.cpp b/gjs/internal.cpp
index 184f2457..1c1a57de 100644
--- a/gjs/internal.cpp
+++ b/gjs/internal.cpp
@@ -28,6 +28,7 @@
 
 #include <codecvt>  // for codecvt_utf8_utf16
 #include <locale>   // for wstring_convert
+#include <memory>   // for unique_ptr
 #include <string>   // for u16string
 #include <vector>
 
@@ -459,3 +460,152 @@ bool gjs_internal_uri_exists(JSContext* cx, unsigned argc, JS::Value* vp) {
     args.rval().setBoolean(g_file_query_exists(file, nullptr));
     return true;
 }
+
+class PromiseData {
+ public:
+    JSContext* cx;
+
+ private:
+    JS::Heap<JSFunction*> m_resolve;
+    JS::Heap<JSFunction*> m_reject;
+
+    JS::HandleFunction resolver() {
+        return JS::HandleFunction::fromMarkedLocation(m_resolve.address());
+    }
+    JS::HandleFunction rejecter() {
+        return JS::HandleFunction::fromMarkedLocation(m_reject.address());
+    }
+
+    static void trace(JSTracer* trc, void* data) {
+        auto* self = PromiseData::from_ptr(data);
+        JS::TraceEdge(trc, &self->m_resolve, "loadResourceOrFileAsync resolve");
+        JS::TraceEdge(trc, &self->m_reject, "loadResourceOrFileAsync reject");
+    }
+
+ public:
+    explicit PromiseData(JSContext* a_cx, JSFunction* resolve,
+                         JSFunction* reject)
+        : cx(a_cx), m_resolve(resolve), m_reject(reject) {
+        JS_AddExtraGCRootsTracer(cx, &PromiseData::trace, this);
+    }
+
+    ~PromiseData() {
+        JS_RemoveExtraGCRootsTracer(cx, &PromiseData::trace, this);
+    }
+
+    static PromiseData* from_ptr(void* ptr) {
+        return static_cast<PromiseData*>(ptr);
+    }
+
+    // Adapted from SpiderMonkey js::RejectPromiseWithPendingError()
+    // 
https://searchfox.org/mozilla-central/rev/95cf843de977805a3951f9137f5ff1930599d94e/js/src/builtin/Promise.cpp#4435
+    void reject_with_pending_exception() {
+        JS::RootedValue exception(cx);
+        bool ok = JS_GetPendingException(cx, &exception);
+        g_assert(ok && "Cannot reject a promise with an uncatchable exception");
+
+        JS::RootedValueArray<1> args(cx);
+        args[0].set(exception);
+        JS::RootedValue ignored_rval(cx);
+        ok = JS_CallFunction(cx, /* this_obj = */ nullptr, rejecter(), args,
+                             &ignored_rval);
+        g_assert(ok && "Failed rejecting promise");
+    }
+
+    void resolve(JS::Value result) {
+        JS::RootedValueArray<1> args(cx);
+        args[0].set(result);
+        JS::RootedValue ignored_rval(cx);
+        bool ok = JS_CallFunction(cx, /* this_obj = */ nullptr, resolver(),
+                                  args, &ignored_rval);
+        g_assert(ok && "Failed resolving promise");
+    }
+};
+
+static void load_async_callback(GObject* file, GAsyncResult* res, void* data) {
+    std::unique_ptr<PromiseData> promise(PromiseData::from_ptr(data));
+
+    char* contents;
+    size_t length;
+    GError* error = nullptr;
+    if (!g_file_load_contents_finish(G_FILE(file), res, &contents, &length,
+                                     /* etag_out = */ nullptr, &error)) {
+        GjsAutoChar uri = g_file_get_uri(G_FILE(file));
+        gjs_throw_custom(promise->cx, JSProto_Error, "ImportError",
+                         "Unable to load file from: %s (%s)", uri.get(),
+                         error->message);
+        g_clear_error(&error);
+        promise->reject_with_pending_exception();
+        return;
+    }
+
+    JS::RootedValue text(promise->cx);
+    bool ok = gjs_string_from_utf8_n(promise->cx, contents, length, &text);
+    g_free(contents);
+    if (!ok) {
+        promise->reject_with_pending_exception();
+        return;
+    }
+
+    promise->resolve(text);
+}
+
+GJS_JSAPI_RETURN_CONVENTION
+static bool load_async_executor(JSContext* cx, unsigned argc, JS::Value* vp) {
+    JS::CallArgs args = CallArgsFromVp(argc, vp);
+
+    g_assert(args.length() == 2 && "Executor called weirdly");
+    g_assert(args[0].isObject() && "Executor called weirdly");
+    g_assert(args[1].isObject() && "Executor called weirdly");
+    g_assert(JS_ObjectIsFunction(&args[0].toObject()) &&
+             "Executor called weirdly");
+    g_assert(JS_ObjectIsFunction(&args[1].toObject()) &&
+             "Executor called weirdly");
+
+    JS::Value priv_value = js::GetFunctionNativeReserved(&args.callee(), 0);
+    g_assert(!priv_value.isNull() && "Executor called twice");
+    GjsAutoUnref<GFile> file = G_FILE(priv_value.toPrivate());
+    g_assert(file && "Executor called twice");
+    // We now own the GFile, and will pass the reference to the GAsyncResult, so
+    // remove it from the executor's private slot so it doesn't become dangling
+    js::SetFunctionNativeReserved(&args.callee(), 0, JS::NullValue());
+
+    auto* data = new PromiseData(cx, JS_GetObjectFunction(&args[0].toObject()),
+                                 JS_GetObjectFunction(&args[1].toObject()));
+    g_file_load_contents_async(file, nullptr, load_async_callback, data);
+
+    args.rval().setUndefined();
+    return true;
+}
+
+bool gjs_internal_load_resource_or_file_async(JSContext* cx, unsigned argc,
+                                              JS::Value* vp) {
+    JS::CallArgs args = CallArgsFromVp(argc, vp);
+
+    g_assert(args.length() == 1 && "loadResourceOrFileAsync(str)");
+    g_assert(args[0].isString() && "loadResourceOrFileAsync(str)");
+
+    JS::RootedString string_arg(cx, args[0].toString());
+    JS::UniqueChars uri = JS_EncodeStringToUTF8(cx, string_arg);
+    if (!uri)
+        return false;
+
+    GjsAutoUnref<GFile> file = g_file_new_for_uri(uri.get());
+
+    JS::RootedObject executor(cx,
+                              JS_GetFunctionObject(js::NewFunctionWithReserved(
+                                  cx, load_async_executor, 2, 0,
+                                  "loadResourceOrFileAsync executor")));
+    if (!executor)
+        return false;
+
+    // Stash the file object for the callback to find later; executor owns it
+    js::SetFunctionNativeReserved(executor, 0, JS::PrivateValue(file.copy()));
+
+    JSObject* promise = JS::NewPromiseObject(cx, executor);
+    if (!promise)
+        return false;
+
+    args.rval().setObject(*promise);
+    return true;
+}
diff --git a/gjs/module.cpp b/gjs/module.cpp
index c810f7c9..63efe59a 100644
--- a/gjs/module.cpp
+++ b/gjs/module.cpp
@@ -21,6 +21,8 @@
 #include <js/Conversions.h>
 #include <js/GCVector.h>  // for RootedVector
 #include <js/Id.h>
+#include <js/Modules.h>
+#include <js/Promise.h>
 #include <js/PropertyDescriptor.h>
 #include <js/RootingAPI.h>
 #include <js/SourceText.h>
@@ -29,6 +31,7 @@
 #include <js/Value.h>
 #include <js/ValueArray.h>
 #include <jsapi.h>  // for JS_DefinePropertyById, ...
+#include <jsfriendapi.h>  // for SetFunctionNativeReserved
 
 #include "gjs/atoms.h"
 #include "gjs/context-private.h"
@@ -484,3 +487,138 @@ JSObject* gjs_module_resolve(JSContext* cx, JS::HandleValue importingModulePriv,
     g_assert(result.isObject() && "resolve hook failed to return an object!");
     return &result.toObject();
 }
+
+// Call JS::FinishDynamicModuleImport() with the values stashed in the function.
+// Can fail in JS::FinishDynamicModuleImport(), but will assert if anything
+// fails in fetching the stashed values, since that would be a serious GJS bug.
+GJS_JSAPI_RETURN_CONVENTION
+static bool finish_import(JSContext* cx, const JS::CallArgs& args) {
+    JS::Value callback_priv = js::GetFunctionNativeReserved(&args.callee(), 0);
+    g_assert(callback_priv.isObject() && "Wrong private value");
+    JS::RootedObject callback_data(cx, &callback_priv.toObject());
+
+    JS::RootedValue importing_module_priv(cx);
+    JS::RootedValue v_specifier(cx);
+    JS::RootedValue v_internal_promise(cx);
+    bool ok =
+        JS_GetProperty(cx, callback_data, "priv", &importing_module_priv) &&
+        JS_GetProperty(cx, callback_data, "promise", &v_internal_promise) &&
+        JS_GetProperty(cx, callback_data, "specifier", &v_specifier);
+    g_assert(ok && "Wrong properties on private value");
+
+    g_assert(v_specifier.isString() && "Wrong type for specifier");
+    g_assert(v_internal_promise.isObject() && "Wrong type for promise");
+
+    JS::RootedString specifier(cx, v_specifier.toString());
+    JS::RootedObject internal_promise(cx, &v_internal_promise.toObject());
+
+    args.rval().setUndefined();
+    return JS::FinishDynamicModuleImport(cx, importing_module_priv, specifier,
+                                         internal_promise);
+}
+
+// Failing a JSAPI function may result either in an exception pending on the
+// context, in which case we must call JS::FinishDynamicModuleImport() to reject
+// the internal promise; or in an uncatchable exception such as OOM, in which
+// case we must not call JS::FinishDynamicModuleImport().
+GJS_JSAPI_RETURN_CONVENTION
+static bool fail_import(JSContext* cx, const JS::CallArgs& args) {
+    if (JS_IsExceptionPending(cx))
+        return finish_import(cx, args);
+    return false;
+}
+
+GJS_JSAPI_RETURN_CONVENTION
+static bool import_rejected(JSContext* cx, unsigned argc, JS::Value* vp) {
+    JS::CallArgs args = JS::CallArgsFromVp(argc, vp);
+
+    gjs_debug(GJS_DEBUG_IMPORTER, "Async import promise rejected");
+
+    // Throw the value that the promise is rejected with, so that
+    // FinishDynamicModuleImport will reject the internal_promise with it.
+    JS_SetPendingException(cx, args.get(0),
+                           JS::ExceptionStackBehavior::DoNotCapture);
+
+    return finish_import(cx, args);
+}
+
+GJS_JSAPI_RETURN_CONVENTION
+static bool import_resolved(JSContext* cx, unsigned argc, JS::Value* vp) {
+    JS::CallArgs args = JS::CallArgsFromVp(argc, vp);
+
+    gjs_debug(GJS_DEBUG_IMPORTER, "Async import promise resolved");
+
+    JS::RootedObject global(cx, gjs_get_import_global(cx));
+    JSAutoRealm ar(cx, global);
+
+    g_assert(args[0].isObject());
+    JS::RootedObject module(cx, &args[0].toObject());
+
+    if (!JS::ModuleInstantiate(cx, module) || !JS::ModuleEvaluate(cx, module))
+        return fail_import(cx, args);
+
+    return finish_import(cx, args);
+}
+
+bool gjs_dynamic_module_resolve(JSContext* cx,
+                                JS::HandleValue importing_module_priv,
+                                JS::HandleString specifier,
+                                JS::HandleObject internal_promise) {
+    g_assert(gjs_global_is_type(cx, GjsGlobalType::DEFAULT) &&
+             "gjs_dynamic_module_resolve can only be called from the default "
+             "global.");
+
+    JS::RootedObject global(cx, JS::CurrentGlobalOrNull(cx));
+    JSAutoRealm ar(cx, global);
+
+    JS::RootedValue v_loader(
+        cx, gjs_get_global_slot(global, GjsGlobalSlot::MODULE_LOADER));
+    g_assert(v_loader.isObject());
+    JS::RootedObject loader(cx, &v_loader.toObject());
+
+    JS::RootedObject callback_data(cx, JS_NewPlainObject(cx));
+    if (!callback_data ||
+        !JS_DefineProperty(cx, callback_data, "specifier", specifier,
+                           JSPROP_PERMANENT) ||
+        !JS_DefineProperty(cx, callback_data, "promise", internal_promise,
+                           JSPROP_PERMANENT) ||
+        !JS_DefineProperty(cx, callback_data, "priv", importing_module_priv,
+                           JSPROP_PERMANENT))
+        return false;
+
+    gjs_debug(GJS_DEBUG_IMPORTER,
+              "Async module resolve hook for module '%s' (relative to %p), "
+              "global %p",
+              gjs_debug_string(specifier).c_str(),
+              &importing_module_priv.toObject(), global.get());
+
+    JS::RootedValueArray<2> args(cx);
+    args[0].set(importing_module_priv);
+    args[1].setString(specifier);
+
+    JS::RootedValue result(cx);
+    if (!JS::Call(cx, loader, "moduleResolveAsyncHook", args, &result))
+        return JS::FinishDynamicModuleImport(cx, importing_module_priv,
+                                             specifier, internal_promise);
+
+    JS::RootedObject resolved(
+        cx, JS_GetFunctionObject(js::NewFunctionWithReserved(
+                cx, import_resolved, 1, 0, "async import resolved")));
+    if (!resolved)
+        return false;
+    JS::RootedObject rejected(
+        cx, JS_GetFunctionObject(js::NewFunctionWithReserved(
+                cx, import_rejected, 1, 0, "async import rejected")));
+    if (!rejected)
+        return false;
+    js::SetFunctionNativeReserved(resolved, 0, JS::ObjectValue(*callback_data));
+    js::SetFunctionNativeReserved(rejected, 0, JS::ObjectValue(*callback_data));
+
+    JS::RootedObject promise(cx, &result.toObject());
+
+    // Calling JS::FinishDynamicModuleImport() at the end of the resolve and
+    // reject handlers will also call the module resolve hook. The module will
+    // already have been resolved, but that is how SpiderMonkey obtains the
+    // module object.
+    return JS::AddPromiseReactions(cx, promise, resolved, rejected);
+}
diff --git a/gjs/module.h b/gjs/module.h
index c754b114..dfe6f18c 100644
--- a/gjs/module.h
+++ b/gjs/module.h
@@ -39,4 +39,10 @@ GJS_JSAPI_RETURN_CONVENTION
 bool gjs_populate_module_meta(JSContext* cx, JS::HandleValue private_ref,
                               JS::HandleObject meta_object);
 
+GJS_JSAPI_RETURN_CONVENTION
+bool gjs_dynamic_module_resolve(JSContext* cx,
+                                JS::HandleValue importing_module_priv,
+                                JS::HandleString specifier,
+                                JS::HandleObject internal_promise);
+
 #endif  // GJS_MODULE_H_
diff --git a/installed-tests/js/.eslintrc.yml b/installed-tests/js/.eslintrc.yml
index 28ec04c6..c1a4c9bd 100644
--- a/installed-tests/js/.eslintrc.yml
+++ b/installed-tests/js/.eslintrc.yml
@@ -35,5 +35,6 @@ overrides:
       - testESModules.js
       - modules/importmeta.js
       - modules/exports.js
+      - modules/say.js
     parserOptions:
       sourceType: module
diff --git a/installed-tests/js/jsunit.gresources.xml b/installed-tests/js/jsunit.gresources.xml
index 3e100b1b..624ac738 100644
--- a/installed-tests/js/jsunit.gresources.xml
+++ b/installed-tests/js/jsunit.gresources.xml
@@ -21,6 +21,7 @@
     <file>modules/mutualImport/a.js</file>
     <file>modules/mutualImport/b.js</file>
     <file>modules/overrides/GIMarshallingTests.js</file>
+    <file>modules/say.js</file>
     <file>modules/subA/subB/__init__.js</file>
     <file>modules/subA/subB/baz.js</file>
     <file>modules/subA/subB/foobar.js</file>
diff --git a/installed-tests/js/modules/say.js b/installed-tests/js/modules/say.js
new file mode 100644
index 00000000..d1c7ab56
--- /dev/null
+++ b/installed-tests/js/modules/say.js
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: MIT OR LGPL-2.0-or-later
+// SPDX-FileCopyrightText: 2020 Philip Chimento <philip chimento gmail com>
+
+export function say(str) {
+    return `<( ${str} )`;
+}
+
+export default function () {
+    return 'default export';
+}
diff --git a/installed-tests/js/testESModules.js b/installed-tests/js/testESModules.js
index d1b7210d..5b0cab6a 100644
--- a/installed-tests/js/testESModules.js
+++ b/installed-tests/js/testESModules.js
@@ -68,6 +68,16 @@ describe('Builtin ES modules', function () {
         expect(typeof N_).toBe('function');
     });
 
+    it('gettext named dynamic import', async function () {
+        const localGettext = await import('gettext');
+        expect(typeof localGettext.ngettext).toEqual('function');
+    });
+
+    it('gettext dynamic import matches static import', async function () {
+        const localGettext = await import('gettext');
+        expect(localGettext.default).toEqual(gettext);
+    });
+
     it('system default import', function () {
         expect(typeof system.exit).toBe('function');
     });
@@ -76,4 +86,38 @@ describe('Builtin ES modules', function () {
         expect(typeof exit).toBe('function');
         expect(exit).toBe(system.exit);
     });
+
+    it('system dynamic import matches static import', async function () {
+        const localSystem = await import('system');
+        expect(localSystem.default).toEqual(system);
+    });
+
+    it('system named dynamic import', async function () {
+        const localSystem = await import('system');
+        expect(typeof localSystem.exit).toBe('function');
+    });
+});
+
+describe('Dynamic imports', function () {
+    let module;
+    beforeEach(async function () {
+        try {
+            module = await import('resource:///org/gjs/jsunit/modules/say.js');
+        } catch (err) {
+            logError(err);
+            fail();
+        }
+    });
+
+    it('default import', function () {
+        expect(module.default()).toEqual('default export');
+    });
+
+    it('named import', function () {
+        expect(module.say('hi')).toEqual('<( hi )');
+    });
+
+    it('dynamic gi import matches static', async function () {
+        expect((await import('gi://Gio')).default).toEqual(Gio);
+    });
 });
diff --git a/modules/internal/.eslintrc.yml b/modules/internal/.eslintrc.yml
index b06bc212..2889cc64 100644
--- a/modules/internal/.eslintrc.yml
+++ b/modules/internal/.eslintrc.yml
@@ -19,6 +19,7 @@ globals:
   compileModule: readonly
   compileInternalModule: readonly
   loadResourceOrFile: readonly
+  loadResourceOrFileAsync: readonly
   parseURI: readonly
   uriExists: readonly
   resolveRelativeResourceOrFile: readonly
diff --git a/modules/internal/loader.js b/modules/internal/loader.js
index fde96eb0..f28814c9 100644
--- a/modules/internal/loader.js
+++ b/modules/internal/loader.js
@@ -122,6 +122,79 @@ class ModuleLoader extends InternalModuleLoader {
 
         return this.resolveBareSpecifier(specifier);
     }
+
+    moduleResolveAsyncHook(importingModulePriv, specifier) {
+        // importingModulePriv may be falsy in the case of gjs_context_eval()
+        if (!importingModulePriv || !importingModulePriv.uri)
+            throw new ImportError('Cannot resolve relative imports from an unknown file.');
+
+        return this.resolveModuleAsync(specifier, importingModulePriv.uri);
+    }
+
+    /**
+     * Resolves a module import with optional handling for relative imports asynchronously.
+     *
+     * @param {string} specifier the specifier (e.g. relative path, root package) to resolve
+     * @param {string | null} importingModuleURI the URI of the module
+     *   triggering this resolve
+     * @returns {import("../types").Module}
+     */
+    async resolveModuleAsync(specifier, importingModuleURI) {
+        const registry = getRegistry(this.global);
+
+        // Check if the module has already been loaded
+        let module = registry.get(specifier);
+        if (module)
+            return module;
+
+        // 1) Resolve path and URI-based imports.
+        const uri = this.resolveSpecifier(specifier, importingModuleURI);
+        if (uri) {
+            module = registry.get(uri.uri);
+
+            // Check if module is already loaded (relative handling)
+            if (module)
+                return module;
+
+            const result = await this.loadURIAsync(uri);
+            if (!result)
+                return null;
+
+            const [text, internal = false] = result;
+
+            const priv = new ModulePrivate(uri.uri, uri.uri, internal);
+            const compiled = this.compileModule(priv, text);
+
+            registry.set(uri.uri, compiled);
+            return compiled;
+        }
+
+        // 2) Resolve internal imports.
+
+        return this.resolveBareSpecifier(specifier);
+    }
+
+    /**
+     * Loads a file or resource URI asynchronously
+     *
+     * @param {Uri} uri the file or resource URI to load
+     * @returns {Promise<[string] | [string, boolean] | null>}
+     */
+    async loadURIAsync(uri) {
+        if (uri.scheme) {
+            const loader = this.schemeHandlers.get(uri.scheme);
+
+            if (loader)
+                return loader.loadAsync(uri);
+        }
+
+        if (uri.scheme === 'file' || uri.scheme === 'resource') {
+            const result = await loadResourceOrFileAsync(uri.uri);
+            return [result];
+        }
+
+        return null;
+    }
 }
 
 const moduleLoader = new ModuleLoader(moduleGlobalThis);
@@ -167,4 +240,11 @@ moduleLoader.registerScheme('gi', {
 
         return [generateGIModule(namespace, version), true];
     },
+    /**
+     * @param {import("./internalLoader.js").Uri} uri the URI to load asynchronously
+     */
+    loadAsync(uri) {
+        // gi: only does string manipulation, so it is safe to use the same code for sync and async.
+        return this.load(uri);
+    },
 });