Make JS calls go even faster

Libraries/LibGC/Heap.cpp

@@ -30,10 +30,18 @@
 
 namespace GC {
 
+static Heap* s_the;
+
+Heap& Heap::the()
+{
+    return *s_the;
+}
+
 Heap::Heap(void* private_data, AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots)
     : HeapBase(private_data)
     , m_gather_embedder_roots(move(gather_embedder_roots))
 {
+    s_the = this;
     static_assert(HeapBlock::min_possible_cell_size <= 32, "Heap Cell tracking uses too much data!");
     m_size_based_cell_allocators.append(make<CellAllocator>(64));
     m_size_based_cell_allocators.append(make<CellAllocator>(96));

Libraries/LibGC/Heap.h

@@ -38,6 +38,8 @@ class GC_API Heap : public HeapBase {
     explicit Heap(void* private_data, AK::Function<void(HashMap<Cell*, GC::HeapRoot>&)> gather_embedder_roots);
     ~Heap();
 
+    static Heap& the();
+
     template<typename T, typename... Args>
     Ref<T> allocate(Args&&... args)
     {

Libraries/LibJS/Bytecode/Interpreter.cpp

@@ -167,12 +167,12 @@ Interpreter::~Interpreter()
 
 ALWAYS_INLINE Value Interpreter::get(Operand op) const
 {
-    return m_running_execution_context->registers_and_constants_and_locals_arguments.data()[op.index()];
+    return m_running_execution_context->registers_and_constants_and_locals_and_arguments()[op.index()];
 }
 
 ALWAYS_INLINE void Interpreter::set(Operand op, Value value)
 {
-    m_running_execution_context->registers_and_constants_and_locals_arguments.data()[op.index()] = value;
+    m_running_execution_context->registers_and_constants_and_locals_and_arguments()[op.index()] = value;
 }
 
 ALWAYS_INLINE Value Interpreter::do_yield(Value value, Optional<Label> continuation)
@@ -311,8 +311,8 @@ NEVER_INLINE Interpreter::HandleExceptionResponse Interpreter::handle_exception(
     auto& handler = handlers->handler_offset;
     auto& finalizer = handlers->finalizer_offset;
 
-    VERIFY(!running_execution_context().unwind_contexts.is_empty());
-    auto& unwind_context = running_execution_context().unwind_contexts.last();
+    auto& unwind_contexts = running_execution_context().ensure_rare_data()->unwind_contexts;
+    auto& unwind_context = unwind_contexts.last();
     VERIFY(unwind_context.executable == &current_executable());
 
     if (handler.has_value()) {
@@ -380,7 +380,10 @@ FLATTEN_ON_CLANG void Interpreter::run_bytecode(size_t entry_point)
 
         handle_End: {
             auto& instruction = *reinterpret_cast<Op::End const*>(&bytecode[program_counter]);
-            reg(Register::return_value()) = get(instruction.value());
+            auto value = get(instruction.value());
+            if (value.is_special_empty_value())
+                value = js_undefined();
+            reg(Register::return_value()) = value;
             return;
         }
 
@@ -485,19 +488,19 @@ FLATTEN_ON_CLANG void Interpreter::run_bytecode(size_t entry_point)
                 do_return(saved_return_value());
                 if (auto handlers = executable.exception_handlers_for_offset(program_counter); handlers.has_value()) {
                     if (auto finalizer = handlers.value().finalizer_offset; finalizer.has_value()) {
-                        VERIFY(!running_execution_context.unwind_contexts.is_empty());
-                        auto& unwind_context = running_execution_context.unwind_contexts.last();
+                        auto& unwind_contexts = running_execution_context.ensure_rare_data()->unwind_contexts;
+                        auto& unwind_context = unwind_contexts.last();
                         VERIFY(unwind_context.executable == &current_executable());
                         reg(Register::saved_return_value()) = reg(Register::return_value());
-                        reg(Register::return_value()) = js_special_empty_value();
+                        reg(Register::return_value()) = js_undefined();
                         program_counter = finalizer.value();
                         // the unwind_context will be pop'ed when entering the finally block
                         goto start;
                     }
                 }
                 return;
             }
-            auto const old_scheduled_jump = running_execution_context.previously_scheduled_jumps.take_last();
+            auto const old_scheduled_jump = running_execution_context.ensure_rare_data()->previously_scheduled_jumps.take_last();
             if (m_running_execution_context->scheduled_jump.has_value()) {
                 program_counter = m_running_execution_context->scheduled_jump.value();
                 m_running_execution_context->scheduled_jump = {};
@@ -699,10 +702,10 @@ FLATTEN_ON_CLANG void Interpreter::run_bytecode(size_t entry_point)
 
 Utf16FlyString const& Interpreter::get_identifier(IdentifierTableIndex index) const
 {
-    return m_running_execution_context->identifier_table.data()[index.value];
+    return m_running_execution_context->identifier_table[index.value];
 }
 
-ThrowCompletionOr<Value> Interpreter::run_executable(ExecutionContext& context, Executable& executable, Optional<size_t> entry_point, Value initial_accumulator_value)
+ThrowCompletionOr<Value> Interpreter::run_executable(ExecutionContext& context, Executable& executable, Optional<size_t> entry_point)
 {
     dbgln_if(JS_BYTECODE_DEBUG, "Bytecode::Interpreter will run unit {}", &executable);
 
@@ -712,14 +715,10 @@ ThrowCompletionOr<Value> Interpreter::run_executable(ExecutionContext& context,
     context.executable = executable;
     context.global_object = realm().global_object();
     context.global_declarative_environment = realm().global_environment().declarative_record();
-    context.identifier_table = executable.identifier_table->identifiers();
+    context.identifier_table = executable.identifier_table->identifiers().data();
 
     ASSERT(executable.registers_and_constants_and_locals_count <= context.registers_and_constants_and_locals_and_arguments_span().size());
 
-    context.registers_and_constants_and_locals_arguments = context.registers_and_constants_and_locals_and_arguments_span();
-
-    reg(Register::accumulator()) = initial_accumulator_value;
-
     // NOTE: We only copy the `this` value from ExecutionContext if it's not already set.
     //       If we are re-entering an async/generator context, the `this` value
     //       may have already been cached by a ResolveThisBinding instruction,
@@ -754,31 +753,28 @@ ThrowCompletionOr<Value> Interpreter::run_executable(ExecutionContext& context,
     if (!exception.is_special_empty_value()) [[unlikely]]
         return throw_completion(exception);
 
-    auto return_value = reg(Register::return_value());
-    if (return_value.is_special_empty_value())
-        return_value = js_undefined();
-    return return_value;
+    return reg(Register::return_value());
 }
 
 void Interpreter::enter_unwind_context()
 {
-    running_execution_context().unwind_contexts.empend(
+    running_execution_context().ensure_rare_data()->unwind_contexts.empend(
         current_executable(),
         running_execution_context().lexical_environment);
-    running_execution_context().previously_scheduled_jumps.append(m_running_execution_context->scheduled_jump);
+    running_execution_context().rare_data()->previously_scheduled_jumps.append(m_running_execution_context->scheduled_jump);
     m_running_execution_context->scheduled_jump = {};
 }
 
 void Interpreter::leave_unwind_context()
 {
-    running_execution_context().unwind_contexts.take_last();
+    running_execution_context().rare_data()->unwind_contexts.take_last();
 }
 
 void Interpreter::catch_exception(Operand dst)
 {
     set(dst, reg(Register::exception()));
     reg(Register::exception()) = js_special_empty_value();
-    auto& context = running_execution_context().unwind_contexts.last();
+    auto& context = running_execution_context().rare_data()->unwind_contexts.last();
     VERIFY(!context.handler_called);
     VERIFY(context.executable == &current_executable());
     context.handler_called = true;
@@ -787,19 +783,19 @@ void Interpreter::catch_exception(Operand dst)
 
 void Interpreter::restore_scheduled_jump()
 {
-    m_running_execution_context->scheduled_jump = running_execution_context().previously_scheduled_jumps.take_last();
+    m_running_execution_context->scheduled_jump = running_execution_context().rare_data()->previously_scheduled_jumps.take_last();
 }
 
 void Interpreter::leave_finally()
 {
     reg(Register::exception()) = js_special_empty_value();
-    m_running_execution_context->scheduled_jump = running_execution_context().previously_scheduled_jumps.take_last();
+    m_running_execution_context->scheduled_jump = running_execution_context().rare_data()->previously_scheduled_jumps.take_last();
 }
 
 void Interpreter::enter_object_environment(Object& object)
 {
     auto& old_environment = running_execution_context().lexical_environment;
-    running_execution_context().saved_lexical_environments.append(old_environment);
+    running_execution_context().ensure_rare_data()->saved_lexical_environments.append(old_environment);
     running_execution_context().lexical_environment = new_object_environment(object, true, old_environment);
 }
 
@@ -1593,7 +1589,7 @@ inline ThrowCompletionOr<ECMAScriptFunctionObject*> new_class(VM& vm, Value supe
 
     // NOTE: NewClass expects classEnv to be active lexical environment
     auto* class_environment = vm.lexical_environment();
-    vm.running_execution_context().lexical_environment = vm.running_execution_context().saved_lexical_environments.take_last();
+    vm.running_execution_context().lexical_environment = vm.running_execution_context().rare_data()->saved_lexical_environments.take_last();
 
     Optional<Utf16FlyString> binding_name;
     Utf16FlyString class_name;
@@ -2416,7 +2412,7 @@ void CreateLexicalEnvironment::execute_impl(Bytecode::Interpreter& interpreter)
         return environment;
     };
     auto& running_execution_context = interpreter.running_execution_context();
-    running_execution_context.saved_lexical_environments.append(make_and_swap_envs(running_execution_context.lexical_environment));
+    running_execution_context.ensure_rare_data()->saved_lexical_environments.append(make_and_swap_envs(running_execution_context.lexical_environment));
     if (m_dst.has_value())
         interpreter.set(*m_dst, running_execution_context.lexical_environment);
 }
@@ -3161,7 +3157,7 @@ ThrowCompletionOr<void> ThrowIfTDZ::execute_impl(Bytecode::Interpreter& interpre
 void LeaveLexicalEnvironment::execute_impl(Bytecode::Interpreter& interpreter) const
 {
     auto& running_execution_context = interpreter.running_execution_context();
-    running_execution_context.lexical_environment = running_execution_context.saved_lexical_environments.take_last();
+    running_execution_context.lexical_environment = running_execution_context.rare_data()->saved_lexical_environments.take_last();
 }
 
 void LeavePrivateEnvironment::execute_impl(Bytecode::Interpreter& interpreter) const

Libraries/LibJS/Bytecode/Interpreter.h

@@ -34,22 +34,23 @@ class JS_API Interpreter {
     ThrowCompletionOr<Value> run(Script&, GC::Ptr<Environment> lexical_environment_override = nullptr);
     ThrowCompletionOr<Value> run(SourceTextModule&);
 
-    ThrowCompletionOr<Value> run(ExecutionContext& context, Executable& executable, Optional<size_t> entry_point = {}, Value initial_accumulator_value = js_special_empty_value())
+    ThrowCompletionOr<Value> run_executable(ExecutionContext&, Executable&, Optional<size_t> entry_point);
+
+    ThrowCompletionOr<Value> run_executable(ExecutionContext& context, Executable& executable, Optional<size_t> entry_point, Value initial_accumulator_value)
     {
-        return run_executable(context, executable, entry_point, initial_accumulator_value);
+        context.registers_and_constants_and_locals_and_arguments_span()[0] = initial_accumulator_value;
+        return run_executable(context, executable, entry_point);
     }
 
-    ThrowCompletionOr<Value> run_executable(ExecutionContext&, Executable&, Optional<size_t> entry_point, Value initial_accumulator_value = js_special_empty_value());
-
     ALWAYS_INLINE Value& accumulator() { return reg(Register::accumulator()); }
     ALWAYS_INLINE Value& saved_return_value() { return reg(Register::saved_return_value()); }
     Value& reg(Register const& r)
     {
-        return m_running_execution_context->registers_and_constants_and_locals_arguments.data()[r.index()];
+        return m_running_execution_context->registers_and_constants_and_locals_and_arguments()[r.index()];
     }
     Value reg(Register const& r) const
     {
-        return m_running_execution_context->registers_and_constants_and_locals_arguments.data()[r.index()];
+        return m_running_execution_context->registers_and_constants_and_locals_and_arguments()[r.index()];
     }
 
     [[nodiscard]] Value get(Operand) const;
@@ -58,6 +59,8 @@ class JS_API Interpreter {
     Value do_yield(Value value, Optional<Label> continuation);
     void do_return(Value value)
     {
+        if (value.is_special_empty_value())
+            value = js_undefined();
         reg(Register::return_value()) = value;
         reg(Register::exception()) = js_special_empty_value();
     }

Libraries/LibJS/Bytecode/Op.h

@@ -1919,6 +1919,7 @@ class Call final : public Instruction {
 
     Call(Operand dst, Operand callee, Operand this_value, ReadonlySpan<ScopedOperand> arguments, Optional<StringTableIndex> expression_string = {})
         : Instruction(Type::Call)
+        , m_length(round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * arguments.size()))
         , m_dst(dst)
         , m_callee(callee)
         , m_this_value(this_value)
@@ -1930,10 +1931,7 @@ class Call final : public Instruction {
     }
 
     size_t length() const { return length_impl(); }
-    size_t length_impl() const
-    {
-        return round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * m_argument_count);
-    }
+    size_t length_impl() const { return m_length; }
 
     Operand dst() const { return m_dst; }
     Operand callee() const { return m_callee; }
@@ -1954,6 +1952,7 @@ class Call final : public Instruction {
     }
 
 private:
+    u32 m_length { 0 };
     Operand m_dst;
     Operand m_callee;
     Operand m_this_value;
@@ -1968,6 +1967,7 @@ class CallBuiltin final : public Instruction {
 
     CallBuiltin(Operand dst, Operand callee, Operand this_value, ReadonlySpan<ScopedOperand> arguments, Builtin builtin, Optional<StringTableIndex> expression_string = {})
         : Instruction(Type::CallBuiltin)
+        , m_length(round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * arguments.size()))
         , m_dst(dst)
         , m_callee(callee)
         , m_this_value(this_value)
@@ -1980,10 +1980,7 @@ class CallBuiltin final : public Instruction {
     }
 
     size_t length() const { return length_impl(); }
-    size_t length_impl() const
-    {
-        return round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * m_argument_count);
-    }
+    size_t length_impl() const { return m_length; }
 
     Operand dst() const { return m_dst; }
     Operand callee() const { return m_callee; }
@@ -2006,6 +2003,7 @@ class CallBuiltin final : public Instruction {
     }
 
 private:
+    u32 m_length { 0 };
     Operand m_dst;
     Operand m_callee;
     Operand m_this_value;
@@ -2021,6 +2019,7 @@ class CallConstruct final : public Instruction {
 
     CallConstruct(Operand dst, Operand callee, ReadonlySpan<ScopedOperand> arguments, Optional<StringTableIndex> expression_string = {})
         : Instruction(Type::CallConstruct)
+        , m_length(round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * arguments.size()))
         , m_dst(dst)
         , m_callee(callee)
         , m_argument_count(arguments.size())
@@ -2031,10 +2030,7 @@ class CallConstruct final : public Instruction {
     }
 
     size_t length() const { return length_impl(); }
-    size_t length_impl() const
-    {
-        return round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * m_argument_count);
-    }
+    size_t length_impl() const { return m_length; }
 
     Operand dst() const { return m_dst; }
     Operand callee() const { return m_callee; }
@@ -2053,6 +2049,7 @@ class CallConstruct final : public Instruction {
     }
 
 private:
+    u32 m_length { 0 };
     Operand m_dst;
     Operand m_callee;
     u32 m_argument_count { 0 };
@@ -2066,6 +2063,7 @@ class CallDirectEval final : public Instruction {
 
     CallDirectEval(Operand dst, Operand callee, Operand this_value, ReadonlySpan<ScopedOperand> arguments, Optional<StringTableIndex> expression_string = {})
         : Instruction(Type::CallDirectEval)
+        , m_length(round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * arguments.size()))
         , m_dst(dst)
         , m_callee(callee)
         , m_this_value(this_value)
@@ -2077,10 +2075,7 @@ class CallDirectEval final : public Instruction {
     }
 
     size_t length() const { return length_impl(); }
-    size_t length_impl() const
-    {
-        return round_up_to_power_of_two(alignof(void*), sizeof(*this) + sizeof(Operand) * m_argument_count);
-    }
+    size_t length_impl() const { return m_length; }
 
     Operand dst() const { return m_dst; }
     Operand callee() const { return m_callee; }
@@ -2101,6 +2096,7 @@ class CallDirectEval final : public Instruction {
     }
 
 private:
+    u32 m_length { 0 };
     Operand m_dst;
     Operand m_callee;
     Operand m_this_value;

Libraries/LibJS/Forward.h

@@ -164,7 +164,7 @@ struct AsyncGeneratorRequest;
 class BigInt;
 class BoundFunction;
 class BuiltinIterator;
-struct CachedSourceRange;
+class CachedSourceRange;
 class Cell;
 class ClassExpression;
 struct ClassFieldDefinition;

Libraries/LibJS/Runtime/Error.h

@@ -21,7 +21,7 @@ struct JS_API TracebackFrame {
     Utf16String function_name;
     [[nodiscard]] SourceRange const& source_range() const;
 
-    RefPtr<CachedSourceRange> cached_source_range;
+    GC::Root<CachedSourceRange> cached_source_range;
 };
 
 enum CompactTraceback {