1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmAirIRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AirCode.h"
32#include "AirGenerate.h"
33#include "AirOpcodeUtils.h"
34#include "AirValidate.h"
35#include "AllowMacroScratchRegisterUsageIf.h"
36#include "B3CCallValue.h"
37#include "B3CheckSpecial.h"
38#include "B3CheckValue.h"
39#include "B3PatchpointSpecial.h"
40#include "B3Procedure.h"
41#include "B3ProcedureInlines.h"
42#include "BinarySwitch.h"
43#include "DisallowMacroScratchRegisterUsage.h"
44#include "JSCInlines.h"
45#include "JSWebAssemblyInstance.h"
46#include "ScratchRegisterAllocator.h"
47#include "VirtualRegister.h"
48#include "WasmCallingConvention.h"
49#include "WasmContextInlines.h"
50#include "WasmExceptionType.h"
51#include "WasmFunctionParser.h"
52#include "WasmInstance.h"
53#include "WasmMemory.h"
54#include "WasmOMGPlan.h"
55#include "WasmOpcodeOrigin.h"
56#include "WasmSignatureInlines.h"
57#include "WasmThunks.h"
58#include <limits>
59#include <wtf/Box.h>
60#include <wtf/Optional.h>
61#include <wtf/StdLibExtras.h>
62
63namespace JSC { namespace Wasm {
64
65using namespace B3::Air;
66
67struct ConstrainedTmp {
68 ConstrainedTmp(Tmp tmp)
69 : ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)
70 { }
71
72 ConstrainedTmp(Tmp tmp, B3::ValueRep rep)
73 : tmp(tmp)
74 , rep(rep)
75 {
76 }
77
78 Tmp tmp;
79 B3::ValueRep rep;
80};
81
82class TypedTmp {
83public:
84 constexpr TypedTmp()
85 : m_tmp()
86 , m_type(Type::Void)
87 { }
88
89 TypedTmp(Tmp tmp, Type type)
90 : m_tmp(tmp)
91 , m_type(type)
92 { }
93
94 TypedTmp(const TypedTmp&) = default;
95 TypedTmp(TypedTmp&&) = default;
96 TypedTmp& operator=(TypedTmp&&) = default;
97 TypedTmp& operator=(const TypedTmp&) = default;
98
99 bool operator==(const TypedTmp& other) const
100 {
101 return m_tmp == other.m_tmp && m_type == other.m_type;
102 }
103 bool operator!=(const TypedTmp& other) const
104 {
105 return !(*this == other);
106 }
107
108 explicit operator bool() const { return !!tmp(); }
109
110 operator Tmp() const { return tmp(); }
111 operator Arg() const { return Arg(tmp()); }
112 Tmp tmp() const { return m_tmp; }
113 Type type() const { return m_type; }
114
115private:
116
117 Tmp m_tmp;
118 Type m_type;
119};
120
121class AirIRGenerator {
122public:
123 struct ControlData {
124 ControlData(B3::Origin origin, Type returnType, TypedTmp resultTmp, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
125 : blockType(type)
126 , continuation(continuation)
127 , special(special)
128 , returnType(returnType)
129 {
130 UNUSED_PARAM(origin); // FIXME: Use origin.
131 if (resultTmp) {
132 ASSERT(returnType != Type::Void);
133 result.append(resultTmp);
134 } else
135 ASSERT(returnType == Type::Void);
136 }
137
138 ControlData()
139 {
140 }
141
142 void dump(PrintStream& out) const
143 {
144 switch (type()) {
145 case BlockType::If:
146 out.print("If: ");
147 break;
148 case BlockType::Block:
149 out.print("Block: ");
150 break;
151 case BlockType::Loop:
152 out.print("Loop: ");
153 break;
154 case BlockType::TopLevel:
155 out.print("TopLevel: ");
156 break;
157 }
158 out.print("Continuation: ", *continuation, ", Special: ");
159 if (special)
160 out.print(*special);
161 else
162 out.print("None");
163 }
164
165 BlockType type() const { return blockType; }
166
167 Type signature() const { return returnType; }
168
169 bool hasNonVoidSignature() const { return result.size(); }
170
171 BasicBlock* targetBlockForBranch()
172 {
173 if (type() == BlockType::Loop)
174 return special;
175 return continuation;
176 }
177
178 void convertIfToBlock()
179 {
180 ASSERT(type() == BlockType::If);
181 blockType = BlockType::Block;
182 special = nullptr;
183 }
184
185 using ResultList = Vector<TypedTmp, 1>;
186
187 ResultList resultForBranch() const
188 {
189 if (type() == BlockType::Loop)
190 return ResultList();
191 return result;
192 }
193
194 private:
195 friend class AirIRGenerator;
196 BlockType blockType;
197 BasicBlock* continuation;
198 BasicBlock* special;
199 ResultList result;
200 Type returnType;
201 };
202
203 using ExpressionType = TypedTmp;
204 using ControlType = ControlData;
205 using ExpressionList = Vector<ExpressionType, 1>;
206 using ResultList = ControlData::ResultList;
207 using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;
208
209 static ExpressionType emptyExpression() { return { }; };
210
211 using ErrorType = String;
212 using UnexpectedResult = Unexpected<ErrorType>;
213 using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;
214 using PartialResult = Expected<void, ErrorType>;
215
216 template <typename ...Args>
217 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
218 {
219 using namespace FailureHelper; // See ADL comment in WasmParser.h.
220 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
221 }
222
223#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
224 if (UNLIKELY(condition)) \
225 return fail(__VA_ARGS__); \
226 } while (0)
227
228 AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
229
230 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
231 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
232 ExpressionType addConstant(Type, uint64_t);
233 ExpressionType addConstant(BasicBlock*, Type, uint64_t);
234
235 // References
236 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
237 PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
238
239 // Tables
240 PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType& index, ExpressionType& result);
241 PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType& index, ExpressionType& value);
242 PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
243 PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType& fill, ExpressionType& delta, ExpressionType& result);
244 PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType& offset, ExpressionType& fill, ExpressionType& count);
245
246 // Locals
247 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
248 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
249
250 // Globals
251 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
252 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
253
254 // Memory
255 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
256 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
257 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
258 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
259
260 // Basic operators
261 template<OpType>
262 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
263 template<OpType>
264 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
265 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
266
267 // Control flow
268 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
269 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
270 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
271 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
272 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
273 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
274
275 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
276 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
277 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
278 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
279 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
280
281 // Calls
282 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
283 PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
284 PartialResult WARN_UNUSED_RETURN addUnreachable();
285
286 PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);
287 PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
288 PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);
289 PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
290
291 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
292 void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
293
294 static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)
295 {
296 Vector<Tmp> result;
297 for (const auto& item : vector)
298 result.append(item.tmp());
299 return result;
300 }
301
302 ALWAYS_INLINE void didKill(const ExpressionType& typedTmp)
303 {
304 Tmp tmp = typedTmp.tmp();
305 if (!tmp)
306 return;
307 if (tmp.isGP())
308 m_freeGPs.append(tmp);
309 else
310 m_freeFPs.append(tmp);
311 }
312
313private:
314 ALWAYS_INLINE void validateInst(Inst& inst)
315 {
316 if (!ASSERT_DISABLED) {
317 if (!inst.isValidForm()) {
318 dataLogLn(inst);
319 CRASH();
320 }
321 }
322 }
323
324 static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }
325 static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }
326 static Arg extractArg(const Arg& arg) { return arg; }
327
328 template<typename... Arguments>
329 void append(BasicBlock* block, Kind kind, Arguments&&... arguments)
330 {
331 // FIXME: Find a way to use origin here.
332 auto& inst = block->append(kind, nullptr, extractArg(arguments)...);
333 validateInst(inst);
334 }
335
336 template<typename... Arguments>
337 void append(Kind kind, Arguments&&... arguments)
338 {
339 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
340 }
341
342 template<typename... Arguments>
343 void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)
344 {
345 Kind kind = op;
346 kind.effects = true;
347 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
348 }
349
350 Tmp newTmp(B3::Bank bank)
351 {
352 switch (bank) {
353 case B3::GP:
354 if (m_freeGPs.size())
355 return m_freeGPs.takeLast();
356 break;
357 case B3::FP:
358 if (m_freeFPs.size())
359 return m_freeFPs.takeLast();
360 break;
361 }
362 return m_code.newTmp(bank);
363 }
364
365 TypedTmp g32() { return { newTmp(B3::GP), Type::I32 }; }
366 TypedTmp g64() { return { newTmp(B3::GP), Type::I64 }; }
367 TypedTmp gAnyref() { return { newTmp(B3::GP), Type::Anyref }; }
368 TypedTmp gFuncref() { return { newTmp(B3::GP), Type::Funcref }; }
369 TypedTmp f32() { return { newTmp(B3::FP), Type::F32 }; }
370 TypedTmp f64() { return { newTmp(B3::FP), Type::F64 }; }
371
372 TypedTmp tmpForType(Type type)
373 {
374 switch (type) {
375 case Type::I32:
376 return g32();
377 case Type::I64:
378 return g64();
379 case Type::Funcref:
380 return gFuncref();
381 case Type::Anyref:
382 return gAnyref();
383 case Type::F32:
384 return f32();
385 case Type::F64:
386 return f64();
387 case Type::Void:
388 return { };
389 default:
390 RELEASE_ASSERT_NOT_REACHED();
391 }
392 }
393
394 B3::PatchpointValue* addPatchpoint(B3::Type type)
395 {
396 return m_proc.add<B3::PatchpointValue>(type, B3::Origin());
397 }
398
399 template <typename ...Args>
400 void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)
401 {
402 emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);
403 }
404
405 template <typename ...Args>
406 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)
407 {
408 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));
409 }
410
411 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)
412 {
413 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp>());
414 }
415
416 template <size_t inlineSize>
417 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Vector<ConstrainedTmp, inlineSize>&& args)
418 {
419 if (!m_patchpointSpecial)
420 m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(std::make_unique<B3::PatchpointSpecial>()));
421
422 Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));
423 Inst resultMov;
424 if (result) {
425 ASSERT(patch->type() != B3::Void);
426 switch (patch->resultConstraint.kind()) {
427 case B3::ValueRep::Register:
428 inst.args.append(Tmp(patch->resultConstraint.reg()));
429 resultMov = Inst(result.isGP() ? Move : MoveDouble, nullptr, Tmp(patch->resultConstraint.reg()), result);
430 break;
431 case B3::ValueRep::SomeRegister:
432 inst.args.append(result);
433 break;
434 default:
435 RELEASE_ASSERT_NOT_REACHED();
436 }
437 } else
438 ASSERT(patch->type() == B3::Void);
439
440 for (ConstrainedTmp& tmp : args) {
441 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
442 // validation. We should abstrcat Patch enough so ValueRep's don't need to be
443 // backed by Values.
444 // https://bugs.webkit.org/show_bug.cgi?id=194040
445 B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);
446 patch->append(dummyValue, tmp.rep);
447 switch (tmp.rep.kind()) {
448 case B3::ValueRep::SomeRegister:
449 inst.args.append(tmp.tmp);
450 break;
451 case B3::ValueRep::Register:
452 patch->earlyClobbered().clear(tmp.rep.reg());
453 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());
454 inst.args.append(Tmp(tmp.rep.reg()));
455 break;
456 case B3::ValueRep::StackArgument: {
457 auto arg = Arg::callArg(tmp.rep.offsetFromSP());
458 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);
459 inst.args.append(arg);
460 break;
461 }
462 default:
463 RELEASE_ASSERT_NOT_REACHED();
464 }
465 }
466
467 if (patch->resultConstraint.isReg())
468 patch->lateClobbered().clear(patch->resultConstraint.reg());
469 for (unsigned i = patch->numGPScratchRegisters; i--;)
470 inst.args.append(g64().tmp());
471 for (unsigned i = patch->numFPScratchRegisters; i--;)
472 inst.args.append(f64().tmp());
473
474 validateInst(inst);
475 basicBlock->append(WTFMove(inst));
476 if (resultMov) {
477 validateInst(resultMov);
478 basicBlock->append(WTFMove(resultMov));
479 }
480 }
481
482 template <typename Branch, typename Generator>
483 void emitCheck(const Branch& makeBranch, const Generator& generator)
484 {
485 // We fail along the truthy edge of 'branch'.
486 Inst branch = makeBranch();
487
488 // FIXME: Make a hashmap of these.
489 B3::CheckSpecial::Key key(branch);
490 B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(std::make_unique<B3::CheckSpecial>(key)));
491
492 // FIXME: Remove the need for dummy values
493 // https://bugs.webkit.org/show_bug.cgi?id=194040
494 B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);
495 B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);
496 checkValue->setGenerator(generator);
497
498 Inst inst(Patch, checkValue, Arg::special(special));
499 inst.args.appendVector(branch.args);
500 m_currentBlock->append(WTFMove(inst));
501 }
502
503 template <typename Func, typename ...Args>
504 void emitCCall(Func func, TypedTmp result, Args... args)
505 {
506 emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);
507 }
508 template <typename Func, typename ...Args>
509 void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)
510 {
511 B3::Type resultType = B3::Void;
512 if (result) {
513 switch (result.type()) {
514 case Type::I32:
515 resultType = B3::Int32;
516 break;
517 case Type::I64:
518 case Type::Anyref:
519 case Type::Funcref:
520 resultType = B3::Int64;
521 break;
522 case Type::F32:
523 resultType = B3::Float;
524 break;
525 case Type::F64:
526 resultType = B3::Double;
527 break;
528 default:
529 RELEASE_ASSERT_NOT_REACHED();
530 }
531 }
532
533 auto makeDummyValue = [&] (Tmp tmp) {
534 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
535 // validation. We should abstrcat CCall enough so we're not reliant on arguments
536 // to the B3::CCallValue.
537 // https://bugs.webkit.org/show_bug.cgi?id=194040
538 if (tmp.isGP())
539 return m_proc.addConstant(B3::Origin(), B3::Int64, 0);
540 return m_proc.addConstant(B3::Origin(), B3::Double, 0);
541 };
542
543 B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));
544 B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);
545
546 Inst inst(CCall, origin);
547
548 Tmp callee = g64();
549 append(block, Move, Arg::immPtr(tagCFunctionPtr<void*>(func, B3CCallPtrTag)), callee);
550 inst.args.append(callee);
551
552 if (result)
553 inst.args.append(result.tmp());
554
555 for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))
556 inst.args.append(tmp);
557
558 block->append(WTFMove(inst));
559 }
560
561 static B3::Air::Opcode moveOpForValueType(Type type)
562 {
563 switch (type) {
564 case Type::I32:
565 return Move32;
566 case Type::I64:
567 case Type::Anyref:
568 case Type::Funcref:
569 return Move;
570 case Type::F32:
571 return MoveFloat;
572 case Type::F64:
573 return MoveDouble;
574 default:
575 RELEASE_ASSERT_NOT_REACHED();
576 }
577 }
578
579 void emitThrowException(CCallHelpers&, ExceptionType);
580
581 void emitTierUpCheck(uint32_t decrementCount, B3::Origin);
582
583 void emitWriteBarrierForJSWrapper();
584 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
585 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
586 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
587
588 void unify(const ExpressionType& dst, const ExpressionType& source);
589 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
590
591 template <typename IntType>
592 void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);
593
594 template <typename IntType>
595 void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
596
597 enum class MinOrMax { Min, Max };
598
599 PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
600
601 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
602
603 void restoreWasmContextInstance(BasicBlock*, TypedTmp);
604 enum class RestoreCachedStackLimit { No, Yes };
605 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);
606
607 B3::Origin origin();
608
609 FunctionParser<AirIRGenerator>* m_parser { nullptr };
610 const ModuleInformation& m_info;
611 const MemoryMode m_mode { MemoryMode::BoundsChecking };
612 const unsigned m_functionIndex { UINT_MAX };
613 const TierUpCount* m_tierUp { nullptr };
614
615 B3::Procedure& m_proc;
616 Code& m_code;
617 BasicBlock* m_currentBlock { nullptr };
618 BasicBlock* m_rootBlock { nullptr };
619 Vector<TypedTmp> m_locals;
620 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
621 GPRReg m_memoryBaseGPR { InvalidGPRReg };
622 GPRReg m_memorySizeGPR { InvalidGPRReg };
623 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
624 bool m_makesCalls { false };
625
626 Vector<Tmp, 8> m_freeGPs;
627 Vector<Tmp, 8> m_freeFPs;
628
629 TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.
630 bool m_usesInstanceValue { false };
631 TypedTmp instanceValue()
632 {
633 m_usesInstanceValue = true;
634 return m_instanceValue;
635 }
636
637 uint32_t m_maxNumJSCallArguments { 0 };
638 unsigned m_numImportFunctions;
639
640 B3::PatchpointSpecial* m_patchpointSpecial { nullptr };
641};
642
643// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
644int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
645{
646 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
647 auto previousPtr = ptr;
648 ptr = g64();
649 auto constant = g64();
650 append(Move, Arg::bigImm(offset), constant);
651 append(Add64, constant, previousPtr, ptr);
652 return 0;
653 }
654 return offset;
655}
656
657void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)
658{
659 if (Context::useFastTLS()) {
660 auto* patchpoint = addPatchpoint(B3::Void);
661 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
662 patchpoint->clobber(RegisterSet::macroScratchRegisters());
663 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
664 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
665 jit.storeWasmContextInstance(params[0].gpr());
666 });
667 emitPatchpoint(block, patchpoint, Tmp(), instance);
668 return;
669 }
670
671 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
672 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
673 auto* patchpoint = addPatchpoint(B3::Void);
674 B3::Effects effects = B3::Effects::none();
675 effects.writesPinned = true;
676 effects.reads = B3::HeapRange::top();
677 patchpoint->effects = effects;
678 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
679 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
680 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {
681 jit.move(param[0].gpr(), wasmContextInstanceGPR);
682 });
683 emitPatchpoint(block, patchpoint, Tmp(), instance);
684}
685
686AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
687 : m_info(info)
688 , m_mode(mode)
689 , m_functionIndex(functionIndex)
690 , m_tierUp(tierUp)
691 , m_proc(procedure)
692 , m_code(m_proc.code())
693 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
694 , m_numImportFunctions(info.importFunctionCount())
695{
696 m_currentBlock = m_code.addBlock();
697 m_rootBlock = m_currentBlock;
698
699 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
700 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
701
702 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
703 m_code.pinRegister(m_memoryBaseGPR);
704
705 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
706 if (!Context::useFastTLS())
707 m_code.pinRegister(m_wasmContextInstanceGPR);
708
709 if (mode != MemoryMode::Signaling) {
710 m_memorySizeGPR = pinnedRegs.sizeRegister;
711 m_code.pinRegister(m_memorySizeGPR);
712 }
713
714 if (throwWasmException)
715 Thunks::singleton().setThrowWasmException(throwWasmException);
716
717 if (info.memory) {
718 switch (m_mode) {
719 case MemoryMode::BoundsChecking:
720 break;
721 case MemoryMode::Signaling:
722 // Most memory accesses in signaling mode don't do an explicit
723 // exception check because they can rely on fault handling to detect
724 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
725 // thunk to exist so that it can jump to that thunk.
726 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
727 CRASH();
728 break;
729 }
730 }
731
732 m_code.setNumEntrypoints(1);
733
734 GPRReg contextInstance = Context::useFastTLS() ? wasmCallingConventionAir().prologueScratch(1) : m_wasmContextInstanceGPR;
735
736 Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
737 AllowMacroScratchRegisterUsage allowScratch(jit);
738 code.emitDefaultPrologue(jit);
739
740 {
741 GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
742 auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
743 jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
744 compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
745 });
746 jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
747 jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
748 }
749
750 {
751 const Checked<int32_t> wasmFrameSize = m_code.frameSize();
752 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
753 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
754 // This allows us to elide stack checks for functions that are terminal nodes in the call
755 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
756 // having any such terminal node have its parent caller include some extra size in its
757 // own check for it. The goal here is twofold:
758 // 1. Emit less code.
759 // 2. Try to speed things up by skipping stack checks.
760 minimumParentCheckSize,
761 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
762 // spill all arguments to the stack, we ensure that a stack check here covers the
763 // stack that such a stub would use.
764 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
765 ));
766 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
767 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
768 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
769
770 // This allows leaf functions to not do stack checks if their frame size is within
771 // certain limits since their caller would have already done the check.
772 if (needsOverflowCheck) {
773 GPRReg scratch = wasmCallingConventionAir().prologueScratch(0);
774
775 if (Context::useFastTLS())
776 jit.loadWasmContextInstance(contextInstance);
777
778 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), GPRInfo::callFrameRegister, scratch);
779 MacroAssembler::JumpList overflow;
780 if (UNLIKELY(needUnderflowCheck))
781 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch, GPRInfo::callFrameRegister));
782 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch, CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit())));
783 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
784 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
785 });
786 } else if (m_usesInstanceValue && Context::useFastTLS()) {
787 // No overflow check is needed, but the instance values still needs to be correct.
788 jit.loadWasmContextInstance(contextInstance);
789 }
790 }
791 });
792
793 m_code.setPrologueForEntrypoint(0, WTFMove(prologueGenerator));
794
795 if (Context::useFastTLS()) {
796 m_instanceValue = g64();
797 // FIXME: Would be nice to only do this if we use instance value.
798 append(Move, Tmp(contextInstance), m_instanceValue);
799 } else
800 m_instanceValue = { Tmp(contextInstance), Type::I64 };
801
802 ASSERT(!m_locals.size());
803 m_locals.grow(signature.argumentCount());
804 for (unsigned i = 0; i < signature.argumentCount(); ++i) {
805 Type type = signature.argument(i);
806 m_locals[i] = tmpForType(type);
807 }
808
809 wasmCallingConventionAir().loadArguments(signature, [&] (const Arg& arg, unsigned i) {
810 switch (signature.argument(i)) {
811 case Type::I32:
812 append(Move32, arg, m_locals[i]);
813 break;
814 case Type::I64:
815 case Type::Anyref:
816 case Type::Funcref:
817 append(Move, arg, m_locals[i]);
818 break;
819 case Type::F32:
820 append(MoveFloat, arg, m_locals[i]);
821 break;
822 case Type::F64:
823 append(MoveDouble, arg, m_locals[i]);
824 break;
825 default:
826 RELEASE_ASSERT_NOT_REACHED();
827 }
828 });
829
830 emitTierUpCheck(TierUpCount::functionEntryDecrement(), B3::Origin());
831}
832
833void AirIRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, TypedTmp instance, BasicBlock* block)
834{
835 restoreWasmContextInstance(block, instance);
836
837 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
838 // The Instance caches the stack limit, but also knows where its canonical location is.
839 static_assert(sizeof(decltype(static_cast<Instance*>(nullptr)->cachedStackLimit())) == sizeof(uint64_t), "");
840
841 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfPointerToActualStackLimit(), B3::Width64));
842 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedStackLimit(), B3::Width64));
843 auto temp = g64();
844 append(block, Move, Arg::addr(instanceValue(), Instance::offsetOfPointerToActualStackLimit()), temp);
845 append(block, Move, Arg::addr(temp), temp);
846 append(block, Move, temp, Arg::addr(instanceValue(), Instance::offsetOfCachedStackLimit()));
847 }
848
849 if (!!memory) {
850 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
851 RegisterSet clobbers;
852 clobbers.set(pinnedRegs->baseMemoryPointer);
853 clobbers.set(pinnedRegs->sizeRegister);
854 if (!isARM64())
855 clobbers.set(RegisterSet::macroScratchRegisters());
856
857 auto* patchpoint = addPatchpoint(B3::Void);
858 B3::Effects effects = B3::Effects::none();
859 effects.writesPinned = true;
860 effects.reads = B3::HeapRange::top();
861 patchpoint->effects = effects;
862 patchpoint->clobber(clobbers);
863 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
864
865 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
866 AllowMacroScratchRegisterUsage allowScratch(jit);
867 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
868 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
869
870 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
871 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
872
873 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
874 });
875
876 emitPatchpoint(block, patchpoint, Tmp(), instance);
877 }
878}
879
880void AirIRGenerator::emitThrowException(CCallHelpers& jit, ExceptionType type)
881{
882 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
883 auto jumpToExceptionStub = jit.jump();
884
885 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
886 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
887 });
888}
889
890auto AirIRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
891{
892 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
893 totalBytesChecked += m_locals.size();
894 uint32_t totalBytes;
895 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
896
897 for (uint32_t i = 0; i < count; ++i) {
898 auto local = tmpForType(type);
899 m_locals.uncheckedAppend(local);
900 switch (type) {
901 case Type::Anyref:
902 case Type::Funcref:
903 append(Move, Arg::imm(JSValue::encode(jsNull())), local);
904 break;
905 case Type::I32:
906 case Type::I64: {
907 append(Xor64, local, local);
908 break;
909 }
910 case Type::F32:
911 case Type::F64: {
912 auto temp = g64();
913 // IEEE 754 "0" is just int32/64 zero.
914 append(Xor64, temp, temp);
915 append(type == Type::F32 ? Move32ToFloat : Move64ToDouble, temp, local);
916 break;
917 }
918 default:
919 RELEASE_ASSERT_NOT_REACHED();
920 }
921 }
922 return { };
923}
924
925auto AirIRGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
926{
927 return addConstant(m_currentBlock, type, value);
928}
929
930auto AirIRGenerator::addConstant(BasicBlock* block, Type type, uint64_t value) -> ExpressionType
931{
932 auto result = tmpForType(type);
933 switch (type) {
934 case Type::I32:
935 case Type::I64:
936 case Type::Anyref:
937 case Type::Funcref:
938 append(block, Move, Arg::bigImm(value), result);
939 break;
940 case Type::F32:
941 case Type::F64: {
942 auto tmp = g64();
943 append(block, Move, Arg::bigImm(value), tmp);
944 append(block, type == Type::F32 ? Move32ToFloat : Move64ToDouble, tmp, result);
945 break;
946 }
947
948 default:
949 RELEASE_ASSERT_NOT_REACHED();
950 }
951
952 return result;
953}
954
955auto AirIRGenerator::addArguments(const Signature& signature) -> PartialResult
956{
957 RELEASE_ASSERT(m_locals.size() == signature.argumentCount()); // We handle arguments in the prologue
958 return { };
959}
960
961auto AirIRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
962{
963 ASSERT(value.tmp());
964 result = tmpForType(Type::I32);
965 auto tmp = g64();
966
967 append(Move, Arg::bigImm(JSValue::encode(jsNull())), tmp);
968 append(Compare64, Arg::relCond(MacroAssembler::Equal), value, tmp, result);
969
970 return { };
971}
972
973auto AirIRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
974{
975 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
976 result = tmpForType(Type::Funcref);
977 emitCCall(&doWasmRefFunc, result, instanceValue(), addConstant(Type::I32, index));
978
979 return { };
980}
981
982auto AirIRGenerator::addTableGet(unsigned tableIndex, ExpressionType& index, ExpressionType& result) -> PartialResult
983{
984 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
985 ASSERT(index.tmp());
986 ASSERT(index.type() == Type::I32);
987 result = tmpForType(m_info.tables[tableIndex].wasmType());
988
989 emitCCall(&getWasmTableElement, result, instanceValue(), addConstant(Type::I32, tableIndex), index);
990 emitCheck([&] {
991 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), result, result);
992 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
993 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
994 });
995
996 return { };
997}
998
999auto AirIRGenerator::addTableSet(unsigned tableIndex, ExpressionType& index, ExpressionType& value) -> PartialResult
1000{
1001 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1002 ASSERT(index.tmp());
1003 ASSERT(index.type() == Type::I32);
1004 ASSERT(value.tmp());
1005
1006 auto shouldThrow = g32();
1007 emitCCall(&setWasmTableElement, shouldThrow, instanceValue(), addConstant(Type::I32, tableIndex), index, value);
1008
1009 emitCheck([&] {
1010 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), shouldThrow, shouldThrow);
1011 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1012 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
1013 });
1014
1015 return { };
1016}
1017
1018auto AirIRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
1019{
1020 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1021 result = tmpForType(Type::I32);
1022
1023 int32_t (*doSize)(Instance*, unsigned) = [] (Instance* instance, unsigned tableIndex) -> int32_t {
1024 return instance->table(tableIndex)->length();
1025 };
1026
1027 emitCCall(doSize, result, instanceValue(), addConstant(Type::I32, tableIndex));
1028
1029 return { };
1030}
1031
1032auto AirIRGenerator::addTableGrow(unsigned tableIndex, ExpressionType& fill, ExpressionType& delta, ExpressionType& result) -> PartialResult
1033{
1034 ASSERT(fill.tmp());
1035 ASSERT(isSubtype(fill.type(), m_info.tables[tableIndex].wasmType()));
1036 ASSERT(delta.tmp());
1037 ASSERT(delta.type() == Type::I32);
1038 result = tmpForType(Type::I32);
1039
1040 emitCCall(&doWasmTableGrow, result, instanceValue(), addConstant(Type::I32, tableIndex), fill, delta);
1041
1042 return { };
1043}
1044
1045auto AirIRGenerator::addTableFill(unsigned tableIndex, ExpressionType& offset, ExpressionType& fill, ExpressionType& count) -> PartialResult
1046{
1047 ASSERT(fill.tmp());
1048 ASSERT(isSubtype(fill.type(), m_info.tables[tableIndex].wasmType()));
1049 ASSERT(offset.tmp());
1050 ASSERT(offset.type() == Type::I32);
1051 ASSERT(count.tmp());
1052 ASSERT(count.type() == Type::I32);
1053
1054 auto result = tmpForType(Type::I32);
1055 emitCCall(&doWasmTableFill, result, instanceValue(), addConstant(Type::I32, tableIndex), offset, fill, count);
1056
1057 emitCheck([&] {
1058 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), result, result);
1059 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1060 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
1061 });
1062
1063 return { };
1064}
1065
1066auto AirIRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
1067{
1068 ASSERT(m_locals[index].tmp());
1069 result = tmpForType(m_locals[index].type());
1070 append(moveOpForValueType(m_locals[index].type()), m_locals[index].tmp(), result);
1071 return { };
1072}
1073
1074auto AirIRGenerator::addUnreachable() -> PartialResult
1075{
1076 B3::PatchpointValue* unreachable = addPatchpoint(B3::Void);
1077 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1078 this->emitThrowException(jit, ExceptionType::Unreachable);
1079 });
1080 unreachable->effects.terminal = true;
1081 emitPatchpoint(unreachable, Tmp());
1082 return { };
1083}
1084
1085auto AirIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
1086{
1087 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
1088 instance->storeTopCallFrame(callFrame);
1089
1090 if (delta < 0)
1091 return -1;
1092
1093 auto grown = instance->memory()->grow(PageCount(delta));
1094 if (!grown) {
1095 switch (grown.error()) {
1096 case Memory::GrowFailReason::InvalidDelta:
1097 case Memory::GrowFailReason::InvalidGrowSize:
1098 case Memory::GrowFailReason::WouldExceedMaximum:
1099 case Memory::GrowFailReason::OutOfMemory:
1100 return -1;
1101 }
1102 RELEASE_ASSERT_NOT_REACHED();
1103 }
1104
1105 return grown.value().pageCount();
1106 };
1107
1108 result = g32();
1109 emitCCall(growMemory, result, TypedTmp { Tmp(GPRInfo::callFrameRegister), Type::I64 }, instanceValue(), delta);
1110 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_currentBlock);
1111
1112 return { };
1113}
1114
1115auto AirIRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
1116{
1117 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
1118
1119 auto temp1 = g64();
1120 auto temp2 = g64();
1121
1122 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedMemorySize(), B3::Width64));
1123 append(Move, Arg::addr(instanceValue(), Instance::offsetOfCachedMemorySize()), temp1);
1124 constexpr uint32_t shiftValue = 16;
1125 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
1126 append(Move, Arg::imm(16), temp2);
1127 addShift(Type::I32, Urshift64, temp1, temp2, result);
1128 append(Move32, result, result);
1129
1130 return { };
1131}
1132
1133auto AirIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
1134{
1135 ASSERT(m_locals[index].tmp());
1136 append(moveOpForValueType(m_locals[index].type()), value, m_locals[index].tmp());
1137 return { };
1138}
1139
1140auto AirIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
1141{
1142 Type type = m_info.globals[index].type;
1143
1144 result = tmpForType(type);
1145
1146 auto temp = g64();
1147
1148 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1149 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1150
1151 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1152 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1153 append(moveOpForValueType(type), Arg::addr(temp, offset), result);
1154 else {
1155 auto temp2 = g64();
1156 append(Move, Arg::bigImm(offset), temp2);
1157 append(Add64, temp2, temp, temp);
1158 append(moveOpForValueType(type), Arg::addr(temp), result);
1159 }
1160 return { };
1161}
1162
1163auto AirIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
1164{
1165 auto temp = g64();
1166
1167 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1168 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1169
1170 Type type = m_info.globals[index].type;
1171
1172 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1173 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1174 append(moveOpForValueType(type), value, Arg::addr(temp, offset));
1175 else {
1176 auto temp2 = g64();
1177 append(Move, Arg::bigImm(offset), temp2);
1178 append(Add64, temp2, temp, temp);
1179 append(moveOpForValueType(type), value, Arg::addr(temp));
1180 }
1181
1182 if (isSubtype(type, Anyref))
1183 emitWriteBarrierForJSWrapper();
1184
1185 return { };
1186}
1187
1188inline void AirIRGenerator::emitWriteBarrierForJSWrapper()
1189{
1190 auto cell = g64();
1191 auto vm = g64();
1192 auto cellState = g32();
1193 auto threshold = g32();
1194
1195 BasicBlock* fenceCheckPath = m_code.addBlock();
1196 BasicBlock* fencePath = m_code.addBlock();
1197 BasicBlock* doSlowPath = m_code.addBlock();
1198 BasicBlock* continuation = m_code.addBlock();
1199
1200 append(Move, Arg::addr(instanceValue(), Instance::offsetOfOwner()), cell);
1201 append(Move, Arg::addr(cell, JSWebAssemblyInstance::offsetOfVM()), vm);
1202 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1203 append(Move32, Arg::addr(vm, VM::offsetOfHeapBarrierThreshold()), threshold);
1204
1205 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, threshold);
1206 m_currentBlock->setSuccessors(continuation, fenceCheckPath);
1207 m_currentBlock = fenceCheckPath;
1208
1209 append(Load8, Arg::addr(vm, VM::offsetOfHeapMutatorShouldBeFenced()), threshold);
1210 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), threshold, threshold);
1211 m_currentBlock->setSuccessors(doSlowPath, fencePath);
1212 m_currentBlock = fencePath;
1213
1214 auto* doFence = addPatchpoint(B3::Void);
1215 doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1216 jit.memoryFence();
1217 });
1218 emitPatchpoint(doFence, Tmp());
1219
1220 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1221 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, Arg::imm(blackThreshold));
1222 m_currentBlock->setSuccessors(continuation, doSlowPath);
1223 m_currentBlock = doSlowPath;
1224
1225 void (*writeBarrier)(JSWebAssemblyInstance*, VM*) = [] (JSWebAssemblyInstance* cell, VM* vm) -> void {
1226 ASSERT(cell);
1227 ASSERT(vm);
1228 vm->heap.writeBarrierSlowPath(cell);
1229 };
1230 emitCCall(writeBarrier, TypedTmp(), cell, vm);
1231 append(Jump);
1232 m_currentBlock->setSuccessors(continuation);
1233 m_currentBlock = continuation;
1234}
1235
1236inline AirIRGenerator::ExpressionType AirIRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1237{
1238 ASSERT(m_memoryBaseGPR);
1239
1240 auto result = g64();
1241 append(Move32, pointer, result);
1242
1243 switch (m_mode) {
1244 case MemoryMode::BoundsChecking: {
1245 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
1246 ASSERT(m_memorySizeGPR);
1247 ASSERT(sizeOfOperation + offset > offset);
1248 auto temp = g64();
1249 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1250 append(Add64, result, temp);
1251
1252 emitCheck([&] {
1253 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, Tmp(m_memorySizeGPR));
1254 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1255 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1256 });
1257 break;
1258 }
1259
1260 case MemoryMode::Signaling: {
1261 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1262 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1263 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1264 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1265 // register + large offset access. We don't think this will be generated frequently.
1266 //
1267 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1268 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1269 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1270 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1271 if (offset >= Memory::fastMappedRedzoneBytes()) {
1272 uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1273 auto temp = g64();
1274 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1275 append(Add64, result, temp);
1276 auto sizeMax = addConstant(Type::I64, maximum);
1277
1278 emitCheck([&] {
1279 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, sizeMax);
1280 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1281 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1282 });
1283 }
1284 break;
1285 }
1286 }
1287
1288 append(Add64, Tmp(m_memoryBaseGPR), result);
1289 return result;
1290}
1291
1292inline uint32_t sizeOfLoadOp(LoadOpType op)
1293{
1294 switch (op) {
1295 case LoadOpType::I32Load8S:
1296 case LoadOpType::I32Load8U:
1297 case LoadOpType::I64Load8S:
1298 case LoadOpType::I64Load8U:
1299 return 1;
1300 case LoadOpType::I32Load16S:
1301 case LoadOpType::I64Load16S:
1302 case LoadOpType::I32Load16U:
1303 case LoadOpType::I64Load16U:
1304 return 2;
1305 case LoadOpType::I32Load:
1306 case LoadOpType::I64Load32S:
1307 case LoadOpType::I64Load32U:
1308 case LoadOpType::F32Load:
1309 return 4;
1310 case LoadOpType::I64Load:
1311 case LoadOpType::F64Load:
1312 return 8;
1313 }
1314 RELEASE_ASSERT_NOT_REACHED();
1315}
1316
1317inline TypedTmp AirIRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1318{
1319 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1320
1321 TypedTmp immTmp;
1322 TypedTmp newPtr;
1323 TypedTmp result;
1324
1325 Arg addrArg;
1326 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfLoadOp(op))))
1327 addrArg = Arg::addr(pointer, offset);
1328 else {
1329 immTmp = g64();
1330 newPtr = g64();
1331 append(Move, Arg::bigImm(offset), immTmp);
1332 append(Add64, immTmp, pointer, newPtr);
1333 addrArg = Arg::addr(newPtr);
1334 }
1335
1336 switch (op) {
1337 case LoadOpType::I32Load8S: {
1338 result = g32();
1339 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1340 break;
1341 }
1342
1343 case LoadOpType::I64Load8S: {
1344 result = g64();
1345 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1346 append(SignExtend32ToPtr, result, result);
1347 break;
1348 }
1349
1350 case LoadOpType::I32Load8U: {
1351 result = g32();
1352 appendEffectful(Load8, addrArg, result);
1353 break;
1354 }
1355
1356 case LoadOpType::I64Load8U: {
1357 result = g64();
1358 appendEffectful(Load8, addrArg, result);
1359 break;
1360 }
1361
1362 case LoadOpType::I32Load16S: {
1363 result = g32();
1364 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1365 break;
1366 }
1367
1368 case LoadOpType::I64Load16S: {
1369 result = g64();
1370 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1371 append(SignExtend32ToPtr, result, result);
1372 break;
1373 }
1374
1375 case LoadOpType::I32Load16U: {
1376 result = g32();
1377 appendEffectful(Load16, addrArg, result);
1378 break;
1379 }
1380
1381 case LoadOpType::I64Load16U: {
1382 result = g64();
1383 appendEffectful(Load16, addrArg, result);
1384 break;
1385 }
1386
1387 case LoadOpType::I32Load:
1388 result = g32();
1389 appendEffectful(Move32, addrArg, result);
1390 break;
1391
1392 case LoadOpType::I64Load32U: {
1393 result = g64();
1394 appendEffectful(Move32, addrArg, result);
1395 break;
1396 }
1397
1398 case LoadOpType::I64Load32S: {
1399 result = g64();
1400 appendEffectful(Move32, addrArg, result);
1401 append(SignExtend32ToPtr, result, result);
1402 break;
1403 }
1404
1405 case LoadOpType::I64Load: {
1406 result = g64();
1407 appendEffectful(Move, addrArg, result);
1408 break;
1409 }
1410
1411 case LoadOpType::F32Load: {
1412 result = f32();
1413 appendEffectful(MoveFloat, addrArg, result);
1414 break;
1415 }
1416
1417 case LoadOpType::F64Load: {
1418 result = f64();
1419 appendEffectful(MoveDouble, addrArg, result);
1420 break;
1421 }
1422 }
1423
1424 return result;
1425}
1426
1427auto AirIRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1428{
1429 ASSERT(pointer.tmp().isGP());
1430
1431 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1432 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1433 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1434 auto* patch = addPatchpoint(B3::Void);
1435 patch->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1436 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1437 });
1438 emitPatchpoint(patch, Tmp());
1439
1440 // We won't reach here, so we just pick a random reg.
1441 switch (op) {
1442 case LoadOpType::I32Load8S:
1443 case LoadOpType::I32Load16S:
1444 case LoadOpType::I32Load:
1445 case LoadOpType::I32Load16U:
1446 case LoadOpType::I32Load8U:
1447 result = g32();
1448 break;
1449 case LoadOpType::I64Load8S:
1450 case LoadOpType::I64Load8U:
1451 case LoadOpType::I64Load16S:
1452 case LoadOpType::I64Load32U:
1453 case LoadOpType::I64Load32S:
1454 case LoadOpType::I64Load:
1455 case LoadOpType::I64Load16U:
1456 result = g64();
1457 break;
1458 case LoadOpType::F32Load:
1459 result = f32();
1460 break;
1461 case LoadOpType::F64Load:
1462 result = f64();
1463 break;
1464 }
1465 } else
1466 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1467
1468 return { };
1469}
1470
1471inline uint32_t sizeOfStoreOp(StoreOpType op)
1472{
1473 switch (op) {
1474 case StoreOpType::I32Store8:
1475 case StoreOpType::I64Store8:
1476 return 1;
1477 case StoreOpType::I32Store16:
1478 case StoreOpType::I64Store16:
1479 return 2;
1480 case StoreOpType::I32Store:
1481 case StoreOpType::I64Store32:
1482 case StoreOpType::F32Store:
1483 return 4;
1484 case StoreOpType::I64Store:
1485 case StoreOpType::F64Store:
1486 return 8;
1487 }
1488 RELEASE_ASSERT_NOT_REACHED();
1489}
1490
1491
1492inline void AirIRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1493{
1494 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1495
1496 TypedTmp immTmp;
1497 TypedTmp newPtr;
1498
1499 Arg addrArg;
1500 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfStoreOp(op))))
1501 addrArg = Arg::addr(pointer, offset);
1502 else {
1503 immTmp = g64();
1504 newPtr = g64();
1505 append(Move, Arg::bigImm(offset), immTmp);
1506 append(Add64, immTmp, pointer, newPtr);
1507 addrArg = Arg::addr(newPtr);
1508 }
1509
1510 switch (op) {
1511 case StoreOpType::I64Store8:
1512 case StoreOpType::I32Store8:
1513 append(Store8, value, addrArg);
1514 return;
1515
1516 case StoreOpType::I64Store16:
1517 case StoreOpType::I32Store16:
1518 append(Store16, value, addrArg);
1519 return;
1520
1521 case StoreOpType::I64Store32:
1522 case StoreOpType::I32Store:
1523 append(Move32, value, addrArg);
1524 return;
1525
1526 case StoreOpType::I64Store:
1527 append(Move, value, addrArg);
1528 return;
1529
1530 case StoreOpType::F32Store:
1531 append(MoveFloat, value, addrArg);
1532 return;
1533
1534 case StoreOpType::F64Store:
1535 append(MoveDouble, value, addrArg);
1536 return;
1537 }
1538
1539 RELEASE_ASSERT_NOT_REACHED();
1540}
1541
1542auto AirIRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1543{
1544 ASSERT(pointer.tmp().isGP());
1545
1546 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1547 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1548 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1549 auto* throwException = addPatchpoint(B3::Void);
1550 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1551 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1552 });
1553 emitPatchpoint(throwException, Tmp());
1554 } else
1555 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1556
1557 return { };
1558}
1559
1560auto AirIRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1561{
1562 ASSERT(nonZero.type() == zero.type());
1563 result = tmpForType(nonZero.type());
1564 append(moveOpForValueType(nonZero.type()), nonZero, result);
1565
1566 BasicBlock* isZero = m_code.addBlock();
1567 BasicBlock* continuation = m_code.addBlock();
1568
1569 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), condition, condition);
1570 m_currentBlock->setSuccessors(isZero, continuation);
1571
1572 append(isZero, moveOpForValueType(zero.type()), zero, result);
1573 append(isZero, Jump);
1574 isZero->setSuccessors(continuation);
1575
1576 m_currentBlock = continuation;
1577
1578 return { };
1579}
1580
1581void AirIRGenerator::emitTierUpCheck(uint32_t decrementCount, B3::Origin origin)
1582{
1583 UNUSED_PARAM(origin);
1584
1585 if (!m_tierUp)
1586 return;
1587
1588 auto countdownPtr = g64();
1589 auto oldCountdown = g64();
1590 auto newCountdown = g64();
1591
1592 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(m_tierUp)), countdownPtr);
1593 append(Move32, Arg::addr(countdownPtr), oldCountdown);
1594
1595 RELEASE_ASSERT(Arg::isValidImmForm(decrementCount));
1596 append(Move32, oldCountdown, newCountdown);
1597 append(Sub32, Arg::imm(decrementCount), newCountdown);
1598 append(Move32, newCountdown, Arg::addr(countdownPtr));
1599
1600 auto* patch = addPatchpoint(B3::Void);
1601 B3::Effects effects = B3::Effects::none();
1602 effects.reads = B3::HeapRange::top();
1603 effects.writes = B3::HeapRange::top();
1604 patch->effects = effects;
1605
1606 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1607 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1608 MacroAssembler::Label tierUpResume = jit.label();
1609
1610 params.addLatePath([=] (CCallHelpers& jit) {
1611 tierUp.link(&jit);
1612
1613 const unsigned extraPaddingBytes = 0;
1614 RegisterSet registersToSpill = { };
1615 registersToSpill.add(GPRInfo::argumentGPR1);
1616 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1617
1618 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1619 MacroAssembler::Call call = jit.nearCall();
1620
1621 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1622 jit.jump(tierUpResume);
1623
1624 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1625 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1626
1627 });
1628 });
1629 });
1630
1631 emitPatchpoint(patch, Tmp(), newCountdown, oldCountdown);
1632}
1633
1634AirIRGenerator::ControlData AirIRGenerator::addLoop(Type signature)
1635{
1636 BasicBlock* body = m_code.addBlock();
1637 BasicBlock* continuation = m_code.addBlock();
1638
1639 append(Jump);
1640 m_currentBlock->setSuccessors(body);
1641
1642 m_currentBlock = body;
1643 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1644
1645 return ControlData(origin(), signature, tmpForType(signature), BlockType::Loop, continuation, body);
1646}
1647
1648AirIRGenerator::ControlData AirIRGenerator::addTopLevel(Type signature)
1649{
1650 return ControlData(B3::Origin(), signature, tmpForType(signature), BlockType::TopLevel, m_code.addBlock());
1651}
1652
1653AirIRGenerator::ControlData AirIRGenerator::addBlock(Type signature)
1654{
1655 return ControlData(origin(), signature, tmpForType(signature), BlockType::Block, m_code.addBlock());
1656}
1657
1658auto AirIRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1659{
1660 BasicBlock* taken = m_code.addBlock();
1661 BasicBlock* notTaken = m_code.addBlock();
1662 BasicBlock* continuation = m_code.addBlock();
1663
1664 // Wasm bools are i32.
1665 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1666 m_currentBlock->setSuccessors(taken, notTaken);
1667
1668 m_currentBlock = taken;
1669 result = ControlData(origin(), signature, tmpForType(signature), BlockType::If, continuation, notTaken);
1670 return { };
1671}
1672
1673auto AirIRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1674{
1675 unifyValuesWithBlock(currentStack, data.result);
1676 append(Jump);
1677 m_currentBlock->setSuccessors(data.continuation);
1678 return addElseToUnreachable(data);
1679}
1680
1681auto AirIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1682{
1683 ASSERT(data.type() == BlockType::If);
1684 m_currentBlock = data.special;
1685 data.convertIfToBlock();
1686 return { };
1687}
1688
1689auto AirIRGenerator::addReturn(const ControlData& data, const ExpressionList& returnValues) -> PartialResult
1690{
1691 ASSERT(returnValues.size() <= 1);
1692 if (returnValues.size()) {
1693 Tmp returnValueGPR = Tmp(GPRInfo::returnValueGPR);
1694 Tmp returnValueFPR = Tmp(FPRInfo::returnValueFPR);
1695 switch (data.signature()) {
1696 case Type::I32:
1697 append(Move32, returnValues[0], returnValueGPR);
1698 append(Ret32, returnValueGPR);
1699 break;
1700 case Type::I64:
1701 case Type::Anyref:
1702 case Type::Funcref:
1703 append(Move, returnValues[0], returnValueGPR);
1704 append(Ret64, returnValueGPR);
1705 break;
1706 case Type::F32:
1707 append(MoveFloat, returnValues[0], returnValueFPR);
1708 append(RetFloat, returnValueFPR);
1709 break;
1710 case Type::F64:
1711 append(MoveDouble, returnValues[0], returnValueFPR);
1712 append(RetFloat, returnValueFPR);
1713 break;
1714 default:
1715 RELEASE_ASSERT_NOT_REACHED();
1716 }
1717 } else
1718 append(RetVoid);
1719 return { };
1720}
1721
1722// NOTE: All branches in Wasm are on 32-bit ints
1723
1724auto AirIRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1725{
1726 unifyValuesWithBlock(returnValues, data.resultForBranch());
1727
1728 BasicBlock* target = data.targetBlockForBranch();
1729 if (condition) {
1730 BasicBlock* continuation = m_code.addBlock();
1731 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1732 m_currentBlock->setSuccessors(target, continuation);
1733 m_currentBlock = continuation;
1734 } else {
1735 append(Jump);
1736 m_currentBlock->setSuccessors(target);
1737 }
1738
1739 return { };
1740}
1741
1742auto AirIRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1743{
1744 auto& successors = m_currentBlock->successors();
1745 ASSERT(successors.isEmpty());
1746 for (const auto& target : targets) {
1747 unifyValuesWithBlock(expressionStack, target->resultForBranch());
1748 successors.append(target->targetBlockForBranch());
1749 }
1750 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1751 successors.append(defaultTarget.targetBlockForBranch());
1752
1753 ASSERT(condition.type() == Type::I32);
1754
1755 // FIXME: We should consider dynamically switching between a jump table
1756 // and a binary switch depending on the number of successors.
1757 // https://bugs.webkit.org/show_bug.cgi?id=194477
1758
1759 size_t numTargets = targets.size();
1760
1761 auto* patchpoint = addPatchpoint(B3::Void);
1762 patchpoint->effects = B3::Effects::none();
1763 patchpoint->effects.terminal = true;
1764 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1765
1766 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1767 AllowMacroScratchRegisterUsage allowScratch(jit);
1768
1769 Vector<int64_t> cases;
1770 cases.reserveInitialCapacity(numTargets);
1771 for (size_t i = 0; i < numTargets; ++i)
1772 cases.uncheckedAppend(i);
1773
1774 GPRReg valueReg = params[0].gpr();
1775 BinarySwitch binarySwitch(valueReg, cases, BinarySwitch::Int32);
1776
1777 Vector<CCallHelpers::Jump> caseJumps;
1778 caseJumps.resize(numTargets);
1779
1780 while (binarySwitch.advance(jit)) {
1781 unsigned value = binarySwitch.caseValue();
1782 unsigned index = binarySwitch.caseIndex();
1783 ASSERT_UNUSED(value, value == index);
1784 ASSERT(index < numTargets);
1785 caseJumps[index] = jit.jump();
1786 }
1787
1788 CCallHelpers::JumpList fallThrough = binarySwitch.fallThrough();
1789
1790 Vector<Box<CCallHelpers::Label>> successorLabels = params.successorLabels();
1791 ASSERT(successorLabels.size() == caseJumps.size() + 1);
1792
1793 params.addLatePath([=, caseJumps = WTFMove(caseJumps), successorLabels = WTFMove(successorLabels)] (CCallHelpers& jit) {
1794 for (size_t i = 0; i < numTargets; ++i)
1795 caseJumps[i].linkTo(*successorLabels[i], &jit);
1796 fallThrough.linkTo(*successorLabels[numTargets], &jit);
1797 });
1798 });
1799
1800 emitPatchpoint(patchpoint, TypedTmp(), condition);
1801
1802 return { };
1803}
1804
1805auto AirIRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1806{
1807 ControlData& data = entry.controlData;
1808
1809 unifyValuesWithBlock(expressionStack, data.result);
1810 append(Jump);
1811 m_currentBlock->setSuccessors(data.continuation);
1812
1813 return addEndToUnreachable(entry);
1814}
1815
1816
1817auto AirIRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1818{
1819 ControlData& data = entry.controlData;
1820 m_currentBlock = data.continuation;
1821
1822 if (data.type() == BlockType::If) {
1823 append(data.special, Jump);
1824 data.special->setSuccessors(m_currentBlock);
1825 }
1826
1827 for (const auto& result : data.result)
1828 entry.enclosedExpressionStack.append(result);
1829
1830 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1831 if (data.type() == BlockType::TopLevel)
1832 return addReturn(data, entry.enclosedExpressionStack);
1833
1834 return { };
1835}
1836
1837auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1838{
1839 ASSERT(signature.argumentCount() == args.size());
1840
1841 m_makesCalls = true;
1842
1843 Type returnType = signature.returnType();
1844 if (returnType != Type::Void)
1845 result = tmpForType(returnType);
1846
1847 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1848
1849 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1850 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1851
1852 auto currentInstance = g64();
1853 append(Move, instanceValue(), currentInstance);
1854
1855 auto targetInstance = g64();
1856
1857 // FIXME: We should have better isel here.
1858 // https://bugs.webkit.org/show_bug.cgi?id=193999
1859 append(Move, Arg::bigImm(Instance::offsetOfTargetInstance(functionIndex)), targetInstance);
1860 append(Add64, instanceValue(), targetInstance);
1861 append(Move, Arg::addr(targetInstance), targetInstance);
1862
1863 BasicBlock* isWasmBlock = m_code.addBlock();
1864 BasicBlock* isEmbedderBlock = m_code.addBlock();
1865 BasicBlock* continuation = m_code.addBlock();
1866
1867 append(BranchTest64, Arg::resCond(MacroAssembler::NonZero), targetInstance, targetInstance);
1868 m_currentBlock->setSuccessors(isWasmBlock, isEmbedderBlock);
1869
1870 {
1871 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1872 patchpoint->effects.writesPinned = true;
1873 patchpoint->effects.readsPinned = true;
1874 // We need to clobber all potential pinned registers since we might be leaving the instance.
1875 // We pessimistically assume we could be calling to something that is bounds checking.
1876 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1877 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1878
1879 Vector<ConstrainedTmp> patchArgs;
1880 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1881 patchArgs.append({ tmp, rep });
1882 });
1883
1884 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1885 AllowMacroScratchRegisterUsage allowScratch(jit);
1886 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1887 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1888 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1889 });
1890 });
1891
1892 emitPatchpoint(isWasmBlock, patchpoint, result, WTFMove(patchArgs));
1893 append(isWasmBlock, Jump);
1894 isWasmBlock->setSuccessors(continuation);
1895 }
1896
1897 {
1898 auto jumpDestination = g64();
1899 append(isEmbedderBlock, Move, Arg::bigImm(Instance::offsetOfWasmToEmbedderStub(functionIndex)), jumpDestination);
1900 append(isEmbedderBlock, Add64, instanceValue(), jumpDestination);
1901 append(isEmbedderBlock, Move, Arg::addr(jumpDestination), jumpDestination);
1902
1903 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1904 patchpoint->effects.writesPinned = true;
1905 patchpoint->effects.readsPinned = true;
1906 // We need to clobber all potential pinned registers since we might be leaving the instance.
1907 // We pessimistically assume we could be calling to something that is bounds checking.
1908 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1909 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1910
1911 Vector<ConstrainedTmp> patchArgs;
1912 patchArgs.append(jumpDestination);
1913
1914 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1915 patchArgs.append({ tmp, rep });
1916 });
1917
1918 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1919 AllowMacroScratchRegisterUsage allowScratch(jit);
1920 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1921 });
1922
1923 emitPatchpoint(isEmbedderBlock, patchpoint, result, WTFMove(patchArgs));
1924 append(isEmbedderBlock, Jump);
1925 isEmbedderBlock->setSuccessors(continuation);
1926 }
1927
1928 m_currentBlock = continuation;
1929 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1930 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, continuation);
1931 } else {
1932 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1933 patchpoint->effects.writesPinned = true;
1934 patchpoint->effects.readsPinned = true;
1935
1936 Vector<ConstrainedTmp> patchArgs;
1937 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1938 patchArgs.append({ tmp, rep });
1939 });
1940
1941 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1942 AllowMacroScratchRegisterUsage allowScratch(jit);
1943 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1944 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1945 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1946 });
1947 });
1948
1949 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(patchArgs));
1950 }
1951
1952 return { };
1953}
1954
1955auto AirIRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1956{
1957 ExpressionType calleeIndex = args.takeLast();
1958 ASSERT(signature.argumentCount() == args.size());
1959 ASSERT(m_info.tableCount() > tableIndex);
1960 ASSERT(m_info.tables[tableIndex].type() == TableElementType::Funcref);
1961
1962 m_makesCalls = true;
1963 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1964 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1965 // can be to the embedder for our stack check calculation.
1966 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1967
1968 auto currentInstance = g64();
1969 append(Move, instanceValue(), currentInstance);
1970
1971 ExpressionType callableFunctionBuffer = g64();
1972 ExpressionType instancesBuffer = g64();
1973 ExpressionType callableFunctionBufferLength = g64();
1974 {
1975 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfFunctions(), B3::Width64));
1976 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfInstances(), B3::Width64));
1977 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfLength(), B3::Width64));
1978
1979 if (UNLIKELY(!Arg::isValidAddrForm(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex), B3::Width64))) {
1980 append(Move, Arg::bigImm(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)), callableFunctionBufferLength);
1981 append(Add64, instanceValue(), callableFunctionBufferLength);
1982 append(Move, Arg::addr(callableFunctionBufferLength), callableFunctionBufferLength);
1983 } else
1984 append(Move, Arg::addr(instanceValue(), Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)), callableFunctionBufferLength);
1985 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfFunctions()), callableFunctionBuffer);
1986 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfInstances()), instancesBuffer);
1987 append(Move32, Arg::addr(callableFunctionBufferLength, Table::offsetOfLength()), callableFunctionBufferLength);
1988 }
1989
1990 append(Move32, calleeIndex, calleeIndex);
1991
1992 // Check the index we are looking for is valid.
1993 emitCheck([&] {
1994 return Inst(Branch32, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), calleeIndex, callableFunctionBufferLength);
1995 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1996 this->emitThrowException(jit, ExceptionType::OutOfBoundsCallIndirect);
1997 });
1998
1999 ExpressionType calleeCode = g64();
2000 {
2001 ExpressionType calleeSignatureIndex = g64();
2002 // Compute the offset in the table index space we are looking for.
2003 append(Move, Arg::imm(sizeof(WasmToWasmImportableFunction)), calleeSignatureIndex);
2004 append(Mul64, calleeIndex, calleeSignatureIndex);
2005 append(Add64, callableFunctionBuffer, calleeSignatureIndex);
2006
2007 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
2008
2009 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
2010 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
2011 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
2012
2013 // FIXME: This seems dumb to do two checks just for a nicer error message.
2014 // We should move just to use a single branch and then figure out what
2015 // error to use in the exception handler.
2016
2017 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfSignatureIndex()), calleeSignatureIndex);
2018
2019 emitCheck([&] {
2020 static_assert(Signature::invalidIndex == 0, "");
2021 return Inst(BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), calleeSignatureIndex, calleeSignatureIndex);
2022 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2023 this->emitThrowException(jit, ExceptionType::NullTableEntry);
2024 });
2025
2026 ExpressionType expectedSignatureIndex = g64();
2027 append(Move, Arg::bigImm(SignatureInformation::get(signature)), expectedSignatureIndex);
2028 emitCheck([&] {
2029 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::NotEqual), calleeSignatureIndex, expectedSignatureIndex);
2030 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2031 this->emitThrowException(jit, ExceptionType::BadSignature);
2032 });
2033 }
2034
2035 // Do a context switch if needed.
2036 {
2037 auto newContextInstance = g64();
2038 append(Move, Arg::index(instancesBuffer, calleeIndex, 8, 0), newContextInstance);
2039
2040 BasicBlock* doContextSwitch = m_code.addBlock();
2041 BasicBlock* continuation = m_code.addBlock();
2042
2043 append(Branch64, Arg::relCond(MacroAssembler::Equal), newContextInstance, instanceValue());
2044 m_currentBlock->setSuccessors(continuation, doContextSwitch);
2045
2046 auto* patchpoint = addPatchpoint(B3::Void);
2047 patchpoint->effects.writesPinned = true;
2048 // We pessimistically assume we're calling something with BoundsChecking memory.
2049 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2050 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2051 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2052 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
2053
2054 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2055 AllowMacroScratchRegisterUsage allowScratch(jit);
2056 GPRReg newContextInstance = params[0].gpr();
2057 GPRReg oldContextInstance = params[1].gpr();
2058 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
2059 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
2060 ASSERT(newContextInstance != baseMemory);
2061 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
2062 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
2063 jit.storeWasmContextInstance(newContextInstance);
2064 // FIXME: We should support more than one memory size register
2065 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
2066 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
2067 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
2068
2069 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
2070 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
2071
2072 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
2073 });
2074
2075 emitPatchpoint(doContextSwitch, patchpoint, Tmp(), newContextInstance, instanceValue());
2076 append(doContextSwitch, Jump);
2077 doContextSwitch->setSuccessors(continuation);
2078
2079 m_currentBlock = continuation;
2080 }
2081
2082 append(Move, Arg::addr(calleeCode), calleeCode);
2083
2084 Type returnType = signature.returnType();
2085 if (returnType != Type::Void)
2086 result = tmpForType(returnType);
2087
2088 auto* patch = addPatchpoint(toB3Type(returnType));
2089 patch->effects.writesPinned = true;
2090 patch->effects.readsPinned = true;
2091 // We need to clobber all potential pinned registers since we might be leaving the instance.
2092 // We pessimistically assume we're always calling something that is bounds checking so
2093 // because the wasm->wasm thunk unconditionally overrides the size registers.
2094 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
2095 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
2096 patch->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2097
2098 Vector<ConstrainedTmp> emitArgs;
2099 emitArgs.append(calleeCode);
2100 wasmCallingConventionAir().setupCall(m_code, returnType, patch, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
2101 emitArgs.append({ tmp, rep });
2102 });
2103 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2104 AllowMacroScratchRegisterUsage allowScratch(jit);
2105 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
2106 });
2107
2108 emitPatchpoint(m_currentBlock, patch, result, WTFMove(emitArgs));
2109
2110 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2111 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, m_currentBlock);
2112
2113 return { };
2114}
2115
2116void AirIRGenerator::unify(const ExpressionType& dst, const ExpressionType& source)
2117{
2118 ASSERT(isSubtype(source.type(), dst.type()));
2119 append(moveOpForValueType(dst.type()), source, dst);
2120}
2121
2122void AirIRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
2123{
2124 ASSERT(result.size() <= resultStack.size());
2125
2126 for (size_t i = 0; i < result.size(); ++i)
2127 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
2128}
2129
2130void AirIRGenerator::dump(const Vector<ControlEntry>&, const ExpressionList*)
2131{
2132}
2133
2134auto AirIRGenerator::origin() -> B3::Origin
2135{
2136 // FIXME: We should implement a way to give Inst's an origin.
2137 return B3::Origin();
2138}
2139
2140Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
2141{
2142 auto result = std::make_unique<InternalFunction>();
2143
2144 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
2145 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
2146
2147 B3::Procedure procedure;
2148 Code& code = procedure.code();
2149
2150 procedure.setOriginPrinter([] (PrintStream& out, B3::Origin origin) {
2151 if (origin.data())
2152 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
2153 });
2154
2155 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
2156 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
2157 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
2158 // optLevel=1.
2159 procedure.setNeedsUsedRegisters(false);
2160
2161 procedure.setOptLevel(Options::webAssemblyBBQOptimizationLevel());
2162
2163 AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
2164 FunctionParser<AirIRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
2165 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
2166
2167
2168 for (BasicBlock* block : code) {
2169 for (size_t i = 0; i < block->numSuccessors(); ++i)
2170 block->successorBlock(i)->addPredecessor(block);
2171 }
2172
2173 {
2174 B3::Air::prepareForGeneration(code);
2175 B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
2176 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
2177 result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
2178 }
2179
2180 return result;
2181}
2182
2183template <typename IntType>
2184void AirIRGenerator::emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right)
2185{
2186 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2187
2188 emitCheck([&] {
2189 return Inst(sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), right, right);
2190 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2191 this->emitThrowException(jit, ExceptionType::DivisionByZero);
2192 });
2193
2194 if (isSignedDiv) {
2195 ASSERT(std::is_signed<IntType>::value);
2196 IntType min = std::numeric_limits<IntType>::min();
2197
2198 // FIXME: Better isel for compare with imms here.
2199 // https://bugs.webkit.org/show_bug.cgi?id=193999
2200 auto minTmp = sizeof(IntType) == 4 ? g32() : g64();
2201 auto negOne = sizeof(IntType) == 4 ? g32() : g64();
2202
2203 B3::Air::Opcode op = sizeof(IntType) == 4 ? Compare32 : Compare64;
2204 append(Move, Arg::bigImm(static_cast<uint64_t>(min)), minTmp);
2205 append(op, Arg::relCond(MacroAssembler::Equal), left, minTmp, minTmp);
2206
2207 append(Move, Arg::imm(-1), negOne);
2208 append(op, Arg::relCond(MacroAssembler::Equal), right, negOne, negOne);
2209
2210 emitCheck([&] {
2211 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), minTmp, negOne);
2212 },
2213 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2214 this->emitThrowException(jit, ExceptionType::IntegerOverflow);
2215 });
2216 }
2217}
2218
2219template <typename IntType>
2220void AirIRGenerator::emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result)
2221{
2222 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2223
2224 result = sizeof(IntType) == 4 ? g32() : g64();
2225
2226 bool isSigned = std::is_signed<IntType>::value;
2227
2228 if (isARM64()) {
2229 B3::Air::Opcode div;
2230 switch (sizeof(IntType)) {
2231 case 4:
2232 div = isSigned ? Div32 : UDiv32;
2233 break;
2234 case 8:
2235 div = isSigned ? Div64 : UDiv64;
2236 break;
2237 }
2238
2239 append(div, lhs, rhs, result);
2240
2241 if (!isDiv) {
2242 append(sizeof(IntType) == 4 ? Mul32 : Mul64, result, rhs, result);
2243 append(sizeof(IntType) == 4 ? Sub32 : Sub64, lhs, result, result);
2244 }
2245
2246 return;
2247 }
2248
2249#if CPU(X86) || CPU(X86_64)
2250 Tmp eax(X86Registers::eax);
2251 Tmp edx(X86Registers::edx);
2252
2253 if (isSigned) {
2254 B3::Air::Opcode convertToDoubleWord;
2255 B3::Air::Opcode div;
2256 switch (sizeof(IntType)) {
2257 case 4:
2258 convertToDoubleWord = X86ConvertToDoubleWord32;
2259 div = X86Div32;
2260 break;
2261 case 8:
2262 convertToDoubleWord = X86ConvertToQuadWord64;
2263 div = X86Div64;
2264 break;
2265 default:
2266 RELEASE_ASSERT_NOT_REACHED();
2267 }
2268
2269 // We implement "res = Div<Chill>/Mod<Chill>(num, den)" as follows:
2270 //
2271 // if (den + 1 <=_unsigned 1) {
2272 // if (!den) {
2273 // res = 0;
2274 // goto done;
2275 // }
2276 // if (num == -2147483648) {
2277 // res = isDiv ? num : 0;
2278 // goto done;
2279 // }
2280 // }
2281 // res = num (/ or %) dev;
2282 // done:
2283
2284 BasicBlock* denIsGood = m_code.addBlock();
2285 BasicBlock* denMayBeBad = m_code.addBlock();
2286 BasicBlock* denNotZero = m_code.addBlock();
2287 BasicBlock* continuation = m_code.addBlock();
2288
2289 auto temp = sizeof(IntType) == 4 ? g32() : g64();
2290 auto one = addConstant(sizeof(IntType) == 4 ? Type::I32 : Type::I64, 1);
2291
2292 append(sizeof(IntType) == 4 ? Add32 : Add64, rhs, one, temp);
2293 append(sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Above), temp, one);
2294 m_currentBlock->setSuccessors(denIsGood, denMayBeBad);
2295
2296 append(denMayBeBad, Xor64, result, result);
2297 append(denMayBeBad, sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, Arg::resCond(MacroAssembler::Zero), rhs, rhs);
2298 denMayBeBad->setSuccessors(continuation, denNotZero);
2299
2300 auto min = addConstant(denNotZero, sizeof(IntType) == 4 ? Type::I32 : Type::I64, std::numeric_limits<IntType>::min());
2301 if (isDiv)
2302 append(denNotZero, sizeof(IntType) == 4 ? Move32 : Move, min, result);
2303 else {
2304 // Result is zero, as set above...
2305 }
2306 append(denNotZero, sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Equal), lhs, min);
2307 denNotZero->setSuccessors(continuation, denIsGood);
2308
2309 auto divResult = isDiv ? eax : edx;
2310 append(denIsGood, Move, lhs, eax);
2311 append(denIsGood, convertToDoubleWord, eax, edx);
2312 append(denIsGood, div, eax, edx, rhs);
2313 append(denIsGood, sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2314 append(denIsGood, Jump);
2315 denIsGood->setSuccessors(continuation);
2316
2317 m_currentBlock = continuation;
2318 return;
2319 }
2320
2321 B3::Air::Opcode div = sizeof(IntType) == 4 ? X86UDiv32 : X86UDiv64;
2322
2323 Tmp divResult = isDiv ? eax : edx;
2324
2325 append(Move, lhs, eax);
2326 append(Xor64, edx, edx);
2327 append(div, eax, edx, rhs);
2328 append(sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2329#else
2330 RELEASE_ASSERT_NOT_REACHED();
2331#endif
2332}
2333
2334template<>
2335auto AirIRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2336{
2337 emitChecksForModOrDiv<int32_t>(true, left, right);
2338 emitModOrDiv<int32_t>(true, left, right, result);
2339 return { };
2340}
2341
2342template<>
2343auto AirIRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2344{
2345 emitChecksForModOrDiv<int32_t>(false, left, right);
2346 emitModOrDiv<int32_t>(false, left, right, result);
2347 return { };
2348}
2349
2350template<>
2351auto AirIRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2352{
2353 emitChecksForModOrDiv<uint32_t>(false, left, right);
2354 emitModOrDiv<uint32_t>(true, left, right, result);
2355 return { };
2356}
2357
2358template<>
2359auto AirIRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2360{
2361 emitChecksForModOrDiv<uint32_t>(false, left, right);
2362 emitModOrDiv<uint32_t>(false, left, right, result);
2363 return { };
2364}
2365
2366template<>
2367auto AirIRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2368{
2369 emitChecksForModOrDiv<int64_t>(true, left, right);
2370 emitModOrDiv<int64_t>(true, left, right, result);
2371 return { };
2372}
2373
2374template<>
2375auto AirIRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2376{
2377 emitChecksForModOrDiv<int64_t>(false, left, right);
2378 emitModOrDiv<int64_t>(false, left, right, result);
2379 return { };
2380}
2381
2382template<>
2383auto AirIRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2384{
2385 emitChecksForModOrDiv<uint64_t>(false, left, right);
2386 emitModOrDiv<uint64_t>(true, left, right, result);
2387 return { };
2388}
2389
2390template<>
2391auto AirIRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2392{
2393 emitChecksForModOrDiv<uint64_t>(false, left, right);
2394 emitModOrDiv<uint64_t>(false, left, right, result);
2395 return { };
2396}
2397
2398template<>
2399auto AirIRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2400{
2401 auto* patchpoint = addPatchpoint(B3::Int32);
2402 patchpoint->effects = B3::Effects::none();
2403 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2404 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2405 });
2406 result = g32();
2407 emitPatchpoint(patchpoint, result, arg);
2408 return { };
2409}
2410
2411template<>
2412auto AirIRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2413{
2414 auto* patchpoint = addPatchpoint(B3::Int64);
2415 patchpoint->effects = B3::Effects::none();
2416 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2417 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2418 });
2419 result = g64();
2420 emitPatchpoint(patchpoint, result, arg);
2421 return { };
2422}
2423
2424template<>
2425auto AirIRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2426{
2427 result = g32();
2428
2429#if CPU(X86_64)
2430 if (MacroAssembler::supportsCountPopulation()) {
2431 auto* patchpoint = addPatchpoint(B3::Int32);
2432 patchpoint->effects = B3::Effects::none();
2433 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2434 jit.countPopulation32(params[1].gpr(), params[0].gpr());
2435 });
2436 emitPatchpoint(patchpoint, result, arg);
2437 return { };
2438 }
2439#endif
2440
2441 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
2442 emitCCall(popcount, result, arg);
2443 return { };
2444}
2445
2446template<>
2447auto AirIRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2448{
2449 result = g64();
2450
2451#if CPU(X86_64)
2452 if (MacroAssembler::supportsCountPopulation()) {
2453 auto* patchpoint = addPatchpoint(B3::Int64);
2454 patchpoint->effects = B3::Effects::none();
2455 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2456 jit.countPopulation64(params[1].gpr(), params[0].gpr());
2457 });
2458 emitPatchpoint(patchpoint, result, arg);
2459 return { };
2460 }
2461#endif
2462
2463 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
2464 emitCCall(popcount, result, arg);
2465 return { };
2466}
2467
2468template<>
2469auto AirIRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2470{
2471 auto* patchpoint = addPatchpoint(B3::Double);
2472 patchpoint->effects = B3::Effects::none();
2473 if (isX86())
2474 patchpoint->numGPScratchRegisters = 1;
2475 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2476 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2477 AllowMacroScratchRegisterUsage allowScratch(jit);
2478#if CPU(X86_64)
2479 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2480#else
2481 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2482#endif
2483 });
2484 result = f64();
2485 emitPatchpoint(patchpoint, result, arg);
2486 return { };
2487}
2488
2489template<>
2490auto AirIRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2491{
2492 auto* patchpoint = addPatchpoint(B3::Float);
2493 patchpoint->effects = B3::Effects::none();
2494 if (isX86())
2495 patchpoint->numGPScratchRegisters = 1;
2496 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2497 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2498 AllowMacroScratchRegisterUsage allowScratch(jit);
2499#if CPU(X86_64)
2500 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2501#else
2502 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2503#endif
2504 });
2505 result = f32();
2506 emitPatchpoint(patchpoint, result, arg);
2507 return { };
2508}
2509
2510template<>
2511auto AirIRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2512{
2513 auto* patchpoint = addPatchpoint(B3::Double);
2514 patchpoint->effects = B3::Effects::none();
2515 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2516 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2517 });
2518 result = f64();
2519 emitPatchpoint(patchpoint, result, arg);
2520 return { };
2521}
2522
2523template<>
2524auto AirIRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2525{
2526 auto* patchpoint = addPatchpoint(B3::Float);
2527 patchpoint->effects = B3::Effects::none();
2528 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2529 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2530 });
2531 result = f32();
2532 emitPatchpoint(patchpoint, result, arg);
2533 return { };
2534}
2535
2536template<>
2537auto AirIRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2538{
2539 auto* patchpoint = addPatchpoint(B3::Double);
2540 patchpoint->effects = B3::Effects::none();
2541 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2542 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2543 });
2544 result = f64();
2545 emitPatchpoint(patchpoint, result, arg);
2546 return { };
2547}
2548
2549template<>
2550auto AirIRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2551{
2552 auto* patchpoint = addPatchpoint(B3::Float);
2553 patchpoint->effects = B3::Effects::none();
2554 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2555 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2556 });
2557 result = f32();
2558 emitPatchpoint(patchpoint, result, arg);
2559 return { };
2560}
2561
2562template<>
2563auto AirIRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2564{
2565 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2566 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
2567
2568 auto temp1 = g32();
2569 auto temp2 = g32();
2570 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2571 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2572 append(Or32, temp1, temp2);
2573
2574 emitCheck([&] {
2575 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2576 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2577 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2578 });
2579
2580 auto* patchpoint = addPatchpoint(B3::Int32);
2581 patchpoint->effects = B3::Effects::none();
2582 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2583 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2584 });
2585 result = g32();
2586 emitPatchpoint(patchpoint, result, arg);
2587
2588 return { };
2589}
2590
2591template<>
2592auto AirIRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2593{
2594 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2595 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2596
2597 auto temp1 = g32();
2598 auto temp2 = g32();
2599 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2600 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2601 append(Or32, temp1, temp2);
2602
2603 emitCheck([&] {
2604 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2605 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2606 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2607 });
2608
2609 auto* patchpoint = addPatchpoint(B3::Int32);
2610 patchpoint->effects = B3::Effects::none();
2611 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2612 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2613 });
2614 result = g32();
2615 emitPatchpoint(patchpoint, result, arg);
2616 return { };
2617}
2618
2619
2620template<>
2621auto AirIRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2622{
2623 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2624 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2625
2626 auto temp1 = g32();
2627 auto temp2 = g32();
2628 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2629 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2630 append(Or32, temp1, temp2);
2631
2632 emitCheck([&] {
2633 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2634 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2635 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2636 });
2637
2638 auto* patchpoint = addPatchpoint(B3::Int32);
2639 patchpoint->effects = B3::Effects::none();
2640 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2641 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2642 });
2643 result = g32();
2644 emitPatchpoint(patchpoint, result, arg);
2645 return { };
2646}
2647
2648template<>
2649auto AirIRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2650{
2651 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2652 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2653
2654 auto temp1 = g32();
2655 auto temp2 = g32();
2656 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2657 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2658 append(Or32, temp1, temp2);
2659
2660 emitCheck([&] {
2661 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2662 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2663 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2664 });
2665
2666 auto* patchpoint = addPatchpoint(B3::Int32);
2667 patchpoint->effects = B3::Effects::none();
2668 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2669 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2670 });
2671 result = g32();
2672 emitPatchpoint(patchpoint, result, arg);
2673 return { };
2674}
2675
2676template<>
2677auto AirIRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2678{
2679 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2680 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2681
2682 auto temp1 = g32();
2683 auto temp2 = g32();
2684 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2685 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2686 append(Or32, temp1, temp2);
2687
2688 emitCheck([&] {
2689 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2690 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2691 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2692 });
2693
2694 auto* patchpoint = addPatchpoint(B3::Int64);
2695 patchpoint->effects = B3::Effects::none();
2696 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2697 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2698 });
2699
2700 result = g64();
2701 emitPatchpoint(patchpoint, result, arg);
2702 return { };
2703}
2704
2705template<>
2706auto AirIRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2707{
2708 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2709 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2710
2711 auto temp1 = g32();
2712 auto temp2 = g32();
2713 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2714 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2715 append(Or32, temp1, temp2);
2716
2717 emitCheck([&] {
2718 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2719 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2720 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2721 });
2722
2723 TypedTmp signBitConstant;
2724 if (isX86())
2725 signBitConstant = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2726
2727 Vector<ConstrainedTmp> args;
2728 auto* patchpoint = addPatchpoint(B3::Int64);
2729 patchpoint->effects = B3::Effects::none();
2730 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2731 args.append(arg);
2732 if (isX86()) {
2733 args.append(signBitConstant);
2734 patchpoint->numFPScratchRegisters = 1;
2735 }
2736 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2737 AllowMacroScratchRegisterUsage allowScratch(jit);
2738 FPRReg scratch = InvalidFPRReg;
2739 FPRReg constant = InvalidFPRReg;
2740 if (isX86()) {
2741 scratch = params.fpScratch(0);
2742 constant = params[2].fpr();
2743 }
2744 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2745 });
2746
2747 result = g64();
2748 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2749 return { };
2750}
2751
2752template<>
2753auto AirIRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2754{
2755 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2756 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2757
2758 auto temp1 = g32();
2759 auto temp2 = g32();
2760 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2761 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2762 append(Or32, temp1, temp2);
2763
2764 emitCheck([&] {
2765 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2766 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2767 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2768 });
2769
2770 auto* patchpoint = addPatchpoint(B3::Int64);
2771 patchpoint->effects = B3::Effects::none();
2772 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2773 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2774 });
2775 result = g64();
2776 emitPatchpoint(patchpoint, result, arg);
2777 return { };
2778}
2779
2780template<>
2781auto AirIRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2782{
2783 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2784 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2785
2786 auto temp1 = g32();
2787 auto temp2 = g32();
2788 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2789 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2790 append(Or32, temp1, temp2);
2791
2792 emitCheck([&] {
2793 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2794 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2795 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2796 });
2797
2798 TypedTmp signBitConstant;
2799 if (isX86())
2800 signBitConstant = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2801
2802 auto* patchpoint = addPatchpoint(B3::Int64);
2803 patchpoint->effects = B3::Effects::none();
2804 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2805 Vector<ConstrainedTmp> args;
2806 args.append(arg);
2807 if (isX86()) {
2808 args.append(signBitConstant);
2809 patchpoint->numFPScratchRegisters = 1;
2810 }
2811 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2812 AllowMacroScratchRegisterUsage allowScratch(jit);
2813 FPRReg scratch = InvalidFPRReg;
2814 FPRReg constant = InvalidFPRReg;
2815 if (isX86()) {
2816 scratch = params.fpScratch(0);
2817 constant = params[2].fpr();
2818 }
2819 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2820 });
2821
2822 result = g64();
2823 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2824
2825 return { };
2826}
2827
2828auto AirIRGenerator::addShift(Type type, B3::Air::Opcode op, ExpressionType value, ExpressionType shift, ExpressionType& result) -> PartialResult
2829{
2830 ASSERT(type == Type::I64 || type == Type::I32);
2831 result = tmpForType(type);
2832
2833 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2834 append(op, value, shift, result);
2835 return { };
2836 }
2837
2838#if CPU(X86_64)
2839 Tmp ecx = Tmp(X86Registers::ecx);
2840 append(Move, value, result);
2841 append(Move, shift, ecx);
2842 append(op, ecx, result);
2843#else
2844 RELEASE_ASSERT_NOT_REACHED();
2845#endif
2846 return { };
2847}
2848
2849auto AirIRGenerator::addIntegerSub(B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2850{
2851 ASSERT(op == Sub32 || op == Sub64);
2852
2853 result = op == Sub32 ? g32() : g64();
2854
2855 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2856 append(op, lhs, rhs, result);
2857 return { };
2858 }
2859
2860 RELEASE_ASSERT(isX86());
2861 // Sub a, b
2862 // means
2863 // b = b Sub a
2864 append(Move, lhs, result);
2865 append(op, rhs, result);
2866 return { };
2867}
2868
2869auto AirIRGenerator::addFloatingPointAbs(B3::Air::Opcode op, ExpressionType value, ExpressionType& result) -> PartialResult
2870{
2871 RELEASE_ASSERT(op == AbsFloat || op == AbsDouble);
2872
2873 result = op == AbsFloat ? f32() : f64();
2874
2875 if (isValidForm(op, Arg::Tmp, Arg::Tmp)) {
2876 append(op, value, result);
2877 return { };
2878 }
2879
2880 RELEASE_ASSERT(isX86());
2881
2882 if (op == AbsFloat) {
2883 auto constant = g32();
2884 append(Move, Arg::imm(static_cast<uint32_t>(~(1ull << 31))), constant);
2885 append(Move32ToFloat, constant, result);
2886 append(AndFloat, value, result);
2887 } else {
2888 auto constant = g64();
2889 append(Move, Arg::bigImm(~(1ull << 63)), constant);
2890 append(Move64ToDouble, constant, result);
2891 append(AndDouble, value, result);
2892 }
2893 return { };
2894}
2895
2896auto AirIRGenerator::addFloatingPointBinOp(Type type, B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2897{
2898 ASSERT(type == Type::F32 || type == Type::F64);
2899 result = tmpForType(type);
2900
2901 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2902 append(op, lhs, rhs, result);
2903 return { };
2904 }
2905
2906 RELEASE_ASSERT(isX86());
2907
2908 // Op a, b
2909 // means
2910 // b = b Op a
2911 append(moveOpForValueType(type), lhs, result);
2912 append(op, rhs, result);
2913 return { };
2914}
2915
2916template<> auto AirIRGenerator::addOp<OpType::F32Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2917{
2918 result = f32();
2919 append(CeilFloat, arg0, result);
2920 return { };
2921}
2922
2923template<> auto AirIRGenerator::addOp<OpType::I32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2924{
2925 result = g32();
2926 append(Mul32, arg0, arg1, result);
2927 return { };
2928}
2929
2930template<> auto AirIRGenerator::addOp<OpType::I32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2931{
2932 return addIntegerSub(Sub32, arg0, arg1, result);
2933}
2934
2935template<> auto AirIRGenerator::addOp<OpType::F64Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2936{
2937 result = g32();
2938 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
2939 return { };
2940}
2941
2942template<> auto AirIRGenerator::addOp<OpType::F32DemoteF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2943{
2944 result = f32();
2945 append(ConvertDoubleToFloat, arg0, result);
2946 return { };
2947}
2948
2949template<> auto AirIRGenerator::addOp<OpType::F32Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2950{
2951 return addFloatingPointMinOrMax(F32, MinOrMax::Min, arg0, arg1, result);
2952}
2953
2954template<> auto AirIRGenerator::addOp<OpType::F64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2955{
2956 result = g32();
2957 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2958 return { };
2959}
2960
2961template<> auto AirIRGenerator::addOp<OpType::F64Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2962{
2963 result = g32();
2964 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
2965 return { };
2966}
2967
2968auto AirIRGenerator::addFloatingPointMinOrMax(Type floatType, MinOrMax minOrMax, ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2969{
2970 ASSERT(floatType == F32 || floatType == F64);
2971 result = tmpForType(floatType);
2972
2973 BasicBlock* isEqual = m_code.addBlock();
2974 BasicBlock* notEqual = m_code.addBlock();
2975 BasicBlock* isLessThan = m_code.addBlock();
2976 BasicBlock* notLessThan = m_code.addBlock();
2977 BasicBlock* isGreaterThan = m_code.addBlock();
2978 BasicBlock* isNaN = m_code.addBlock();
2979 BasicBlock* continuation = m_code.addBlock();
2980
2981 auto branchOp = floatType == F32 ? BranchFloat : BranchDouble;
2982 append(m_currentBlock, branchOp, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1);
2983 m_currentBlock->setSuccessors(isEqual, notEqual);
2984
2985 append(notEqual, branchOp, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1);
2986 notEqual->setSuccessors(isLessThan, notLessThan);
2987
2988 append(notLessThan, branchOp, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1);
2989 notLessThan->setSuccessors(isGreaterThan, isNaN);
2990
2991 auto andOp = floatType == F32 ? AndFloat : AndDouble;
2992 auto orOp = floatType == F32 ? OrFloat : OrDouble;
2993 append(isEqual, minOrMax == MinOrMax::Max ? andOp : orOp, arg0, arg1, result);
2994 append(isEqual, Jump);
2995 isEqual->setSuccessors(continuation);
2996
2997 auto isLessThanResult = minOrMax == MinOrMax::Max ? arg1 : arg0;
2998 append(isLessThan, moveOpForValueType(floatType), isLessThanResult, result);
2999 append(isLessThan, Jump);
3000 isLessThan->setSuccessors(continuation);
3001
3002 auto isGreaterThanResult = minOrMax == MinOrMax::Max ? arg0 : arg1;
3003 append(isGreaterThan, moveOpForValueType(floatType), isGreaterThanResult, result);
3004 append(isGreaterThan, Jump);
3005 isGreaterThan->setSuccessors(continuation);
3006
3007 auto addOp = floatType == F32 ? AddFloat : AddDouble;
3008 append(isNaN, addOp, arg0, arg1, result);
3009 append(isNaN, Jump);
3010 isNaN->setSuccessors(continuation);
3011
3012 m_currentBlock = continuation;
3013
3014 return { };
3015}
3016
3017template<> auto AirIRGenerator::addOp<OpType::F32Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3018{
3019 return addFloatingPointMinOrMax(F32, MinOrMax::Max, arg0, arg1, result);
3020}
3021
3022template<> auto AirIRGenerator::addOp<OpType::F64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3023{
3024 return addFloatingPointBinOp(Type::F64, MulDouble, arg0, arg1, result);
3025}
3026
3027template<> auto AirIRGenerator::addOp<OpType::F32Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3028{
3029 return addFloatingPointBinOp(Type::F32, DivFloat, arg0, arg1, result);
3030}
3031
3032template<> auto AirIRGenerator::addOp<OpType::I32Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3033{
3034 result = g32();
3035 append(CountLeadingZeros32, arg0, result);
3036 return { };
3037}
3038
3039template<> auto AirIRGenerator::addOp<OpType::F32Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3040{
3041 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3042 // https://bugs.webkit.org/show_bug.cgi?id=193999
3043 result = f32();
3044 auto temp1 = g32();
3045 auto sign = g32();
3046 auto value = g32();
3047
3048 // FIXME: Try to use Imm where possible:
3049 // https://bugs.webkit.org/show_bug.cgi?id=193999
3050 append(MoveFloatTo32, arg1, temp1);
3051 append(Move, Arg::bigImm(0x80000000), sign);
3052 append(And32, temp1, sign, sign);
3053
3054 append(MoveDoubleTo64, arg0, temp1);
3055 append(Move, Arg::bigImm(0x7fffffff), value);
3056 append(And32, temp1, value, value);
3057
3058 append(Or32, sign, value, value);
3059 append(Move32ToFloat, value, result);
3060
3061 return { };
3062}
3063
3064template<> auto AirIRGenerator::addOp<OpType::F64ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3065{
3066 result = f64();
3067 auto temp = g64();
3068 append(Move32, arg0, temp);
3069 append(ConvertInt64ToDouble, temp, result);
3070 return { };
3071}
3072
3073template<> auto AirIRGenerator::addOp<OpType::F32ReinterpretI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3074{
3075 result = f32();
3076 append(Move32ToFloat, arg0, result);
3077 return { };
3078}
3079
3080template<> auto AirIRGenerator::addOp<OpType::I64And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3081{
3082 result = g64();
3083 append(And64, arg0, arg1, result);
3084 return { };
3085}
3086
3087template<> auto AirIRGenerator::addOp<OpType::F32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3088{
3089 result = g32();
3090 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
3091 return { };
3092}
3093
3094template<> auto AirIRGenerator::addOp<OpType::F64Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3095{
3096 result = g32();
3097 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3098 return { };
3099}
3100
3101template<> auto AirIRGenerator::addOp<OpType::F32Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3102{
3103 result = f32();
3104 append(SqrtFloat, arg0, result);
3105 return { };
3106}
3107
3108template<> auto AirIRGenerator::addOp<OpType::F64Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3109{
3110 result = g32();
3111 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3112 return { };
3113}
3114
3115template<> auto AirIRGenerator::addOp<OpType::I64GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3116{
3117 result = g32();
3118 append(Compare64, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3119 return { };
3120}
3121
3122template<> auto AirIRGenerator::addOp<OpType::I64GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3123{
3124 result = g32();
3125 append(Compare64, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3126 return { };
3127}
3128
3129template<> auto AirIRGenerator::addOp<OpType::I64Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3130{
3131 result = g32();
3132 append(Test64, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3133 return { };
3134}
3135
3136template<> auto AirIRGenerator::addOp<OpType::F64Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3137{
3138 return addFloatingPointBinOp(Type::F64, DivDouble, arg0, arg1, result);
3139}
3140
3141template<> auto AirIRGenerator::addOp<OpType::F32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3142{
3143 result = f32();
3144 append(AddFloat, arg0, arg1, result);
3145 return { };
3146}
3147
3148template<> auto AirIRGenerator::addOp<OpType::I64Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3149{
3150 result = g64();
3151 append(Or64, arg0, arg1, result);
3152 return { };
3153}
3154
3155template<> auto AirIRGenerator::addOp<OpType::I32LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3156{
3157 result = g32();
3158 append(Compare32, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3159 return { };
3160}
3161
3162template<> auto AirIRGenerator::addOp<OpType::I32LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3163{
3164 result = g32();
3165 append(Compare32, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3166 return { };
3167}
3168
3169template<> auto AirIRGenerator::addOp<OpType::I64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3170{
3171 result = g32();
3172 append(Compare64, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3173 return { };
3174}
3175
3176template<> auto AirIRGenerator::addOp<OpType::I64Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3177{
3178 result = g64();
3179 append(CountLeadingZeros64, arg0, result);
3180 return { };
3181}
3182
3183template<> auto AirIRGenerator::addOp<OpType::F32Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3184{
3185 result = f32();
3186 if (isValidForm(NegateFloat, Arg::Tmp, Arg::Tmp))
3187 append(NegateFloat, arg0, result);
3188 else {
3189 auto constant = addConstant(Type::I32, bitwise_cast<uint32_t>(static_cast<float>(-0.0)));
3190 auto temp = g32();
3191 append(MoveFloatTo32, arg0, temp);
3192 append(Xor32, constant, temp);
3193 append(Move32ToFloat, temp, result);
3194 }
3195 return { };
3196}
3197
3198template<> auto AirIRGenerator::addOp<OpType::I32And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3199{
3200 result = g32();
3201 append(And32, arg0, arg1, result);
3202 return { };
3203}
3204
3205template<> auto AirIRGenerator::addOp<OpType::I32LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3206{
3207 result = g32();
3208 append(Compare32, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3209 return { };
3210}
3211
3212template<> auto AirIRGenerator::addOp<OpType::I64Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3213{
3214 return addShift(Type::I64, RotateRight64, arg0, arg1, result);
3215}
3216
3217template<> auto AirIRGenerator::addOp<OpType::F64Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3218{
3219 return addFloatingPointAbs(AbsDouble, arg0, result);
3220}
3221
3222template<> auto AirIRGenerator::addOp<OpType::I32LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3223{
3224 result = g32();
3225 append(Compare32, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3226 return { };
3227}
3228
3229template<> auto AirIRGenerator::addOp<OpType::I32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3230{
3231 result = g32();
3232 append(Compare32, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3233 return { };
3234}
3235
3236template<> auto AirIRGenerator::addOp<OpType::F64Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3237{
3238 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3239 // https://bugs.webkit.org/show_bug.cgi?id=193999
3240 result = f64();
3241 auto temp1 = g64();
3242 auto sign = g64();
3243 auto value = g64();
3244
3245 append(MoveDoubleTo64, arg1, temp1);
3246 append(Move, Arg::bigImm(0x8000000000000000), sign);
3247 append(And64, temp1, sign, sign);
3248
3249 append(MoveDoubleTo64, arg0, temp1);
3250 append(Move, Arg::bigImm(0x7fffffffffffffff), value);
3251 append(And64, temp1, value, value);
3252
3253 append(Or64, sign, value, value);
3254 append(Move64ToDouble, value, result);
3255
3256 return { };
3257}
3258
3259template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3260{
3261 result = f32();
3262 append(ConvertInt64ToFloat, arg0, result);
3263 return { };
3264}
3265
3266template<> auto AirIRGenerator::addOp<OpType::I64Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3267{
3268 if (isARM64()) {
3269 // ARM64 doesn't have a rotate left.
3270 auto newShift = g64();
3271 append(Move, arg1, newShift);
3272 append(Neg64, newShift);
3273 return addShift(Type::I64, RotateRight64, arg0, newShift, result);
3274 } else
3275 return addShift(Type::I64, RotateLeft64, arg0, arg1, result);
3276}
3277
3278template<> auto AirIRGenerator::addOp<OpType::F32Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3279{
3280 result = g32();
3281 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3282 return { };
3283}
3284
3285template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3286{
3287 result = f64();
3288 append(ConvertInt32ToDouble, arg0, result);
3289 return { };
3290}
3291
3292template<> auto AirIRGenerator::addOp<OpType::F64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3293{
3294 result = g32();
3295 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3296 return { };
3297}
3298
3299template<> auto AirIRGenerator::addOp<OpType::F32Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3300{
3301 result = g32();
3302 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3303 return { };
3304}
3305
3306template<> auto AirIRGenerator::addOp<OpType::F32Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3307{
3308 result = g32();
3309 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3310 return { };
3311}
3312
3313template<> auto AirIRGenerator::addOp<OpType::I32ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3314{
3315 return addShift(Type::I32, Urshift32, arg0, arg1, result);
3316}
3317
3318template<> auto AirIRGenerator::addOp<OpType::F32ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3319{
3320 result = f32();
3321 auto temp = g64();
3322 append(Move32, arg0, temp);
3323 append(ConvertInt64ToFloat, temp, result);
3324 return { };
3325}
3326
3327template<> auto AirIRGenerator::addOp<OpType::I32ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3328{
3329 return addShift(Type::I32, Rshift32, arg0, arg1, result);
3330}
3331
3332template<> auto AirIRGenerator::addOp<OpType::I32GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3333{
3334 result = g32();
3335 append(Compare32, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3336 return { };
3337}
3338
3339template<> auto AirIRGenerator::addOp<OpType::F64Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3340{
3341 result = f64();
3342 append(CeilDouble, arg0, result);
3343 return { };
3344}
3345
3346template<> auto AirIRGenerator::addOp<OpType::I32GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3347{
3348 result = g32();
3349 append(Compare32, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3350 return { };
3351}
3352
3353template<> auto AirIRGenerator::addOp<OpType::I32Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3354{
3355 return addShift(Type::I32, Lshift32, arg0, arg1, result);
3356}
3357
3358template<> auto AirIRGenerator::addOp<OpType::F64Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3359{
3360 result = f64();
3361 append(FloorDouble, arg0, result);
3362 return { };
3363}
3364
3365template<> auto AirIRGenerator::addOp<OpType::I32Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3366{
3367 result = g32();
3368 append(Xor32, arg0, arg1, result);
3369 return { };
3370}
3371
3372template<> auto AirIRGenerator::addOp<OpType::F32Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3373{
3374 return addFloatingPointAbs(AbsFloat, arg0, result);
3375}
3376
3377template<> auto AirIRGenerator::addOp<OpType::F64Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3378{
3379 return addFloatingPointMinOrMax(F64, MinOrMax::Min, arg0, arg1, result);
3380}
3381
3382template<> auto AirIRGenerator::addOp<OpType::F32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3383{
3384 result = f32();
3385 append(MulFloat, arg0, arg1, result);
3386 return { };
3387}
3388
3389template<> auto AirIRGenerator::addOp<OpType::I64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3390{
3391 return addIntegerSub(Sub64, arg0, arg1, result);
3392}
3393
3394template<> auto AirIRGenerator::addOp<OpType::I32ReinterpretF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3395{
3396 result = g32();
3397 append(MoveFloatTo32, arg0, result);
3398 return { };
3399}
3400
3401template<> auto AirIRGenerator::addOp<OpType::I32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3402{
3403 result = g32();
3404 append(Add32, arg0, arg1, result);
3405 return { };
3406}
3407
3408template<> auto AirIRGenerator::addOp<OpType::F64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3409{
3410 return addFloatingPointBinOp(Type::F64, SubDouble, arg0, arg1, result);
3411}
3412
3413template<> auto AirIRGenerator::addOp<OpType::I32Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3414{
3415 result = g32();
3416 append(Or32, arg0, arg1, result);
3417 return { };
3418}
3419
3420template<> auto AirIRGenerator::addOp<OpType::I64LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3421{
3422 result = g32();
3423 append(Compare64, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3424 return { };
3425}
3426
3427template<> auto AirIRGenerator::addOp<OpType::I64LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3428{
3429 result = g32();
3430 append(Compare64, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3431 return { };
3432}
3433
3434template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3435{
3436 result = f64();
3437 append(ConvertInt64ToDouble, arg0, result);
3438 return { };
3439}
3440
3441template<> auto AirIRGenerator::addOp<OpType::I64Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3442{
3443 result = g64();
3444 append(Xor64, arg0, arg1, result);
3445 return { };
3446}
3447
3448template<> auto AirIRGenerator::addOp<OpType::I64GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3449{
3450 result = g32();
3451 append(Compare64, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3452 return { };
3453}
3454
3455template<> auto AirIRGenerator::addOp<OpType::I64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3456{
3457 result = g64();
3458 append(Mul64, arg0, arg1, result);
3459 return { };
3460}
3461
3462template<> auto AirIRGenerator::addOp<OpType::F32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3463{
3464 result = f32();
3465 if (isValidForm(SubFloat, Arg::Tmp, Arg::Tmp, Arg::Tmp))
3466 append(SubFloat, arg0, arg1, result);
3467 else {
3468 RELEASE_ASSERT(isX86());
3469 append(MoveFloat, arg0, result);
3470 append(SubFloat, arg1, result);
3471 }
3472 return { };
3473}
3474
3475template<> auto AirIRGenerator::addOp<OpType::F64PromoteF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3476{
3477 result = f64();
3478 append(ConvertFloatToDouble, arg0, result);
3479 return { };
3480}
3481
3482template<> auto AirIRGenerator::addOp<OpType::F64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3483{
3484 result = f64();
3485 append(AddDouble, arg0, arg1, result);
3486 return { };
3487}
3488
3489template<> auto AirIRGenerator::addOp<OpType::I64GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3490{
3491 result = g32();
3492 append(Compare64, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3493 return { };
3494}
3495
3496template<> auto AirIRGenerator::addOp<OpType::I64ExtendUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3497{
3498 result = g64();
3499 append(Move32, arg0, result);
3500 return { };
3501}
3502
3503template<> auto AirIRGenerator::addOp<OpType::I32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3504{
3505 result = g32();
3506 RELEASE_ASSERT(arg0 && arg1);
3507 append(Compare32, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3508 return { };
3509}
3510
3511template<> auto AirIRGenerator::addOp<OpType::F64ReinterpretI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3512{
3513 result = f64();
3514 append(Move64ToDouble, arg0, result);
3515 return { };
3516}
3517
3518template<> auto AirIRGenerator::addOp<OpType::F32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3519{
3520 result = g32();
3521 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3522 return { };
3523}
3524
3525template<> auto AirIRGenerator::addOp<OpType::I64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3526{
3527 result = g32();
3528 append(Compare64, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3529 return { };
3530}
3531
3532template<> auto AirIRGenerator::addOp<OpType::F32Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3533{
3534 result = f32();
3535 append(FloorFloat, arg0, result);
3536 return { };
3537}
3538
3539template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3540{
3541 result = f32();
3542 append(ConvertInt32ToFloat, arg0, result);
3543 return { };
3544}
3545
3546template<> auto AirIRGenerator::addOp<OpType::I32Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3547{
3548 result = g32();
3549 append(Test32, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3550 return { };
3551}
3552
3553template<> auto AirIRGenerator::addOp<OpType::I64ReinterpretF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3554{
3555 result = g64();
3556 append(MoveDoubleTo64, arg0, result);
3557 return { };
3558}
3559
3560template<> auto AirIRGenerator::addOp<OpType::I64ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3561{
3562 return addShift(Type::I64, Rshift64, arg0, arg1, result);
3563}
3564
3565template<> auto AirIRGenerator::addOp<OpType::I64ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3566{
3567 return addShift(Type::I64, Urshift64, arg0, arg1, result);
3568}
3569
3570template<> auto AirIRGenerator::addOp<OpType::F64Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3571{
3572 result = f64();
3573 append(SqrtDouble, arg0, result);
3574 return { };
3575}
3576
3577template<> auto AirIRGenerator::addOp<OpType::I64Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3578{
3579 return addShift(Type::I64, Lshift64, arg0, arg1, result);
3580}
3581
3582template<> auto AirIRGenerator::addOp<OpType::F32Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3583{
3584 result = g32();
3585 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3586 return { };
3587}
3588
3589template<> auto AirIRGenerator::addOp<OpType::I32WrapI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3590{
3591 result = g32();
3592 append(Move32, arg0, result);
3593 return { };
3594}
3595
3596template<> auto AirIRGenerator::addOp<OpType::I32Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3597{
3598 if (isARM64()) {
3599 // ARM64 doesn't have a rotate left.
3600 auto newShift = g64();
3601 append(Move, arg1, newShift);
3602 append(Neg64, newShift);
3603 return addShift(Type::I32, RotateRight32, arg0, newShift, result);
3604 } else
3605 return addShift(Type::I32, RotateLeft32, arg0, arg1, result);
3606}
3607
3608template<> auto AirIRGenerator::addOp<OpType::I32Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3609{
3610 return addShift(Type::I32, RotateRight32, arg0, arg1, result);
3611}
3612
3613template<> auto AirIRGenerator::addOp<OpType::I32GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3614{
3615 result = g32();
3616 append(Compare32, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3617 return { };
3618}
3619
3620template<> auto AirIRGenerator::addOp<OpType::I64ExtendSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3621{
3622 result = g64();
3623 append(SignExtend32ToPtr, arg0, result);
3624 return { };
3625}
3626
3627template<> auto AirIRGenerator::addOp<OpType::I32GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3628{
3629 result = g32();
3630 append(Compare32, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3631 return { };
3632}
3633
3634template<> auto AirIRGenerator::addOp<OpType::F64Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3635{
3636 result = f64();
3637 if (isValidForm(NegateDouble, Arg::Tmp, Arg::Tmp))
3638 append(NegateDouble, arg0, result);
3639 else {
3640 auto constant = addConstant(Type::I64, bitwise_cast<uint64_t>(static_cast<double>(-0.0)));
3641 auto temp = g64();
3642 append(MoveDoubleTo64, arg0, temp);
3643 append(Xor64, constant, temp);
3644 append(Move64ToDouble, temp, result);
3645 }
3646 return { };
3647}
3648
3649template<> auto AirIRGenerator::addOp<OpType::F64Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3650{
3651 return addFloatingPointMinOrMax(F64, MinOrMax::Max, arg0, arg1, result);
3652}
3653
3654template<> auto AirIRGenerator::addOp<OpType::I64LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3655{
3656 result = g32();
3657 append(Compare64, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3658 return { };
3659}
3660
3661template<> auto AirIRGenerator::addOp<OpType::I64LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3662{
3663 result = g32();
3664 append(Compare64, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3665 return { };
3666}
3667
3668template<> auto AirIRGenerator::addOp<OpType::I64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3669{
3670 result = g64();
3671 append(Add64, arg0, arg1, result);
3672 return { };
3673}
3674
3675} } // namespace JSC::Wasm
3676
3677#endif // ENABLE(WEBASSEMBLY)
3678