1/*
2 * Copyright (C) 2011-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#pragma once
27
28#if ENABLE(JIT)
29
30#include "AssemblyHelpers.h"
31#include "FPRInfo.h"
32#include "GPRInfo.h"
33#include "StackAlignment.h"
34#include <wtf/FunctionTraits.h>
35
36namespace JSC {
37
38#if CPU(MIPS) || (OS(WINDOWS) && CPU(X86_64))
39#define POKE_ARGUMENT_OFFSET 4
40#else
41#define POKE_ARGUMENT_OFFSET 0
42#endif
43
44class CallFrame;
45class Structure;
46namespace DFG {
47class RegisteredStructure;
48};
49
50class CCallHelpers : public AssemblyHelpers {
51public:
52 CCallHelpers(CodeBlock* codeBlock = 0)
53 : AssemblyHelpers(codeBlock)
54 {
55 }
56
57 // Wrapper to encode JSCell GPR into JSValue.
58 class CellValue {
59 public:
60 explicit CellValue(GPRReg gpr)
61 : m_gpr(gpr)
62 {
63 }
64
65 GPRReg gpr() const { return m_gpr; }
66
67 private:
68 GPRReg m_gpr;
69 };
70
71 // The most general helper for setting arguments that fit in a GPR, if you can compute each
72 // argument without using any argument registers. You usually want one of the setupArguments*()
73 // methods below instead of this. This thing is most useful if you have *a lot* of arguments.
74 template<typename Functor>
75 void setupArgument(unsigned argumentIndex, const Functor& functor)
76 {
77 unsigned numberOfRegs = GPRInfo::numberOfArgumentRegisters; // Disguise the constant from clang's tautological compare warning.
78 if (argumentIndex < numberOfRegs) {
79 functor(GPRInfo::toArgumentRegister(argumentIndex));
80 return;
81 }
82
83 functor(GPRInfo::nonArgGPR0);
84 poke(GPRInfo::nonArgGPR0, POKE_ARGUMENT_OFFSET + argumentIndex - GPRInfo::numberOfArgumentRegisters);
85 }
86
87private:
88
89 template<unsigned NumberOfRegisters, typename RegType>
90 ALWAYS_INLINE void setupStubArgs(std::array<RegType, NumberOfRegisters> destinations, std::array<RegType, NumberOfRegisters> sources)
91 {
92 if (!ASSERT_DISABLED) {
93 RegisterSet set;
94 for (RegType dest : destinations)
95 set.set(dest);
96 ASSERT_WITH_MESSAGE(set.numberOfSetRegisters() == NumberOfRegisters, "Destinations should not be aliased.");
97 }
98
99 typedef std::pair<RegType, RegType> RegPair;
100 Vector<RegPair, NumberOfRegisters> pairs;
101
102 for (unsigned i = 0; i < NumberOfRegisters; ++i) {
103 if (sources[i] != destinations[i])
104 pairs.append(std::make_pair(sources[i], destinations[i]));
105 }
106
107#if !ASSERT_DISABLED
108 auto numUniqueSources = [&] () -> unsigned {
109 RegisterSet set;
110 for (auto& pair : pairs) {
111 RegType source = pair.first;
112 set.set(source);
113 }
114 return set.numberOfSetRegisters();
115 };
116
117 auto numUniqueDests = [&] () -> unsigned {
118 RegisterSet set;
119 for (auto& pair : pairs) {
120 RegType dest = pair.second;
121 set.set(dest);
122 }
123 return set.numberOfSetRegisters();
124 };
125#endif
126
127 while (pairs.size()) {
128 RegisterSet freeDestinations;
129 for (auto& pair : pairs) {
130 RegType dest = pair.second;
131 freeDestinations.set(dest);
132 }
133 for (auto& pair : pairs) {
134 RegType source = pair.first;
135 freeDestinations.clear(source);
136 }
137
138 if (freeDestinations.numberOfSetRegisters()) {
139 bool madeMove = false;
140 for (unsigned i = 0; i < pairs.size(); i++) {
141 auto& pair = pairs[i];
142 RegType source = pair.first;
143 RegType dest = pair.second;
144 if (freeDestinations.get(dest)) {
145 move(source, dest);
146 pairs.remove(i);
147 madeMove = true;
148 break;
149 }
150 }
151 ASSERT_UNUSED(madeMove, madeMove);
152 continue;
153 }
154
155 ASSERT(numUniqueDests() == numUniqueSources());
156 ASSERT(numUniqueDests() == pairs.size());
157 // The set of source and destination registers are equivalent sets. This means we don't have
158 // any free destination registers that won't also clobber a source. We get around this by
159 // exchanging registers.
160
161 RegType source = pairs[0].first;
162 RegType dest = pairs[0].second;
163 swap(source, dest);
164 pairs.remove(0);
165
166 RegType newSource = source;
167 for (auto& pair : pairs) {
168 RegType source = pair.first;
169 if (source == dest) {
170 pair.first = newSource;
171 break;
172 }
173 }
174
175 // We may have introduced pairs that have the same source and destination. Remove those now.
176 for (unsigned i = 0; i < pairs.size(); i++) {
177 auto& pair = pairs[i];
178 if (pair.first == pair.second) {
179 pairs.remove(i);
180 i--;
181 }
182 }
183 }
184 }
185
186#if CPU(MIPS) || (CPU(ARM_THUMB2) && !CPU(ARM_HARDFP))
187 template<unsigned NumCrossSources, unsigned NumberOfRegisters>
188 ALWAYS_INLINE void setupStubCrossArgs(std::array<GPRReg, NumberOfRegisters> destinations, std::array<FPRReg, NumberOfRegisters> sources) {
189 for (unsigned i = 0; i < NumCrossSources; i++) {
190 GPRReg dest = destinations[i];
191 FPRReg source = sources[i];
192
193 moveDouble(source, dest);
194 }
195 }
196#endif
197
198 template<typename RegType>
199 using InfoTypeForReg = decltype(toInfoFromReg(RegType(-1)));
200
201 // extraGPRArgs is used to track 64-bit argument types passed in register on 32-bit architectures.
202 // extraPoke is used to track 64-bit argument types passed on the stack.
203 template<unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke>
204 struct ArgCollection {
205 ArgCollection()
206 {
207 gprSources.fill(InvalidGPRReg);
208 gprDestinations.fill(InvalidGPRReg);
209 fprSources.fill(InvalidFPRReg);
210 fprDestinations.fill(InvalidFPRReg);
211 crossSources.fill(InvalidFPRReg);
212 crossDestinations.fill(InvalidGPRReg);
213 }
214
215 template<unsigned a, unsigned b, unsigned c, unsigned d, unsigned e, unsigned f, unsigned g>
216 ArgCollection(ArgCollection<a, b, c, d, e, f, g>& other)
217 {
218 gprSources = other.gprSources;
219 gprDestinations = other.gprDestinations;
220 fprSources = other.fprSources;
221 fprDestinations = other.fprDestinations;
222 crossSources = other.crossSources;
223 crossDestinations = other.crossDestinations;
224 }
225
226 ArgCollection<numGPRArgs + 1, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> pushRegArg(GPRReg argument, GPRReg destination)
227 {
228 ArgCollection<numGPRArgs + 1, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> result(*this);
229
230 result.gprSources[numGPRSources] = argument;
231 result.gprDestinations[numGPRSources] = destination;
232 return result;
233 }
234
235 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources + 1, numCrossSources, extraGPRArgs, extraPoke> pushRegArg(FPRReg argument, FPRReg destination)
236 {
237 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources + 1, numCrossSources, extraGPRArgs, extraPoke> result(*this);
238
239 result.fprSources[numFPRSources] = argument;
240 result.fprDestinations[numFPRSources] = destination;
241 return result;
242 }
243
244 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources + 1, extraGPRArgs, extraPoke> pushRegArg(FPRReg argument, GPRReg destination)
245 {
246 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources + 1, extraGPRArgs, extraPoke> result(*this);
247
248 result.crossSources[numCrossSources] = argument;
249 result.crossDestinations[numCrossSources] = destination;
250 return result;
251 }
252
253 ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, extraPoke> pushExtraRegArg(GPRReg argument, GPRReg destination)
254 {
255 ArgCollection<numGPRArgs, numGPRSources + 1, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, extraPoke> result(*this);
256
257 result.gprSources[numGPRSources] = argument;
258 result.gprDestinations[numGPRSources] = destination;
259 return result;
260 }
261
262 ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> addGPRArg()
263 {
264 return ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke>(*this);
265 }
266
267 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, extraPoke> addGPRExtraArg()
268 {
269 return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs + 1, extraPoke>(*this);
270 }
271
272 ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> addStackArg(GPRReg)
273 {
274 return ArgCollection<numGPRArgs + 1, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke>(*this);
275 }
276
277 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> addStackArg(FPRReg)
278 {
279 return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs + 1, numFPRSources, numCrossSources, extraGPRArgs, extraPoke>(*this);
280 }
281
282 ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke + 1> addPoke()
283 {
284 return ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke + 1>(*this);
285 }
286
287#if OS(WINDOWS) && CPU(X86_64)
288 unsigned argCount(GPRReg) { return numGPRArgs + numFPRArgs; }
289 unsigned argCount(FPRReg) { return numGPRArgs + numFPRArgs; }
290#else
291 unsigned argCount(GPRReg) { return numGPRArgs + extraGPRArgs; }
292 unsigned argCount(FPRReg) { return numFPRArgs; }
293#endif
294
295 // store GPR -> GPR assignments
296 std::array<GPRReg, GPRInfo::numberOfRegisters> gprSources;
297 std::array<GPRReg, GPRInfo::numberOfRegisters> gprDestinations;
298
299 // store FPR -> FPR assignments
300 std::array<FPRReg, FPRInfo::numberOfRegisters> fprSources;
301 std::array<FPRReg, FPRInfo::numberOfRegisters> fprDestinations;
302
303 // store FPR -> GPR assignments
304 std::array<FPRReg, GPRInfo::numberOfRegisters> crossSources;
305 std::array<GPRReg, GPRInfo::numberOfRegisters> crossDestinations;
306 };
307
308 template<unsigned TargetSize, typename RegType>
309 std::array<RegType, TargetSize> clampArrayToSize(std::array<RegType, InfoTypeForReg<RegType>::numberOfRegisters> sourceArray)
310 {
311 static_assert(TargetSize <= sourceArray.size(), "TargetSize is bigger than source.size()");
312 RELEASE_ASSERT(TargetSize <= InfoTypeForReg<RegType>::numberOfRegisters);
313
314 std::array<RegType, TargetSize> result { };
315
316 for (unsigned i = 0; i < TargetSize; i++) {
317 ASSERT(sourceArray[i] != static_cast<int32_t>(InfoTypeForReg<RegType>::InvalidIndex));
318 result[i] = sourceArray[i];
319 }
320
321 return result;
322 }
323
324 ALWAYS_INLINE unsigned calculatePokeOffset(unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke)
325 {
326 // Clang claims that it cannot find the symbol for FPRReg/GPRReg::numberOfArgumentRegisters when they are passed directly to std::max... seems like a bug
327 unsigned numberOfFPArgumentRegisters = FPRInfo::numberOfArgumentRegisters;
328 unsigned numberOfGPArgumentRegisters = GPRInfo::numberOfArgumentRegisters;
329
330 currentGPRArgument += extraGPRArgs;
331 currentFPRArgument -= numCrossSources;
332 IGNORE_WARNINGS_BEGIN("type-limits")
333 ASSERT(currentGPRArgument >= GPRInfo::numberOfArgumentRegisters || currentFPRArgument >= FPRInfo::numberOfArgumentRegisters);
334 IGNORE_WARNINGS_END
335
336 unsigned pokeOffset = POKE_ARGUMENT_OFFSET + extraPoke;
337 pokeOffset += std::max(currentGPRArgument, numberOfGPArgumentRegisters) - numberOfGPArgumentRegisters;
338 pokeOffset += std::max(currentFPRArgument, numberOfFPArgumentRegisters) - numberOfFPArgumentRegisters;
339 return pokeOffset;
340 }
341
342 template<typename ArgType>
343 ALWAYS_INLINE void pokeForArgument(ArgType arg, unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke)
344 {
345 unsigned pokeOffset = calculatePokeOffset(currentGPRArgument, currentFPRArgument, numCrossSources, extraGPRArgs, extraPoke);
346 poke(arg, pokeOffset);
347 }
348
349 ALWAYS_INLINE bool stackAligned(unsigned currentGPRArgument, unsigned currentFPRArgument, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke)
350 {
351 unsigned pokeOffset = calculatePokeOffset(currentGPRArgument, currentFPRArgument, numCrossSources, extraGPRArgs, extraPoke);
352 return !(pokeOffset & 1);
353 }
354
355 // In the auto-calling convention code below the order of operations is:
356 // 1) spill arguments to stack slots
357 // 2) shuffle incomming argument values in registers to argument registers
358 // 3) fill immediate values to argument registers
359 // To do this, we recurse forwards through our args collecting argument values in registers and spilling stack slots.
360 // when we run out of args we then run our shuffling code to relocate registers. Finally, as we unwind from our
361 // recursion we can fill immediates.
362
363#define CURRENT_ARGUMENT_TYPE typename FunctionTraits<OperationType>::template ArgumentType<numGPRArgs + numFPRArgs>
364#define RESULT_TYPE typename FunctionTraits<OperationType>::ResultType
365
366#if USE(JSVALUE64)
367
368 // Avoid MSVC optimization time explosion associated with __forceinline in recursive templates.
369 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename RegType, typename... Args>
370 ALWAYS_INLINE_EXCEPT_MSVC void marshallArgumentRegister(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, RegType arg, Args... args)
371 {
372 using InfoType = InfoTypeForReg<RegType>;
373 unsigned numArgRegisters = InfoType::numberOfArgumentRegisters;
374#if OS(WINDOWS) && CPU(X86_64)
375 unsigned currentArgCount = argSourceRegs.argCount(arg) + (std::is_same<RESULT_TYPE, SlowPathReturnType>::value ? 1 : 0);
376#else
377 unsigned currentArgCount = argSourceRegs.argCount(arg);
378#endif
379 if (currentArgCount < numArgRegisters) {
380 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, InfoType::toArgumentRegister(currentArgCount));
381 setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);
382 return;
383 }
384
385 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke);
386 setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg), args...);
387 }
388
389 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
390 ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, FPRReg arg, Args... args)
391 {
392 static_assert(std::is_same<CURRENT_ARGUMENT_TYPE, double>::value, "We should only be passing FPRRegs to a double");
393 marshallArgumentRegister<OperationType>(argSourceRegs, arg, args...);
394 }
395
396 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
397 ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, GPRReg arg, Args... args)
398 {
399 marshallArgumentRegister<OperationType>(argSourceRegs, arg, args...);
400 }
401
402 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
403 ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, JSValueRegs arg, Args... args)
404 {
405 marshallArgumentRegister<OperationType>(argSourceRegs, arg.gpr(), args...);
406 }
407
408 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
409 ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, CellValue arg, Args... args)
410 {
411 marshallArgumentRegister<OperationType>(argSourceRegs, arg.gpr(), args...);
412 }
413
414#else // USE(JSVALUE64)
415#if CPU(ARM_THUMB2) || CPU(MIPS)
416
417 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
418 void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, FPRReg arg, Args... args)
419 {
420 static_assert(std::is_same<CURRENT_ARGUMENT_TYPE, double>::value, "We should only be passing FPRRegs to a double");
421
422 // MIPS and ARM-hardfp pass FP arguments in FP registers.
423#if CPU(MIPS)
424 unsigned numberOfFPArgumentRegisters = FPRInfo::numberOfArgumentRegisters;
425 unsigned currentFPArgCount = argSourceRegs.argCount(arg);
426
427 // MIPS can only use FP argument registers if it isn't preceeded by any GP argument.
428 if (currentFPArgCount < numberOfFPArgumentRegisters && !numGPRArgs) {
429 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, FPRInfo::toArgumentRegister(currentFPArgCount));
430 setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg().addGPRExtraArg(), args...);
431 return;
432 }
433#elif CPU(ARM_THUMB2) && CPU(ARM_HARDFP)
434 unsigned numberOfFPArgumentRegisters = FPRInfo::numberOfArgumentRegisters;
435 unsigned currentFPArgCount = argSourceRegs.argCount(arg);
436
437 if (currentFPArgCount < numberOfFPArgumentRegisters) {
438 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, FPRInfo::toArgumentRegister(currentFPArgCount));
439 setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);
440 return;
441 }
442#endif
443
444#if CPU(MIPS) || (CPU(ARM_THUMB2) && !CPU(ARM_HARDFP))
445 // On MIPS and ARM-softfp FP arguments can be passed in GP registers.
446 unsigned numberOfGPArgumentRegisters = GPRInfo::numberOfArgumentRegisters;
447 unsigned currentGPArgCount = argSourceRegs.argCount(GPRInfo::regT0);
448 unsigned alignedGPArgCount = roundUpToMultipleOf<2>(currentGPArgCount);
449
450 if (alignedGPArgCount + 1 < numberOfGPArgumentRegisters) {
451 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, GPRInfo::toArgumentRegister(alignedGPArgCount));
452
453 if (alignedGPArgCount > currentGPArgCount)
454 setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg().addGPRExtraArg().addGPRExtraArg(), args...);
455 else
456 setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg().addGPRExtraArg(), args...);
457
458 return;
459 }
460
461 if (currentGPArgCount < numberOfGPArgumentRegisters) {
462 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs + 1, extraPoke);
463 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRExtraArg().addStackArg(arg).addPoke(), args...);
464 return;
465 }
466#endif
467
468 // Otherwise pass FP argument on stack.
469 if (stackAligned(numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke)) {
470 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke);
471 setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg).addPoke(), args...);
472 } else {
473 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke + 1);
474 setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg).addPoke().addPoke(), args...);
475 }
476 }
477
478 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
479 std::enable_if_t<sizeof(CURRENT_ARGUMENT_TYPE) <= 4>
480 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, GPRReg arg, Args... args)
481 {
482 unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;
483 unsigned currentArgCount = argSourceRegs.argCount(arg);
484 if (currentArgCount < numArgRegisters) {
485 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(arg, GPRInfo::toArgumentRegister(currentArgCount));
486 setupArgumentsImpl<OperationType>(updatedArgSourceRegs, args...);
487 return;
488 }
489
490 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke);
491 setupArgumentsImpl<OperationType>(argSourceRegs.addStackArg(arg), args...);
492 }
493
494 template<typename OperationType, typename Arg1, typename Arg2, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
495 void pokeArgumentsAligned(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg1 arg1, Arg2 arg2, Args... args)
496 {
497 unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;
498 unsigned currentArgCount = argSourceRegs.argCount(GPRInfo::regT0);
499
500 if (currentArgCount + 1 == numArgRegisters) {
501 pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs + 1, extraPoke);
502 pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs + 1, extraPoke + 1);
503 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRExtraArg().addGPRArg().addPoke(), args...);
504 } else if (stackAligned(numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke)) {
505 pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke);
506 pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke + 1);
507 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg().addPoke(), args...);
508 } else {
509 pokeForArgument(arg1, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke + 1);
510 pokeForArgument(arg2, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke + 2);
511 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg().addPoke().addPoke(), args...);
512 }
513 }
514
515 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
516 std::enable_if_t<std::is_same<CURRENT_ARGUMENT_TYPE, EncodedJSValue>::value>
517 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, CellValue payload, Args... args)
518 {
519 unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;
520 unsigned currentArgCount = argSourceRegs.argCount(payload.gpr());
521 unsigned alignedArgCount = roundUpToMultipleOf<2>(currentArgCount);
522
523 if (alignedArgCount + 1 < numArgRegisters) {
524 auto updatedArgSourceRegs = argSourceRegs.pushRegArg(payload.gpr(), GPRInfo::toArgumentRegister(alignedArgCount));
525
526 if (alignedArgCount > currentArgCount)
527 setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg().addGPRExtraArg(), args...);
528 else
529 setupArgumentsImpl<OperationType>(updatedArgSourceRegs.addGPRExtraArg(), args...);
530
531 move(TrustedImm32(JSValue::CellTag), GPRInfo::toArgumentRegister(alignedArgCount + 1));
532
533 } else
534 pokeArgumentsAligned<OperationType>(argSourceRegs, payload.gpr(), TrustedImm32(JSValue::CellTag), args...);
535 }
536
537 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename... Args>
538 std::enable_if_t<std::is_same<CURRENT_ARGUMENT_TYPE, EncodedJSValue>::value>
539 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, JSValueRegs arg, Args... args)
540 {
541 unsigned numArgRegisters = GPRInfo::numberOfArgumentRegisters;
542 unsigned currentArgCount = argSourceRegs.argCount(arg.tagGPR());
543 unsigned alignedArgCount = roundUpToMultipleOf<2>(currentArgCount);
544
545 if (alignedArgCount + 1 < numArgRegisters) {
546 // JSValueRegs is passed in two 32-bit registers on these architectures. Increase both numGPRArgs and extraGPRArgs by 1.
547 // We can't just add 2 to numGPRArgs, since it is used for CURRENT_ARGUMENT_TYPE. Adding 2 would lead to a skipped argument.
548 auto updatedArgSourceRegs1 = argSourceRegs.pushRegArg(arg.payloadGPR(), GPRInfo::toArgumentRegister(alignedArgCount));
549 auto updatedArgSourceRegs2 = updatedArgSourceRegs1.pushExtraRegArg(arg.tagGPR(), GPRInfo::toArgumentRegister(alignedArgCount + 1));
550
551 if (alignedArgCount > currentArgCount)
552 setupArgumentsImpl<OperationType>(updatedArgSourceRegs2.addGPRExtraArg(), args...);
553 else
554 setupArgumentsImpl<OperationType>(updatedArgSourceRegs2, args...);
555 } else
556 pokeArgumentsAligned<OperationType>(argSourceRegs, arg.payloadGPR(), arg.tagGPR(), args...);
557 }
558
559#endif // CPU(ARM_THUMB2) || CPU(MIPS)
560#endif // USE(JSVALUE64)
561
562 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename Arg, typename... Args>
563 ALWAYS_INLINE std::enable_if_t<
564 std::is_base_of<TrustedImm, Arg>::value
565 || std::is_convertible<Arg, TrustedImm>::value> // We have this since DFGSpeculativeJIT has it's own implementation of TrustedImmPtr
566 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg arg, Args... args)
567 {
568 // Right now this only supports non-floating point immediate arguments since we never call operations with non-register values.
569 // If we ever needed to support immediate floating point arguments we would need to duplicate this logic for both types, which sounds
570 // gross so it's probably better to do that marshalling before the call operation...
571 static_assert(!std::is_floating_point<CURRENT_ARGUMENT_TYPE>::value, "We don't support immediate floats/doubles in setupArguments");
572 auto numArgRegisters = GPRInfo::numberOfArgumentRegisters;
573#if OS(WINDOWS) && CPU(X86_64)
574 auto currentArgCount = numGPRArgs + numFPRArgs + (std::is_same<RESULT_TYPE, SlowPathReturnType>::value ? 1 : 0);
575#else
576 auto currentArgCount = numGPRArgs + extraGPRArgs;
577#endif
578 if (currentArgCount < numArgRegisters) {
579 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);
580 move(arg, GPRInfo::toArgumentRegister(currentArgCount));
581 return;
582 }
583
584 pokeForArgument(arg, numGPRArgs, numFPRArgs, numCrossSources, extraGPRArgs, extraPoke);
585 setupArgumentsImpl<OperationType>(argSourceRegs.addGPRArg(), args...);
586 }
587
588 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename Arg, typename... Args>
589 ALWAYS_INLINE std::enable_if_t<
590 std::is_same<CURRENT_ARGUMENT_TYPE, Arg>::value
591 && std::is_integral<CURRENT_ARGUMENT_TYPE>::value
592 && (sizeof(CURRENT_ARGUMENT_TYPE) <= 4)>
593 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg arg, Args... args)
594 {
595 setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImm32(arg), args...);
596 }
597
598 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename Arg, typename... Args>
599 ALWAYS_INLINE std::enable_if_t<
600 std::is_same<CURRENT_ARGUMENT_TYPE, Arg>::value
601 && std::is_integral<CURRENT_ARGUMENT_TYPE>::value
602 && (sizeof(CURRENT_ARGUMENT_TYPE) == 8)>
603 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg arg, Args... args)
604 {
605 setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImm64(arg), args...);
606 }
607
608 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename Arg, typename... Args>
609 ALWAYS_INLINE std::enable_if_t<
610 std::is_pointer<CURRENT_ARGUMENT_TYPE>::value
611 && ((std::is_pointer<Arg>::value && std::is_convertible<std::remove_const_t<std::remove_pointer_t<Arg>>*, CURRENT_ARGUMENT_TYPE>::value)
612 || std::is_same<Arg, std::nullptr_t>::value)>
613 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg arg, Args... args)
614 {
615 setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImmPtr(arg), args...);
616 }
617
618 // Special case DFG::RegisteredStructure because it's really annoying to deal with otherwise...
619 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke, typename Arg, typename... Args>
620 ALWAYS_INLINE std::enable_if_t<
621 std::is_same<CURRENT_ARGUMENT_TYPE, Structure*>::value
622 && std::is_same<Arg, DFG::RegisteredStructure>::value>
623 setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs, Arg arg, Args... args)
624 {
625 setupArgumentsImpl<OperationType>(argSourceRegs, TrustedImmPtr(arg.get()), args...);
626 }
627
628#undef CURRENT_ARGUMENT_TYPE
629#undef RESULT_TYPE
630
631 // Base case; set up the argument registers.
632 template<typename OperationType, unsigned numGPRArgs, unsigned numGPRSources, unsigned numFPRArgs, unsigned numFPRSources, unsigned numCrossSources, unsigned extraGPRArgs, unsigned extraPoke>
633 ALWAYS_INLINE void setupArgumentsImpl(ArgCollection<numGPRArgs, numGPRSources, numFPRArgs, numFPRSources, numCrossSources, extraGPRArgs, extraPoke> argSourceRegs)
634 {
635 static_assert(FunctionTraits<OperationType>::arity == numGPRArgs + numFPRArgs, "One last sanity check");
636#if USE(JSVALUE64)
637 static_assert(FunctionTraits<OperationType>::cCallArity() == numGPRArgs + numFPRArgs + extraPoke, "Check the CCall arity");
638#endif
639 setupStubArgs<numGPRSources, GPRReg>(clampArrayToSize<numGPRSources, GPRReg>(argSourceRegs.gprDestinations), clampArrayToSize<numGPRSources, GPRReg>(argSourceRegs.gprSources));
640#if CPU(MIPS) || (CPU(ARM_THUMB2) && !CPU(ARM_HARDFP))
641 setupStubCrossArgs<numCrossSources>(argSourceRegs.crossDestinations, argSourceRegs.crossSources);
642#else
643 static_assert(!numCrossSources, "shouldn't be used on this architecture.");
644#endif
645 setupStubArgs<numFPRSources, FPRReg>(clampArrayToSize<numFPRSources, FPRReg>(argSourceRegs.fprDestinations), clampArrayToSize<numFPRSources, FPRReg>(argSourceRegs.fprSources));
646 }
647
648public:
649
650#define FIRST_ARGUMENT_TYPE typename FunctionTraits<OperationType>::template ArgumentType<0>
651
652 template<typename OperationType, typename... Args>
653 ALWAYS_INLINE std::enable_if_t<std::is_same<FIRST_ARGUMENT_TYPE, CallFrame*>::value> setupArguments(Args... args)
654 {
655#if USE(JSVALUE64)
656 // This only really works for 64-bit since jsvalue regs mess things up for 32-bit...
657 static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args) + 1, "Basic sanity check");
658#endif
659 setupArgumentsImpl<OperationType, 0, 0, 0, 0, 0, 0, 0>(ArgCollection<0, 0, 0, 0, 0, 0, 0>(), GPRInfo::callFrameRegister, args...);
660 }
661
662 template<typename OperationType, typename... Args>
663 ALWAYS_INLINE std::enable_if_t<!std::is_same<FIRST_ARGUMENT_TYPE, CallFrame*>::value> setupArguments(Args... args)
664 {
665#if USE(JSVALUE64)
666 // This only really works for 64-bit since jsvalue regs mess things up for 32-bit...
667 static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Basic sanity check");
668#endif
669 setupArgumentsImpl<OperationType, 0, 0, 0, 0, 0, 0, 0>(ArgCollection<0, 0, 0, 0, 0, 0, 0>(), args...);
670 }
671
672#undef FIRST_ARGUMENT_TYPE
673
674 void setupResults(GPRReg destA, GPRReg destB)
675 {
676 GPRReg srcA = GPRInfo::returnValueGPR;
677 GPRReg srcB = GPRInfo::returnValueGPR2;
678
679 if (destA == InvalidGPRReg)
680 move(srcB, destB);
681 else if (destB == InvalidGPRReg)
682 move(srcA, destA);
683 else if (srcB != destA) {
684 // Handle the easy cases - two simple moves.
685 move(srcA, destA);
686 move(srcB, destB);
687 } else if (srcA != destB) {
688 // Handle the non-swap case - just put srcB in place first.
689 move(srcB, destB);
690 move(srcA, destA);
691 } else
692 swap(destA, destB);
693 }
694
695 void setupResults(JSValueRegs regs)
696 {
697#if USE(JSVALUE64)
698 move(GPRInfo::returnValueGPR, regs.gpr());
699#else
700 setupResults(regs.payloadGPR(), regs.tagGPR());
701#endif
702 }
703
704 void jumpToExceptionHandler(VM& vm)
705 {
706 // genericUnwind() leaves the handler CallFrame* in vm->callFrameForCatch,
707 // and the address of the handler in vm->targetMachinePCForThrow.
708 loadPtr(&vm.targetMachinePCForThrow, GPRInfo::regT1);
709 farJump(GPRInfo::regT1, ExceptionHandlerPtrTag);
710 }
711
712 void prepareForTailCallSlow(GPRReg calleeGPR = InvalidGPRReg)
713 {
714 GPRReg temp1 = calleeGPR == GPRInfo::regT0 ? GPRInfo::regT3 : GPRInfo::regT0;
715 GPRReg temp2 = calleeGPR == GPRInfo::regT1 ? GPRInfo::regT3 : GPRInfo::regT1;
716 GPRReg temp3 = calleeGPR == GPRInfo::regT2 ? GPRInfo::regT3 : GPRInfo::regT2;
717
718 GPRReg newFramePointer = temp1;
719 GPRReg newFrameSizeGPR = temp2;
720 {
721 // The old frame size is its number of arguments (or number of
722 // parameters in case of arity fixup), plus the frame header size,
723 // aligned
724 GPRReg oldFrameSizeGPR = temp2;
725 {
726 GPRReg argCountGPR = oldFrameSizeGPR;
727 load32(Address(framePointerRegister, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), argCountGPR);
728
729 {
730 GPRReg numParametersGPR = temp1;
731 {
732 GPRReg codeBlockGPR = numParametersGPR;
733 loadPtr(Address(framePointerRegister, CallFrameSlot::codeBlock * static_cast<int>(sizeof(Register))), codeBlockGPR);
734 load32(Address(codeBlockGPR, CodeBlock::offsetOfNumParameters()), numParametersGPR);
735 }
736
737 ASSERT(numParametersGPR != argCountGPR);
738 Jump argumentCountWasNotFixedUp = branch32(BelowOrEqual, numParametersGPR, argCountGPR);
739 move(numParametersGPR, argCountGPR);
740 argumentCountWasNotFixedUp.link(this);
741 }
742
743 add32(TrustedImm32(stackAlignmentRegisters() + CallFrame::headerSizeInRegisters - 1), argCountGPR, oldFrameSizeGPR);
744 and32(TrustedImm32(-stackAlignmentRegisters()), oldFrameSizeGPR);
745 // We assume < 2^28 arguments
746 mul32(TrustedImm32(sizeof(Register)), oldFrameSizeGPR, oldFrameSizeGPR);
747 }
748
749 // The new frame pointer is at framePointer + oldFrameSize - newFrameSize
750 ASSERT(newFramePointer != oldFrameSizeGPR);
751 addPtr(framePointerRegister, oldFrameSizeGPR, newFramePointer);
752
753 // The new frame size is just the number of arguments plus the
754 // frame header size, aligned
755 ASSERT(newFrameSizeGPR != newFramePointer);
756 load32(Address(stackPointerRegister, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)),
757 newFrameSizeGPR);
758 add32(TrustedImm32(stackAlignmentRegisters() + CallFrame::headerSizeInRegisters - 1), newFrameSizeGPR);
759 and32(TrustedImm32(-stackAlignmentRegisters()), newFrameSizeGPR);
760 // We assume < 2^28 arguments
761 mul32(TrustedImm32(sizeof(Register)), newFrameSizeGPR, newFrameSizeGPR);
762 }
763
764 GPRReg tempGPR = temp3;
765 ASSERT(tempGPR != newFramePointer && tempGPR != newFrameSizeGPR);
766
767 // We don't need the current frame beyond this point. Masquerade as our
768 // caller.
769#if CPU(ARM_THUMB2) || CPU(ARM64)
770 loadPtr(Address(framePointerRegister, CallFrame::returnPCOffset()), linkRegister);
771 subPtr(TrustedImm32(2 * sizeof(void*)), newFrameSizeGPR);
772#if CPU(ARM64E)
773 addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), MacroAssembler::framePointerRegister, tempGPR);
774 untagPtr(tempGPR, linkRegister);
775#endif
776#elif CPU(MIPS)
777 loadPtr(Address(framePointerRegister, sizeof(void*)), returnAddressRegister);
778 subPtr(TrustedImm32(2 * sizeof(void*)), newFrameSizeGPR);
779#elif CPU(X86_64)
780 loadPtr(Address(framePointerRegister, sizeof(void*)), tempGPR);
781 push(tempGPR);
782 subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);
783#else
784 UNREACHABLE_FOR_PLATFORM();
785#endif
786 subPtr(newFrameSizeGPR, newFramePointer);
787 loadPtr(Address(framePointerRegister), framePointerRegister);
788
789
790 // We need to move the newFrameSizeGPR slots above the stack pointer by
791 // newFramePointer registers. We use pointer-sized chunks.
792 MacroAssembler::Label copyLoop(label());
793
794 subPtr(TrustedImm32(sizeof(void*)), newFrameSizeGPR);
795 loadPtr(BaseIndex(stackPointerRegister, newFrameSizeGPR, TimesOne), tempGPR);
796 storePtr(tempGPR, BaseIndex(newFramePointer, newFrameSizeGPR, TimesOne));
797
798 branchTest32(MacroAssembler::NonZero, newFrameSizeGPR).linkTo(copyLoop, this);
799
800 // Ready for a jump!
801 move(newFramePointer, stackPointerRegister);
802 }
803
804 // These operations clobber all volatile registers. They assume that there is room on the top of
805 // stack to marshall call arguments.
806 void logShadowChickenProloguePacket(GPRReg shadowPacket, GPRReg scratch1, GPRReg scope);
807 void logShadowChickenTailPacket(GPRReg shadowPacket, JSValueRegs thisRegs, GPRReg scope, CodeBlock*, CallSiteIndex);
808 // Leaves behind a pointer to the Packet we should write to in shadowPacket.
809 void ensureShadowChickenPacket(VM&, GPRReg shadowPacket, GPRReg scratch1NonArgGPR, GPRReg scratch2);
810};
811
812} // namespace JSC
813
814#endif // ENABLE(JIT)
815