1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "AirLowerAfterRegAlloc.h"
28
29#if ENABLE(B3_JIT)
30
31#include "AirArgInlines.h"
32#include "AirCCallingConvention.h"
33#include "AirCode.h"
34#include "AirEmitShuffle.h"
35#include "AirInsertionSet.h"
36#include "AirInstInlines.h"
37#include "AirPadInterference.h"
38#include "AirRegLiveness.h"
39#include "AirPhaseScope.h"
40#include "B3CCallValue.h"
41#include "B3ValueInlines.h"
42#include "RegisterSet.h"
43#include <wtf/HashMap.h>
44#include <wtf/ListDump.h>
45
46namespace JSC { namespace B3 { namespace Air {
47
48namespace {
49
50namespace AirLowerAfterRegAllocInternal {
51static constexpr bool verbose = false;
52}
53
54} // anonymous namespace
55
56void lowerAfterRegAlloc(Code& code)
57{
58 PhaseScope phaseScope(code, "lowerAfterRegAlloc");
59
60 if (AirLowerAfterRegAllocInternal::verbose)
61 dataLog("Code before lowerAfterRegAlloc:\n", code);
62
63 auto isRelevant = [] (Inst& inst) -> bool {
64 return inst.kind.opcode == Shuffle || inst.kind.opcode == ColdCCall;
65 };
66
67 bool haveAnyRelevant = false;
68 for (BasicBlock* block : code) {
69 for (Inst& inst : *block) {
70 if (isRelevant(inst)) {
71 haveAnyRelevant = true;
72 break;
73 }
74 }
75 if (haveAnyRelevant)
76 break;
77 }
78 if (!haveAnyRelevant)
79 return;
80
81 padInterference(code);
82
83 HashMap<Inst*, RegisterSet> usedRegisters;
84
85 RegLiveness liveness(code);
86 for (BasicBlock* block : code) {
87 RegLiveness::LocalCalc localCalc(liveness, block);
88
89 for (unsigned instIndex = block->size(); instIndex--;) {
90 Inst& inst = block->at(instIndex);
91
92 RegisterSet set;
93
94 if (isRelevant(inst)) {
95 for (Reg reg : localCalc.live())
96 set.set(reg);
97 }
98
99 localCalc.execute(instIndex);
100
101 if (isRelevant(inst))
102 usedRegisters.add(&inst, set);
103 }
104 }
105
106 std::array<std::array<StackSlot*, 2>, numBanks> slots;
107 forEachBank(
108 [&] (Bank bank) {
109 for (unsigned i = 0; i < 2; ++i)
110 slots[bank][i] = nullptr;
111 });
112
113 // If we run after stack allocation then we cannot use those callee saves that aren't in
114 // the callee save list. Note that we are only run after stack allocation in -O1, so this
115 // kind of slop is OK.
116 RegisterSet blacklistedCalleeSaves;
117 if (code.stackIsAllocated()) {
118 blacklistedCalleeSaves = RegisterSet::calleeSaveRegisters();
119 blacklistedCalleeSaves.exclude(code.calleeSaveRegisters());
120 }
121
122 auto getScratches = [&] (RegisterSet set, Bank bank) -> std::array<Arg, 2> {
123 std::array<Arg, 2> result;
124 for (unsigned i = 0; i < 2; ++i) {
125 bool found = false;
126 for (Reg reg : code.regsInPriorityOrder(bank)) {
127 if (!set.get(reg) && !blacklistedCalleeSaves.get(reg)) {
128 result[i] = Tmp(reg);
129 set.set(reg);
130 found = true;
131 break;
132 }
133 }
134 if (!found) {
135 StackSlot*& slot = slots[bank][i];
136 if (!slot)
137 slot = code.addStackSlot(bytes(conservativeWidth(bank)), StackSlotKind::Spill);
138 result[i] = Arg::stack(slots[bank][i]);
139 }
140 }
141 return result;
142 };
143
144 // Now transform the code.
145 InsertionSet insertionSet(code);
146 for (BasicBlock* block : code) {
147 for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
148 Inst& inst = block->at(instIndex);
149
150 switch (inst.kind.opcode) {
151 case Shuffle: {
152 RegisterSet set = usedRegisters.get(&inst);
153 Vector<ShufflePair> pairs;
154 for (unsigned i = 0; i < inst.args.size(); i += 3) {
155 Arg src = inst.args[i + 0];
156 Arg dst = inst.args[i + 1];
157 Width width = inst.args[i + 2].width();
158
159 // The used register set contains things live after the shuffle. But
160 // emitShuffle() wants a scratch register that is not just dead but also does not
161 // interfere with either sources or destinations.
162 auto excludeRegisters = [&] (Tmp tmp) {
163 if (tmp.isReg())
164 set.set(tmp.reg());
165 };
166 src.forEachTmpFast(excludeRegisters);
167 dst.forEachTmpFast(excludeRegisters);
168
169 pairs.append(ShufflePair(src, dst, width));
170 }
171 std::array<Arg, 2> gpScratch = getScratches(set, GP);
172 std::array<Arg, 2> fpScratch = getScratches(set, FP);
173 insertionSet.insertInsts(
174 instIndex, emitShuffle(code, pairs, gpScratch, fpScratch, inst.origin));
175 inst = Inst();
176 break;
177 }
178
179 case ColdCCall: {
180 CCallValue* value = inst.origin->as<CCallValue>();
181 Kind oldKind = inst.kind;
182
183 RegisterSet liveRegs = usedRegisters.get(&inst);
184 RegisterSet regsToSave = liveRegs;
185 regsToSave.exclude(RegisterSet::calleeSaveRegisters());
186 regsToSave.exclude(RegisterSet::stackRegisters());
187 regsToSave.exclude(RegisterSet::reservedHardwareRegisters());
188
189 RegisterSet preUsed = liveRegs;
190 Vector<Arg> destinations = computeCCallingConvention(code, value);
191 Tmp result = cCallResult(value->type());
192 Arg originalResult = result ? inst.args[1] : Arg();
193
194 Vector<ShufflePair> pairs;
195 for (unsigned i = 0; i < destinations.size(); ++i) {
196 Value* child = value->child(i);
197 Arg src = inst.args[result ? (i >= 1 ? i + 1 : i) : i ];
198 Arg dst = destinations[i];
199 Width width = widthForType(child->type());
200 pairs.append(ShufflePair(src, dst, width));
201
202 auto excludeRegisters = [&] (Tmp tmp) {
203 if (tmp.isReg())
204 preUsed.set(tmp.reg());
205 };
206 src.forEachTmpFast(excludeRegisters);
207 dst.forEachTmpFast(excludeRegisters);
208 }
209
210 std::array<Arg, 2> gpScratch = getScratches(preUsed, GP);
211 std::array<Arg, 2> fpScratch = getScratches(preUsed, FP);
212
213 // Also need to save all live registers. Don't need to worry about the result
214 // register.
215 if (originalResult.isReg())
216 regsToSave.clear(originalResult.reg());
217 Vector<StackSlot*> stackSlots;
218 regsToSave.forEach(
219 [&] (Reg reg) {
220 Tmp tmp(reg);
221 Arg arg(tmp);
222 Width width = conservativeWidth(arg.bank());
223 StackSlot* stackSlot =
224 code.addStackSlot(bytes(width), StackSlotKind::Spill);
225 pairs.append(ShufflePair(arg, Arg::stack(stackSlot), width));
226 stackSlots.append(stackSlot);
227 });
228
229 if (AirLowerAfterRegAllocInternal::verbose)
230 dataLog("Pre-call pairs for ", inst, ": ", listDump(pairs), "\n");
231
232 insertionSet.insertInsts(
233 instIndex, emitShuffle(code, pairs, gpScratch, fpScratch, inst.origin));
234
235 inst = buildCCall(code, inst.origin, destinations);
236 if (oldKind.effects)
237 inst.kind.effects = true;
238
239 // Now we need to emit code to restore registers.
240 pairs.shrink(0);
241 unsigned stackSlotIndex = 0;
242 regsToSave.forEach(
243 [&] (Reg reg) {
244 Tmp tmp(reg);
245 Arg arg(tmp);
246 Width width = conservativeWidth(arg.bank());
247 StackSlot* stackSlot = stackSlots[stackSlotIndex++];
248 pairs.append(ShufflePair(Arg::stack(stackSlot), arg, width));
249 });
250 if (result) {
251 ShufflePair pair(result, originalResult, widthForType(value->type()));
252 pairs.append(pair);
253 }
254
255 // For finding scratch registers, we need to account for the possibility that
256 // the result is dead.
257 if (originalResult.isReg())
258 liveRegs.set(originalResult.reg());
259
260 gpScratch = getScratches(liveRegs, GP);
261 fpScratch = getScratches(liveRegs, FP);
262
263 insertionSet.insertInsts(
264 instIndex + 1, emitShuffle(code, pairs, gpScratch, fpScratch, inst.origin));
265 break;
266 }
267
268 default:
269 break;
270 }
271 }
272
273 insertionSet.execute(block);
274
275 block->insts().removeAllMatching(
276 [&] (Inst& inst) -> bool {
277 return !inst;
278 });
279 }
280
281 if (AirLowerAfterRegAllocInternal::verbose)
282 dataLog("Code after lowerAfterRegAlloc:\n", code);
283}
284
285} } } // namespace JSC::B3::Air
286
287#endif // ENABLE(B3_JIT)
288
289