1/*
2 * Copyright (C) 2009-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#pragma once
27
28#if ENABLE(ASSEMBLER)
29
30#define DUMP_LINK_STATISTICS 0
31#define DUMP_CODE 0
32
33#define GLOBAL_THUNK_ID reinterpret_cast<void*>(static_cast<intptr_t>(-1))
34#define REGEXP_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-2))
35#define CSS_CODE_ID reinterpret_cast<void*>(static_cast<intptr_t>(-3))
36
37#include "JITCompilationEffort.h"
38#include "MacroAssembler.h"
39#include "MacroAssemblerCodeRef.h"
40#include <wtf/DataLog.h>
41#include <wtf/FastMalloc.h>
42#include <wtf/Noncopyable.h>
43
44namespace JSC {
45
46namespace Wasm {
47enum class CompilationMode : uint8_t;
48}
49
50class CodeBlock;
51
52// LinkBuffer:
53//
54// This class assists in linking code generated by the macro assembler, once code generation
55// has been completed, and the code has been copied to is final location in memory. At this
56// time pointers to labels within the code may be resolved, and relative offsets to external
57// addresses may be fixed.
58//
59// Specifically:
60// * Jump objects may be linked to external targets,
61// * The address of Jump objects may taken, such that it can later be relinked.
62// * The return address of a Call may be acquired.
63// * The address of a Label pointing into the code may be resolved.
64// * The value referenced by a DataLabel may be set.
65//
66class LinkBuffer {
67 WTF_MAKE_NONCOPYABLE(LinkBuffer); WTF_MAKE_FAST_ALLOCATED;
68
69 template<PtrTag tag> using CodePtr = MacroAssemblerCodePtr<tag>;
70 template<PtrTag tag> using CodeRef = MacroAssemblerCodeRef<tag>;
71 typedef MacroAssembler::Label Label;
72 typedef MacroAssembler::Jump Jump;
73 typedef MacroAssembler::PatchableJump PatchableJump;
74 typedef MacroAssembler::JumpList JumpList;
75 typedef MacroAssembler::Call Call;
76 typedef MacroAssembler::DataLabelCompact DataLabelCompact;
77 typedef MacroAssembler::DataLabel32 DataLabel32;
78 typedef MacroAssembler::DataLabelPtr DataLabelPtr;
79 typedef MacroAssembler::ConvertibleLoadLabel ConvertibleLoadLabel;
80#if ENABLE(BRANCH_COMPACTION)
81 typedef MacroAssembler::LinkRecord LinkRecord;
82 typedef MacroAssembler::JumpLinkType JumpLinkType;
83#endif
84
85public:
86 LinkBuffer(MacroAssembler& macroAssembler, void* ownerUID, JITCompilationEffort effort = JITCompilationMustSucceed)
87 : m_size(0)
88 , m_didAllocate(false)
89#ifndef NDEBUG
90 , m_completed(false)
91#endif
92 {
93 linkCode(macroAssembler, ownerUID, effort);
94 }
95
96 template<PtrTag tag>
97 LinkBuffer(MacroAssembler& macroAssembler, MacroAssemblerCodePtr<tag> code, size_t size, JITCompilationEffort effort = JITCompilationMustSucceed, bool shouldPerformBranchCompaction = true)
98 : m_size(size)
99 , m_didAllocate(false)
100#ifndef NDEBUG
101 , m_completed(false)
102#endif
103 , m_code(code.template retagged<LinkBufferPtrTag>())
104 {
105#if ENABLE(BRANCH_COMPACTION)
106 m_shouldPerformBranchCompaction = shouldPerformBranchCompaction;
107#else
108 UNUSED_PARAM(shouldPerformBranchCompaction);
109#endif
110 linkCode(macroAssembler, 0, effort);
111 }
112
113 ~LinkBuffer()
114 {
115 }
116
117 bool didFailToAllocate() const
118 {
119 return !m_didAllocate;
120 }
121
122 bool isValid() const
123 {
124 return !didFailToAllocate();
125 }
126
127 // These methods are used to link or set values at code generation time.
128
129 template<PtrTag tag, typename Func, typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func>::type>::value>>
130 void link(Call call, Func funcName)
131 {
132 FunctionPtr<tag> function(funcName);
133 link(call, function);
134 }
135
136 template<PtrTag tag>
137 void link(Call call, FunctionPtr<tag> function)
138 {
139 ASSERT(call.isFlagSet(Call::Linkable));
140 call.m_label = applyOffset(call.m_label);
141 MacroAssembler::linkCall(code(), call, function);
142 }
143
144 template<PtrTag tag>
145 void link(Call call, CodeLocationLabel<tag> label)
146 {
147 link(call, FunctionPtr<tag>(label));
148 }
149
150 template<PtrTag tag>
151 void link(Jump jump, CodeLocationLabel<tag> label)
152 {
153 jump.m_label = applyOffset(jump.m_label);
154 MacroAssembler::linkJump(code(), jump, label);
155 }
156
157 template<PtrTag tag>
158 void link(const JumpList& list, CodeLocationLabel<tag> label)
159 {
160 for (const Jump& jump : list.jumps())
161 link(jump, label);
162 }
163
164 void patch(DataLabelPtr label, void* value)
165 {
166 AssemblerLabel target = applyOffset(label.m_label);
167 MacroAssembler::linkPointer(code(), target, value);
168 }
169
170 template<PtrTag tag>
171 void patch(DataLabelPtr label, CodeLocationLabel<tag> value)
172 {
173 AssemblerLabel target = applyOffset(label.m_label);
174 MacroAssembler::linkPointer(code(), target, value);
175 }
176
177 // These methods are used to obtain handles to allow the code to be relinked / repatched later.
178
179 template<PtrTag tag>
180 CodeLocationLabel<tag> entrypoint()
181 {
182 return CodeLocationLabel<tag>(tagCodePtr<tag>(code()));
183 }
184
185 template<PtrTag tag>
186 CodeLocationCall<tag> locationOf(Call call)
187 {
188 ASSERT(call.isFlagSet(Call::Linkable));
189 ASSERT(!call.isFlagSet(Call::Near));
190 return CodeLocationCall<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(call.m_label)));
191 }
192
193 template<PtrTag tag>
194 CodeLocationNearCall<tag> locationOfNearCall(Call call)
195 {
196 ASSERT(call.isFlagSet(Call::Linkable));
197 ASSERT(call.isFlagSet(Call::Near));
198 return CodeLocationNearCall<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(call.m_label)),
199 call.isFlagSet(Call::Tail) ? NearCallMode::Tail : NearCallMode::Regular);
200 }
201
202 template<PtrTag tag>
203 CodeLocationLabel<tag> locationOf(PatchableJump jump)
204 {
205 return CodeLocationLabel<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(jump.m_jump.m_label)));
206 }
207
208 template<PtrTag tag>
209 CodeLocationLabel<tag> locationOf(Label label)
210 {
211 return CodeLocationLabel<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(label.m_label)));
212 }
213
214 template<PtrTag tag>
215 CodeLocationDataLabelPtr<tag> locationOf(DataLabelPtr label)
216 {
217 return CodeLocationDataLabelPtr<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(label.m_label)));
218 }
219
220 template<PtrTag tag>
221 CodeLocationDataLabel32<tag> locationOf(DataLabel32 label)
222 {
223 return CodeLocationDataLabel32<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(label.m_label)));
224 }
225
226 template<PtrTag tag>
227 CodeLocationDataLabelCompact<tag> locationOf(DataLabelCompact label)
228 {
229 return CodeLocationDataLabelCompact<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(label.m_label)));
230 }
231
232 template<PtrTag tag>
233 CodeLocationConvertibleLoad<tag> locationOf(ConvertibleLoadLabel label)
234 {
235 return CodeLocationConvertibleLoad<tag>(MacroAssembler::getLinkerAddress<tag>(code(), applyOffset(label.m_label)));
236 }
237
238 // This method obtains the return address of the call, given as an offset from
239 // the start of the code.
240 unsigned returnAddressOffset(Call call)
241 {
242 call.m_label = applyOffset(call.m_label);
243 return MacroAssembler::getLinkerCallReturnOffset(call);
244 }
245
246 uint32_t offsetOf(Label label)
247 {
248 return applyOffset(label.m_label).m_offset;
249 }
250
251 unsigned offsetOf(PatchableJump jump)
252 {
253 return applyOffset(jump.m_jump.m_label).m_offset;
254 }
255
256 // Upon completion of all patching 'FINALIZE_CODE()' should be called once to
257 // complete generation of the code. Alternatively, call
258 // finalizeCodeWithoutDisassembly() directly if you have your own way of
259 // displaying disassembly.
260
261 template<PtrTag tag>
262 CodeRef<tag> finalizeCodeWithoutDisassembly()
263 {
264 return finalizeCodeWithoutDisassemblyImpl().template retagged<tag>();
265 }
266
267 template<PtrTag tag, typename... Args>
268 CodeRef<tag> finalizeCodeWithDisassembly(bool dumpDisassembly, const char* format, Args... args)
269 {
270 ALLOW_NONLITERAL_FORMAT_BEGIN
271 IGNORE_WARNINGS_BEGIN("format-security")
272 return finalizeCodeWithDisassemblyImpl(dumpDisassembly, format, args...).template retagged<tag>();
273 IGNORE_WARNINGS_END
274 ALLOW_NONLITERAL_FORMAT_END
275 }
276
277 template<PtrTag tag>
278 CodePtr<tag> trampolineAt(Label label)
279 {
280 return CodePtr<tag>(MacroAssembler::AssemblerType_T::getRelocatedAddress(code(), applyOffset(label.m_label)));
281 }
282
283 void* debugAddress()
284 {
285 return m_code.dataLocation();
286 }
287
288 size_t size() const { return m_size; }
289
290 bool wasAlreadyDisassembled() const { return m_alreadyDisassembled; }
291 void didAlreadyDisassemble() { m_alreadyDisassembled = true; }
292
293private:
294 JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithoutDisassemblyImpl();
295 JS_EXPORT_PRIVATE CodeRef<LinkBufferPtrTag> finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, const char* format, ...) WTF_ATTRIBUTE_PRINTF(3, 4);
296
297#if ENABLE(BRANCH_COMPACTION)
298 int executableOffsetFor(int location)
299 {
300 // Returning 0 in this case works because at location <
301 // sizeof(int32_t), no compaction could have happened before this
302 // point as the assembler could not have placed a branch instruction
303 // within this space that required compaction.
304 if (location < static_cast<int>(sizeof(int32_t)))
305 return 0;
306 return bitwise_cast<int32_t*>(m_assemblerStorage.buffer())[location / sizeof(int32_t) - 1];
307 }
308#endif
309
310 template <typename T> T applyOffset(T src)
311 {
312#if ENABLE(BRANCH_COMPACTION)
313 src.m_offset -= executableOffsetFor(src.m_offset);
314#endif
315 return src;
316 }
317
318 // Keep this private! - the underlying code should only be obtained externally via finalizeCode().
319 void* code()
320 {
321 return m_code.dataLocation();
322 }
323
324 void allocate(MacroAssembler&, void* ownerUID, JITCompilationEffort);
325
326 JS_EXPORT_PRIVATE void linkCode(MacroAssembler&, void* ownerUID, JITCompilationEffort);
327#if ENABLE(BRANCH_COMPACTION)
328 template <typename InstructionType>
329 void copyCompactAndLinkCode(MacroAssembler&, void* ownerUID, JITCompilationEffort);
330#endif
331
332 void performFinalization();
333
334#if DUMP_LINK_STATISTICS
335 static void dumpLinkStatistics(void* code, size_t initialSize, size_t finalSize);
336#endif
337
338#if DUMP_CODE
339 static void dumpCode(void* code, size_t);
340#endif
341
342 RefPtr<ExecutableMemoryHandle> m_executableMemory;
343 size_t m_size;
344#if ENABLE(BRANCH_COMPACTION)
345 AssemblerData m_assemblerStorage;
346 bool m_shouldPerformBranchCompaction { true };
347#endif
348 bool m_didAllocate;
349#ifndef NDEBUG
350 bool m_completed;
351#endif
352 bool m_alreadyDisassembled { false };
353 MacroAssemblerCodePtr<LinkBufferPtrTag> m_code;
354 Vector<RefPtr<SharedTask<void(LinkBuffer&)>>> m_linkTasks;
355};
356
357#if OS(LINUX)
358#define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
359 (UNLIKELY((condition)) \
360 ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>(true, __VA_ARGS__) \
361 : (UNLIKELY(JSC::Options::logJITCodeForPerf()) \
362 ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>(false, __VA_ARGS__) \
363 : (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>()))
364#else
365#define FINALIZE_CODE_IF(condition, linkBufferReference, resultPtrTag, ...) \
366 (UNLIKELY((condition)) \
367 ? (linkBufferReference).finalizeCodeWithDisassembly<resultPtrTag>(true, __VA_ARGS__) \
368 : (linkBufferReference).finalizeCodeWithoutDisassembly<resultPtrTag>())
369#endif
370
371bool shouldDumpDisassemblyFor(CodeBlock*);
372
373#define FINALIZE_CODE_FOR(codeBlock, linkBufferReference, resultPtrTag, ...) \
374 FINALIZE_CODE_IF((shouldDumpDisassemblyFor(codeBlock) || Options::asyncDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)
375
376// Use this to finalize code, like so:
377//
378// CodeRef code = FINALIZE_CODE(linkBuffer, tag, "my super thingy number %d", number);
379//
380// Which, in disassembly mode, will print:
381//
382// Generated JIT code for my super thingy number 42:
383// Code at [0x123456, 0x234567]:
384// 0x123456: mov $0, 0
385// 0x12345a: ret
386//
387// ... and so on.
388//
389// Note that the format string and print arguments are only evaluated when dumpDisassembly
390// is true, so you can hide expensive disassembly-only computations inside there.
391
392#define FINALIZE_CODE(linkBufferReference, resultPtrTag, ...) \
393 FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)
394
395#define FINALIZE_DFG_CODE(linkBufferReference, resultPtrTag, ...) \
396 FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpDFGDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)
397
398#define FINALIZE_REGEXP_CODE(linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading) \
399 FINALIZE_CODE_IF(JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpRegExpDisassembly(), linkBufferReference, resultPtrTag, dataLogFArgumentsForHeading)
400
401bool shouldDumpDisassemblyFor(Wasm::CompilationMode);
402
403#define FINALIZE_WASM_CODE(linkBufferReference, resultPtrTag, ...) \
404 FINALIZE_CODE_IF((JSC::Options::asyncDisassembly() || JSC::Options::dumpDisassembly() || Options::dumpWasmDisassembly()), linkBufferReference, resultPtrTag, __VA_ARGS__)
405
406#define FINALIZE_WASM_CODE_FOR_MODE(mode, linkBufferReference, resultPtrTag, ...) \
407 FINALIZE_CODE_IF(shouldDumpDisassemblyFor(mode), linkBufferReference, resultPtrTag, __VA_ARGS__)
408
409
410
411} // namespace JSC
412
413#endif // ENABLE(ASSEMBLER)
414