1/*
2 * Copyright (C) 2008-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#pragma once
27
28#if ENABLE(ASSEMBLER)
29
30#include "ExecutableAllocator.h"
31#include "JITCompilationEffort.h"
32#include "stdint.h"
33#include <string.h>
34#include <wtf/Assertions.h>
35#include <wtf/FastMalloc.h>
36#if CPU(ARM64E)
37#include <wtf/PtrTag.h>
38#endif
39#include <wtf/StdLibExtras.h>
40#include <wtf/UnalignedAccess.h>
41
42namespace JSC {
43
44 class LinkBuffer;
45
46 struct AssemblerLabel {
47 AssemblerLabel()
48 : m_offset(std::numeric_limits<uint32_t>::max())
49 {
50 }
51
52 explicit AssemblerLabel(uint32_t offset)
53 : m_offset(offset)
54 {
55 }
56
57 bool isSet() const { return (m_offset != std::numeric_limits<uint32_t>::max()); }
58
59 AssemblerLabel labelAtOffset(int offset) const
60 {
61 return AssemblerLabel(m_offset + offset);
62 }
63
64 bool operator==(const AssemblerLabel& other) const { return m_offset == other.m_offset; }
65
66 uint32_t m_offset;
67 };
68
69 class AssemblerData {
70 WTF_MAKE_NONCOPYABLE(AssemblerData);
71 static constexpr size_t InlineCapacity = 128;
72 public:
73 AssemblerData()
74 : m_buffer(m_inlineBuffer)
75 , m_capacity(InlineCapacity)
76 {
77 }
78
79 AssemblerData(size_t initialCapacity)
80 {
81 if (initialCapacity <= InlineCapacity) {
82 m_capacity = InlineCapacity;
83 m_buffer = m_inlineBuffer;
84 } else {
85 m_capacity = initialCapacity;
86 m_buffer = static_cast<char*>(fastMalloc(m_capacity));
87 }
88 }
89
90 AssemblerData(AssemblerData&& other)
91 {
92 if (other.isInlineBuffer()) {
93 ASSERT(other.m_capacity == InlineCapacity);
94 memcpy(m_inlineBuffer, other.m_inlineBuffer, InlineCapacity);
95 m_buffer = m_inlineBuffer;
96 } else
97 m_buffer = other.m_buffer;
98 m_capacity = other.m_capacity;
99
100 other.m_buffer = nullptr;
101 other.m_capacity = 0;
102 }
103
104 AssemblerData& operator=(AssemblerData&& other)
105 {
106 if (m_buffer && !isInlineBuffer())
107 fastFree(m_buffer);
108
109 if (other.isInlineBuffer()) {
110 ASSERT(other.m_capacity == InlineCapacity);
111 memcpy(m_inlineBuffer, other.m_inlineBuffer, InlineCapacity);
112 m_buffer = m_inlineBuffer;
113 } else
114 m_buffer = other.m_buffer;
115 m_capacity = other.m_capacity;
116
117 other.m_buffer = nullptr;
118 other.m_capacity = 0;
119 return *this;
120 }
121
122 ~AssemblerData()
123 {
124 if (m_buffer && !isInlineBuffer())
125 fastFree(m_buffer);
126 }
127
128 char* buffer() const { return m_buffer; }
129
130 unsigned capacity() const { return m_capacity; }
131
132 void grow(unsigned extraCapacity = 0)
133 {
134 m_capacity = m_capacity + m_capacity / 2 + extraCapacity;
135 if (isInlineBuffer()) {
136 m_buffer = static_cast<char*>(fastMalloc(m_capacity));
137 memcpy(m_buffer, m_inlineBuffer, InlineCapacity);
138 } else
139 m_buffer = static_cast<char*>(fastRealloc(m_buffer, m_capacity));
140 }
141
142 private:
143 bool isInlineBuffer() const { return m_buffer == m_inlineBuffer; }
144 char* m_buffer;
145 char m_inlineBuffer[InlineCapacity];
146 unsigned m_capacity;
147 };
148
149#if CPU(ARM64E)
150 class ARM64EHash {
151 public:
152 ARM64EHash() = default;
153 ALWAYS_INLINE void update(uint32_t value)
154 {
155 uint64_t input = value ^ m_hash;
156 uint64_t a = static_cast<uint32_t>(tagInt(input, static_cast<PtrTag>(0)) >> 39);
157 uint64_t b = tagInt(input, static_cast<PtrTag>(0xb7e151628aed2a6a)) >> 23;
158 m_hash = a ^ b;
159 }
160 uint32_t finalHash() const
161 {
162 uint64_t hash = m_hash;
163 uint64_t a = static_cast<uint32_t>(tagInt(hash, static_cast<PtrTag>(0xbf7158809cf4f3c7)) >> 39);
164 uint64_t b = tagInt(hash, static_cast<PtrTag>(0x62e7160f38b4da56)) >> 23;
165 return static_cast<uint32_t>(a ^ b);
166 }
167 private:
168 uint32_t m_hash { 0 };
169 };
170#endif
171
172 class AssemblerBuffer {
173 public:
174 AssemblerBuffer()
175 : m_storage()
176 , m_index(0)
177 {
178 }
179
180 bool isAvailable(unsigned space)
181 {
182 return m_index + space <= m_storage.capacity();
183 }
184
185 void ensureSpace(unsigned space)
186 {
187 while (!isAvailable(space))
188 outOfLineGrow();
189 }
190
191 bool isAligned(int alignment) const
192 {
193 return !(m_index & (alignment - 1));
194 }
195
196#if !CPU(ARM64)
197 void putByteUnchecked(int8_t value) { putIntegralUnchecked(value); }
198 void putByte(int8_t value) { putIntegral(value); }
199 void putShortUnchecked(int16_t value) { putIntegralUnchecked(value); }
200 void putShort(int16_t value) { putIntegral(value); }
201 void putInt64Unchecked(int64_t value) { putIntegralUnchecked(value); }
202 void putInt64(int64_t value) { putIntegral(value); }
203#endif
204 void putIntUnchecked(int32_t value) { putIntegralUnchecked(value); }
205 void putInt(int32_t value) { putIntegral(value); }
206
207 size_t codeSize() const
208 {
209 return m_index;
210 }
211
212#if !CPU(ARM64)
213 void setCodeSize(size_t index)
214 {
215 // Warning: Only use this if you know exactly what you are doing.
216 // For example, say you want 40 bytes of nops, it's ok to grow
217 // and then fill 40 bytes of nops using bigger instructions.
218 m_index = index;
219 ASSERT(m_index <= m_storage.capacity());
220 }
221#endif
222
223 AssemblerLabel label() const
224 {
225 return AssemblerLabel(m_index);
226 }
227
228 unsigned debugOffset() { return m_index; }
229
230 AssemblerData&& releaseAssemblerData() { return WTFMove(m_storage); }
231
232 // LocalWriter is a trick to keep the storage buffer and the index
233 // in memory while issuing multiple Stores.
234 // It is created in a block scope and its attribute can stay live
235 // between writes.
236 //
237 // LocalWriter *CANNOT* be mixed with other types of access to AssemblerBuffer.
238 // AssemblerBuffer cannot be used until its LocalWriter goes out of scope.
239#if !CPU(ARM64) // If we ever need to use this on arm64e, we would need to make the checksum aware of this.
240 class LocalWriter {
241 public:
242 LocalWriter(AssemblerBuffer& buffer, unsigned requiredSpace)
243 : m_buffer(buffer)
244 {
245 buffer.ensureSpace(requiredSpace);
246 m_storageBuffer = buffer.m_storage.buffer();
247 m_index = buffer.m_index;
248#if !defined(NDEBUG)
249 m_initialIndex = m_index;
250 m_requiredSpace = requiredSpace;
251#endif
252 }
253
254 ~LocalWriter()
255 {
256 ASSERT(m_index - m_initialIndex <= m_requiredSpace);
257 ASSERT(m_buffer.m_index == m_initialIndex);
258 ASSERT(m_storageBuffer == m_buffer.m_storage.buffer());
259 m_buffer.m_index = m_index;
260 }
261
262 void putByteUnchecked(int8_t value) { putIntegralUnchecked(value); }
263 void putShortUnchecked(int16_t value) { putIntegralUnchecked(value); }
264 void putIntUnchecked(int32_t value) { putIntegralUnchecked(value); }
265 void putInt64Unchecked(int64_t value) { putIntegralUnchecked(value); }
266 private:
267 template<typename IntegralType>
268 void putIntegralUnchecked(IntegralType value)
269 {
270 ASSERT(m_index + sizeof(IntegralType) <= m_buffer.m_storage.capacity());
271 WTF::unalignedStore<IntegralType>(m_storageBuffer + m_index, value);
272 m_index += sizeof(IntegralType);
273 }
274 AssemblerBuffer& m_buffer;
275 char* m_storageBuffer;
276 unsigned m_index;
277#if !defined(NDEBUG)
278 unsigned m_initialIndex;
279 unsigned m_requiredSpace;
280#endif
281 };
282#endif // !CPU(ARM64)
283
284#if CPU(ARM64E)
285 ARM64EHash hash() const { return m_hash; }
286#endif
287
288#if !CPU(ARM64) // If we were to define this on arm64e, we'd need a way to update the hash as we write directly into the buffer.
289 void* data() const { return m_storage.buffer(); }
290#endif
291
292
293 protected:
294 template<typename IntegralType>
295 void putIntegral(IntegralType value)
296 {
297 unsigned nextIndex = m_index + sizeof(IntegralType);
298 if (UNLIKELY(nextIndex > m_storage.capacity()))
299 outOfLineGrow();
300 putIntegralUnchecked<IntegralType>(value);
301 }
302
303 template<typename IntegralType>
304 void putIntegralUnchecked(IntegralType value)
305 {
306#if CPU(ARM64)
307 static_assert(sizeof(value) == 4, "");
308#if CPU(ARM64E)
309 m_hash.update(value);
310#endif
311#endif
312 ASSERT(isAvailable(sizeof(IntegralType)));
313 WTF::unalignedStore<IntegralType>(m_storage.buffer() + m_index, value);
314 m_index += sizeof(IntegralType);
315 }
316
317 private:
318 void grow(int extraCapacity = 0)
319 {
320 m_storage.grow(extraCapacity);
321 }
322
323 NEVER_INLINE void outOfLineGrow()
324 {
325 m_storage.grow();
326 }
327
328#if !CPU(ARM64)
329 friend LocalWriter;
330#endif
331 friend LinkBuffer;
332
333 AssemblerData m_storage;
334 unsigned m_index;
335#if CPU(ARM64E)
336 ARM64EHash m_hash;
337#endif
338 };
339
340} // namespace JSC
341
342#endif // ENABLE(ASSEMBLER)
343