1/*
2 * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmMemory.h"
28#include "WasmInstance.h"
29
30#if ENABLE(WEBASSEMBLY)
31
32#include "Options.h"
33#include <wtf/DataLog.h>
34#include <wtf/Gigacage.h>
35#include <wtf/Lock.h>
36#include <wtf/OSAllocator.h>
37#include <wtf/PageBlock.h>
38#include <wtf/Platform.h>
39#include <wtf/PrintStream.h>
40#include <wtf/RAMSize.h>
41#include <wtf/Vector.h>
42
43#include <cstring>
44#include <mutex>
45
46namespace JSC { namespace Wasm {
47
48// FIXME: We could be smarter about memset / mmap / madvise. https://bugs.webkit.org/show_bug.cgi?id=170343
49// FIXME: Give up some of the cached fast memories if the GC determines it's easy to get them back, and they haven't been used in a while. https://bugs.webkit.org/show_bug.cgi?id=170773
50// FIXME: Limit slow memory size. https://bugs.webkit.org/show_bug.cgi?id=170825
51
52namespace {
53
54constexpr bool verbose = false;
55
56NEVER_INLINE NO_RETURN_DUE_TO_CRASH void webAssemblyCouldntGetFastMemory() { CRASH(); }
57
58struct MemoryResult {
59 enum Kind {
60 Success,
61 SuccessAndNotifyMemoryPressure,
62 SyncTryToReclaimMemory
63 };
64
65 static const char* toString(Kind kind)
66 {
67 switch (kind) {
68 case Success:
69 return "Success";
70 case SuccessAndNotifyMemoryPressure:
71 return "SuccessAndNotifyMemoryPressure";
72 case SyncTryToReclaimMemory:
73 return "SyncTryToReclaimMemory";
74 }
75 RELEASE_ASSERT_NOT_REACHED();
76 return nullptr;
77 }
78
79 MemoryResult() { }
80
81 MemoryResult(void* basePtr, Kind kind)
82 : basePtr(basePtr)
83 , kind(kind)
84 {
85 }
86
87 void dump(PrintStream& out) const
88 {
89 out.print("{basePtr = ", RawPointer(basePtr), ", kind = ", toString(kind), "}");
90 }
91
92 void* basePtr;
93 Kind kind;
94};
95
96class MemoryManager {
97 WTF_MAKE_FAST_ALLOCATED;
98 WTF_MAKE_NONCOPYABLE(MemoryManager);
99public:
100 MemoryManager()
101 : m_maxFastMemoryCount(Options::maxNumWebAssemblyFastMemories())
102 {
103 }
104
105 MemoryResult tryAllocateFastMemory()
106 {
107 MemoryResult result = [&] {
108 auto holder = holdLock(m_lock);
109 if (m_fastMemories.size() >= m_maxFastMemoryCount)
110 return MemoryResult(nullptr, MemoryResult::SyncTryToReclaimMemory);
111
112 void* result = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, Memory::fastMappedBytes());
113 if (!result)
114 return MemoryResult(nullptr, MemoryResult::SyncTryToReclaimMemory);
115
116 m_fastMemories.append(result);
117
118 return MemoryResult(
119 result,
120 m_fastMemories.size() >= m_maxFastMemoryCount / 2 ? MemoryResult::SuccessAndNotifyMemoryPressure : MemoryResult::Success);
121 }();
122
123 if (Options::logWebAssemblyMemory())
124 dataLog("Allocated virtual: ", result, "; state: ", *this, "\n");
125
126 return result;
127 }
128
129 void freeFastMemory(void* basePtr)
130 {
131 {
132 auto holder = holdLock(m_lock);
133 Gigacage::freeVirtualPages(Gigacage::Primitive, basePtr, Memory::fastMappedBytes());
134 m_fastMemories.removeFirst(basePtr);
135 }
136
137 if (Options::logWebAssemblyMemory())
138 dataLog("Freed virtual; state: ", *this, "\n");
139 }
140
141 bool isAddressInFastMemory(void* address)
142 {
143 // NOTE: This can be called from a signal handler, but only after we proved that we're in JIT code.
144 auto holder = holdLock(m_lock);
145 for (void* memory : m_fastMemories) {
146 char* start = static_cast<char*>(memory);
147 if (start <= address && address <= start + Memory::fastMappedBytes())
148 return true;
149 }
150 return false;
151 }
152
153 // We allow people to "commit" more wasm memory than there is on the system since most of the time
154 // people don't actually write to most of that memory. There is some chance that this gets us
155 // JetSammed but that's possible anyway.
156 inline size_t memoryLimit() const { return ramSize() * 3; }
157
158 // FIXME: Ideally, bmalloc would have this kind of mechanism. Then, we would just forward to that
159 // mechanism here.
160 MemoryResult::Kind tryAllocatePhysicalBytes(size_t bytes)
161 {
162 MemoryResult::Kind result = [&] {
163 auto holder = holdLock(m_lock);
164 if (m_physicalBytes + bytes > memoryLimit())
165 return MemoryResult::SyncTryToReclaimMemory;
166
167 m_physicalBytes += bytes;
168
169 if (m_physicalBytes >= memoryLimit() / 2)
170 return MemoryResult::SuccessAndNotifyMemoryPressure;
171
172 return MemoryResult::Success;
173 }();
174
175 if (Options::logWebAssemblyMemory())
176 dataLog("Allocated physical: ", bytes, ", ", MemoryResult::toString(result), "; state: ", *this, "\n");
177
178 return result;
179 }
180
181 void freePhysicalBytes(size_t bytes)
182 {
183 {
184 auto holder = holdLock(m_lock);
185 m_physicalBytes -= bytes;
186 }
187
188 if (Options::logWebAssemblyMemory())
189 dataLog("Freed physical: ", bytes, "; state: ", *this, "\n");
190 }
191
192 void dump(PrintStream& out) const
193 {
194 out.print("fast memories = ", m_fastMemories.size(), "/", m_maxFastMemoryCount, ", bytes = ", m_physicalBytes, "/", memoryLimit());
195 }
196
197private:
198 Lock m_lock;
199 unsigned m_maxFastMemoryCount { 0 };
200 Vector<void*> m_fastMemories;
201 size_t m_physicalBytes { 0 };
202};
203
204static MemoryManager& memoryManager()
205{
206 static std::once_flag onceFlag;
207 static MemoryManager* manager;
208 std::call_once(
209 onceFlag,
210 [] {
211 manager = new MemoryManager();
212 });
213 return *manager;
214}
215
216template<typename Func>
217bool tryAllocate(const Func& allocate, const WTF::Function<void(Memory::NotifyPressure)>& notifyMemoryPressure, const WTF::Function<void(Memory::SyncTryToReclaim)>& syncTryToReclaimMemory)
218{
219 unsigned numTries = 2;
220 bool done = false;
221 for (unsigned i = 0; i < numTries && !done; ++i) {
222 switch (allocate()) {
223 case MemoryResult::Success:
224 done = true;
225 break;
226 case MemoryResult::SuccessAndNotifyMemoryPressure:
227 if (notifyMemoryPressure)
228 notifyMemoryPressure(Memory::NotifyPressureTag);
229 done = true;
230 break;
231 case MemoryResult::SyncTryToReclaimMemory:
232 if (i + 1 == numTries)
233 break;
234 if (syncTryToReclaimMemory)
235 syncTryToReclaimMemory(Memory::SyncTryToReclaimTag);
236 break;
237 }
238 }
239 return done;
240}
241
242} // anonymous namespace
243
244Memory::Memory()
245{
246}
247
248Memory::Memory(PageCount initial, PageCount maximum, Function<void(NotifyPressure)>&& notifyMemoryPressure, Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
249 : m_initial(initial)
250 , m_maximum(maximum)
251 , m_notifyMemoryPressure(WTFMove(notifyMemoryPressure))
252 , m_syncTryToReclaimMemory(WTFMove(syncTryToReclaimMemory))
253 , m_growSuccessCallback(WTFMove(growSuccessCallback))
254{
255 ASSERT(!initial.bytes());
256 ASSERT(m_mode == MemoryMode::BoundsChecking);
257 dataLogLnIf(verbose, "Memory::Memory allocating ", *this);
258 ASSERT(!memory());
259}
260
261Memory::Memory(void* memory, PageCount initial, PageCount maximum, size_t mappedCapacity, MemoryMode mode, Function<void(NotifyPressure)>&& notifyMemoryPressure, Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
262 : m_memory(memory, initial.bytes())
263 , m_size(initial.bytes())
264 , m_initial(initial)
265 , m_maximum(maximum)
266 , m_mappedCapacity(mappedCapacity)
267 , m_mode(mode)
268 , m_notifyMemoryPressure(WTFMove(notifyMemoryPressure))
269 , m_syncTryToReclaimMemory(WTFMove(syncTryToReclaimMemory))
270 , m_growSuccessCallback(WTFMove(growSuccessCallback))
271{
272 dataLogLnIf(verbose, "Memory::Memory allocating ", *this);
273}
274
275Ref<Memory> Memory::create()
276{
277 return adoptRef(*new Memory());
278}
279
280RefPtr<Memory> Memory::tryCreate(PageCount initial, PageCount maximum, WTF::Function<void(NotifyPressure)>&& notifyMemoryPressure, WTF::Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
281{
282 ASSERT(initial);
283 RELEASE_ASSERT(!maximum || maximum >= initial); // This should be guaranteed by our caller.
284
285 const size_t initialBytes = initial.bytes();
286 const size_t maximumBytes = maximum ? maximum.bytes() : 0;
287
288 if (initialBytes > MAX_ARRAY_BUFFER_SIZE)
289 return nullptr; // Client will throw OOMError.
290
291 if (maximum && !maximumBytes) {
292 // User specified a zero maximum, initial size must also be zero.
293 RELEASE_ASSERT(!initialBytes);
294 return adoptRef(new Memory(initial, maximum, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
295 }
296
297 bool done = tryAllocate(
298 [&] () -> MemoryResult::Kind {
299 return memoryManager().tryAllocatePhysicalBytes(initialBytes);
300 }, notifyMemoryPressure, syncTryToReclaimMemory);
301 if (!done)
302 return nullptr;
303
304 char* fastMemory = nullptr;
305 if (Options::useWebAssemblyFastMemory()) {
306 tryAllocate(
307 [&] () -> MemoryResult::Kind {
308 auto result = memoryManager().tryAllocateFastMemory();
309 fastMemory = bitwise_cast<char*>(result.basePtr);
310 return result.kind;
311 }, notifyMemoryPressure, syncTryToReclaimMemory);
312 }
313
314 if (fastMemory) {
315
316 if (mprotect(fastMemory + initialBytes, Memory::fastMappedBytes() - initialBytes, PROT_NONE)) {
317 dataLog("mprotect failed: ", strerror(errno), "\n");
318 RELEASE_ASSERT_NOT_REACHED();
319 }
320
321 return adoptRef(new Memory(fastMemory, initial, maximum, Memory::fastMappedBytes(), MemoryMode::Signaling, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
322 }
323
324 if (UNLIKELY(Options::crashIfWebAssemblyCantFastMemory()))
325 webAssemblyCouldntGetFastMemory();
326
327 if (!initialBytes)
328 return adoptRef(new Memory(initial, maximum, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
329
330 void* slowMemory = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, initialBytes);
331 if (!slowMemory) {
332 memoryManager().freePhysicalBytes(initialBytes);
333 return nullptr;
334 }
335 return adoptRef(new Memory(slowMemory, initial, maximum, initialBytes, MemoryMode::BoundsChecking, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
336}
337
338Memory::~Memory()
339{
340 if (m_memory) {
341 memoryManager().freePhysicalBytes(m_size);
342 switch (m_mode) {
343 case MemoryMode::Signaling:
344 if (mprotect(memory(), Memory::fastMappedBytes(), PROT_READ | PROT_WRITE)) {
345 dataLog("mprotect failed: ", strerror(errno), "\n");
346 RELEASE_ASSERT_NOT_REACHED();
347 }
348 memoryManager().freeFastMemory(memory());
349 break;
350 case MemoryMode::BoundsChecking:
351 Gigacage::freeVirtualPages(Gigacage::Primitive, memory(), m_size);
352 break;
353 }
354 }
355}
356
357size_t Memory::fastMappedRedzoneBytes()
358{
359 return static_cast<size_t>(PageCount::pageSize) * Options::webAssemblyFastMemoryRedzonePages();
360}
361
362size_t Memory::fastMappedBytes()
363{
364 static_assert(sizeof(uint64_t) == sizeof(size_t), "We rely on allowing the maximum size of Memory we map to be 2^32 + redzone which is larger than fits in a 32-bit integer that we'd pass to mprotect if this didn't hold.");
365 return static_cast<size_t>(std::numeric_limits<uint32_t>::max()) + fastMappedRedzoneBytes();
366}
367
368bool Memory::addressIsInActiveFastMemory(void* address)
369{
370 return memoryManager().isAddressInFastMemory(address);
371}
372
373Expected<PageCount, Memory::GrowFailReason> Memory::grow(PageCount delta)
374{
375 const Wasm::PageCount oldPageCount = sizeInPages();
376
377 if (!delta.isValid())
378 return makeUnexpected(GrowFailReason::InvalidDelta);
379
380 const Wasm::PageCount newPageCount = oldPageCount + delta;
381 if (!newPageCount || !newPageCount.isValid())
382 return makeUnexpected(GrowFailReason::InvalidGrowSize);
383 if (newPageCount.bytes() > MAX_ARRAY_BUFFER_SIZE)
384 return makeUnexpected(GrowFailReason::OutOfMemory);
385
386 auto success = [&] () {
387 m_growSuccessCallback(GrowSuccessTag, oldPageCount, newPageCount);
388 // Update cache for instance
389 for (auto& instance : m_instances) {
390 if (instance.get() != nullptr)
391 instance.get()->updateCachedMemory();
392 }
393 return oldPageCount;
394 };
395
396 if (delta.pageCount() == 0)
397 return success();
398
399 dataLogLnIf(verbose, "Memory::grow(", delta, ") to ", newPageCount, " from ", *this);
400 RELEASE_ASSERT(newPageCount > PageCount::fromBytes(m_size));
401
402 if (maximum() && newPageCount > maximum())
403 return makeUnexpected(GrowFailReason::WouldExceedMaximum);
404
405 size_t desiredSize = newPageCount.bytes();
406 RELEASE_ASSERT(desiredSize <= MAX_ARRAY_BUFFER_SIZE);
407 RELEASE_ASSERT(desiredSize > m_size);
408 size_t extraBytes = desiredSize - m_size;
409 RELEASE_ASSERT(extraBytes);
410 bool allocationSuccess = tryAllocate(
411 [&] () -> MemoryResult::Kind {
412 return memoryManager().tryAllocatePhysicalBytes(extraBytes);
413 }, m_notifyMemoryPressure, m_syncTryToReclaimMemory);
414 if (!allocationSuccess)
415 return makeUnexpected(GrowFailReason::OutOfMemory);
416
417 switch (mode()) {
418 case MemoryMode::BoundsChecking: {
419 RELEASE_ASSERT(maximum().bytes() != 0);
420
421 void* newMemory = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, desiredSize);
422 if (!newMemory)
423 return makeUnexpected(GrowFailReason::OutOfMemory);
424
425 memcpy(newMemory, memory(), m_size);
426 if (m_memory)
427 Gigacage::freeVirtualPages(Gigacage::Primitive, memory(), m_size);
428 m_memory = CagedMemory(newMemory, desiredSize);
429 m_mappedCapacity = desiredSize;
430 m_size = desiredSize;
431 ASSERT(memory() == newMemory);
432 return success();
433 }
434 case MemoryMode::Signaling: {
435 RELEASE_ASSERT(memory());
436 // Signaling memory must have been pre-allocated virtually.
437 uint8_t* startAddress = static_cast<uint8_t*>(memory()) + m_size;
438
439 dataLogLnIf(verbose, "Marking WebAssembly memory's ", RawPointer(memory()), " as read+write in range [", RawPointer(startAddress), ", ", RawPointer(startAddress + extraBytes), ")");
440 if (mprotect(startAddress, extraBytes, PROT_READ | PROT_WRITE)) {
441 dataLog("mprotect failed: ", strerror(errno), "\n");
442 RELEASE_ASSERT_NOT_REACHED();
443 }
444 m_memory.recage(m_size, desiredSize);
445 m_size = desiredSize;
446 return success();
447 }
448 }
449
450 RELEASE_ASSERT_NOT_REACHED();
451 return oldPageCount;
452}
453
454void Memory::registerInstance(Instance* instance)
455{
456 size_t count = m_instances.size();
457 for (size_t index = 0; index < count; index++) {
458 if (m_instances.at(index).get() == nullptr) {
459 m_instances.at(index) = makeWeakPtr(*instance);
460 return;
461 }
462 }
463 m_instances.append(makeWeakPtr(*instance));
464}
465
466void Memory::dump(PrintStream& out) const
467{
468 out.print("Memory at ", RawPointer(memory()), ", size ", m_size, "B capacity ", m_mappedCapacity, "B, initial ", m_initial, " maximum ", m_maximum, " mode ", makeString(m_mode));
469}
470
471} // namespace JSC
472
473} // namespace Wasm
474
475#endif // ENABLE(WEBASSEMBLY)
476