1/*
2 * Copyright (C) 2017-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WebResourceLoadStatisticsTelemetry.h"
28
29#if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31#include "ResourceLoadStatisticsMemoryStore.h"
32#include "WebPageProxy.h"
33#include "WebProcessPool.h"
34#include "WebProcessProxy.h"
35#include <WebCore/DiagnosticLoggingKeys.h>
36#include <WebCore/ResourceLoadStatistics.h>
37#include <wtf/MainThread.h>
38#include <wtf/NeverDestroyed.h>
39#include <wtf/RunLoop.h>
40#include <wtf/text/StringConcatenateNumbers.h>
41
42namespace WebKit {
43using namespace WebCore;
44
45const unsigned minimumPrevalentResourcesForTelemetry = 3;
46const unsigned significantFiguresForLoggedValues = 3;
47static bool notifyPagesWhenTelemetryWasCaptured = false;
48
49struct PrevalentResourceTelemetry {
50 unsigned numberOfTimesDataRecordsRemoved;
51 bool hasHadUserInteraction;
52 unsigned daysSinceUserInteraction;
53 unsigned subframeUnderTopFrameOrigins;
54 unsigned subresourceUnderTopFrameOrigins;
55 unsigned subresourceUniqueRedirectsTo;
56 unsigned timesAccessedAsFirstPartyDueToUserInteraction;
57 unsigned timesAccessedAsFirstPartyDueToStorageAccessAPI;
58};
59
60static Vector<PrevalentResourceTelemetry> sortedPrevalentResourceTelemetry(const ResourceLoadStatisticsMemoryStore& store)
61{
62 ASSERT(!RunLoop::isMain());
63 Vector<PrevalentResourceTelemetry> sorted;
64 store.processStatistics([&sorted] (auto& statistic) {
65 if (!statistic.isPrevalentResource)
66 return;
67
68 unsigned daysSinceUserInteraction = statistic.mostRecentUserInteractionTime <= WallTime() ? 0 : std::floor((WallTime::now() - statistic.mostRecentUserInteractionTime) / 24_h);
69 sorted.append(PrevalentResourceTelemetry {
70 statistic.dataRecordsRemoved,
71 statistic.hadUserInteraction,
72 daysSinceUserInteraction,
73 statistic.subframeUnderTopFrameDomains.size(),
74 statistic.subresourceUnderTopFrameDomains.size(),
75 statistic.subresourceUniqueRedirectsTo.size(),
76 statistic.timesAccessedAsFirstPartyDueToUserInteraction,
77 statistic.timesAccessedAsFirstPartyDueToStorageAccessAPI
78 });
79 });
80
81 if (sorted.size() < minimumPrevalentResourcesForTelemetry)
82 return { };
83
84 std::sort(sorted.begin(), sorted.end(), [](const PrevalentResourceTelemetry& a, const PrevalentResourceTelemetry& b) {
85 return a.subframeUnderTopFrameOrigins + a.subresourceUnderTopFrameOrigins + a.subresourceUniqueRedirectsTo >
86 b.subframeUnderTopFrameOrigins + b.subresourceUnderTopFrameOrigins + b.subresourceUniqueRedirectsTo;
87 });
88
89 return sorted;
90}
91
92static unsigned numberOfResourcesWithUserInteraction(const Vector<PrevalentResourceTelemetry>& resources, size_t begin, size_t end)
93{
94 if (resources.isEmpty() || resources.size() < begin + 1 || resources.size() < end + 1)
95 return 0;
96
97 unsigned result = 0;
98 for (size_t i = begin; i < end; ++i) {
99 if (resources[i].hasHadUserInteraction)
100 ++result;
101 }
102
103 return result;
104}
105
106static unsigned median(const Vector<unsigned>& v)
107{
108 if (v.isEmpty())
109 return 0;
110 if (v.size() == 1)
111 return v[0];
112
113 auto size = v.size();
114 auto middle = size / 2;
115 if (size % 2)
116 return v[middle];
117 return (v[middle - 1] + v[middle]) / 2;
118}
119
120static unsigned median(const Vector<PrevalentResourceTelemetry>& v, unsigned begin, unsigned end, const WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)>& statisticGetter)
121{
122 if (v.isEmpty() || v.size() < begin + 1 || v.size() < end + 1)
123 return 0;
124
125 Vector<unsigned> part;
126 part.reserveInitialCapacity(end - begin + 1);
127 for (unsigned i = begin; i <= end; ++i)
128 part.uncheckedAppend(statisticGetter(v[i]));
129
130 return median(part);
131}
132
133static void submitTopList(unsigned numberOfResourcesFromTheTop, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, const WebResourceLoadStatisticsStore& store)
134{
135 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subframeUnderTopFrameOriginsGetter = [] (auto& t) {
136 return t.subframeUnderTopFrameOrigins;
137 };
138 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subresourceUnderTopFrameOriginsGetter = [] (auto& t) {
139 return t.subresourceUnderTopFrameOrigins;
140 };
141 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subresourceUniqueRedirectsToGetter = [] (auto& t) {
142 return t.subresourceUniqueRedirectsTo;
143 };
144 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> numberOfTimesDataRecordsRemovedGetter = [] (auto& t) {
145 return t.numberOfTimesDataRecordsRemoved;
146 };
147 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> numberOfTimesAccessedAsFirstPartyDueToUserInteractionGetter = [] (auto& t) {
148 return t.timesAccessedAsFirstPartyDueToUserInteraction;
149 };
150 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> numberOfTimesAccessedAsFirstPartyDueToStorageAccessAPIGetter = [] (auto& t) {
151 return t.timesAccessedAsFirstPartyDueToStorageAccessAPI;
152 };
153
154 unsigned topPrevalentResourcesWithUserInteraction = numberOfResourcesWithUserInteraction(sortedPrevalentResources, 0, numberOfResourcesFromTheTop - 1);
155 unsigned topSubframeUnderTopFrameOrigins = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subframeUnderTopFrameOriginsGetter);
156 unsigned topSubresourceUnderTopFrameOrigins = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subresourceUnderTopFrameOriginsGetter);
157 unsigned topSubresourceUniqueRedirectsTo = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subresourceUniqueRedirectsToGetter);
158 unsigned topNumberOfTimesDataRecordsRemoved = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, numberOfTimesDataRecordsRemovedGetter);
159 unsigned topNumberOfTimesAccessedAsFirstPartyDueToUserInteraction = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, numberOfTimesAccessedAsFirstPartyDueToUserInteractionGetter);
160 unsigned topNumberOfTimesAccessedAsFirstPartyDueToStorageAccessAPI = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, numberOfTimesAccessedAsFirstPartyDueToStorageAccessAPIGetter);
161
162 String descriptionPreamble = makeString("top", numberOfResourcesFromTheTop);
163
164 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "PrevalentResourcesWithUserInteraction",
165 topPrevalentResourcesWithUserInteraction, significantFiguresForLoggedValues, ShouldSample::No);
166 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubframeUnderTopFrameOrigins",
167 topSubframeUnderTopFrameOrigins, significantFiguresForLoggedValues, ShouldSample::No);
168 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubresourceUnderTopFrameOrigins",
169 topSubresourceUnderTopFrameOrigins, significantFiguresForLoggedValues, ShouldSample::No);
170 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubresourceUniqueRedirectsTo",
171 topSubresourceUniqueRedirectsTo, significantFiguresForLoggedValues, ShouldSample::No);
172 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "NumberOfTimesDataRecordsRemoved",
173 topNumberOfTimesDataRecordsRemoved, significantFiguresForLoggedValues, ShouldSample::No);
174 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "NumberOfTimesAccessedAsFirstPartyDueToUserInteraction",
175 topNumberOfTimesAccessedAsFirstPartyDueToUserInteraction, significantFiguresForLoggedValues, ShouldSample::No);
176 store.sendDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "NumberOfTimesAccessedAsFirstPartyDueToStorageAccessAPI",
177 topNumberOfTimesAccessedAsFirstPartyDueToStorageAccessAPI, significantFiguresForLoggedValues, ShouldSample::No);
178}
179
180static void submitTopLists(const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, const WebResourceLoadStatisticsStore& store)
181{
182 submitTopList(1, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, store);
183
184 if (sortedPrevalentResourcesWithoutUserInteraction.size() < 3)
185 return;
186 submitTopList(3, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, store);
187
188 if (sortedPrevalentResourcesWithoutUserInteraction.size() < 10)
189 return;
190 submitTopList(10, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, store);
191
192 if (sortedPrevalentResourcesWithoutUserInteraction.size() < 50)
193 return;
194 submitTopList(50, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, store);
195
196 if (sortedPrevalentResourcesWithoutUserInteraction.size() < 100)
197 return;
198 submitTopList(100, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, store);
199}
200
201// This function is for testing purposes.
202void static notifyPages(unsigned totalPrevalentResources, unsigned totalPrevalentResourcesWithUserInteraction, unsigned top3SubframeUnderTopFrameOrigins, const WebResourceLoadStatisticsStore& store)
203{
204 RunLoop::main().dispatch([totalPrevalentResources, totalPrevalentResourcesWithUserInteraction, top3SubframeUnderTopFrameOrigins, store = makeRef(store)] {
205 store->notifyPageStatisticsTelemetryFinished(totalPrevalentResources, totalPrevalentResourcesWithUserInteraction, top3SubframeUnderTopFrameOrigins);
206 });
207}
208
209// This function is for testing purposes.
210void static notifyPages(const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, unsigned totalNumberOfPrevalentResourcesWithUserInteraction, const WebResourceLoadStatisticsStore& store)
211{
212 WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subframeUnderTopFrameOriginsGetter = [] (const PrevalentResourceTelemetry& t) {
213 return t.subframeUnderTopFrameOrigins;
214 };
215
216 notifyPages(sortedPrevalentResources.size(), totalNumberOfPrevalentResourcesWithUserInteraction, median(sortedPrevalentResourcesWithoutUserInteraction, 0, 2, subframeUnderTopFrameOriginsGetter), store);
217}
218
219void WebResourceLoadStatisticsTelemetry::calculateAndSubmit(const ResourceLoadStatisticsMemoryStore& resourceLoadStatisticsStore)
220{
221 ASSERT(!RunLoop::isMain());
222
223 auto sortedPrevalentResources = sortedPrevalentResourceTelemetry(resourceLoadStatisticsStore);
224 if (notifyPagesWhenTelemetryWasCaptured && sortedPrevalentResources.isEmpty()) {
225 notifyPages(0, 0, 0, resourceLoadStatisticsStore.store());
226 return;
227 }
228
229 Vector<PrevalentResourceTelemetry> sortedPrevalentResourcesWithoutUserInteraction;
230 sortedPrevalentResourcesWithoutUserInteraction.reserveInitialCapacity(sortedPrevalentResources.size());
231 Vector<unsigned> prevalentResourcesDaysSinceUserInteraction;
232
233 for (auto& prevalentResource : sortedPrevalentResources) {
234 if (prevalentResource.hasHadUserInteraction)
235 prevalentResourcesDaysSinceUserInteraction.append(prevalentResource.daysSinceUserInteraction);
236 else
237 sortedPrevalentResourcesWithoutUserInteraction.uncheckedAppend(prevalentResource);
238 }
239
240 // Dispatch on the main thread to make sure the WebPageProxy we're using doesn't go away.
241 RunLoop::main().dispatch([sortedPrevalentResources = WTFMove(sortedPrevalentResources), sortedPrevalentResourcesWithoutUserInteraction = WTFMove(sortedPrevalentResourcesWithoutUserInteraction), prevalentResourcesDaysSinceUserInteraction = WTFMove(prevalentResourcesDaysSinceUserInteraction), resourceLoadStatisticsStore = makeWeakPtr(resourceLoadStatisticsStore)] () {
242 if (!resourceLoadStatisticsStore)
243 return;
244
245 auto webPageProxy = WebPageProxy::nonEphemeralWebPageProxy();
246 if (!webPageProxy) {
247 if (notifyPagesWhenTelemetryWasCaptured)
248 notifyPages(0, 0, 0, resourceLoadStatisticsStore->store());
249 return;
250 }
251
252 if (notifyPagesWhenTelemetryWasCaptured) {
253 notifyPages(sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, prevalentResourcesDaysSinceUserInteraction.size(), resourceLoadStatisticsStore->store());
254 // The notify pages function is for testing so we don't need to do an actual submission.
255 return;
256 }
257
258 webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), "totalNumberOfPrevalentResources"_s, sortedPrevalentResources.size(), significantFiguresForLoggedValues, ShouldSample::No);
259 webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), "totalNumberOfPrevalentResourcesWithUserInteraction"_s, prevalentResourcesDaysSinceUserInteraction.size(), significantFiguresForLoggedValues, ShouldSample::No);
260
261 if (prevalentResourcesDaysSinceUserInteraction.size() > 0)
262 webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), "topPrevalentResourceWithUserInteractionDaysSinceUserInteraction"_s, prevalentResourcesDaysSinceUserInteraction[0], significantFiguresForLoggedValues, ShouldSample::No);
263 if (prevalentResourcesDaysSinceUserInteraction.size() > 1)
264 webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), "medianPrevalentResourcesWithUserInteractionDaysSinceUserInteraction"_s, median(prevalentResourcesDaysSinceUserInteraction), significantFiguresForLoggedValues, ShouldSample::No);
265
266 submitTopLists(sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, resourceLoadStatisticsStore->store());
267 });
268}
269
270void WebResourceLoadStatisticsTelemetry::setNotifyPagesWhenTelemetryWasCaptured(bool always)
271{
272 notifyPagesWhenTelemetryWasCaptured = always;
273}
274
275}
276
277#endif
278