Skip to content
10 changes: 5 additions & 5 deletions .benchmark/REPORT.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ Time from `preloadAllPaywalls()` until every paywall reaches `PaywallLoadingStat

| Tier | Paywalls | Mean | Baseline | Δ | Limit | Cold mean | Warm mean | Median | StdDev | CV | Samples | Status |
|------|----------|------|----------|---|-------|-----------|-----------|--------|--------|----|---------|--------|
| LOW | 10 | 5.95s | 6.34s | -6.2% | +30% | 8.08s | — | 5.95s | 3.30s | 40.8% | 10 (10×1) | 🟢 improved |
| MID | 10 | 7.08s | 8.78s | -19.3% | +15% | 7.20s | — | 7.08s | 1.62s | 22.5% | 10 (10×1) | 🟢 improved |
| HIGH | 10 | 6.39s | 6.72s | -4.9% | +20% | 7.87s | — | 6.39s | 3.16s | 40.2% | 10 (10×1) | 🟢 improved |
| LOW | 10 | 6.13s | 5.95s | +3.2% | +30% | 6.46s | — | 6.13s | 877ms | 13.6% | 10 (10×1) | ✅ OK |
| MID | 10 | 4.98s | 7.08s | -29.7% | +15% | 5.42s | — | 4.98s | 1.00s | 18.5% | 10 (10×1) | 🟢 improved |
| HIGH | 10 | 7.28s | 6.39s | +13.9% | +20% | 8.13s | — | 7.28s | 3.13s | 38.4% | 10 (10×1) | ✅ OK |

### Devices

| CI tier | Model | API | CPU cores | SDK-classified tier |
|---------|-------|-----|-----------|---------------------|
| LOW | sdk_gphone64_x86_64 | 34 | 2 | mid |
| MID | sdk_gphone64_x86_64 | 34 | 3 | low |
| HIGH | sdk_gphone64_x86_64 | 34 | 4 | low |
| MID | sdk_gphone64_x86_64 | 34 | 3 | mid |
| HIGH | sdk_gphone64_x86_64 | 34 | 4 | mid |

- A tier fails when its mean exceeds the baseline by more than the delta limit.
- The SDK-classified tier column shows what `DeviceClassifier` thinks each emulator is — if it drifts from the CI tier label, adjust the emulator cores/RAM in the workflow matrix.
Expand Down
276 changes: 138 additions & 138 deletions .benchmark/results/preload-benchmark-high.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,233 +4,233 @@
"cpuCores": 4,
"fingerprint": "google/sdk_gphone64_x86_64/emu64xa:14/UE1A.230829.036.A4/12096271:user/release-keys",
"model": "sdk_gphone64_x86_64",
"sdkClassifiedTier": "low",
"sdkClassifiedTier": "mid",
"sdkInt": 34
},
"iterationsPerRun": 1,
"runCount": 10,
"samples": [
{
"allReadyMs": 15685,
"allReadyMs": 12970,
"cold": true,
"configureMs": 2613,
"configureMs": 3327,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 15685,
"data-v3-0ed9-2024-04-04": 14353,
"free-trial-v3-d8d7-2024-04-04": 15124,
"inimai---open-app-copy-c50c-2024-11-28": 11154,
"localized-paywall-e901-2024-04-04": 14506,
"price-tester-1b8e-2024-04-04": 14767,
"products-v3-60c5-2024-04-04": 9961,
"superwall-template-9f9f-2024-05-21": 10738,
"url-paywall-v3-3810-2024-04-04": 12089,
"video-v3-72b4-2024-04-04": 11782
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 12970,
"data-v3-0ed9-2024-04-04": 9393,
"free-trial-v3-d8d7-2024-04-04": 10372,
"inimai---open-app-copy-c50c-2024-11-28": 7239,
"localized-paywall-e901-2024-04-04": 8828,
"price-tester-1b8e-2024-04-04": 9550,
"products-v3-60c5-2024-04-04": 9753,
"superwall-template-9f9f-2024-05-21": 7752,
"url-paywall-v3-3810-2024-04-04": 8267,
"video-v3-72b4-2024-04-04": 10007
},
"preloadCompleteEventMs": 1489,
"preloadCompleteEventMs": 1523,
"run": 1
},
{
"allReadyMs": 7912,
"allReadyMs": 13067,
"cold": true,
"configureMs": 1537,
"configureMs": 3630,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 6791,
"data-v3-0ed9-2024-04-04": 7403,
"free-trial-v3-d8d7-2024-04-04": 6232,
"inimai---open-app-copy-c50c-2024-11-28": 4704,
"localized-paywall-e901-2024-04-04": 7912,
"price-tester-1b8e-2024-04-04": 5365,
"products-v3-60c5-2024-04-04": 5979,
"superwall-template-9f9f-2024-05-21": 5060,
"url-paywall-v3-3810-2024-04-04": 6385,
"video-v3-72b4-2024-04-04": 4603
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 11493,
"data-v3-0ed9-2024-04-04": 13067,
"free-trial-v3-d8d7-2024-04-04": 10260,
"inimai---open-app-copy-c50c-2024-11-28": 9327,
"localized-paywall-e901-2024-04-04": 9683,
"price-tester-1b8e-2024-04-04": 10462,
"products-v3-60c5-2024-04-04": 10922,
"superwall-template-9f9f-2024-05-21": 12203,
"url-paywall-v3-3810-2024-04-04": 8969,
"video-v3-72b4-2024-04-04": 10719
},
"preloadCompleteEventMs": 1153,
"preloadCompleteEventMs": 2354,
"run": 2
},
{
"allReadyMs": 8799,
"allReadyMs": 8170,
"cold": true,
"configureMs": 1896,
"configureMs": 1435,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 8799,
"data-v3-0ed9-2024-04-04": 5728,
"free-trial-v3-d8d7-2024-04-04": 4860,
"inimai---open-app-copy-c50c-2024-11-28": 5063,
"localized-paywall-e901-2024-04-04": 6695,
"price-tester-1b8e-2024-04-04": 5317,
"products-v3-60c5-2024-04-04": 4657,
"superwall-template-9f9f-2024-05-21": 6542,
"url-paywall-v3-3810-2024-04-04": 4297,
"video-v3-72b4-2024-04-04": 6948
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 7917,
"data-v3-0ed9-2024-04-04": 7511,
"free-trial-v3-d8d7-2024-04-04": 6842,
"inimai---open-app-copy-c50c-2024-11-28": 4942,
"localized-paywall-e901-2024-04-04": 6945,
"price-tester-1b8e-2024-04-04": 5767,
"products-v3-60c5-2024-04-04": 8170,
"superwall-template-9f9f-2024-05-21": 5455,
"url-paywall-v3-3810-2024-04-04": 6079,
"video-v3-72b4-2024-04-04": 6639
},
"preloadCompleteEventMs": 1171,
"preloadCompleteEventMs": 1087,
"run": 3
},
{
"allReadyMs": 10317,
"allReadyMs": 10477,
"cold": true,
"configureMs": 2148,
"configureMs": 2896,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 9503,
"data-v3-0ed9-2024-04-04": 10317,
"free-trial-v3-d8d7-2024-04-04": 8227,
"inimai---open-app-copy-c50c-2024-11-28": 7349,
"localized-paywall-e901-2024-04-04": 8329,
"price-tester-1b8e-2024-04-04": 6686,
"products-v3-60c5-2024-04-04": 8840,
"superwall-template-9f9f-2024-05-21": 7145,
"url-paywall-v3-3810-2024-04-04": 8635,
"video-v3-72b4-2024-04-04": 9707
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 8595,
"data-v3-0ed9-2024-04-04": 10018,
"free-trial-v3-d8d7-2024-04-04": 10477,
"inimai---open-app-copy-c50c-2024-11-28": 6602,
"localized-paywall-e901-2024-04-04": 7826,
"price-tester-1b8e-2024-04-04": 7674,
"products-v3-60c5-2024-04-04": 8850,
"superwall-template-9f9f-2024-05-21": 6912,
"url-paywall-v3-3810-2024-04-04": 6496,
"video-v3-72b4-2024-04-04": 8029
},
"preloadCompleteEventMs": 2077,
"preloadCompleteEventMs": 3942,
"run": 4
},
{
"allReadyMs": 6060,
"allReadyMs": 8688,
"cold": true,
"configureMs": 2361,
"configureMs": 2191,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5299,
"data-v3-0ed9-2024-04-04": 6060,
"free-trial-v3-d8d7-2024-04-04": 4536,
"inimai---open-app-copy-c50c-2024-11-28": 4080,
"localized-paywall-e901-2024-04-04": 4637,
"price-tester-1b8e-2024-04-04": 5705,
"products-v3-60c5-2024-04-04": 3978,
"superwall-template-9f9f-2024-05-21": 4282,
"url-paywall-v3-3810-2024-04-04": 3826,
"video-v3-72b4-2024-04-04": 4943
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 6317,
"data-v3-0ed9-2024-04-04": 8688,
"free-trial-v3-d8d7-2024-04-04": 5454,
"inimai---open-app-copy-c50c-2024-11-28": 4787,
"localized-paywall-e901-2024-04-04": 5555,
"price-tester-1b8e-2024-04-04": 5301,
"products-v3-60c5-2024-04-04": 6732,
"superwall-template-9f9f-2024-05-21": 5098,
"url-paywall-v3-3810-2024-04-04": 4528,
"video-v3-72b4-2024-04-04": 6476
},
"preloadCompleteEventMs": 1405,
"preloadCompleteEventMs": 1358,
"run": 5
},
{
"allReadyMs": 5798,
"allReadyMs": 6386,
"cold": true,
"configureMs": 1946,
"configureMs": 1964,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 4681,
"data-v3-0ed9-2024-04-04": 5798,
"free-trial-v3-d8d7-2024-04-04": 5341,
"inimai---open-app-copy-c50c-2024-11-28": 3867,
"localized-paywall-e901-2024-04-04": 5443,
"price-tester-1b8e-2024-04-04": 4835,
"products-v3-60c5-2024-04-04": 4172,
"superwall-template-9f9f-2024-05-21": 5037,
"url-paywall-v3-3810-2024-04-04": 4070,
"video-v3-72b4-2024-04-04": 4323
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 6133,
"data-v3-0ed9-2024-04-04": 6386,
"free-trial-v3-d8d7-2024-04-04": 5778,
"inimai---open-app-copy-c50c-2024-11-28": 4659,
"localized-paywall-e901-2024-04-04": 5066,
"price-tester-1b8e-2024-04-04": 4557,
"products-v3-60c5-2024-04-04": 5269,
"superwall-template-9f9f-2024-05-21": 4914,
"url-paywall-v3-3810-2024-04-04": 5371,
"video-v3-72b4-2024-04-04": 5879
},
"preloadCompleteEventMs": 1076,
"preloadCompleteEventMs": 1425,
"run": 6
},
{
"allReadyMs": 5850,
"allReadyMs": 4893,
"cold": true,
"configureMs": 1926,
"configureMs": 1597,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5850,
"data-v3-0ed9-2024-04-04": 5291,
"free-trial-v3-d8d7-2024-04-04": 4734,
"inimai---open-app-copy-c50c-2024-11-28": 3611,
"localized-paywall-e901-2024-04-04": 4068,
"price-tester-1b8e-2024-04-04": 3509,
"products-v3-60c5-2024-04-04": 4379,
"superwall-template-9f9f-2024-05-21": 3864,
"url-paywall-v3-3810-2024-04-04": 4937,
"video-v3-72b4-2024-04-04": 4835
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 4893,
"data-v3-0ed9-2024-04-04": 4691,
"free-trial-v3-d8d7-2024-04-04": 3871,
"inimai---open-app-copy-c50c-2024-11-28": 3466,
"localized-paywall-e901-2024-04-04": 3921,
"price-tester-1b8e-2024-04-04": 4387,
"products-v3-60c5-2024-04-04": 4184,
"superwall-template-9f9f-2024-05-21": 3668,
"url-paywall-v3-3810-2024-04-04": 3351,
"video-v3-72b4-2024-04-04": 4337
},
"preloadCompleteEventMs": 676,
"preloadCompleteEventMs": 846,
"run": 7
},
{
"allReadyMs": 6342,
"allReadyMs": 4954,
"cold": true,
"configureMs": 1691,
"configureMs": 1189,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 6342,
"data-v3-0ed9-2024-04-04": 4510,
"free-trial-v3-d8d7-2024-04-04": 5481,
"inimai---open-app-copy-c50c-2024-11-28": 4054,
"localized-paywall-e901-2024-04-04": 5278,
"price-tester-1b8e-2024-04-04": 3649,
"products-v3-60c5-2024-04-04": 5024,
"superwall-template-9f9f-2024-05-21": 3902,
"url-paywall-v3-3810-2024-04-04": 4872,
"video-v3-72b4-2024-04-04": 5177
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 4599,
"data-v3-0ed9-2024-04-04": 4292,
"free-trial-v3-d8d7-2024-04-04": 3785,
"inimai---open-app-copy-c50c-2024-11-28": 3480,
"localized-paywall-e901-2024-04-04": 3582,
"price-tester-1b8e-2024-04-04": 3328,
"products-v3-60c5-2024-04-04": 3937,
"superwall-template-9f9f-2024-05-21": 4853,
"url-paywall-v3-3810-2024-04-04": 4038,
"video-v3-72b4-2024-04-04": 4954
},
"preloadCompleteEventMs": 1032,
"preloadCompleteEventMs": 881,
"run": 8
},
{
"allReadyMs": 6436,
"allReadyMs": 5487,
"cold": true,
"configureMs": 1749,
"configureMs": 1487,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5472,
"data-v3-0ed9-2024-04-04": 4205,
"free-trial-v3-d8d7-2024-04-04": 4965,
"inimai---open-app-copy-c50c-2024-11-28": 6436,
"localized-paywall-e901-2024-04-04": 5117,
"price-tester-1b8e-2024-04-04": 5219,
"products-v3-60c5-2024-04-04": 4813,
"superwall-template-9f9f-2024-05-21": 4509,
"url-paywall-v3-3810-2024-04-04": 3646,
"video-v3-72b4-2024-04-04": 3901
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5487,
"data-v3-0ed9-2024-04-04": 4675,
"free-trial-v3-d8d7-2024-04-04": 4320,
"inimai---open-app-copy-c50c-2024-11-28": 3912,
"localized-paywall-e901-2024-04-04": 4421,
"price-tester-1b8e-2024-04-04": 4014,
"products-v3-60c5-2024-04-04": 3708,
"superwall-template-9f9f-2024-05-21": 3505,
"url-paywall-v3-3810-2024-04-04": 3861,
"video-v3-72b4-2024-04-04": 4828
},
"preloadCompleteEventMs": 978,
"preloadCompleteEventMs": 865,
"run": 9
},
{
"allReadyMs": 5456,
"allReadyMs": 6209,
"cold": true,
"configureMs": 1853,
"configureMs": 1698,
"index": 0,
"paywallCount": 10,
"perPaywallReadyMs": {
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5355,
"data-v3-0ed9-2024-04-04": 4646,
"free-trial-v3-d8d7-2024-04-04": 4190,
"inimai---open-app-copy-c50c-2024-11-28": 3478,
"localized-paywall-e901-2024-04-04": 5001,
"price-tester-1b8e-2024-04-04": 3529,
"products-v3-60c5-2024-04-04": 4342,
"superwall-template-9f9f-2024-05-21": 3783,
"url-paywall-v3-3810-2024-04-04": 4037,
"video-v3-72b4-2024-04-04": 5456
"based-on-teleprompter-bigvu-template-2b72-2024-04-04": 5701,
"data-v3-0ed9-2024-04-04": 6057,
"free-trial-v3-d8d7-2024-04-04": 4785,
"inimai---open-app-copy-c50c-2024-11-28": 4175,
"localized-paywall-e901-2024-04-04": 4938,
"price-tester-1b8e-2024-04-04": 5243,
"products-v3-60c5-2024-04-04": 4379,
"superwall-template-9f9f-2024-05-21": 4022,
"url-paywall-v3-3810-2024-04-04": 5141,
"video-v3-72b4-2024-04-04": 6209
},
"preloadCompleteEventMs": 710,
"preloadCompleteEventMs": 655,
"run": 10
}
],
"schemaVersion": 1,
"stats": {
"coefficientOfVariationPct": 40.158560805785584,
"coldMeanMs": 7865.5,
"maxMs": 15685,
"meanMs": 7865.5,
"medianMs": 6389.0,
"minMs": 5456,
"coefficientOfVariationPct": 38.44018590825072,
"coldMeanMs": 8130.1,
"maxMs": 13067,
"meanMs": 8130.1,
"medianMs": 7278.0,
"minMs": 4893,
"sampleCount": 10,
"stdDevMs": 3158.6716001790655,
"stdDevMs": 3125.2255545266917,
"warmMeanMs": null
},
"tier": "HIGH"
Expand Down
Loading