diff --git a/docs/config-app.md b/docs/config-app.md index a661f5a74a2..5c5fbaa95e2 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -457,6 +457,8 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value. - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list. +- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. +- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2. - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour). - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional. diff --git a/docs/metrics.md b/docs/metrics.md index c07e0660598..e5844413032 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version +- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA) - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 517fbb36bd5..e533ba53636 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) { updatePrivacyTcfVendorListMetric(version, MetricName.fallback); } + public void updatePrivacyTcfVendorListLatestOkMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.ok); + } + + public void updatePrivacyTcfVendorListLatestErrorMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.err); + } + private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) { final TcfMetrics tcfMetrics = privacy().tcf(); tcfMetrics.fromVersion(version).vendorList().incCounter(metricName); diff --git a/src/main/java/org/prebid/server/metric/TcfMetrics.java b/src/main/java/org/prebid/server/metric/TcfMetrics.java index 9fd5a811562..ba5d8ed86f3 100644 --- a/src/main/java/org/prebid/server/metric/TcfMetrics.java +++ b/src/main/java/org/prebid/server/metric/TcfMetrics.java @@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics { private final TcfVersionMetrics tcfVersion1Metrics; private final TcfVersionMetrics tcfVersion2Metrics; + private final VendorListLatestMetrics vendorListLatestMetrics; TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( @@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics { tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1"); tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2"); + vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); } TcfVersionMetrics fromVersion(int version) { @@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) { }; } + VendorListLatestMetrics vendorListLatest() { + return vendorListLatestMetrics; + } + private static String createTcfPrefix(String prefix) { return prefix + ".tcf"; } @@ -87,4 +93,22 @@ private static Function nameCreator(String prefix) { return metricName -> "%s.%s".formatted(prefix, metricName); } } + + static class VendorListLatestMetrics extends UpdatableMetrics { + + VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + super( + metricRegistry, + counterType, + nameCreator(createLatestPrefix(prefix))); + } + + private static String createLatestPrefix(String prefix) { + return prefix + ".vendorlist.latest"; + } + + private static Function nameCreator(String prefix) { + return metricName -> "%s.%s".formatted(prefix, metricName); + } + } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java new file mode 100644 index 00000000000..dc3c8b62ed2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -0,0 +1,122 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import org.apache.commons.collections4.CollectionUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.util.HttpUtil; +import org.prebid.server.vertx.Initializable; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class LiveVendorListService implements Initializable { + + private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class); + + private final String liveGvlUrl; + private final long refreshPeriodMs; + private final int defaultTimeoutMs; + private final Vertx vertx; + private final HttpClient httpClient; + private final JacksonMapper mapper; + private final Metrics metrics; + private final Clock clock; + + private volatile Set deletedVendorIds = Set.of(); + + public LiveVendorListService(String liveGvlUrl, + long refreshPeriodMs, + int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + JacksonMapper mapper, + Metrics metrics, + Clock clock) { + + this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl)); + this.refreshPeriodMs = refreshPeriodMs; + this.defaultTimeoutMs = defaultTimeoutMs; + this.vertx = Objects.requireNonNull(vertx); + this.httpClient = Objects.requireNonNull(httpClient); + this.mapper = Objects.requireNonNull(mapper); + this.metrics = Objects.requireNonNull(metrics); + this.clock = Objects.requireNonNull(clock); + } + + public boolean isDeleted(Integer id) { + final Set ids = deletedVendorIds; + return !ids.isEmpty() && ids.contains(id); + } + + @Override + public void initialize(Promise initializePromise) { + vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh()); + + initializePromise.tryComplete(); + } + + void refresh() { + httpClient.get(liveGvlUrl, defaultTimeoutMs) + .map(this::processResponse) + .map(this::extractDeletedVendorIds) + .map(this::updateDeletedVendorIds) + .otherwise(this::handleError); + } + + private VendorList processResponse(HttpClientResponse response) { + final int statusCode = response.getStatusCode(); + if (statusCode != 200) { + throw new PreBidException("HTTP status code " + statusCode); + } + + final String body = response.getBody(); + try { + return mapper.mapper().readValue(body, VendorList.class); + } catch (IOException e) { + throw new PreBidException("Cannot parse live vendor list: " + body, e); + } + } + + Set extractDeletedVendorIds(VendorList vendorList) { + final Instant now = clock.instant(); + return vendorList.getVendors().values().stream() + .filter(vendor -> isDeletedAt(vendor, now)) + .map(Vendor::getId) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + private static boolean isDeletedAt(Vendor vendor, Instant now) { + final Instant deletedDate = vendor.getDeletedDate(); + return deletedDate != null && deletedDate.isBefore(now); + } + + private Void updateDeletedVendorIds(Set ids) { + if (CollectionUtils.isEmpty(ids)) { + throw new PreBidException("Live GVL response has no deleted vendors"); + } + + deletedVendorIds = ids; + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + return null; + } + + private Void handleError(Throwable exception) { + logger.warn("Error occurred while fetching live GVL", exception); + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + return null; + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 5e261d9b6b4..dada968604d 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -6,23 +6,37 @@ import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class VersionedVendorListService { private final VendorListService vendorListServiceV2; private final VendorListService vendorListServiceV3; + private final LiveVendorListService liveVendorListService; + + public VersionedVendorListService(VendorListService vendorListServiceV2, + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService) { - public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3) { this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2); this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3); + this.liveVendorListService = Objects.requireNonNull(liveVendorListService); } public Future> forConsent(TCString consent) { final int tcfPolicyVersion = consent.getTcfPolicyVersion(); final int vendorListVersion = consent.getVendorListVersion(); - return tcfPolicyVersion < 4 + final Future> vendorListFuture = tcfPolicyVersion < 4 ? vendorListServiceV2.forVersion(vendorListVersion) : vendorListServiceV3.forVersion(vendorListVersion); + + return vendorListFuture.map(this::filterDeletedVendors); + } + + private Map filterDeletedVendors(Map vendors) { + return vendors.entrySet().stream() + .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java index 6bb2be9dddb..336a5d4cdae 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.time.Instant; import java.util.EnumSet; @AllArgsConstructor @@ -16,6 +17,9 @@ public class Vendor { Integer id; + @JsonProperty("deletedDate") + Instant deletedDate; + EnumSet purposes; @JsonProperty("legIntPurposes") diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index b8175828c3b..70d5b672cab 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -37,6 +37,7 @@ import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.typestrategies.PurposeTwoBasicEnforcePurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesOneStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler; import org.prebid.server.privacy.gdpr.vendorlist.VendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; @@ -135,11 +136,35 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { return new VendorListServiceConfigurationProperties(); } + @Bean + LiveVendorListService liveVendorListService( + @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl, + @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs, + @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + JacksonMapper mapper, + Metrics metrics, + Clock clock) { + + return new LiveVendorListService( + liveGvlUrl, + refreshPeriodMs, + defaultTimeoutMs, + vertx, + httpClient, + mapper, + metrics, + clock); + } + @Bean VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2, - VendorListService vendorListServiceV3) { + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService) { - return new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + return new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService); } @Bean diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 55822047954..28fc07808f8 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -207,6 +207,8 @@ gdpr: eea-countries: at,bg,be,cy,cz,dk,ee,fi,fr,de,gr,hu,ie,it,lv,lt,lu,mt,nl,pl,pt,ro,sk,si,es,se,gb,is,no,li,ai,aw,pt,bm,aq,io,vg,ic,ky,fk,re,mw,gp,gf,yt,pf,tf,gl,pt,ms,an,bq,cw,sx,nc,pn,sh,pm,gs,tc,uk,wf vendorlist: default-timeout-ms: 2000 + live-gvl-refresh-period-ms: 86400000 + live-gvl-url: https://vendor-list.consensu.org/v3/vendor-list.json v2: http-endpoint-template: https://vendor-list.consensu.org/v2/archives/vendor-list-v{VERSION}.json refresh-missing-list-period-ms: 3600000 diff --git a/src/main/resources/metrics-config/prometheus-labels.yaml b/src/main/resources/metrics-config/prometheus-labels.yaml index b40139d6a8a..6941b1d7cb0 100644 --- a/src/main/resources/metrics-config/prometheus-labels.yaml +++ b/src/main/resources/metrics-config/prometheus-labels.yaml @@ -85,6 +85,10 @@ mappers: labels: tcf: ${0} status: ${1} + - match: privacy.tcf.vendorlist.latest.* + name: privacy.tcf.vendorlist.latest + labels: + status: ${0} - match: privacy.tcf.*.* name: privacy.tcf.${1} labels: diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 8a56e279d30..1df98cedd68 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1004,6 +1004,24 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() { assertThat(metricRegistry.counter("privacy.tcf.v1.vendorlist.fallback").getCount()).isEqualTo(1); } + @Test + public void updatePrivacyTcfVendorListLatestOkMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.ok").getCount()).isOne(); + } + + @Test + public void updatePrivacyTcfVendorListLatestErrorMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.err").getCount()).isOne(); + } + @Test public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java new file mode 100644 index 00000000000..f2f05c8b88b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -0,0 +1,229 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +public class LiveVendorListServiceTest extends VertxTest { + + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private static final String LIVE_GVL_URL = "https://example.com"; + + @Mock + private Vertx vertx; + @Mock + private HttpClient httpClient; + @Mock + private Metrics metrics; + + private LiveVendorListService target; + + @BeforeEach + public void setUp() { + target = new LiveVendorListService( + LIVE_GVL_URL, + 0, + 1000, + vertx, + httpClient, + jacksonMapper, + metrics, + Clock.fixed(NOW, ZoneOffset.UTC)); + } + + @Test + public void isDeletedShouldReturnFalseWhenFetchNeverSucceeded() { + // when and then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(null)).isFalse(); + } + + @Test + public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throws JsonProcessingException { + // given + final String responseBody = givenLiveGvlJson(Map.of(42, "2024-01-01T00:00:00Z")); + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(200, null, responseBody))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(42)).isTrue(); + assertThat(target.isDeleted(99)).isFalse(); + } + + @Test + public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() { + // given + final VendorList vendorList = givenVendorList(Map.of( + 1, givenVendor(1, "2024-01-01T00:00:00Z"), + 2, givenVendor(2, null), + 3, givenVendor(3, "2025-01-01T00:00:00Z"), + 4, givenVendor(4, "2024-06-01T12:00:00Z"))); + + // when + final var deletedIds = target.extractDeletedVendorIds(vendorList); + + // then + assertThat(deletedIds).containsExactly(1); + } + + @Test + public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws JsonProcessingException { + // given + givenHttpClientReturnsResponse(200, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z"))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() throws JsonProcessingException { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(2, "2024-02-01T00:00:00Z"))))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(2)).isTrue(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnHttpFailure() { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnNonOkStatus() { + // given + givenHttpClientReturnsResponse(503, "{}"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidJson() { + // given + givenHttpClientReturnsResponse(200, "invalid-json"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricWhenNoDeletedVendorsInResponse() throws JsonProcessingException { + // given + givenHttpClientReturnsResponse(200, givenLiveGvlJson(emptyMap())); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + } + + @Test + public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), + Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + private void givenHttpClientReturnsResponse(int statusCode, String response) { + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); + } + + private String givenLiveGvlJson(Map vendorIdToDeletedDate) throws JsonProcessingException { + final Map vendors = new HashMap<>(); + for (Map.Entry entry : vendorIdToDeletedDate.entrySet()) { + vendors.put(entry.getKey(), Map.of("id", entry.getKey(), "deletedDate", entry.getValue())); + } + return mapper.writeValueAsString(Map.of("vendors", vendors)); + } + + private static Vendor givenVendor(int id, String deletedDate) { + return Vendor.builder() + .id(id) + .deletedDate(deletedDate != null ? Instant.parse(deletedDate) : null) + .build(); + } + + private static VendorList givenVendorList(Map vendors) { + return VendorList.of(1, Date.from(Instant.parse("2020-08-20T16:05:24Z")), vendors); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 8437698b9f0..3ec04ec5602 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -2,29 +2,40 @@ import com.iabtcf.decoder.TCString; import com.iabtcf.encoder.TCStringEncoder; +import io.vertx.core.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; +import static org.prebid.server.assertion.FutureAssertion.assertThat; @ExtendWith(MockitoExtension.class) public class VersionedVendorListServiceTest { - private VersionedVendorListService versionedVendorListService; + private VersionedVendorListService target; @Mock private VendorListService vendorListServiceV2; @Mock private VendorListService vendorListServiceV3; + @Mock + private LiveVendorListService liveVendorListService; @BeforeEach public void setUp() { - versionedVendorListService = new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + target = new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService); } @Test @@ -36,9 +47,10 @@ public void versionedVendorListServiceShouldTreatTcfPolicyLessThanFourAsVendorLi .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV2).forVersion(12); @@ -53,11 +65,38 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV3.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV3).forVersion(12); } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.isDeleted(1)).willReturn(true); + given(liveVendorListService.isDeleted(52)).willReturn(false); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result).containsOnlyKeys(52); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } }