Handle GVL deletions

docs/config-app.md

@@ -457,6 +457,8 @@ If not defined in config all other Health Checkers would be disabled and endpoin
 - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value.
 - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow.
 - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list.
+- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`.
+- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours).
 - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2.
 - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour).
 - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional.

docs/metrics.md

@@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det
 - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version 
 - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version
 - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version
+- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection
 - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA)
 - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules
 - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag

src/main/java/org/prebid/server/metric/Metrics.java

@@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) {
         updatePrivacyTcfVendorListMetric(version, MetricName.fallback);
     }
 
+    public void updatePrivacyTcfVendorListLatestOkMetric() {
+        privacy().tcf().vendorListLatest().incCounter(MetricName.ok);
+    }
+
+    public void updatePrivacyTcfVendorListLatestErrorMetric() {
+        privacy().tcf().vendorListLatest().incCounter(MetricName.err);
+    }
+
     private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) {
         final TcfMetrics tcfMetrics = privacy().tcf();
         tcfMetrics.fromVersion(version).vendorList().incCounter(metricName);

src/main/java/org/prebid/server/metric/TcfMetrics.java

@@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics {
 
     private final TcfVersionMetrics tcfVersion1Metrics;
     private final TcfVersionMetrics tcfVersion2Metrics;
+    private final VendorListLatestMetrics vendorListLatestMetrics;
 
     TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
         super(
@@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics {
 
         tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1");
         tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2");
+        vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix));
     }
 
     TcfVersionMetrics fromVersion(int version) {
@@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) {
         };
     }
 
+    VendorListLatestMetrics vendorListLatest() {
+        return vendorListLatestMetrics;
+    }
+
     private static String createTcfPrefix(String prefix) {
         return prefix + ".tcf";
     }
@@ -87,4 +93,22 @@ private static Function<MetricName, String> nameCreator(String prefix) {
             return metricName -> "%s.%s".formatted(prefix, metricName);
         }
     }
+
+    static class VendorListLatestMetrics extends UpdatableMetrics {
+
+        VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+            super(
+                    metricRegistry,
+                    counterType,
+                    nameCreator(createLatestPrefix(prefix)));
+        }
+
+        private static String createLatestPrefix(String prefix) {
+            return prefix + ".vendorlist.latest";
+        }
+
+        private static Function<MetricName, String> nameCreator(String prefix) {
+            return metricName -> "%s.%s".formatted(prefix, metricName);
+        }
+    }
 }

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java

@@ -0,0 +1,122 @@
+package org.prebid.server.privacy.gdpr.vendorlist;
+
+import io.vertx.core.Promise;
+import io.vertx.core.Vertx;
+import org.apache.commons.collections4.CollectionUtils;
+import org.prebid.server.exception.PreBidException;
+import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.log.Logger;
+import org.prebid.server.log.LoggerFactory;
+import org.prebid.server.metric.Metrics;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList;
+import org.prebid.server.util.HttpUtil;
+import org.prebid.server.vertx.Initializable;
+import org.prebid.server.vertx.httpclient.HttpClient;
+import org.prebid.server.vertx.httpclient.model.HttpClientResponse;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LiveVendorListService implements Initializable {
+
+    private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class);
+
+    private final String liveGvlUrl;
+    private final long refreshPeriodMs;
+    private final int defaultTimeoutMs;
+    private final Vertx vertx;
+    private final HttpClient httpClient;
+    private final JacksonMapper mapper;
+    private final Metrics metrics;
+    private final Clock clock;
+
+    private volatile Set<Integer> deletedVendorIds = Set.of();
+
+    public LiveVendorListService(String liveGvlUrl,
+                                 long refreshPeriodMs,
+                                 int defaultTimeoutMs,
+                                 Vertx vertx,
+                                 HttpClient httpClient,
+                                 JacksonMapper mapper,
+                                 Metrics metrics,
+                                 Clock clock) {
+
+        this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl));
+        this.refreshPeriodMs = refreshPeriodMs;
+        this.defaultTimeoutMs = defaultTimeoutMs;
+        this.vertx = Objects.requireNonNull(vertx);
+        this.httpClient = Objects.requireNonNull(httpClient);
+        this.mapper = Objects.requireNonNull(mapper);
+        this.metrics = Objects.requireNonNull(metrics);
+        this.clock = Objects.requireNonNull(clock);
+    }
+
+    public boolean isDeleted(Integer id) {
+        final Set<Integer> ids = deletedVendorIds;
+        return !ids.isEmpty() && ids.contains(id);
+    }
+
+    @Override
+    public void initialize(Promise<Void> initializePromise) {
+        vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh());
+
+        initializePromise.tryComplete();
+    }
+
+    void refresh() {
+        httpClient.get(liveGvlUrl, defaultTimeoutMs)
+                .map(this::processResponse)
+                .map(this::extractDeletedVendorIds)
+                .map(this::updateDeletedVendorIds)
+                .otherwise(this::handleError);
+    }
+
+    private VendorList processResponse(HttpClientResponse response) {
+        final int statusCode = response.getStatusCode();
+        if (statusCode != 200) {
+            throw new PreBidException("HTTP status code " + statusCode);
+        }
+
+        final String body = response.getBody();
+        try {
+            return mapper.mapper().readValue(body, VendorList.class);
+        } catch (IOException e) {
+            throw new PreBidException("Cannot parse live vendor list: " + body, e);
+        }
+    }
+
+    Set<Integer> extractDeletedVendorIds(VendorList vendorList) {
+        final Instant now = clock.instant();
+        return vendorList.getVendors().values().stream()
+                .filter(vendor -> isDeletedAt(vendor, now))
+                .map(Vendor::getId)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+    private static boolean isDeletedAt(Vendor vendor, Instant now) {
+        final Instant deletedDate = vendor.getDeletedDate();
+        return deletedDate != null && deletedDate.isBefore(now);
+    }
+
+    private Void updateDeletedVendorIds(Set<Integer> ids) {
+        if (CollectionUtils.isEmpty(ids)) {
+            throw new PreBidException("Live GVL response has no deleted vendors");
+        }
+
+        deletedVendorIds = ids;
+        metrics.updatePrivacyTcfVendorListLatestOkMetric();
+        return null;
+    }
+
+    private Void handleError(Throwable exception) {
+        logger.warn("Error occurred while fetching live GVL", exception);
+        metrics.updatePrivacyTcfVendorListLatestErrorMetric();
+        return null;
+    }
+}

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java

@@ -6,23 +6,37 @@
 
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 public class VersionedVendorListService {
 
     private final VendorListService vendorListServiceV2;
     private final VendorListService vendorListServiceV3;
+    private final LiveVendorListService liveVendorListService;
+
+    public VersionedVendorListService(VendorListService vendorListServiceV2,
+                                      VendorListService vendorListServiceV3,
+                                      LiveVendorListService liveVendorListService) {
 
-    public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3) {
         this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2);
         this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3);
+        this.liveVendorListService = Objects.requireNonNull(liveVendorListService);
     }
 
     public Future<Map<Integer, Vendor>> forConsent(TCString consent) {
         final int tcfPolicyVersion = consent.getTcfPolicyVersion();
         final int vendorListVersion = consent.getVendorListVersion();
 
-        return tcfPolicyVersion < 4
+        final Future<Map<Integer, Vendor>> vendorListFuture = tcfPolicyVersion < 4
                 ? vendorListServiceV2.forVersion(vendorListVersion)
                 : vendorListServiceV3.forVersion(vendorListVersion);
+
+        return vendorListFuture.map(this::filterDeletedVendors);
+    }
+
+    private Map<Integer, Vendor> filterDeletedVendors(Map<Integer, Vendor> vendors) {
+        return vendors.entrySet().stream()
+                .filter(entry -> !liveVendorListService.isDeleted(entry.getKey()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 }

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java

@@ -6,6 +6,7 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import java.time.Instant;
 import java.util.EnumSet;
 
 @AllArgsConstructor
@@ -16,6 +17,9 @@ public class Vendor {
 
     Integer id;
 
+    @JsonProperty("deletedDate")
+    Instant deletedDate;
+
     EnumSet<PurposeCode> purposes;
 
     @JsonProperty("legIntPurposes")

src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java

@@ -37,6 +37,7 @@
 import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.typestrategies.PurposeTwoBasicEnforcePurposeStrategy;
 import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesOneStrategy;
 import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy;
+import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService;
 import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler;
 import org.prebid.server.privacy.gdpr.vendorlist.VendorListService;
 import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService;
@@ -135,11 +136,35 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() {
         return new VendorListServiceConfigurationProperties();
     }
 
+    @Bean
+    LiveVendorListService liveVendorListService(
+            @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl,
+            @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs,
+            @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs,
+            Vertx vertx,
+            HttpClient httpClient,
+            JacksonMapper mapper,
+            Metrics metrics,
+            Clock clock) {
+
+        return new LiveVendorListService(
+                liveGvlUrl,
+                refreshPeriodMs,
+                defaultTimeoutMs,
+                vertx,
+                httpClient,
+                mapper,
+                metrics,
+                clock);
+    }
+
     @Bean
     VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2,
-                                                          VendorListService vendorListServiceV3) {
+                                                          VendorListService vendorListServiceV3,
+                                                          LiveVendorListService liveVendorListService) {
 
-        return new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3);
+        return new VersionedVendorListService(
+                vendorListServiceV2, vendorListServiceV3, liveVendorListService);
     }
 
     @Bean

src/main/resources/application.yaml

@@ -207,6 +207,8 @@ gdpr:
   eea-countries: at,bg,be,cy,cz,dk,ee,fi,fr,de,gr,hu,ie,it,lv,lt,lu,mt,nl,pl,pt,ro,sk,si,es,se,gb,is,no,li,ai,aw,pt,bm,aq,io,vg,ic,ky,fk,re,mw,gp,gf,yt,pf,tf,gl,pt,ms,an,bq,cw,sx,nc,pn,sh,pm,gs,tc,uk,wf
   vendorlist:
     default-timeout-ms: 2000
+    live-gvl-refresh-period-ms: 86400000
+    live-gvl-url: https://vendor-list.consensu.org/v3/vendor-list.json
     v2:
       http-endpoint-template: https://vendor-list.consensu.org/v2/archives/vendor-list-v{VERSION}.json
       refresh-missing-list-period-ms: 3600000

src/main/resources/metrics-config/prometheus-labels.yaml

@@ -85,6 +85,10 @@ mappers:
     labels:
       tcf: ${0}
       status: ${1}
+  - match: privacy.tcf.vendorlist.latest.*
+    name: privacy.tcf.vendorlist.latest
+    labels:
+      status: ${0}
   - match: privacy.tcf.*.*
     name: privacy.tcf.${1}
     labels:

src/test/java/org/prebid/server/metric/MetricsTest.java

@@ -1004,6 +1004,24 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() {
         assertThat(metricRegistry.counter("privacy.tcf.v1.vendorlist.fallback").getCount()).isEqualTo(1);
     }
 
+    @Test
+    public void updatePrivacyTcfVendorListLatestOkMetricShouldIncrementMetric() {
+        // when
+        metrics.updatePrivacyTcfVendorListLatestOkMetric();
+
+        // then
+        assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.ok").getCount()).isOne();
+    }
+
+    @Test
+    public void updatePrivacyTcfVendorListLatestErrorMetricShouldIncrementMetric() {
+        // when
+        metrics.updatePrivacyTcfVendorListLatestErrorMetric();
+
+        // then
+        assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.err").getCount()).isOne();
+    }
+
     @Test
     public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() {
         // given