From 99c7b9095a3545d32dfb6e20b5deccbdefc969c0 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 2 Jul 2026 15:02:01 -0500 Subject: [PATCH] Fix float truncation in min/max derived field scripts The min/max derived field painless function coerced every incoming Number (and the stored value) to a long before comparing, to avoid a ClassCastException on mixed Integer/Long values. That truncated floating point values: a Float-sourced min_value/max_value field compared truncated values (so 2.4 vs 2.9 looked equal) and stored the truncated result (2.9 became 2), silently corrupting derived indices. Replace the coerce-then-Collections.min/max approach with a loop that tracks the winning element and a numeric-aware comparison function: integral values are compared via Long.compare (preserving full precision beyond 2^53), comparisons involving a floating point value use Double.compare (preserving fractional parts, including when JSON produces mixed Integer/Double values for the same field), and non-numeric values keep using their natural compareTo. The original (un-coerced) winning value is what gets stored. Also add a Float-sourced min/max derived field to the test schema and acceptance coverage for fractional and mixed integral/fractional values. --- config/schema/artifacts/data_warehouse.yaml | 5 +- config/schema/artifacts/datastore_config.yaml | 96 +++++++----- config/schema/artifacts/json_schemas.yaml | 5 + .../artifacts/json_schemas_by_version/v1.yaml | 8 + config/schema/artifacts/runtime_metadata.yaml | 101 ++++++++++++- config/schema/artifacts/schema.graphql | 138 ++++++++++++++++++ .../artifacts_with_apollo/data_warehouse.yaml | 5 +- .../datastore_config.yaml | 96 +++++++----- .../artifacts_with_apollo/json_schemas.yaml | 5 + .../json_schemas_by_version/v1.yaml | 8 + .../runtime_metadata.yaml | 101 ++++++++++++- .../artifacts_with_apollo/schema.graphql | 138 ++++++++++++++++++ config/schema/widgets.rb | 5 + .../acceptance/derived_indexing_types_spec.rb | 38 +++++ .../derived_fields/min_or_max_value.rb | 59 +++++--- .../derived_fields/min_or_max_value.rbs | 4 + .../update_scripts/min_or_max_value_spec.rb | 3 + .../schema_definition/update_scripts_spec.rb | 1 + .../derived_fields/min_or_max_value_spec.rb | 68 +++++++++ .../spec_support/factories/widgets.rb | 1 + 20 files changed, 781 insertions(+), 104 deletions(-) create mode 100644 elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value_spec.rb diff --git a/config/schema/artifacts/data_warehouse.yaml b/config/schema/artifacts/data_warehouse.yaml index ade2602f7..67b7547fd 100644 --- a/config/schema/artifacts/data_warehouse.yaml +++ b/config/schema/artifacts/data_warehouse.yaml @@ -144,7 +144,9 @@ tables: widget_options STRUCT, colors ARRAY>, nested_fields STRUCT, oldest_widget_created_at TIMESTAMP, - max_weight_in_ng BIGINT + max_weight_in_ng BIGINT, + min_weight_in_grams FLOAT, + max_weight_in_grams FLOAT ) widget_records: table_schema: |- @@ -177,6 +179,7 @@ tables: named_inventor STRUCT, weight_in_ng_str BIGINT, weight_in_ng BIGINT, + weight_in_grams FLOAT, tags ARRAY, amounts ARRAY, fees ARRAY>, diff --git a/config/schema/artifacts/datastore_config.yaml b/config/schema/artifacts/datastore_config.yaml index 6cf3519ac..901b6888b 100644 --- a/config/schema/artifacts/datastore_config.yaml +++ b/config/schema/artifacts/datastore_config.yaml @@ -1322,6 +1322,10 @@ index_templates: format: strict_date_time max_weight_in_ng: type: long + min_weight_in_grams: + type: double + max_weight_in_grams: + type: double __counts: properties: widget_names2: @@ -1469,6 +1473,8 @@ index_templates: type: long weight_in_ng: type: long + weight_in_grams: + type: double tags: type: keyword amounts: @@ -1984,11 +1990,31 @@ indices: index.number_of_shards: 1 index.max_result_window: 10000 scripts: - update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c: + update_WidgetCurrency_from_Widget_514248e200403f9ab1766130a517439a: context: update script: lang: painless source: |- + // Compares two values of a min or max field, coercing numeric values as needed. + // + // Different `Number` implementations (e.g. `Integer` vs `Long`, or `Long` vs `Double`) cannot be + // compared with `compareTo` (it throws a `ClassCastException`), and JSON parsing can produce any + // of them for the same field (e.g. `2` parses as an `Integer` while `2.9` parses as a `Double`). + // Integral values are compared as `long`s to preserve full precision (a `double` cannot exactly + // represent all integral values above 2^53), while comparisons involving a floating point value + // are performed as `double`s to avoid truncating fractional parts. + int minOrMaxValue_compareValues(def value1, def value2) { + if (value1 instanceof Number && value2 instanceof Number) { + if (value1 instanceof Float || value1 instanceof Double || value2 instanceof Float || value2 instanceof Double) { + return Double.compare(((Number)value1).doubleValue(), ((Number)value2).doubleValue()); + } + + return Long.compare(((Number)value1).longValue(), ((Number)value2).longValue()); + } + + return value1.compareTo(value2); + } + // Idempotently inserts the given value in the `sortedList`, returning `true` if the list was updated. boolean appendOnlySet_idempotentlyInsertValue(def value, List sortedList) { // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): @@ -2074,58 +2100,46 @@ scripts: boolean maxValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { def currentFieldValue = parentObject[fieldName]; - // Normalize incoming list numerics to long to avoid Integer/Long class cast issues - List coercedValues = new ArrayList(); - for (def v : values) { - if (v != null) { - if (v instanceof Number) { - coercedValues.add(((Number)v).longValue()); - } else { - coercedValues.add(v); - } - } - } - def maxNewValue = coercedValues.isEmpty() ? null : Collections.max(coercedValues); - def coercedCurrentFieldValue = null; - if (currentFieldValue != null) { - coercedCurrentFieldValue = (currentFieldValue instanceof Number) ? ((Number)currentFieldValue).longValue() : currentFieldValue; + // Track the winning value itself (rather than a coerced copy of it) so that the exact value + // from the source event or existing document gets stored, with no coercion applied. + def maxValue = currentFieldValue; + boolean updated = false; + + for (def value : values) { + if (value != null && (maxValue == null || minOrMaxValue_compareValues(value, maxValue) > 0)) { + maxValue = value; + updated = true; + } } - if (maxNewValue != null && (coercedCurrentFieldValue == null || maxNewValue.compareTo(coercedCurrentFieldValue) > 0)) { - parentObject[fieldName] = maxNewValue; - return true; + if (updated) { + parentObject[fieldName] = maxValue; } - return false; + return updated; } boolean minValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { def currentFieldValue = parentObject[fieldName]; - // Normalize incoming list numerics to long to avoid Integer/Long class cast issues - List coercedValues = new ArrayList(); - for (def v : values) { - if (v != null) { - if (v instanceof Number) { - coercedValues.add(((Number)v).longValue()); - } else { - coercedValues.add(v); - } - } - } - def minNewValue = coercedValues.isEmpty() ? null : Collections.min(coercedValues); - def coercedCurrentFieldValue = null; - if (currentFieldValue != null) { - coercedCurrentFieldValue = (currentFieldValue instanceof Number) ? ((Number)currentFieldValue).longValue() : currentFieldValue; + // Track the winning value itself (rather than a coerced copy of it) so that the exact value + // from the source event or existing document gets stored, with no coercion applied. + def minValue = currentFieldValue; + boolean updated = false; + + for (def value : values) { + if (value != null && (minValue == null || minOrMaxValue_compareValues(value, minValue) < 0)) { + minValue = value; + updated = true; + } } - if (minNewValue != null && (coercedCurrentFieldValue == null || minNewValue.compareTo(coercedCurrentFieldValue) < 0)) { - parentObject[fieldName] = minNewValue; - return true; + if (updated) { + parentObject[fieldName] = minValue; } - return false; + return updated; } Map data = params.topLevelFields; @@ -2159,7 +2173,9 @@ scripts: boolean details__symbol_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_symbol"], ctx._source.details, "details.symbol", "symbol", true, true); boolean details__unit_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_unit"], ctx._source.details, "details.unit", "unit", false, false); boolean introduced_on_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_introduced_on"], ctx._source, "introduced_on", "introduced_on", true, false); + boolean max_weight_in_grams_was_noop = !maxValue_idempotentlyUpdateValue(data["weight_in_grams"], ctx._source, "max_weight_in_grams"); boolean max_weight_in_ng_was_noop = !maxValue_idempotentlyUpdateValue(data["weight_in_ng"], ctx._source, "max_weight_in_ng"); + boolean min_weight_in_grams_was_noop = !minValue_idempotentlyUpdateValue(data["weight_in_grams"], ctx._source, "min_weight_in_grams"); boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", true, false); boolean nested_fields__max_widget_cost_was_noop = !maxValue_idempotentlyUpdateValue(data["cost.amount_cents"], ctx._source.nested_fields, "max_widget_cost"); boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); @@ -2176,7 +2192,7 @@ scripts: // For records with no new values to index, only skip the update if the document itself doesn't already exist. // Otherwise create an (empty) document to reflect the fact that the id has been seen. - if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && max_weight_in_ng_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { + if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && max_weight_in_grams_was_noop && max_weight_in_ng_was_noop && min_weight_in_grams_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { ctx.op = 'none'; } else { // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly diff --git a/config/schema/artifacts/json_schemas.yaml b/config/schema/artifacts/json_schemas.yaml index 1073b9508..e78df5ce8 100644 --- a/config/schema/artifacts/json_schemas.yaml +++ b/config/schema/artifacts/json_schemas.yaml @@ -1194,6 +1194,10 @@ json_schema_version: 1 "$ref": "#/$defs/LongString" weight_in_ng: "$ref": "#/$defs/JsonSafeLong" + weight_in_grams: + anyOf: + - "$ref": "#/$defs/Float" + - type: 'null' tags: type: array items: @@ -1247,6 +1251,7 @@ json_schema_version: 1 - named_inventor - weight_in_ng_str - weight_in_ng + - weight_in_grams - tags - amounts - fees diff --git a/config/schema/artifacts/json_schemas_by_version/v1.yaml b/config/schema/artifacts/json_schemas_by_version/v1.yaml index 0bfe3ea8a..504c819a4 100644 --- a/config/schema/artifacts/json_schemas_by_version/v1.yaml +++ b/config/schema/artifacts/json_schemas_by_version/v1.yaml @@ -1644,6 +1644,13 @@ json_schema_version: 1 ElasticGraph: type: JsonSafeLong! nameInIndex: weight_in_ng + weight_in_grams: + anyOf: + - "$ref": "#/$defs/Float" + - type: 'null' + ElasticGraph: + type: Float + nameInIndex: weight_in_grams tags: type: array items: @@ -1709,6 +1716,7 @@ json_schema_version: 1 - named_inventor - weight_in_ng_str - weight_in_ng + - weight_in_grams - tags - amounts - fees diff --git a/config/schema/artifacts/runtime_metadata.yaml b/config/schema/artifacts/runtime_metadata.yaml index 2d4fdb104..407fcb978 100644 --- a/config/schema/artifacts/runtime_metadata.yaml +++ b/config/schema/artifacts/runtime_metadata.yaml @@ -815,6 +815,14 @@ enum_types_by_name: sort_field: direction: desc field_path: voltage + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -1169,6 +1177,14 @@ enum_types_by_name: sort_field: direction: desc field_path: introduced_on + max_weight_in_grams_ASC: + sort_field: + direction: asc + field_path: max_weight_in_grams + max_weight_in_grams_DESC: + sort_field: + direction: desc + field_path: max_weight_in_grams max_weight_in_ng_ASC: sort_field: direction: asc @@ -1177,6 +1193,14 @@ enum_types_by_name: sort_field: direction: desc field_path: max_weight_in_ng + min_weight_in_grams_ASC: + sort_field: + direction: asc + field_path: min_weight_in_grams + min_weight_in_grams_DESC: + sort_field: + direction: desc + field_path: min_weight_in_grams name_ASC: sort_field: direction: asc @@ -1507,6 +1531,14 @@ enum_types_by_name: sort_field: direction: desc field_path: timestamps.created_at + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -1821,6 +1853,14 @@ enum_types_by_name: sort_field: direction: desc field_path: the_opts.the_sighs + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -3059,8 +3099,12 @@ index_definitions_by_name: source: __self introduced_on: source: __self + max_weight_in_grams: + source: __self max_weight_in_ng: source: __self + min_weight_in_grams: + source: __self name: source: __self nested_fields.max_widget_cost: @@ -3206,6 +3250,8 @@ index_definitions_by_name: source: __self the_opts.the_sighs: source: __self + weight_in_grams: + source: __self weight_in_ng: source: __self weight_in_ng_str: @@ -5479,6 +5525,9 @@ object_types_by_name: voltage: resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -5631,6 +5680,9 @@ object_types_by_name: voltage: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -5842,6 +5894,9 @@ object_types_by_name: voltage: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -8392,6 +8447,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -8411,7 +8469,7 @@ object_types_by_name: - id_source: cost.currency rollover_timestamp_value_source: cost_currency_introduced_on routing_value_source: cost_currency_primary_continent - script_id: update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c + script_id: update_WidgetCurrency_from_Widget_514248e200403f9ab1766130a517439a top_level_fields_params: cost.amount_cents: cardinality: many @@ -8437,6 +8495,8 @@ object_types_by_name: cardinality: many tags: cardinality: many + weight_in_grams: + cardinality: many weight_in_ng: cardinality: many type: WidgetCurrency @@ -8513,6 +8573,8 @@ object_types_by_name: cardinality: one the_opts: cardinality: one + weight_in_grams: + cardinality: one weight_in_ng: cardinality: one weight_in_ng_str: @@ -8644,6 +8706,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -8717,9 +8782,15 @@ object_types_by_name: introduced_on: resolver: name: get_record_field_value + max_weight_in_grams: + resolver: + name: get_record_field_value max_weight_in_ng: resolver: name: get_record_field_value + min_weight_in_grams: + resolver: + name: get_record_field_value name: resolver: name: get_record_field_value @@ -8769,8 +8840,12 @@ object_types_by_name: cardinality: one introduced_on: cardinality: one + max_weight_in_grams: + cardinality: one max_weight_in_ng: cardinality: one + min_weight_in_grams: + cardinality: one name: cardinality: one nested_fields: @@ -8799,9 +8874,15 @@ object_types_by_name: introduced_on: resolver: name: object_with_lookahead + max_weight_in_grams: + resolver: + name: object_with_lookahead max_weight_in_ng: resolver: name: object_with_lookahead + min_weight_in_grams: + resolver: + name: object_with_lookahead name: resolver: name: object_with_lookahead @@ -8903,9 +8984,15 @@ object_types_by_name: introduced_on: resolver: name: object_with_lookahead + max_weight_in_grams: + resolver: + name: object_with_lookahead max_weight_in_ng: resolver: name: object_with_lookahead + min_weight_in_grams: + resolver: + name: object_with_lookahead name: resolver: name: object_with_lookahead @@ -9081,6 +9168,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -9391,6 +9481,9 @@ object_types_by_name: timestamps: resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -9509,6 +9602,9 @@ object_types_by_name: timestamps: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -9696,6 +9792,9 @@ object_types_by_name: timestamps: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead diff --git a/config/schema/artifacts/schema.graphql b/config/schema/artifacts/schema.graphql index fc8aba5b4..ffd4b26ac 100644 --- a/config/schema/artifacts/schema.graphql +++ b/config/schema/artifacts/schema.graphql @@ -7078,6 +7078,11 @@ type NamedEntityAggregatedValues { """ voltage: IntAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -7533,6 +7538,13 @@ input NamedEntityFilterInput { """ voltage: IntFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -7782,6 +7794,11 @@ type NamedEntityGroupedBy { """ voltage: Int + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -8392,6 +8409,16 @@ enum NamedEntitySortOrderInput { """ voltage_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ @@ -17049,6 +17076,7 @@ type Widget implements NamedEntity { size: Size tags: [String!]! the_options: WidgetOptions + weight_in_grams: Float weight_in_ng: JsonSafeLong! weight_in_ng_str: LongString! workspace_id: ID @@ -17204,6 +17232,11 @@ type WidgetAggregatedValues { """ the_options: WidgetOptionsAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -17323,7 +17356,9 @@ type WidgetCurrency { details: CurrencyDetails id: ID! introduced_on: Date + max_weight_in_grams: Float max_weight_in_ng: JsonSafeLong + min_weight_in_grams: Float name: String nested_fields: WidgetCurrencyNestedFields oldest_widget_created_at: DateTime @@ -17391,11 +17426,21 @@ type WidgetCurrencyAggregatedValues { """ introduced_on: DateAggregatedValues + """ + Computed aggregate values for the `max_weight_in_grams` field. + """ + max_weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `max_weight_in_ng` field. """ max_weight_in_ng: JsonSafeLongAggregatedValues + """ + Computed aggregate values for the `min_weight_in_grams` field. + """ + min_weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `name` field. """ @@ -17606,6 +17651,13 @@ input WidgetCurrencyFilterInput { """ introduced_on: DateFilterInput + """ + Used to filter on the `max_weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + max_weight_in_grams: FloatFilterInput + """ Used to filter on the `max_weight_in_ng` field. @@ -17613,6 +17665,13 @@ input WidgetCurrencyFilterInput { """ max_weight_in_ng: JsonSafeLongFilterInput + """ + Used to filter on the `min_weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + min_weight_in_grams: FloatFilterInput + """ Used to filter on the `name` field. @@ -17692,11 +17751,21 @@ type WidgetCurrencyGroupedBy { """ introduced_on: DateGroupedBy + """ + The `max_weight_in_grams` field value for this group. + """ + max_weight_in_grams: Float + """ The `max_weight_in_ng` field value for this group. """ max_weight_in_ng: JsonSafeLong + """ + The `min_weight_in_grams` field value for this group. + """ + min_weight_in_grams: Float + """ The `name` field value for this group. """ @@ -17885,6 +17954,16 @@ enum WidgetCurrencySortOrderInput { """ introduced_on_DESC + """ + Sorts ascending by the `max_weight_in_grams` field. + """ + max_weight_in_grams_ASC + + """ + Sorts descending by the `max_weight_in_grams` field. + """ + max_weight_in_grams_DESC + """ Sorts ascending by the `max_weight_in_ng` field. """ @@ -17895,6 +17974,16 @@ enum WidgetCurrencySortOrderInput { """ max_weight_in_ng_DESC + """ + Sorts ascending by the `min_weight_in_grams` field. + """ + min_weight_in_grams_ASC + + """ + Sorts descending by the `min_weight_in_grams` field. + """ + min_weight_in_grams_DESC + """ Sorts ascending by the `name` field. """ @@ -18218,6 +18307,13 @@ input WidgetFilterInput { """ the_options: WidgetOptionsFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -18412,6 +18508,11 @@ type WidgetGroupedBy { """ the_options: WidgetOptionsGroupedBy + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -19013,6 +19114,11 @@ type WidgetOrAddressAggregatedValues { """ timestamps: AddressTimestampsAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -19436,6 +19542,13 @@ input WidgetOrAddressFilterInput { """ timestamps: AddressTimestampsFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -19640,6 +19753,11 @@ type WidgetOrAddressGroupedBy { """ timestamps: AddressTimestampsGroupedBy + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -20150,6 +20268,16 @@ enum WidgetOrAddressSortOrderInput { """ timestamps_created_at_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ @@ -20545,6 +20673,16 @@ enum WidgetSortOrderInput { """ the_options_the_size_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ diff --git a/config/schema/artifacts_with_apollo/data_warehouse.yaml b/config/schema/artifacts_with_apollo/data_warehouse.yaml index ade2602f7..67b7547fd 100644 --- a/config/schema/artifacts_with_apollo/data_warehouse.yaml +++ b/config/schema/artifacts_with_apollo/data_warehouse.yaml @@ -144,7 +144,9 @@ tables: widget_options STRUCT, colors ARRAY>, nested_fields STRUCT, oldest_widget_created_at TIMESTAMP, - max_weight_in_ng BIGINT + max_weight_in_ng BIGINT, + min_weight_in_grams FLOAT, + max_weight_in_grams FLOAT ) widget_records: table_schema: |- @@ -177,6 +179,7 @@ tables: named_inventor STRUCT, weight_in_ng_str BIGINT, weight_in_ng BIGINT, + weight_in_grams FLOAT, tags ARRAY, amounts ARRAY, fees ARRAY>, diff --git a/config/schema/artifacts_with_apollo/datastore_config.yaml b/config/schema/artifacts_with_apollo/datastore_config.yaml index 6cf3519ac..901b6888b 100644 --- a/config/schema/artifacts_with_apollo/datastore_config.yaml +++ b/config/schema/artifacts_with_apollo/datastore_config.yaml @@ -1322,6 +1322,10 @@ index_templates: format: strict_date_time max_weight_in_ng: type: long + min_weight_in_grams: + type: double + max_weight_in_grams: + type: double __counts: properties: widget_names2: @@ -1469,6 +1473,8 @@ index_templates: type: long weight_in_ng: type: long + weight_in_grams: + type: double tags: type: keyword amounts: @@ -1984,11 +1990,31 @@ indices: index.number_of_shards: 1 index.max_result_window: 10000 scripts: - update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c: + update_WidgetCurrency_from_Widget_514248e200403f9ab1766130a517439a: context: update script: lang: painless source: |- + // Compares two values of a min or max field, coercing numeric values as needed. + // + // Different `Number` implementations (e.g. `Integer` vs `Long`, or `Long` vs `Double`) cannot be + // compared with `compareTo` (it throws a `ClassCastException`), and JSON parsing can produce any + // of them for the same field (e.g. `2` parses as an `Integer` while `2.9` parses as a `Double`). + // Integral values are compared as `long`s to preserve full precision (a `double` cannot exactly + // represent all integral values above 2^53), while comparisons involving a floating point value + // are performed as `double`s to avoid truncating fractional parts. + int minOrMaxValue_compareValues(def value1, def value2) { + if (value1 instanceof Number && value2 instanceof Number) { + if (value1 instanceof Float || value1 instanceof Double || value2 instanceof Float || value2 instanceof Double) { + return Double.compare(((Number)value1).doubleValue(), ((Number)value2).doubleValue()); + } + + return Long.compare(((Number)value1).longValue(), ((Number)value2).longValue()); + } + + return value1.compareTo(value2); + } + // Idempotently inserts the given value in the `sortedList`, returning `true` if the list was updated. boolean appendOnlySet_idempotentlyInsertValue(def value, List sortedList) { // As per https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/Collections.html#binarySearch(java.util.List,java.lang.Object): @@ -2074,58 +2100,46 @@ scripts: boolean maxValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { def currentFieldValue = parentObject[fieldName]; - // Normalize incoming list numerics to long to avoid Integer/Long class cast issues - List coercedValues = new ArrayList(); - for (def v : values) { - if (v != null) { - if (v instanceof Number) { - coercedValues.add(((Number)v).longValue()); - } else { - coercedValues.add(v); - } - } - } - def maxNewValue = coercedValues.isEmpty() ? null : Collections.max(coercedValues); - def coercedCurrentFieldValue = null; - if (currentFieldValue != null) { - coercedCurrentFieldValue = (currentFieldValue instanceof Number) ? ((Number)currentFieldValue).longValue() : currentFieldValue; + // Track the winning value itself (rather than a coerced copy of it) so that the exact value + // from the source event or existing document gets stored, with no coercion applied. + def maxValue = currentFieldValue; + boolean updated = false; + + for (def value : values) { + if (value != null && (maxValue == null || minOrMaxValue_compareValues(value, maxValue) > 0)) { + maxValue = value; + updated = true; + } } - if (maxNewValue != null && (coercedCurrentFieldValue == null || maxNewValue.compareTo(coercedCurrentFieldValue) > 0)) { - parentObject[fieldName] = maxNewValue; - return true; + if (updated) { + parentObject[fieldName] = maxValue; } - return false; + return updated; } boolean minValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { def currentFieldValue = parentObject[fieldName]; - // Normalize incoming list numerics to long to avoid Integer/Long class cast issues - List coercedValues = new ArrayList(); - for (def v : values) { - if (v != null) { - if (v instanceof Number) { - coercedValues.add(((Number)v).longValue()); - } else { - coercedValues.add(v); - } - } - } - def minNewValue = coercedValues.isEmpty() ? null : Collections.min(coercedValues); - def coercedCurrentFieldValue = null; - if (currentFieldValue != null) { - coercedCurrentFieldValue = (currentFieldValue instanceof Number) ? ((Number)currentFieldValue).longValue() : currentFieldValue; + // Track the winning value itself (rather than a coerced copy of it) so that the exact value + // from the source event or existing document gets stored, with no coercion applied. + def minValue = currentFieldValue; + boolean updated = false; + + for (def value : values) { + if (value != null && (minValue == null || minOrMaxValue_compareValues(value, minValue) < 0)) { + minValue = value; + updated = true; + } } - if (minNewValue != null && (coercedCurrentFieldValue == null || minNewValue.compareTo(coercedCurrentFieldValue) < 0)) { - parentObject[fieldName] = minNewValue; - return true; + if (updated) { + parentObject[fieldName] = minValue; } - return false; + return updated; } Map data = params.topLevelFields; @@ -2159,7 +2173,9 @@ scripts: boolean details__symbol_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_symbol"], ctx._source.details, "details.symbol", "symbol", true, true); boolean details__unit_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_unit"], ctx._source.details, "details.unit", "unit", false, false); boolean introduced_on_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_introduced_on"], ctx._source, "introduced_on", "introduced_on", true, false); + boolean max_weight_in_grams_was_noop = !maxValue_idempotentlyUpdateValue(data["weight_in_grams"], ctx._source, "max_weight_in_grams"); boolean max_weight_in_ng_was_noop = !maxValue_idempotentlyUpdateValue(data["weight_in_ng"], ctx._source, "max_weight_in_ng"); + boolean min_weight_in_grams_was_noop = !minValue_idempotentlyUpdateValue(data["weight_in_grams"], ctx._source, "min_weight_in_grams"); boolean name_was_noop = !immutableValue_idempotentlyUpdateValue(scriptErrors, data["cost_currency_name"], ctx._source, "name", "name", true, false); boolean nested_fields__max_widget_cost_was_noop = !maxValue_idempotentlyUpdateValue(data["cost.amount_cents"], ctx._source.nested_fields, "max_widget_cost"); boolean oldest_widget_created_at_was_noop = !minValue_idempotentlyUpdateValue(data["created_at"], ctx._source, "oldest_widget_created_at"); @@ -2176,7 +2192,7 @@ scripts: // For records with no new values to index, only skip the update if the document itself doesn't already exist. // Otherwise create an (empty) document to reflect the fact that the id has been seen. - if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && max_weight_in_ng_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { + if (ctx._source.id != null && details__symbol_was_noop && details__unit_was_noop && introduced_on_was_noop && max_weight_in_grams_was_noop && max_weight_in_ng_was_noop && min_weight_in_grams_was_noop && name_was_noop && nested_fields__max_widget_cost_was_noop && oldest_widget_created_at_was_noop && primary_continent_was_noop && widget_fee_currencies_was_noop && widget_names2_was_noop && widget_options__colors_was_noop && widget_options__sizes_was_noop && widget_tags_was_noop) { ctx.op = 'none'; } else { // Here we set `_source.id` because if we don't, it'll never be set, making these docs subtly diff --git a/config/schema/artifacts_with_apollo/json_schemas.yaml b/config/schema/artifacts_with_apollo/json_schemas.yaml index 1073b9508..e78df5ce8 100644 --- a/config/schema/artifacts_with_apollo/json_schemas.yaml +++ b/config/schema/artifacts_with_apollo/json_schemas.yaml @@ -1194,6 +1194,10 @@ json_schema_version: 1 "$ref": "#/$defs/LongString" weight_in_ng: "$ref": "#/$defs/JsonSafeLong" + weight_in_grams: + anyOf: + - "$ref": "#/$defs/Float" + - type: 'null' tags: type: array items: @@ -1247,6 +1251,7 @@ json_schema_version: 1 - named_inventor - weight_in_ng_str - weight_in_ng + - weight_in_grams - tags - amounts - fees diff --git a/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml b/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml index 0bfe3ea8a..504c819a4 100644 --- a/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml +++ b/config/schema/artifacts_with_apollo/json_schemas_by_version/v1.yaml @@ -1644,6 +1644,13 @@ json_schema_version: 1 ElasticGraph: type: JsonSafeLong! nameInIndex: weight_in_ng + weight_in_grams: + anyOf: + - "$ref": "#/$defs/Float" + - type: 'null' + ElasticGraph: + type: Float + nameInIndex: weight_in_grams tags: type: array items: @@ -1709,6 +1716,7 @@ json_schema_version: 1 - named_inventor - weight_in_ng_str - weight_in_ng + - weight_in_grams - tags - amounts - fees diff --git a/config/schema/artifacts_with_apollo/runtime_metadata.yaml b/config/schema/artifacts_with_apollo/runtime_metadata.yaml index 7f890421c..789a7c48d 100644 --- a/config/schema/artifacts_with_apollo/runtime_metadata.yaml +++ b/config/schema/artifacts_with_apollo/runtime_metadata.yaml @@ -815,6 +815,14 @@ enum_types_by_name: sort_field: direction: desc field_path: voltage + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -1169,6 +1177,14 @@ enum_types_by_name: sort_field: direction: desc field_path: introduced_on + max_weight_in_grams_ASC: + sort_field: + direction: asc + field_path: max_weight_in_grams + max_weight_in_grams_DESC: + sort_field: + direction: desc + field_path: max_weight_in_grams max_weight_in_ng_ASC: sort_field: direction: asc @@ -1177,6 +1193,14 @@ enum_types_by_name: sort_field: direction: desc field_path: max_weight_in_ng + min_weight_in_grams_ASC: + sort_field: + direction: asc + field_path: min_weight_in_grams + min_weight_in_grams_DESC: + sort_field: + direction: desc + field_path: min_weight_in_grams name_ASC: sort_field: direction: asc @@ -1507,6 +1531,14 @@ enum_types_by_name: sort_field: direction: desc field_path: timestamps.created_at + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -1821,6 +1853,14 @@ enum_types_by_name: sort_field: direction: desc field_path: the_opts.the_sighs + weight_in_grams_ASC: + sort_field: + direction: asc + field_path: weight_in_grams + weight_in_grams_DESC: + sort_field: + direction: desc + field_path: weight_in_grams weight_in_ng_ASC: sort_field: direction: asc @@ -3088,8 +3128,12 @@ index_definitions_by_name: source: __self introduced_on: source: __self + max_weight_in_grams: + source: __self max_weight_in_ng: source: __self + min_weight_in_grams: + source: __self name: source: __self nested_fields.max_widget_cost: @@ -3235,6 +3279,8 @@ index_definitions_by_name: source: __self the_opts.the_sighs: source: __self + weight_in_grams: + source: __self weight_in_ng: source: __self weight_in_ng_str: @@ -5602,6 +5648,9 @@ object_types_by_name: voltage: resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -5754,6 +5803,9 @@ object_types_by_name: voltage: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -5965,6 +6017,9 @@ object_types_by_name: voltage: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -8521,6 +8576,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -8540,7 +8598,7 @@ object_types_by_name: - id_source: cost.currency rollover_timestamp_value_source: cost_currency_introduced_on routing_value_source: cost_currency_primary_continent - script_id: update_WidgetCurrency_from_Widget_e72813bd1a8767343222b77bf53be31c + script_id: update_WidgetCurrency_from_Widget_514248e200403f9ab1766130a517439a top_level_fields_params: cost.amount_cents: cardinality: many @@ -8566,6 +8624,8 @@ object_types_by_name: cardinality: many tags: cardinality: many + weight_in_grams: + cardinality: many weight_in_ng: cardinality: many type: WidgetCurrency @@ -8642,6 +8702,8 @@ object_types_by_name: cardinality: one the_opts: cardinality: one + weight_in_grams: + cardinality: one weight_in_ng: cardinality: one weight_in_ng_str: @@ -8773,6 +8835,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -8846,9 +8911,15 @@ object_types_by_name: introduced_on: resolver: name: get_record_field_value + max_weight_in_grams: + resolver: + name: get_record_field_value max_weight_in_ng: resolver: name: get_record_field_value + min_weight_in_grams: + resolver: + name: get_record_field_value name: resolver: name: get_record_field_value @@ -8898,8 +8969,12 @@ object_types_by_name: cardinality: one introduced_on: cardinality: one + max_weight_in_grams: + cardinality: one max_weight_in_ng: cardinality: one + min_weight_in_grams: + cardinality: one name: cardinality: one nested_fields: @@ -8928,9 +9003,15 @@ object_types_by_name: introduced_on: resolver: name: object_with_lookahead + max_weight_in_grams: + resolver: + name: object_with_lookahead max_weight_in_ng: resolver: name: object_with_lookahead + min_weight_in_grams: + resolver: + name: object_with_lookahead name: resolver: name: object_with_lookahead @@ -9032,9 +9113,15 @@ object_types_by_name: introduced_on: resolver: name: object_with_lookahead + max_weight_in_grams: + resolver: + name: object_with_lookahead max_weight_in_ng: resolver: name: object_with_lookahead + min_weight_in_grams: + resolver: + name: object_with_lookahead name: resolver: name: object_with_lookahead @@ -9210,6 +9297,9 @@ object_types_by_name: name_in_index: the_opts resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -9520,6 +9610,9 @@ object_types_by_name: timestamps: resolver: name: get_record_field_value + weight_in_grams: + resolver: + name: get_record_field_value weight_in_ng: resolver: name: get_record_field_value @@ -9638,6 +9731,9 @@ object_types_by_name: timestamps: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead @@ -9825,6 +9921,9 @@ object_types_by_name: timestamps: resolver: name: object_with_lookahead + weight_in_grams: + resolver: + name: object_with_lookahead weight_in_ng: resolver: name: object_with_lookahead diff --git a/config/schema/artifacts_with_apollo/schema.graphql b/config/schema/artifacts_with_apollo/schema.graphql index e1b23ad7b..967fdf721 100644 --- a/config/schema/artifacts_with_apollo/schema.graphql +++ b/config/schema/artifacts_with_apollo/schema.graphql @@ -7360,6 +7360,11 @@ type NamedEntityAggregatedValues { """ voltage: IntAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -7815,6 +7820,13 @@ input NamedEntityFilterInput { """ voltage: IntFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -8064,6 +8076,11 @@ type NamedEntityGroupedBy { """ voltage: Int + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -8674,6 +8691,16 @@ enum NamedEntitySortOrderInput { """ voltage_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ @@ -17372,6 +17399,7 @@ type Widget implements NamedEntity @key(fields: "id") { size: Size tags: [String!]! the_options: WidgetOptions + weight_in_grams: Float weight_in_ng: JsonSafeLong! weight_in_ng_str: LongString! workspace_id: ID @@ -17527,6 +17555,11 @@ type WidgetAggregatedValues { """ the_options: WidgetOptionsAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -17646,7 +17679,9 @@ type WidgetCurrency @key(fields: "id") { details: CurrencyDetails id: ID! introduced_on: Date + max_weight_in_grams: Float max_weight_in_ng: JsonSafeLong + min_weight_in_grams: Float name: String nested_fields: WidgetCurrencyNestedFields oldest_widget_created_at: DateTime @@ -17714,11 +17749,21 @@ type WidgetCurrencyAggregatedValues { """ introduced_on: DateAggregatedValues + """ + Computed aggregate values for the `max_weight_in_grams` field. + """ + max_weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `max_weight_in_ng` field. """ max_weight_in_ng: JsonSafeLongAggregatedValues + """ + Computed aggregate values for the `min_weight_in_grams` field. + """ + min_weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `name` field. """ @@ -17929,6 +17974,13 @@ input WidgetCurrencyFilterInput { """ introduced_on: DateFilterInput + """ + Used to filter on the `max_weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + max_weight_in_grams: FloatFilterInput + """ Used to filter on the `max_weight_in_ng` field. @@ -17936,6 +17988,13 @@ input WidgetCurrencyFilterInput { """ max_weight_in_ng: JsonSafeLongFilterInput + """ + Used to filter on the `min_weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + min_weight_in_grams: FloatFilterInput + """ Used to filter on the `name` field. @@ -18015,11 +18074,21 @@ type WidgetCurrencyGroupedBy { """ introduced_on: DateGroupedBy + """ + The `max_weight_in_grams` field value for this group. + """ + max_weight_in_grams: Float + """ The `max_weight_in_ng` field value for this group. """ max_weight_in_ng: JsonSafeLong + """ + The `min_weight_in_grams` field value for this group. + """ + min_weight_in_grams: Float + """ The `name` field value for this group. """ @@ -18208,6 +18277,16 @@ enum WidgetCurrencySortOrderInput { """ introduced_on_DESC + """ + Sorts ascending by the `max_weight_in_grams` field. + """ + max_weight_in_grams_ASC + + """ + Sorts descending by the `max_weight_in_grams` field. + """ + max_weight_in_grams_DESC + """ Sorts ascending by the `max_weight_in_ng` field. """ @@ -18218,6 +18297,16 @@ enum WidgetCurrencySortOrderInput { """ max_weight_in_ng_DESC + """ + Sorts ascending by the `min_weight_in_grams` field. + """ + min_weight_in_grams_ASC + + """ + Sorts descending by the `min_weight_in_grams` field. + """ + min_weight_in_grams_DESC + """ Sorts ascending by the `name` field. """ @@ -18541,6 +18630,13 @@ input WidgetFilterInput { """ the_options: WidgetOptionsFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -18735,6 +18831,11 @@ type WidgetGroupedBy { """ the_options: WidgetOptionsGroupedBy + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -19336,6 +19437,11 @@ type WidgetOrAddressAggregatedValues { """ timestamps: AddressTimestampsAggregatedValues + """ + Computed aggregate values for the `weight_in_grams` field. + """ + weight_in_grams: FloatAggregatedValues + """ Computed aggregate values for the `weight_in_ng` field. """ @@ -19759,6 +19865,13 @@ input WidgetOrAddressFilterInput { """ timestamps: AddressTimestampsFilterInput + """ + Used to filter on the `weight_in_grams` field. + + When `null` or an empty object is passed, matches all documents. + """ + weight_in_grams: FloatFilterInput + """ Used to filter on the `weight_in_ng` field. @@ -19963,6 +20076,11 @@ type WidgetOrAddressGroupedBy { """ timestamps: AddressTimestampsGroupedBy + """ + The `weight_in_grams` field value for this group. + """ + weight_in_grams: Float + """ The `weight_in_ng` field value for this group. """ @@ -20473,6 +20591,16 @@ enum WidgetOrAddressSortOrderInput { """ timestamps_created_at_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ @@ -20868,6 +20996,16 @@ enum WidgetSortOrderInput { """ the_options_the_size_DESC + """ + Sorts ascending by the `weight_in_grams` field. + """ + weight_in_grams_ASC + + """ + Sorts descending by the `weight_in_grams` field. + """ + weight_in_grams_DESC + """ Sorts ascending by the `weight_in_ng` field. """ diff --git a/config/schema/widgets.rb b/config/schema/widgets.rb index 490f85408..6112ec50b 100644 --- a/config/schema/widgets.rb +++ b/config/schema/widgets.rb @@ -135,6 +135,7 @@ t.field "named_inventor", "NamedInventor" t.field "weight_in_ng_str", "LongString!" # Weight in nanograms, to exercise Long support. t.field "weight_in_ng", "JsonSafeLong!" # Weight in nanograms, to exercise Long support. + t.field "weight_in_grams", "Float" # Weight in grams, to exercise Float support (e.g. in min/max derived fields). t.field "tags", "[String!]!", sortable: false, singular: "tag" t.field "amounts", "[Int!]!", sortable: false do |f| f.mapping index: false @@ -187,6 +188,8 @@ derive.append_only_set "widget_fee_currencies", from: "fees.currency" derive.max_value "nested_fields.max_widget_cost", from: "cost.amount_cents" derive.max_value "max_weight_in_ng", from: "weight_in_ng" + derive.min_value "min_weight_in_grams", from: "weight_in_grams" + derive.max_value "max_weight_in_grams", from: "weight_in_grams" derive.min_value "oldest_widget_created_at", from: "created_at" end end @@ -219,6 +222,8 @@ t.field "nested_fields", "WidgetCurrencyNestedFields" t.field "oldest_widget_created_at", "DateTime" t.field "max_weight_in_ng", "JsonSafeLong" + t.field "min_weight_in_grams", "Float" + t.field "max_weight_in_grams", "Float" t.index "widget_currencies" do |i| i.rollover :yearly, "introduced_on" i.route_with "primary_continent" diff --git a/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb b/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb index 8a7202e4a..0c041408f 100644 --- a/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb +++ b/elasticgraph-indexer/spec/acceptance/derived_indexing_types_spec.rb @@ -291,6 +291,44 @@ module ElasticGraph expect(doc.dig("nested_fields", "max_widget_cost")).to eq(large_value) end + it "derives min/max `Float` values without truncating their fractional parts" do + light = widget("SMALL", "RED", "USD", name: "light", tags: [], weight_in_grams: 2.9) + heavy = widget("LARGE", "RED", "USD", name: "heavy", tags: [], weight_in_grams: 2.4) + + index_records(light) + index_records(heavy) + + doc = fetch_from_index("WidgetCurrency", "USD") + expect(doc.fetch("min_weight_in_grams")).to eq(2.4) + expect(doc.fetch("max_weight_in_grams")).to eq(2.9) + end + + it "derives min/max `Float` values when an integral value is indexed before a fractional value" do + # JSON `2` parses as an `Integer` while JSON `2.9` parses as a `Double`, so this exercises + # comparisons between mixed numeric types (which cannot be compared with `compareTo`). + integral = widget("SMALL", "RED", "USD", name: "integral", tags: [], weight_in_grams: 2) + fractional = widget("LARGE", "RED", "USD", name: "fractional", tags: [], weight_in_grams: 2.9) + + index_records(integral) + index_records(fractional) + + doc = fetch_from_index("WidgetCurrency", "USD") + expect(doc.fetch("min_weight_in_grams")).to eq(2) + expect(doc.fetch("max_weight_in_grams")).to eq(2.9) + end + + it "derives min/max `Float` values when a fractional value is indexed before an integral value" do + integral = widget("SMALL", "RED", "USD", name: "integral", tags: [], weight_in_grams: 2) + fractional = widget("LARGE", "RED", "USD", name: "fractional", tags: [], weight_in_grams: 2.9) + + index_records(fractional) + index_records(integral) + + doc = fetch_from_index("WidgetCurrency", "USD") + expect(doc.fetch("min_weight_in_grams")).to eq(2) + expect(doc.fetch("max_weight_in_grams")).to eq(2.9) + end + def expect_payload_from_lookup_and_search(payload) doc = fetch_from_index("WidgetCurrency", payload.fetch("id")) expect(doc).to include(payload) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb index 5b54ca729..0f0423ac7 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rb @@ -33,7 +33,7 @@ def setup_statements # @return [Array] painless functions required by a min or max value field. def function_definitions - [MinOrMaxValue.function_def(min_or_max)] + [COMPARE_VALUES, MinOrMaxValue.function_def(min_or_max)] end # @param min_or_max [:min, :max] which type of function to generate. @@ -44,33 +44,52 @@ def self.function_def(min_or_max) <<~EOS boolean #{min_or_max}Value_idempotentlyUpdateValue(List values, def parentObject, String fieldName) { def currentFieldValue = parentObject[fieldName]; - // Normalize incoming list numerics to long to avoid Integer/Long class cast issues - List coercedValues = new ArrayList(); - for (def v : values) { - if (v != null) { - if (v instanceof Number) { - coercedValues.add(((Number)v).longValue()); - } else { - coercedValues.add(v); - } - } - } - def #{min_or_max}NewValue = coercedValues.isEmpty() ? null : Collections.#{min_or_max}(coercedValues); - def coercedCurrentFieldValue = null; - if (currentFieldValue != null) { - coercedCurrentFieldValue = (currentFieldValue instanceof Number) ? ((Number)currentFieldValue).longValue() : currentFieldValue; + // Track the winning value itself (rather than a coerced copy of it) so that the exact value + // from the source event or existing document gets stored, with no coercion applied. + def #{min_or_max}Value = currentFieldValue; + boolean updated = false; + + for (def value : values) { + if (value != null && (#{min_or_max}Value == null || minOrMaxValue_compareValues(value, #{min_or_max}Value) #{operator} 0)) { + #{min_or_max}Value = value; + updated = true; + } } - if (#{min_or_max}NewValue != null && (coercedCurrentFieldValue == null || #{min_or_max}NewValue.compareTo(coercedCurrentFieldValue) #{operator} 0)) { - parentObject[fieldName] = #{min_or_max}NewValue; - return true; + if (updated) { + parentObject[fieldName] = #{min_or_max}Value; } - return false; + return updated; } EOS end + + private + + # Painless function which compares two values of a min or max field, coercing numeric values as needed. + COMPARE_VALUES = <<~EOS + // Compares two values of a min or max field, coercing numeric values as needed. + // + // Different `Number` implementations (e.g. `Integer` vs `Long`, or `Long` vs `Double`) cannot be + // compared with `compareTo` (it throws a `ClassCastException`), and JSON parsing can produce any + // of them for the same field (e.g. `2` parses as an `Integer` while `2.9` parses as a `Double`). + // Integral values are compared as `long`s to preserve full precision (a `double` cannot exactly + // represent all integral values above 2^53), while comparisons involving a floating point value + // are performed as `double`s to avoid truncating fractional parts. + int minOrMaxValue_compareValues(def value1, def value2) { + if (value1 instanceof Number && value2 instanceof Number) { + if (value1 instanceof Float || value1 instanceof Double || value2 instanceof Float || value2 instanceof Double) { + return Double.compare(((Number)value1).doubleValue(), ((Number)value2).doubleValue()); + } + + return Long.compare(((Number)value1).longValue(), ((Number)value2).longValue()); + } + + return value1.compareTo(value2); + } + EOS end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs index c2a9a7ee6..3d58b6853 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value.rbs @@ -8,6 +8,10 @@ module ElasticGraph def self.new: (::String, ::String, :min | :max) -> instance def self.function_def: (:min | :max) -> ::String + + private + + COMPARE_VALUES: ::String end end end diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb index 8e32258fb..8d49f6b11 100644 --- a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts/min_or_max_value_spec.rb @@ -16,6 +16,7 @@ module SchemaDefinition context "for just a `max_value` field" do include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue::COMPARE_VALUES, Indexing::DerivedFields::MinOrMaxValue.function_def(:max) ] @@ -64,6 +65,7 @@ module SchemaDefinition context "for just a `min_value` field" do include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue::COMPARE_VALUES, Indexing::DerivedFields::MinOrMaxValue.function_def(:min) ] @@ -112,6 +114,7 @@ module SchemaDefinition context "for both a `min_value` and `max_value` field" do include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue::COMPARE_VALUES, Indexing::DerivedFields::MinOrMaxValue.function_def(:max), Indexing::DerivedFields::MinOrMaxValue.function_def(:min) ] diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb index e69d96022..beea804be 100644 --- a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/update_scripts_spec.rb @@ -16,6 +16,7 @@ module SchemaDefinition describe "for a derived indexing type" do include_context "widget currency script support", expected_function_defs: [ + Indexing::DerivedFields::MinOrMaxValue::COMPARE_VALUES, Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUE, Indexing::DerivedFields::AppendOnlySet::IDEMPOTENTLY_INSERT_VALUES, Indexing::DerivedFields::ImmutableValue::IDEMPOTENTLY_SET_VALUE, diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value_spec.rb new file mode 100644 index 000000000..81642b869 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value_spec.rb @@ -0,0 +1,68 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_definition/indexing/derived_fields/min_or_max_value" + +module ElasticGraph + module SchemaDefinition + module Indexing + module DerivedFields + RSpec.describe MinOrMaxValue do + describe ".function_def" do + it "compares candidate values with `<` for a `min` field" do + function_def = MinOrMaxValue.function_def(:min) + + expect(function_def).to include("boolean minValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName)") + expect(function_def).to include("minOrMaxValue_compareValues(value, minValue) < 0") + end + + it "compares candidate values with `>` for a `max` field" do + function_def = MinOrMaxValue.function_def(:max) + + expect(function_def).to include("boolean maxValue_idempotentlyUpdateValue(List values, def parentObject, String fieldName)") + expect(function_def).to include("minOrMaxValue_compareValues(value, maxValue) > 0") + end + + it "stores the original winning value without coercing it, to avoid corrupting the indexed value" do + expect(MinOrMaxValue.function_def(:min)).to include("parentObject[fieldName] = minValue;").and exclude("longValue", "doubleValue") + expect(MinOrMaxValue.function_def(:max)).to include("parentObject[fieldName] = maxValue;").and exclude("longValue", "doubleValue") + end + end + + describe "the `minOrMaxValue_compareValues` painless function" do + it "compares integral values as `long`s to preserve full precision while avoiding `ClassCastException` on mixed `Integer`/`Long` values" do + expect(MinOrMaxValue::COMPARE_VALUES).to include("Long.compare(((Number)value1).longValue(), ((Number)value2).longValue())") + end + + it "compares as `double`s when either value is a floating point number, to avoid truncating fractional parts" do + expect(MinOrMaxValue::COMPARE_VALUES).to include( + "value1 instanceof Float || value1 instanceof Double || value2 instanceof Float || value2 instanceof Double", + "Double.compare(((Number)value1).doubleValue(), ((Number)value2).doubleValue())" + ) + end + + it "compares non-numeric values (e.g. `Date`/`DateTime`/`LocalTime` strings) via their natural `compareTo`" do + expect(MinOrMaxValue::COMPARE_VALUES).to include("return value1.compareTo(value2);") + end + end + + describe "#function_definitions" do + it "includes the comparison function so that the min/max function can use it" do + field = MinOrMaxValue.new("min_cost", "cost", :min) + + expect(field.function_definitions).to eq [ + MinOrMaxValue::COMPARE_VALUES, + MinOrMaxValue.function_def(:min) + ] + end + end + end + end + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb b/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb index df0b6f379..039a872fd 100644 --- a/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb +++ b/spec_support/lib/elastic_graph/spec_support/factories/widgets.rb @@ -122,6 +122,7 @@ named_inventor { inventor } weight_in_ng { Faker::Number.between(from: 2**51, to: (2**53) - 1) } weight_in_ng_str { Faker::Number.between(from: 2**60, to: 2**61) } + weight_in_grams { Faker::Number.between(from: 1.0, to: 100.0) } tags { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Alphanumeric.alpha(number: 6) } } amounts { Array.new(Faker::Number.between(from: 0, to: 4)) { Faker::Number.between(from: 100, to: 10000) } } fees { build_list(:money, Faker::Number.between(from: 0, to: 4)) }