diff --git a/CODEBASE_OVERVIEW.md b/CODEBASE_OVERVIEW.md index 8cc77e3b8..a7f4b58ea 100644 --- a/CODEBASE_OVERVIEW.md +++ b/CODEBASE_OVERVIEW.md @@ -111,7 +111,6 @@ graph LR; rackup["rackup"]; rake["rake"]; webrick["webrick"]; - elasticgraph-json_ingestion["eg-json_ingestion"]; elasticgraph-schema_artifacts["eg-schema_artifacts"]; graphql["graphql"]; elasticgraph --> elasticgraph-support; @@ -126,7 +125,6 @@ graph LR; elasticgraph-local --> webrick; elasticgraph-schema_definition --> elasticgraph-graphql; elasticgraph-schema_definition --> elasticgraph-indexer; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; elasticgraph-schema_definition --> elasticgraph-schema_artifacts; elasticgraph-schema_definition --> elasticgraph-support; elasticgraph-schema_definition --> graphql; @@ -143,7 +141,6 @@ graph LR; class rackup externalGemCatStyle; class rake externalGemCatStyle; class webrick externalGemCatStyle; - class elasticgraph-json_ingestion otherEgGemStyle; class elasticgraph-schema_artifacts otherEgGemStyle; class graphql externalGemCatStyle; click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank; diff --git a/Gemfile.lock b/Gemfile.lock index 1786c742b..9e18f46ec 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -198,7 +198,6 @@ PATH elasticgraph-schema_definition (1.2.1.pre) elasticgraph-graphql (= 1.2.1.pre) elasticgraph-indexer (= 1.2.1.pre) - elasticgraph-json_ingestion (= 1.2.1.pre) elasticgraph-schema_artifacts (= 1.2.1.pre) elasticgraph-support (= 1.2.1.pre) graphql (~> 2.6.2) diff --git a/Rakefile b/Rakefile index 831dc68b3..0e3dbc045 100644 --- a/Rakefile +++ b/Rakefile @@ -8,9 +8,10 @@ require "delegate" require "elastic_graph/apollo/schema_definition/api_extension" -require "elastic_graph/warehouse/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/schema_definition/rake_tasks" +require "elastic_graph/warehouse/schema_definition/api_extension" require "yaml" project_root = File.expand_path(__dir__) @@ -49,6 +50,7 @@ configure_local_rake_tasks = ->(tasks) do tasks.index_document_sizes = true tasks.env_port_mapping = {test: test_port} tasks.output = schema_def_output + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension tasks.define_fake_data_batch_for(:widgets) do require "rspec/core" # the factories file expects RSpec to be loaded, so load it. diff --git a/config/site/Rakefile b/config/site/Rakefile index c3eec0216..b76f8eb7b 100644 --- a/config/site/Rakefile +++ b/config/site/Rakefile @@ -433,6 +433,7 @@ module ElasticGraph # When we are releasing gems, these dependencies aren't available. unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile") + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" @@ -460,6 +461,7 @@ module ElasticGraph # Here we provide a "default Rakefile" for examples that don't have any specialized needs and that aren't trying # to demonstrate any `Rakefile` APIs. ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks| + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension tasks.opensearch_versions = [] end end diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb index f322eb599..1ae388eeb 100644 --- a/config/site/support/doctest_helper.rb +++ b/config/site/support/doctest_helper.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" require "elastic_graph/schema_definition/api" require "elastic_graph/schema_definition/schema_artifact_manager" @@ -51,6 +52,7 @@ module ElasticGraph descriptions_needing_schema_def_api_and_extension_modules = { "ElasticGraph.define_schema" => [], "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension], + "ElasticGraph::JSONIngestion::SchemaDefinition" => [], "ElasticGraph::SchemaDefinition" => [], "ElasticGraph::Warehouse::SchemaDefinition" => [ElasticGraph::Warehouse::SchemaDefinition::APIExtension] } @@ -60,7 +62,7 @@ module ElasticGraph @api = SchemaDefinition::API.new( SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}), true, - extension_modules: extension_modules + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension] + extension_modules ) # This is required in all schemas, but we don't want to have to put in all our examples, @@ -90,11 +92,27 @@ module ElasticGraph end end + doctest.before "ElasticGraph::JSONIngestion::SchemaDefinition" do + @api = SchemaDefinition::API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}), + true, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension] + ) + + @api.json_schema_version 1 + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = @api + end + + doctest.after "ElasticGraph::JSONIngestion::SchemaDefinition" do + ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = nil + end + doctest.before "ElasticGraph::SchemaDefinition::API#json_schema_version" do ElasticGraph.define_schema do |schema| # `schema.json_schema_version` raises an error when the version is set more than once. # By default we set it above. Here we clear it to allow our example to set it. schema.state.json_schema_version = nil + schema.state.json_schema_version_setter_location = nil end end diff --git a/elasticgraph-admin/elasticgraph-admin.gemspec b/elasticgraph-admin/elasticgraph-admin.gemspec index 722116506..cc90e68a4 100644 --- a/elasticgraph-admin/elasticgraph-admin.gemspec +++ b/elasticgraph-admin/elasticgraph-admin.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_dependency "rake", "~> 13.4", ">= 13.4.2" spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-apollo/README.md b/elasticgraph-apollo/README.md index 2503b0377..e5aeb759b 100644 --- a/elasticgraph-apollo/README.md +++ b/elasticgraph-apollo/README.md @@ -49,29 +49,30 @@ index 4a5ef1e..5c16c2b 100644 ``` -Finally, update `Rakefile` so that `ElasticGraph::GraphQL::Apollo::SchemaDefinition::APIExtension` is -used as one of the `extension_modules`: +Finally, update `Rakefile` so that `ElasticGraph::Apollo::SchemaDefinition::APIExtension` is +used as one of the `schema_definition_extension_modules`: ```diff diff --git a/Rakefile b/Rakefile index 2943335..26633c3 100644 --- a/Rakefile +++ b/Rakefile -@@ -1,5 +1,6 @@ +@@ -1,6 +1,7 @@ project_root = File.expand_path(__dir__) +require "elastic_graph/apollo/schema_definition/api_extension" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" require "rspec/core/rake_task" -@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new( - local_config_yaml: settings_file, - path_to_schema: "#{project_root}/config/schema.rb" - ) do |tasks| -+ tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension -+ +@@ -16,5 +17,6 @@ ElasticGraph::Local::RakeTasks.new( # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension +- ++ tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension ++ + # Customizes the names of fields generated by ElasticGraph. ``` That's it! diff --git a/elasticgraph-apollo/apollo_tests_implementation/Rakefile b/elasticgraph-apollo/apollo_tests_implementation/Rakefile index befade088..7dc607f3c 100644 --- a/elasticgraph-apollo/apollo_tests_implementation/Rakefile +++ b/elasticgraph-apollo/apollo_tests_implementation/Rakefile @@ -6,8 +6,9 @@ # # frozen_string_literal: true -require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/rake_tasks" require "pathname" project_root = Pathname.new(__dir__) @@ -17,5 +18,8 @@ ElasticGraph::SchemaDefinition::RakeTasks.new( index_document_sizes: false, path_to_schema: project_root / "config/products_schema.rb", schema_artifacts_directory: project_root / "config/schema/artifacts", - extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + extension_modules: [ + ElasticGraph::Apollo::SchemaDefinition::APIExtension, + ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension + ] ) diff --git a/elasticgraph-apollo/elasticgraph-apollo.gemspec b/elasticgraph-apollo/elasticgraph-apollo.gemspec index d94ab6c37..f1b8a970b 100644 --- a/elasticgraph-apollo/elasticgraph-apollo.gemspec +++ b/elasticgraph-apollo/elasticgraph-apollo.gemspec @@ -52,6 +52,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb index 915c79c34..a8130e4eb 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/spec_support/schema_definition_helpers" module ElasticGraph @@ -552,7 +553,7 @@ def self.with_both_casing_forms(&block) end def define_schema(&block) - extension_modules = [SchemaDefinition::APIExtension] + extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, SchemaDefinition::APIExtension] super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) end end diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb index 740bc003e..34970bb06 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/spec_support/runtime_metadata_support" require "elastic_graph/spec_support/schema_definition_helpers" require "graphql" @@ -1431,7 +1432,8 @@ def expect_identifiable_type_tagging_of_token(&type_def_for) end def define_schema(with_apollo: true, &block) - extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : [] + extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] + extension_modules += [SchemaDefinition::APIExtension] if with_apollo super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block) end end diff --git a/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec index 05ee33e5f..89fd32983 100644 --- a/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec +++ b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec @@ -46,6 +46,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-graphql/elasticgraph-graphql.gemspec b/elasticgraph-graphql/elasticgraph-graphql.gemspec index 8bc5ab51b..eba63698a 100644 --- a/elasticgraph-graphql/elasticgraph-graphql.gemspec +++ b/elasticgraph-graphql/elasticgraph-graphql.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION diff --git a/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb index 494152ebf..05d18f5ab 100644 --- a/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb +++ b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb @@ -8,6 +8,7 @@ require "elastic_graph/graphql" require "elastic_graph/indexer" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "support/graphql" @@ -65,6 +66,7 @@ def dump_schema_artifacts(json_schema_version:, team_extras: "") run_rake "schema_artifacts:dump" do |output| SchemaDefinition::RakeTasks.new( schema_element_name_form: :snake_case, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", diff --git a/elasticgraph-indexer/elasticgraph-indexer.gemspec b/elasticgraph-indexer/elasticgraph-indexer.gemspec index 9cd116162..9760addb5 100644 --- a/elasticgraph-indexer/elasticgraph-indexer.gemspec +++ b/elasticgraph-indexer/elasticgraph-indexer.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb index a7f241102..baa04ec86 100644 --- a/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb +++ b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/indexer" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" module ElasticGraph @@ -401,6 +402,7 @@ def dump_artifacts run_rake "schema_artifacts:dump" do |output| SchemaDefinition::RakeTasks.new( schema_element_name_form: :snake_case, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb index 66eb66525..1ecc94fc3 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb @@ -576,7 +576,11 @@ def build_preparer_for_old_json_schema_version(v1_def:, v2_def:) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end end diff --git a/elasticgraph-json_ingestion/README.md b/elasticgraph-json_ingestion/README.md index c5f2fae01..db1eb3988 100644 --- a/elasticgraph-json_ingestion/README.md +++ b/elasticgraph-json_ingestion/README.md @@ -2,8 +2,8 @@ JSON Schema ingestion support for ElasticGraph. -This gem contains the JSON Schema helper code used by schema definition to generate indexing -event schemas and merge ElasticGraph metadata into versioned schema artifacts. +This gem provides the schema-definition extension that generates JSON Schema artifacts for indexing +events and validates JSON-ingestion-specific schema options. ## Dependency Diagram @@ -17,7 +17,4 @@ graph LR; elasticgraph-support["elasticgraph-support"]; elasticgraph-json_ingestion --> elasticgraph-support; class elasticgraph-support otherEgGemStyle; - elasticgraph-schema_definition["elasticgraph-schema_definition"]; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; - class elasticgraph-schema_definition otherEgGemStyle; ``` diff --git a/elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec b/elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec index cdade27c0..37387b193 100644 --- a/elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec +++ b/elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec @@ -36,8 +36,8 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION - # This gem's schema-definition extension code references `elasticgraph-schema_definition`, but - # applications load it through schema-definition tasks after `elasticgraph-schema_definition` is already - # available. Keeping this as a development dependency avoids a runtime dependency cycle. + # This gem's extension code references `elasticgraph-schema_definition`, but applications load it + # through schema-definition tasks after `elasticgraph-schema_definition` is already available. + # Keeping it as a development dependency avoids pulling schema definition into runtime bundles. spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb new file mode 100644 index 000000000..fc8197269 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb @@ -0,0 +1,178 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones" +require "elastic_graph/json_ingestion/schema_definition/factory_extension" +require "elastic_graph/json_ingestion/schema_definition/state_extension" + +module ElasticGraph + module JSONIngestion + # Namespace for all JSON Schema schema definition support. + # + # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module. + module SchemaDefinition + # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance + # to add JSON Schema artifact generation capabilities. + module APIExtension + # Default JSON schema options applied to ElasticGraph's built-in scalar types when this extension + # is loaded. Keyed by the un-overridden type name; the lookup at runtime maps each key through + # `type_name_overrides` so renamed built-ins still receive the right options. + BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME = { + "Boolean" => {type: "boolean"}, + "Float" => {type: "number"}, + "ID" => {type: "string"}, + "Int" => {type: "integer", minimum: INT_MIN, maximum: INT_MAX}, + "String" => {type: "string"}, + "Cursor" => {type: "string"}, + "Date" => {type: "string", format: "date"}, + "DateTime" => {type: "string", format: "date-time"}, + "LocalTime" => {type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN}, + "TimeZone" => {type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a.freeze}, + "Untyped" => {type: ["array", "boolean", "integer", "number", "object", "string"].freeze}, + "JsonSafeLong" => {type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX}, + "LongString" => {type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX} + }.freeze + + # Wires up the JSON ingestion extensions when this module is extended onto an API instance. + # + # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend + # @return [void] + # @api private + def self.extended(api) + state = api.state.extend(StateExtension) # : ElasticGraph::SchemaDefinition::State & StateExtension + state.reserved_type_names << EVENT_ENVELOPE_JSON_SCHEMA_NAME + api.factory.extend FactoryExtension + + # Build a lookup from final (post-`type_name_overrides`) names to JSON schema options. We can't + # key directly on `type.name` because users may have overridden the names of built-in scalars + # (e.g. `Cursor` → `PreCursor`); the keys in `BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME` are + # always the un-overridden names. + options_by_final_name = BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME.to_h do |name, options| + [api.state.type_ref(name).to_final_form.name, options] + end + + api.on_built_in_types do |type| + if (options = options_by_final_name[type.name]) + scalar_type = type # : ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension + scalar_type.json_schema(**options) + elsif type.name == api.state.type_ref("GeoLocation").to_final_form.name + # @type var geo_location_type: ElasticGraph::SchemaDefinition::SchemaElements::TypeWithSubfields & SchemaElements::ObjectInterfaceExtension + geo_location_type = _ = type + names = api.state.schema_elements + + # We use `nullable: false` because `GeoLocation` is indexed as a single `geo_point` field, + # and therefore can't support a `latitude` without a `longitude` or vice-versa. + latitude = geo_location_type.graphql_fields_by_name.fetch(names.latitude) # : ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension + longitude = geo_location_type.graphql_fields_by_name.fetch(names.longitude) # : ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension + latitude.json_schema minimum: -90, maximum: 90, nullable: false + longitude.json_schema minimum: -180, maximum: 180, nullable: false + end + end + end + + # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema + # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique + # version number. The publisher will then include this version number in published events to identify the version of the schema it + # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync. + # + # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying to constantly + # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this + # on every JSON schema change with {#enforce_json_schema_version}. + # + # @param version [Integer] current version number of the JSON schema artifact + # @return [void] + # @see #enforce_json_schema_version + def json_schema_version(version) + state = json_ingestion_state + + if !version.is_a?(Integer) || version < 1 + raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}" + end + + if state.json_schema_version + raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{state.json_schema_version}" + end + + state.json_schema_version = version + state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first + nil + end + + # Configures whether JSON schema artifact dumping enforces the requirement that the JSON schema version is incremented every time + # dumping the JSON schemas results in a changed artifact. Defaults to `true`. + # + # @note Generally speaking, you will want this to be `true` for any ElasticGraph application that is in + # production as the versioning of JSON schemas is what supports safe schema evolution as it allows + # ElasticGraph to identify which version of the JSON schema the publishing system was operating on + # when it published an event. + # + # It can be useful to set it to `false` before your application is in production, as you do not want + # to be forced to bump the version after every single schema change while you are building an initial + # prototype. + # + # @param value [Boolean] whether to require `json_schema_version` to be incremented on changes that impact `json_schemas.yaml` + # @return [void] + # @see #json_schema_version + # + # @example Disable enforcement during initial prototyping + # ElasticGraph.define_schema do |schema| + # # TODO: remove this once we're past the prototyping stage + # schema.enforce_json_schema_version false + # end + def enforce_json_schema_version(value) + unless value == true || value == false + raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}" + end + + json_ingestion_state.enforce_json_schema_version = value + nil + end + + # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the + # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to + # configure this behavior. + # + # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events. + # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events. + # @return [void] + # + # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled + # field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher + # accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId` + # is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of + # these to `true` (or none). + def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true) + state = json_ingestion_state + + unless [true, false].include?(allow_omitted_fields) + raise Errors::SchemaError, "`allow_omitted_fields` must be true or false" + end + + unless [true, false].include?(allow_extra_fields) + raise Errors::SchemaError, "`allow_extra_fields` must be true or false" + end + + state.allow_omitted_json_schema_fields = allow_omitted_fields + state.allow_extra_json_schema_fields = allow_extra_fields + nil + end + + private + + # Returns the API's `state` narrowed to include this gem's `StateExtension`. Centralizes + # the Steep cast that's needed because Steep can't see the `extend(StateExtension)` applied + # at runtime in `extended`. + def json_ingestion_state + state # : ElasticGraph::SchemaDefinition::State & StateExtension + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/factory_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/factory_extension.rb new file mode 100644 index 000000000..a62400a82 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/factory_extension.rb @@ -0,0 +1,129 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum" +require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/object" +require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar" +require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/union" +require "elastic_graph/json_ingestion/schema_definition/indexing/index_extension" +require "elastic_graph/json_ingestion/schema_definition/results_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension" +require "elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::Factory` to wire up + # JSON Schema support on Results and SchemaArtifactManager instances. + # + # @api private + module FactoryExtension + # @private + def new_enum_type(name) + super(name) do |type| + type.extend SchemaElements::EnumTypeExtension + yield type if block_given? + end + end + + # @private + def new_enum_indexing_field_type(...) + Indexing::FieldType::Enum.new(super) + end + + # @private + def new_field(**kwargs, &block) + super(**kwargs) do |field| + field.extend SchemaElements::FieldExtension + block&.call(field) + end + end + + # @private + def new_index(name, settings, type, &block) + super(name, settings, type) do |index| + index.extend Indexing::IndexExtension + index.require_id_in_json_schema + block&.call(index) + end + end + + # @private + def new_interface_type(name) + super(name) do |type| + type.extend SchemaElements::ObjectInterfaceExtension + yield type if block_given? + end + end + + # @private + def new_object_type(name) + super(name) do |type| + type.extend SchemaElements::ObjectInterfaceExtension + yield type if block_given? + end + end + + # @private + def new_object_indexing_field_type(type_name:, subfields:, mapping_options:, doc_comment:, json_schema_options: {}) + Indexing::FieldType::Object.new( + super( + type_name: type_name, + subfields: subfields, + mapping_options: mapping_options, + doc_comment: doc_comment + ), + json_schema_options: json_schema_options + ) + end + + # @private + def new_scalar_type(name) + super(name) do |type| + type.extend SchemaElements::ScalarTypeExtension + yield type if block_given? + type.validate_json_schema_configuration! unless state.initially_registered_built_in_types.empty? + end + end + + # @private + def new_scalar_indexing_field_type(...) + Indexing::FieldType::Scalar.new(super) + end + + # @private + def new_type_reference(name) + super(name).extend(SchemaElements::TypeReferenceExtension) + end + + # @private + def new_union_indexing_field_type(...) + Indexing::FieldType::Union.new(super) + end + + # Creates a new Results instance with JSON Schema extensions. + # + # @return [ElasticGraph::SchemaDefinition::Results] the created results instance + def new_results + super.extend(ResultsExtension) + end + + # Creates a new SchemaArtifactManager instance with JSON Schema extensions. + # + # @return [ElasticGraph::SchemaDefinition::SchemaArtifactManager] the created artifact manager + def new_schema_artifact_manager(...) + super.extend(SchemaArtifactManagerExtension) + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb new file mode 100644 index 000000000..850bbd0f4 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb @@ -0,0 +1,168 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata" +require "elastic_graph/schema_definition/indexing/field" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Namespace for JSON-schema-aware indexing components. + module Indexing + # Wraps an indexing field with JSON schema generation behavior. + # + # @api private + class Field < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::Field) + # @dynamic __getobj__, json_schema_layers, json_schema_customizations + # @return [Array] JSON schema wrapper layers from the field type reference + attr_reader :json_schema_layers + + # @return [Hash] user-defined JSON schema customizations + attr_reader :json_schema_customizations + + # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema + # validation will reject values which cannot be indexed into fields of a specific mapping type. + # + # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation + # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type). + # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values. + JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = { + "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1}, + "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1}, + "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, + "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} + } + + # @param field [ElasticGraph::SchemaDefinition::Indexing::Field] the indexing field to wrap + # @param json_schema_layers [Array] JSON schema wrapper layers from the field type reference + # @param json_schema_customizations [Hash] user-defined JSON schema customizations + def initialize(field, json_schema_layers:, json_schema_customizations:) + super(field) + @json_schema_layers = json_schema_layers + @json_schema_customizations = json_schema_customizations + end + + # Returns the JSON schema definition for this field. + # + # @return [Hash] the JSON schema hash + def json_schema + @json_schema ||= + json_schema_layers + .reverse # resolve layers from innermost to outermost wrappings + .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) } + .merge(outer_json_schema_customizations) + .merge({"description" => doc_comment}.compact) + .then { |hash| Support::HashUtil.stringify_keys(hash) } + end + + # @return [JSONSchemaFieldMetadata] metadata about this field for inclusion in the JSON schema + def json_schema_metadata + JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) + end + + def nullable? + json_schema_layers.include?(:nullable) + end + + # Compares fields, including JSON schema metadata tracked by this wrapper. + # + # @param other [Object] the object to compare against + # @return [Boolean] true when the field and JSON schema metadata match + def ==(other) + case other + when Field + __getobj__ == other.__getobj__ && + json_schema_layers == other.json_schema_layers && + json_schema_customizations == other.json_schema_customizations + else + super + end + end + + def eql?(other) + self == other + end + + # Returns a hash code based on the wrapped field and JSON schema metadata. + # + # @return [Integer] the hash code + def hash + [__getobj__, json_schema_layers, json_schema_customizations].hash + end + + private + + def inner_json_schema + user_specified_customizations = + if user_specified_json_schema_customizations_go_on_outside? + {} # : ::Hash[::String, untyped] + else + Support::HashUtil.stringify_keys(json_schema_customizations) + end + + customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {} + customizations = customizations_from_mapping.merge(user_specified_customizations) + # @type var field_type: _JSONFieldType + field_type = _ = indexing_field_type + customizations = field_type.format_field_json_schema_customizations(customizations) + + ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"} + return ref if customizations.empty? + + # Combine any customizations with the type ref under an "allOf" subschema: + # all of these properties must hold true for the type to be valid. + # + # Note that if we simply combine the customizations with the `$ref` + # at the same level, it will not work, because other subschema + # properties are ignored when they are in the same object as a `$ref`: + # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168 + {"allOf" => [ref, customizations]} + end + + def outer_json_schema_customizations + return {} unless user_specified_json_schema_customizations_go_on_outside? + Support::HashUtil.stringify_keys(json_schema_customizations) + end + + # Indicates if the user-specified JSON schema customizations should go on the inside + # (where they normally go) or on the outside. They only go on the outside when it's + # an array field, because then they apply to the array itself instead of the items in the + # array. + def user_specified_json_schema_customizations_go_on_outside? + json_schema_layers.include?(:array) + end + + def process_layer(layer, schema) + case layer + when :nullable + # Here we use "anyOf" to ensure that JSON can either match the schema OR null. + # + # (Using "oneOf" would mean that if we had a schema that also allowed null, + # null would never be allowed, since "oneOf" must match exactly one subschema). + { + "anyOf" => [ + schema, + {"type" => "null"} + ] + } + when :array + {"type" => "array", "items" => schema} + else + # :nocov: - layer is only ever `:nullable` or `:array` so we never get here. + schema + # :nocov: + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb new file mode 100644 index 000000000..c25e73ead --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb @@ -0,0 +1,79 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/json_ingestion/schema_definition/indexing/field" +require "elastic_graph/schema_definition/indexing/field_reference" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + # Wraps an indexing field reference with JSON schema state needed when resolving fields. + # + # @api private + class FieldReference < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldReference) + # @dynamic __getobj__, json_schema_layers, json_schema_customizations + # @return [Array] JSON schema wrapper layers from the field type reference + attr_reader :json_schema_layers + + # @return [Hash] user-defined JSON schema customizations + attr_reader :json_schema_customizations + + # @param field_reference [ElasticGraph::SchemaDefinition::Indexing::FieldReference] the field reference to wrap + # @param json_schema_layers [Array] JSON schema wrapper layers from the field type reference + # @param json_schema_customizations [Hash] user-defined JSON schema customizations + def initialize(field_reference, json_schema_layers:, json_schema_customizations:) + super(field_reference) + @json_schema_layers = json_schema_layers + @json_schema_customizations = json_schema_customizations + end + + # Resolves this reference to a JSON-schema-aware indexing field. + # + # @return [Field, nil] the resolved field, or nil when the type is unresolved + def resolve + return nil unless (resolved_field = super) + + Field.new( + resolved_field, + json_schema_layers: json_schema_layers, + json_schema_customizations: json_schema_customizations + ) + end + + # Compares field references, including JSON schema metadata tracked by this wrapper. + # + # @param other [Object] the object to compare against + # @return [Boolean] true when the field reference and JSON schema metadata match + def ==(other) + case other + when FieldReference + __getobj__ == other.__getobj__ && + json_schema_layers == other.json_schema_layers && + json_schema_customizations == other.json_schema_customizations + else + super + end + end + + def eql?(other) + self == other + end + + # Returns a hash code based on the wrapped field reference and JSON schema metadata. + # + # @return [Integer] the hash code + def hash + [__getobj__, json_schema_layers, json_schema_customizations].hash + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rb new file mode 100644 index 000000000..8ff361ee2 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rb @@ -0,0 +1,55 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/indexing/field_type/enum" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + # Namespace for JSON-schema-aware indexing field types. + module FieldType + # Wraps enum indexing field types with JSON schema serialization. + # + # @private + class Enum < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldType::Enum) + # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this enum type. + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] JSON schema customizations + # @return [Hash] formatted customizations. + def format_field_json_schema_customizations(customizations) + # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep + # other customizations (such as the `maxLength` field customization EG automatically applies to fields + # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway). + # + # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema + # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found + # that the presence of extra field customizations inhibits the library's ability to generate code in the way + # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to + # being wrapped in an `allOf`). + # + # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed + # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to + # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR). + customizations.slice("enum") + end + + # @return [Hash] the JSON schema for this enum type. + def to_json_schema + {"type" => "string", "enum" => enum_value_names} + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb new file mode 100644 index 000000000..54b647665 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb @@ -0,0 +1,131 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/errors" +require "elastic_graph/schema_definition/indexing/field_type/object" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + # Wraps object/interface indexing field types with JSON schema serialization. + # + # @private + class Object < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldType::Object) + # @dynamic __getobj__, json_schema_options + # @return [Hash] JSON schema options for this object type + attr_reader :json_schema_options + + # @param field_type [ElasticGraph::SchemaDefinition::Indexing::FieldType::Object] the object field type to wrap + # @param json_schema_options [Hash] JSON schema options for this object type + def initialize(field_type, json_schema_options:) + super(field_type) + @json_schema_options = json_schema_options + end + + # @return [Hash] field metadata keyed by field name + def json_schema_field_metadata_by_field_name + # @type var json_subfields: ::Array[Indexing::Field] + json_subfields = _ = subfields + json_subfields.to_h { |field| [field.name, field.json_schema_metadata] } + end + + # @param customizations [Hash] the customizations to format + # @return [Hash] the formatted customizations + def format_field_json_schema_customizations(customizations) + customizations + end + + # @return [Hash] the JSON schema definition for this object type + def to_json_schema + # @type var state: ElasticGraph::SchemaDefinition::State & StateExtension + state = _ = schema_def_state + # @type var json_subfields: ::Array[Indexing::Field] + json_subfields = _ = subfields + + @to_json_schema ||= + if json_schema_options.empty? + # Fields that are `sourced_from` an alternate type must not be included in this type's JSON schema, + # since events of this type won't include them. + other_source_subfields, json_schema_candidate_subfields = json_subfields.partition(&:source) + validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) + json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script) + required_fields = json_schema_subfields + required_fields = required_fields.reject(&:nullable?) if state.allow_omitted_json_schema_fields + + { + "type" => "object", + "properties" => json_schema_subfields.to_h { |field| [field.name, field.json_schema] }.merge(json_schema_typename_field), + # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present + # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want + # to require it in the context of a union type. The union's JSON schema requires the field. + "required" => required_fields.map(&:name).freeze, + "additionalProperties" => (false unless state.allow_extra_json_schema_fields), + "description" => doc_comment + }.compact.freeze + else + Support::HashUtil.stringify_keys(json_schema_options) + end + end + + def ==(other) + case other + when Object + __getobj__ == other.__getobj__ && + json_schema_options == other.json_schema_options + else + super + end + end + + def eql?(other) + self == other + end + + def hash + [__getobj__, json_schema_options].hash + end + + private + + # Returns a `__typename` property which we use for union types. + # + # This must always be set to the name of the type (thus the const value). + # + # We also add a "default" value. This does not impact validation, but rather + # aids tools like our Kotlin codegen to save publishers from having to set the + # property explicitly when creating events. + def json_schema_typename_field + { + "__typename" => { + "type" => "string", + "const" => type_name, + "default" => type_name + } + } + end + + def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) + problem_fields = other_source_subfields.reject { |field| field.json_schema_customizations.empty? } + return if problem_fields.empty? + + field_descriptions = problem_fields.map(&:name).sort.map { |field| "`#{field}`" }.join(", ") + raise Errors::SchemaError, + "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \ + "another type and also have JSON schema customizations. Instead, put the JSON schema " \ + "customizations on the source type's field definitions." + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb new file mode 100644 index 000000000..558113f20 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb @@ -0,0 +1,45 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/indexing/field_type/scalar" +require "elastic_graph/support/hash_util" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + # Wraps scalar indexing field types with JSON schema serialization. + # + # @private + class Scalar < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldType::Scalar) + # @return [Hash] empty hash, as scalar types have no subfields + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] the customizations to format + # @return [Hash] the formatted customizations + def format_field_json_schema_customizations(customizations) + customizations + end + + # @return [Hash] the JSON schema definition for this scalar type + def to_json_schema + json_scalar_type = scalar_type # : ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension + json_scalar_type.validate_json_schema_configuration! + + Support::HashUtil.stringify_keys(json_scalar_type.json_schema_options) + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rb new file mode 100644 index 000000000..526467cda --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rb @@ -0,0 +1,55 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "delegate" +require "elastic_graph/schema_definition/indexing/field_type/union" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + # Wraps union indexing field types with JSON schema serialization. + # + # @private + class Union < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldType::Union) + # @return [Hash] empty hash, as union types have no subfields + def json_schema_field_metadata_by_field_name + {} + end + + # @param customizations [Hash] the customizations to format + # @return [Hash] the formatted customizations + def format_field_json_schema_customizations(customizations) + customizations + end + + # @return [Hash] the JSON schema definition for this union type + def to_json_schema + subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} } + + # A union type can represent multiple subtypes, referenced by the "anyOf" clause below. + # We also add a requirement for the presence of __typename to indicate which type + # is being referenced (this property is pre-defined on the type itself as a constant). + # + # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas + # to form a union, and validate equivalently when no object can satisfy multiple of the + # subschemas (which is the case here given the __typename requirements are mutually + # exclusive), we chose to use "oneOf" here because it works better with this library: + # https://github.com/pwall567/json-kotlin-schema-codegen + { + "required" => %w[__typename], + "oneOf" => subtype_json_schemas + } + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rb new file mode 100644 index 000000000..94a4b47e6 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rb @@ -0,0 +1,53 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + # Extends indices with JSON-schema-specific event requirements. + module IndexExtension + # @private + def require_id_in_json_schema + schema_def_state.after_user_definition_complete do + # @type var routing_field_path: _JSONFieldPath + routing_field_path = _ = self.routing_field_path + routing_field_path.last_part.json_schema nullable: false + end + end + + # @private + def rollover(frequency, timestamp_field_path_name) + super + + schema_def_state.after_user_definition_complete do + # @type var timestamp_field_path: _JSONFieldPath + timestamp_field_path = _ = public_field_path(timestamp_field_path_name, explanation: "it is referenced as an index `rollover` field") + timestamp_field_path + .path_parts + .each { |field| field.json_schema nullable: false } + end + end + + # @private + def route_with(routing_field_path_name) + super + + schema_def_state.after_user_definition_complete do + # @type var routing_field_path: _JSONFieldPath + routing_field_path = _ = public_field_path(routing_field_path_name, explanation: "it is referenced as an index `route_with` field") + + routing_field_path.path_parts.take(routing_field_path.path_parts.size - 1).each { |field| field.json_schema nullable: false } + routing_field_path.last_part.json_schema nullable: false, pattern: ElasticGraph::SchemaDefinition::Indexing::Index::HAS_NON_WHITE_SPACE_REGEX + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rb new file mode 100644 index 000000000..2ce6148b7 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rb @@ -0,0 +1,36 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/support/hash_util" +require "elastic_graph/support/json_schema/meta_schema_validator" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Validates JSON-schema-specific configuration supplied through schema definition APIs. + # + # @api private + module JSONSchemaOptionValidator + # Validates JSON schema options against the JSON meta-schema. + # + # @param schema_element [Object] the schema element being configured (used in error messages) + # @param options [Hash] the JSON schema options to validate + # @raise [Errors::SchemaError] if the options are invalid + # @return [void] + def self.validate!(schema_element, options) + validatable_json_schema = Support::HashUtil.stringify_keys(options) + + if (error_msg = Support::JSONSchema.strict_meta_schema_validator.validate_with_error_message(validatable_json_schema)) + raise Errors::SchemaError, "Invalid JSON schema options set on #{schema_element}:\n\n#{error_msg}" + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb new file mode 100644 index 000000000..b776cb097 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb @@ -0,0 +1,132 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/errors" +require "elastic_graph/json_ingestion/schema_definition/indexing/event_envelope" +require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::Results} that adds + # JSON Schema generation support. + # + # @private + module ResultsExtension + # @param version [Integer] desired JSON schema version + # @return [Hash] the JSON schema for the requested version, if available + # @raise [Errors::NotFoundError] if the requested JSON schema version is not available + def json_schemas_for(version) + unless available_json_schema_versions.include?(version) + raise Errors::NotFoundError, "The requested json schema version (#{version}) is not available. Available versions: #{available_json_schema_versions.to_a.join(", ")}." + end + + @latest_versioned_json_schema ||= merge_field_metadata_into_json_schema(current_public_json_schema).json_schema + end + + # @return [Set] set of available JSON schema versions + def available_json_schema_versions + @available_json_schema_versions ||= Set[latest_json_schema_version] + end + + # @return [Integer] the current JSON schema version + def latest_json_schema_version + current_public_json_schema[JSON_SCHEMA_VERSION_KEY] + end + + # @private + def json_schema_version_setter_location + json_ingestion_state.json_schema_version_setter_location + end + + # @private + def json_schema_field_metadata_by_type_and_field_name + @json_schema_field_metadata_by_type_and_field_name ||= json_schema_indexing_field_types_by_name + .transform_values(&:json_schema_field_metadata_by_field_name) + end + + # @private + def current_public_json_schema + @current_public_json_schema ||= build_public_json_schema + end + + # @private + def merge_field_metadata_into_json_schema(json_schema) + json_schema_with_metadata_merger.merge_metadata_into(json_schema) + end + + # @private + def unused_deprecated_elements + json_schema_with_metadata_merger.unused_deprecated_elements + end + + private + + # Returns the wrapped state narrowed to include this gem's `StateExtension`. Centralizes + # the Steep cast that's needed because Steep can't see the `extend(StateExtension)` applied + # at runtime in {APIExtension.extended}. + def json_ingestion_state + state # : ElasticGraph::SchemaDefinition::State & StateExtension + end + + def json_schema_with_metadata_merger + @json_schema_with_metadata_merger ||= Indexing::JSONSchemaWithMetadata::Merger.new(self) + end + + def build_public_json_schema + json_schema_version = json_ingestion_state.json_schema_version + if json_schema_version.nil? + raise Errors::SchemaError, "`json_schema_version` must be specified in the schema. To resolve, add `schema.json_schema_version 1` in a schema definition block." + end + + root_document_type_names = state.object_types_by_name.values + .select { |type| type.root_document_type? && !type.abstract? } + .reject { |type| derived_indexing_type_names.include?(type.name) } + .map(&:name) + + definitions_by_name = json_schema_indexing_field_types_by_name + .transform_values(&:to_json_schema) + .compact + + { + "$schema" => JSON_META_SCHEMA, + JSON_SCHEMA_VERSION_KEY => json_schema_version, + "$defs" => { + "ElasticGraphEventEnvelope" => Indexing::EventEnvelope.json_schema(root_document_type_names, json_schema_version) + }.merge(definitions_by_name) + } + end + + def json_schema_indexing_field_types_by_name + # Force `all_types` to materialize before iterating `state.types_by_name`. Reading `all_types` + # runs the `on_built_in_types` callbacks (registered by `APIExtension.extended`) that attach + # `json_schema_options` onto the built-in scalar types — those options must be set before we + # call `to_indexing_field_type` on any of them or the resulting JSON schema is incomplete. + all_types + + @json_schema_indexing_field_types_by_name ||= state + .types_by_name + .except("Query") + .values + .reject do |t| + derived_indexing_type_names.include?(t.name) || + # Skip graphql framework types + t.graphql_only? + end + .sort_by(&:name) + .to_h do |type| + # @type var indexing_field_type: Indexing::_JSONFieldType + indexing_field_type = _ = type.to_indexing_field_type + [type.name, indexing_field_type] + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb new file mode 100644 index 000000000..c4c0f6ef4 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb @@ -0,0 +1,267 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/json_schema_pruner" +require "yaml" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::SchemaArtifactManager} that adds + # JSON Schema artifact generation support. + # + # @private + module SchemaArtifactManagerExtension + # Overrides `dump_artifacts` to add JSON schema version bump checking before dumping. + def dump_artifacts + schema_results = json_ingestion_schema_definition_results + state = json_ingestion_state + + check_if_needs_json_schema_version_bump do |recommended_json_schema_version| + if state.enforce_json_schema_version + # @type var setter_location: ::Thread::Backtrace::Location + # We use `_ =` because while `json_schema_version_setter_location` can be nil, + # it'll never be nil if we get here and we want the type to be non-nilable. + setter_location = _ = schema_results.json_schema_version_setter_location + setter_location_path = ::Pathname.new(setter_location.absolute_path.to_s).relative_path_from(::Dir.pwd) + + abort "A change has been attempted to `json_schemas.yaml`, but the `json_schema_version` has not been correspondingly incremented. Please " \ + "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \ + "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ + " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ + "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \ + "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." + else + @output.puts <<~EOS + WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. + This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`. + EOS + end + end + + super + end + + private + + # Returns the wrapped {ElasticGraph::SchemaDefinition::Results} narrowed to include this + # gem's `ResultsExtension`. Centralizes the Steep cast that's needed because Steep can't + # see the `extend(ResultsExtension)` applied at runtime. + def json_ingestion_schema_definition_results + schema_definition_results # : ElasticGraph::SchemaDefinition::Results & ResultsExtension + end + + def json_ingestion_state + json_ingestion_schema_definition_results.state # : ElasticGraph::SchemaDefinition::State & StateExtension + end + + # Overrides the base `artifacts_from_schema_def` method to add JSON schema artifacts. + def artifacts_from_schema_def + base_artifacts = super + + versioned_artifacts = build_desired_versioned_json_schemas(json_schemas_artifact.desired_contents).values.map do |versioned_schema| + new_versioned_json_schema_artifact(versioned_schema) + end + + base_artifacts + [json_schemas_artifact] + versioned_artifacts + end + + def json_schemas_artifact + schema_results = json_ingestion_schema_definition_results + + @json_schemas_artifact ||= new_yaml_artifact( + JSON_SCHEMAS_FILE, + JSONSchemaPruner.prune(schema_results.current_public_json_schema), + extra_comment_lines: [ + "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", + "they can perform code generation and event validation." + ] + ) + end + + def check_if_needs_json_schema_version_bump(&block) + if json_schemas_artifact.out_of_date? + existing_schema_version = json_schemas_artifact.existing_dumped_contents&.dig(JSON_SCHEMA_VERSION_KEY) || -1 + desired_schema_version = json_schemas_artifact.desired_contents[JSON_SCHEMA_VERSION_KEY] + + if existing_schema_version >= desired_schema_version + yield existing_schema_version + 1 + end + end + end + + def build_desired_versioned_json_schemas(current_public_json_schema) + schema_results = json_ingestion_schema_definition_results + versioned_parsed_yamls = ::Dir.glob(::File.join(@schema_artifacts_directory, JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v*.yaml")).map do |file| + ::YAML.safe_load_file(file) + end + [current_public_json_schema] + + results_by_json_schema_version = versioned_parsed_yamls.to_h do |parsed_yaml| + merged_schema = schema_results.merge_field_metadata_into_json_schema(parsed_yaml) + [merged_schema.json_schema_version, merged_schema] + end + + report_json_schema_merge_errors(results_by_json_schema_version.values) + report_json_schema_merge_warnings + + results_by_json_schema_version.transform_values(&:json_schema) + end + + def report_json_schema_merge_errors(merged_results) + json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] + json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] + json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] + + merged_results.each do |result| + result.missing_fields.each do |field| + json_schema_versions_by_missing_field[field] << result.json_schema_version + end + + result.missing_types.each do |type| + json_schema_versions_by_missing_type[type] << result.json_schema_version + end + + result.missing_necessary_fields.each do |missing_necessary_field| + json_schema_versions_by_missing_necessary_field[missing_necessary_field] << result.json_schema_version + end + end + + missing_field_errors = json_schema_versions_by_missing_field.map do |field, json_schema_versions| + missing_field_error_for(field, json_schema_versions) + end + + missing_type_errors = json_schema_versions_by_missing_type.map do |type, json_schema_versions| + missing_type_error_for(type, json_schema_versions) + end + + missing_necessary_field_errors = json_schema_versions_by_missing_necessary_field.map do |field, json_schema_versions| + missing_necessary_field_error_for(field, json_schema_versions) + end + + definition_conflict_errors = merged_results + .flat_map { |result| result.definition_conflicts.to_a } + .group_by(&:name) + .map do |name, deprecated_elements| + <<~EOS + The schema definition of `#{name}` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: + + #{format_deprecated_elements(deprecated_elements)} + EOS + end + + errors = missing_field_errors + missing_type_errors + missing_necessary_field_errors + definition_conflict_errors + return if errors.empty? + + abort errors.join("\n\n") + end + + def report_json_schema_merge_warnings + schema_results = json_ingestion_schema_definition_results + unused_elements = schema_results.unused_deprecated_elements + return if unused_elements.empty? + + @output.puts <<~EOS + The schema definition has #{unused_elements.size} unneeded reference(s) to deprecated schema elements. These can all be safely deleted: + + #{format_deprecated_elements(unused_elements)} + + EOS + end + + def format_deprecated_elements(deprecated_elements) + descriptions = deprecated_elements + .sort_by { |e| [e.defined_at.path, e.defined_at.lineno] } + .map(&:description) + .uniq + + descriptions.each.with_index(1).map { |desc, idx| "#{idx}. #{desc}" }.join("\n") + end + + def missing_field_error_for(qualified_field, json_schema_versions) + type, field = qualified_field.split(".") + + <<~EOS + The `#{qualified_field}` field (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this field's data when ingesting events at #{old_versions(json_schema_versions)}. + To continue, do one of the following: + + 1. If the `#{qualified_field}` field has been renamed, indicate this by calling `field.renamed_from "#{field}"` on the renamed field. + 2. If the `#{qualified_field}` field has been dropped, indicate this by calling `type.deleted_field "#{field}"` on the `#{type}` type. + 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def missing_type_error_for(type, json_schema_versions) + <<~EOS + The `#{type}` type (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. + ElasticGraph cannot guess what it should do with this type's data when ingesting events at #{old_versions(json_schema_versions)}. + To continue, do one of the following: + + 1. If the `#{type}` type has been renamed, indicate this by calling `type.renamed_from "#{type}"` on the renamed type. + 2. If the `#{type}` type has been dropped, indicate this by calling `schema.deleted_type "#{type}"` on the schema. + 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def missing_necessary_field_error_for(field, json_schema_versions) + path = field.fully_qualified_path.split(".").last + # :nocov: -- we only cover one side of this ternary. + has_or_have = (json_schema_versions.size == 1) ? "has" : "have" + # :nocov: + + <<~EOS + #{describe_json_schema_versions(json_schema_versions, "and")} #{has_or_have} no field that maps to the #{field.field_type} field path of `#{field.fully_qualified_path}`. + Since the field path is required for #{field.field_type}, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: + + 1. If the `#{field.fully_qualified_path}` field has been renamed, indicate this by calling `field.renamed_from "#{path}"` on the renamed field rather than using `deleted_field`. + 2. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. + EOS + end + + def describe_json_schema_versions(json_schema_versions, conjunction) + json_schema_versions = json_schema_versions.sort + + case json_schema_versions.size + when 1 + "JSON schema version #{json_schema_versions.first}" + when 2 + "JSON schema versions #{json_schema_versions.first} #{conjunction} #{json_schema_versions.last}" + else + versions = json_schema_versions.take(json_schema_versions.size - 1) + "JSON schema versions #{versions.join(", ")}, #{conjunction} #{json_schema_versions.last}" + end + end + + def old_versions(json_schema_versions) + return "this old version" if json_schema_versions.size == 1 + "these old versions" + end + + def files_noun_phrase(json_schema_versions) + return "its file" if json_schema_versions.size == 1 + "their files" + end + + def new_versioned_json_schema_artifact(desired_contents) + # File name depends on the schema_version field in the json schema. + schema_version = desired_contents[JSON_SCHEMA_VERSION_KEY] + + new_yaml_artifact( + ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{schema_version}.yaml"), + desired_contents, + extra_comment_lines: [ + "This JSON schema file contains internal ElasticGraph metadata and should be considered private.", + "The unversioned JSON schema file is public and intended to be provided to publishers." + ] + ) + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rb new file mode 100644 index 000000000..68eb12abd --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rb @@ -0,0 +1,24 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + # Extends enum types with JSON schema behavior. + module EnumTypeExtension + # @private + def configure_derived_scalar_type(scalar_type) + super + scalar_type.json_schema type: "string" + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb new file mode 100644 index 000000000..74b1a1669 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb @@ -0,0 +1,99 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/json_ingestion/schema_definition/indexing/field_reference" +require "elastic_graph/json_ingestion/schema_definition/json_schema_option_validator" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Namespace for schema-element extensions that contribute JSON schema generation and validation. + module SchemaElements + # Extends schema-definition fields with JSON schema validation behavior. + module FieldExtension + # @return [Hash] JSON schema options for this field + def json_schema_options + @json_schema_options ||= {} + end + + # @return [Boolean] whether this field has been marked as non-nullable in the JSON schema + def non_nullable_in_json_schema? + @non_nullable_in_json_schema || false + end + + # Sets whether this field is non-nullable in the JSON schema. + # @param value [Boolean] true to make the field non-nullable + attr_writer :non_nullable_in_json_schema + + # Defines JSON Schema validations for this field. Validations defined here will be included in the + # generated `json_schemas.yaml` artifact, which ElasticGraph uses to validate events before indexing. + # Publishers may also use the artifact for code generation and validation before publishing events. + # + # Can be called multiple times; each call merges options into the existing set. + # + # Use these validations sparingly. Events that violate JSON Schema validation fail to index, so we + # recommend limiting them to constraints that ElasticGraph needs in order to operate correctly. + # `nullable: false` is also supported to disallow `null` values in indexed data while keeping the + # GraphQL field nullable. + # + # @param nullable [Boolean, nil] set to `false` to make this field non-nullable in the JSON schema + # @param options [Hash] additional JSON schema options + # @return [void] + # + # @example Define additional validations on a field + # ElasticGraph.define_schema do |schema| + # schema.object_type "Card" do |t| + # t.field "id", "ID!" + # t.field "expYear", "Int" do |f| + # f.json_schema minimum: 2000, maximum: 2099 + # end + # + # t.index "cards" + # end + # end + def json_schema(nullable: nil, **options) + if options.key?(:type) + raise Errors::SchemaError, "Cannot override JSON schema type of field `#{name}` with `#{options.fetch(:type)}`" + end + + case nullable + when true + raise Errors::SchemaError, "`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead." + when false + @non_nullable_in_json_schema = true + end + + JSONSchemaOptionValidator.validate!(self, options) + json_schema_options.update(options) + end + + # @private + def to_indexing_field_reference + reference = super + return nil unless reference + + Indexing::FieldReference.new( + reference.with(type: type_for_json_schema), + json_schema_layers: type_for_json_schema.json_schema_layers, + json_schema_customizations: json_schema_options + ) + end + + private + + def type_for_json_schema + (non_nullable_in_json_schema? ? type.wrap_non_null : type).then do |type_ref| + type_ref # : ElasticGraph::SchemaDefinition::SchemaElements::TypeReference & TypeReferenceExtension + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rb new file mode 100644 index 000000000..5be155fca --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rb @@ -0,0 +1,46 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/json_ingestion/schema_definition/json_schema_option_validator" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + # Extends object and interface types with JSON schema behavior. + module ObjectInterfaceExtension + # @return [Hash] JSON schema options for this type + def json_schema_options + @json_schema_options ||= {} + end + + # Configures JSON Schema options for this object or interface type. + # + # Can be called multiple times; each call merges options into the existing set. When options are present, + # they replace ElasticGraph's generated object schema for this type. + # + # @param options [Hash] JSON schema options + # @return [void] + def json_schema(**options) + JSONSchemaOptionValidator.validate!(self, options) + json_schema_options.update(options) + end + + # @private + def to_indexing_field_type + # @type var field_type: Indexing::FieldType::Object | Indexing::FieldType::Union + field_type = _ = super + return field_type unless field_type.is_a?(Indexing::FieldType::Object) + + Indexing::FieldType::Object.new(field_type.__getobj__, json_schema_options: json_schema_options) + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb new file mode 100644 index 000000000..99ef5d199 --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb @@ -0,0 +1,83 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/json_ingestion/schema_definition/json_schema_option_validator" + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + # Extends scalar types with JSON schema validation and serialization behavior. + module ScalarTypeExtension + # @return [Hash] JSON schema options for this scalar type + def json_schema_options + @json_schema_options ||= {} + end + + # Defines the JSON Schema representation of this scalar type. This is required for custom scalar types + # because ElasticGraph cannot infer how arbitrary scalars are represented in indexing events. + # + # Can be called multiple times; each call merges options into the existing set. + # + # @param options [Hash] JSON schema options + # @return [void] + # + # @example Define the JSON Schema representation of a custom scalar type + # ElasticGraph.define_schema do |schema| + # schema.scalar_type "URL" do |t| + # t.mapping type: "keyword" + # t.json_schema type: "string", format: "uri" + # end + # end + def json_schema(**options) + JSONSchemaOptionValidator.validate!(self, options) + json_schema_options.update(options) + self.runtime_metadata = runtime_metadata.with(grouping_missing_value_placeholder: inferred_grouping_missing_value_placeholder) unless grouping_missing_value_placeholder_overridden + end + + # Validates that json_schema has been configured on this scalar type. + # + # @raise [Errors::SchemaError] if json_schema has not been configured + # @return [void] + def validate_json_schema_configuration! + return unless json_schema_options.empty? + + raise Errors::SchemaError, "Scalar types require `json_schema` to be configured, but `#{name}` lacks `json_schema`." + end + + private + + def inferred_grouping_missing_value_placeholder + case mapping_type + when "long" + # It is only safe to use NaN for a long when the long's range is safe to coerce to a float + # without loss of precision. This is because using NaN as the missing value will cause + # the datastore to coerce the other bucket keys to float. + # JSON schema min/max only constrains newly indexed values, not existing data that may fall + # outside the range before the constraints were added. In that edge case, users can set + # `grouping_missing_value_placeholder` to `nil`. + if (json_schema_options[:minimum] || LONG_STRING_MIN) >= JSON_SAFE_LONG_MIN && + (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX + inferred_numeric_placeholder_for_integer_type + end + when "unsigned_long" + # Similar to the checks above for long except we only need to check the max + # (since the min is zero even if not specified). + if (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX + inferred_numeric_placeholder_for_integer_type + end + else + super + end + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rb new file mode 100644 index 000000000..599a3df4d --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rb @@ -0,0 +1,49 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + # Extends schema-definition type references with JSON schema layer calculation. + # + # @private + module TypeReferenceExtension + # Returns all JSON schema array/nullable layers of a type, from outermost to innermost. + # For example, `[[Int]]` returns `[:nullable, :array, :nullable, :array, :nullable]`. + def json_schema_layers + @json_schema_layers ||= begin + layers, inner_type = peel_json_schema_layers_once + + if layers.empty? || inner_type == self + layers + else + layers + inner_type.json_schema_layers + end + end + end + + private + + def peel_json_schema_layers_once + if list? + inner_type = unwrap_list # : ElasticGraph::SchemaDefinition::SchemaElements::TypeReference & TypeReferenceExtension + return [[:array], inner_type] if non_null? + return [[:nullable, :array], inner_type] + end + + no_layers = [] # : ::Array[ElasticGraph::SchemaDefinition::jsonSchemaLayer] + inner_type = unwrap_non_null # : ElasticGraph::SchemaDefinition::SchemaElements::TypeReference & TypeReferenceExtension + return [no_layers, inner_type] if non_null? + [[:nullable], self] + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb new file mode 100644 index 000000000..d072bb02d --- /dev/null +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/state_extension.rb @@ -0,0 +1,33 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module JSONIngestion + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::State` to support JSON ingestion state. + # + # @private + module StateExtension + # @dynamic json_schema_version, json_schema_version= + # @dynamic json_schema_version_setter_location, json_schema_version_setter_location= + # @dynamic enforce_json_schema_version, enforce_json_schema_version= + # @dynamic allow_omitted_json_schema_fields, allow_omitted_json_schema_fields= + # @dynamic allow_extra_json_schema_fields, allow_extra_json_schema_fields= + attr_accessor :json_schema_version, :json_schema_version_setter_location, :enforce_json_schema_version, :allow_omitted_json_schema_fields, :allow_extra_json_schema_fields + + def self.extended(state) + state.json_schema_version = nil + state.json_schema_version_setter_location = nil + state.enforce_json_schema_version = true + state.allow_omitted_json_schema_fields = false + state.allow_extra_json_schema_fields = true + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs new file mode 100644 index 000000000..12f309219 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/api_extension.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module APIExtension: ::ElasticGraph::SchemaDefinition::API + BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME: ::Hash[::String, ::Hash[::Symbol, untyped]] + + def json_schema_version: (::Integer) -> void + def enforce_json_schema_version: (bool) -> void + def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void + + def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void + + private + + def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension) + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs new file mode 100644 index 000000000..8a14b01b5 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs @@ -0,0 +1,27 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module FactoryExtension: ::ElasticGraph::SchemaDefinition::Factory + def new_enum_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) + def new_enum_indexing_field_type: (::Array[::String]) -> Indexing::FieldType::Enum + def new_field: (**untyped) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension) + def new_index: (::String, ::Hash[::String, untyped], ::ElasticGraph::SchemaDefinition::indexableType) ?{ (::ElasticGraph::SchemaDefinition::Indexing::Index & Indexing::IndexExtension) -> void } -> (::ElasticGraph::SchemaDefinition::Indexing::Index & Indexing::IndexExtension) + def new_interface_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceExtension) + def new_object_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceExtension) + def new_object_indexing_field_type: ( + type_name: ::String, + subfields: ::Array[Indexing::Field], + mapping_options: ::ElasticGraph::SchemaDefinition::Mixins::HasTypeInfo::optionsHash, + doc_comment: ::String?, + ?json_schema_options: ::Hash[::Symbol, untyped] + ) -> Indexing::FieldType::Object + def new_scalar_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) + def new_scalar_indexing_field_type: (scalar_type: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> Indexing::FieldType::Scalar + def new_type_reference: (::String) -> ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference + def new_union_indexing_field_type: (::Hash[::String, Indexing::FieldType::Object]) -> Indexing::FieldType::Union + def new_results: () -> ::ElasticGraph::SchemaDefinition::Results + def new_schema_artifact_manager: (**untyped) -> ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs new file mode 100644 index 000000000..d3ff6d0e0 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs @@ -0,0 +1,41 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + class FieldSupertype < ::ElasticGraph::SchemaDefinition::Indexing::Field + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::Field) -> void + end + + class Field < FieldSupertype + attr_reader json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray + attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] + + @json_schema: ::Hash[::String, untyped]? + + JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE: ::Hash[::String, ::Hash[::String, untyped]] + + def initialize: ( + ::ElasticGraph::SchemaDefinition::Indexing::Field field, + json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray, + json_schema_customizations: ::Hash[::Symbol, untyped] + ) -> void + + def json_schema: () -> ::Hash[::String, untyped] + def json_schema_metadata: () -> JSONSchemaFieldMetadata + def nullable?: () -> bool + def ==: (untyped other) -> bool + def eql?: (untyped other) -> bool + def hash: () -> ::Integer + def __getobj__: () -> ::ElasticGraph::SchemaDefinition::Indexing::Field + + private + + def inner_json_schema: () -> ::Hash[::String, untyped] + def outer_json_schema_customizations: () -> ::Hash[::String, untyped] + def user_specified_json_schema_customizations_go_on_outside?: () -> bool + def process_layer: (::Symbol layer, ::Hash[::String, untyped] schema) -> ::Hash[::String, untyped] + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs new file mode 100644 index 000000000..84c7faed8 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + class FieldReferenceSupertype < ::ElasticGraph::SchemaDefinition::Indexing::FieldReference + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::FieldReference) -> void + end + + class FieldReference < FieldReferenceSupertype + attr_reader json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray + attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] + + def initialize: ( + ::ElasticGraph::SchemaDefinition::Indexing::FieldReference field_reference, + json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray, + json_schema_customizations: ::Hash[::Symbol, untyped] + ) -> void + + def resolve: () -> Field? + def ==: (untyped other) -> bool + def eql?: (untyped other) -> bool + def hash: () -> ::Integer + def __getobj__: () -> ::ElasticGraph::SchemaDefinition::Indexing::FieldReference + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type.rbs new file mode 100644 index 000000000..ab74a224d --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + interface _JSONFieldType + def to_mapping: () -> ::Hash[::String, untyped] + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rbs new file mode 100644 index 000000000..da9a29da1 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + class EnumSupertype < ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Enum + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::FieldType::Enum) -> void + end + + class Enum < EnumSupertype + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs new file mode 100644 index 000000000..6d157abaf --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs @@ -0,0 +1,37 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + class ObjectSupertype < ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Object + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::FieldType::Object) -> void + end + + class Object < ObjectSupertype + attr_reader json_schema_options: ::Hash[::Symbol, untyped] + + @to_json_schema: ::Hash[::String, untyped]? + + def initialize: ( + ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Object field_type, + json_schema_options: ::Hash[::Symbol, untyped] + ) -> void + + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + def ==: (untyped other) -> bool + def eql?: (untyped other) -> bool + def hash: () -> ::Integer + def __getobj__: () -> ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Object + + private + + def json_schema_typename_field: () -> ::Hash[::String, untyped] + def validate_sourced_fields_have_no_json_schema_overrides: (::Array[Field]) -> void + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rbs new file mode 100644 index 000000000..ce85b0b8c --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + class ScalarSupertype < ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Scalar + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::FieldType::Scalar) -> void + end + + class Scalar < ScalarSupertype + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rbs new file mode 100644 index 000000000..5fbae3383 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + module FieldType + class UnionSupertype < ::ElasticGraph::SchemaDefinition::Indexing::FieldType::Union + def initialize: (::ElasticGraph::SchemaDefinition::Indexing::FieldType::Union) -> void + end + + class Union < UnionSupertype + def self.new: (::ElasticGraph::SchemaDefinition::Indexing::FieldType::Union) -> instance + + def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONSchemaFieldMetadata] + def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] + def to_json_schema: () -> ::Hash[::String, untyped] + end + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rbs new file mode 100644 index 000000000..8084031eb --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/index_extension.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module Indexing + interface _JSONFieldPath + def last_part: () -> (::ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension) + def path_parts: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::Field & SchemaElements::FieldExtension] + def type: () -> ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference + def full_description: () -> ::String + end + + module IndexExtension: ::ElasticGraph::SchemaDefinition::Indexing::Index + def require_id_in_json_schema: () -> void + def rollover: (::Symbol, ::String) -> void + def route_with: (::String) -> void + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs index f78578800..4ddde82f6 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs @@ -40,7 +40,7 @@ module ElasticGraph attr_reader unused_deprecated_elements: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] - def initialize: (ElasticGraph::SchemaDefinition::Results) -> void + def initialize: ((ElasticGraph::SchemaDefinition::Results & ResultsExtension)) -> void def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata private diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rbs new file mode 100644 index 000000000..c4e10cb4d --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_option_validator.rbs @@ -0,0 +1,9 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module JSONSchemaOptionValidator + def self.validate!: (untyped, ::Hash[::Symbol, untyped]) -> void + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs new file mode 100644 index 000000000..bd617f7d5 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs @@ -0,0 +1,30 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module ResultsExtension : ::ElasticGraph::SchemaDefinition::Results + def json_schemas_for: (::Integer) -> ::Hash[::String, untyped] + def available_json_schema_versions: () -> ::Set[::Integer] + def latest_json_schema_version: () -> ::Integer + def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? + def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]] + def current_public_json_schema: () -> ::Hash[::String, untyped] + def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> Indexing::JSONSchemaWithMetadata + def unused_deprecated_elements: () -> ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] + + private + + @latest_versioned_json_schema: ::Hash[::String, untyped]? + @available_json_schema_versions: ::Set[::Integer]? + @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]? + @current_public_json_schema: ::Hash[::String, untyped]? + @json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger? + @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_JSONFieldType]? + + def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension) + def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger + def build_public_json_schema: () -> ::Hash[::String, untyped] + def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_JSONFieldType] + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs new file mode 100644 index 000000000..107769d01 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs @@ -0,0 +1,28 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaArtifactManagerExtension : ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + private + + @json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]? + + def json_ingestion_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension) + def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension) + def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]] + def json_schemas_artifact: () -> ::ElasticGraph::SchemaDefinition::SchemaArtifact[::Hash[::String, untyped]] + def check_if_needs_json_schema_version_bump: () { (::Integer) -> void } -> void + def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] + def report_json_schema_merge_errors: (::Array[Indexing::JSONSchemaWithMetadata]) -> void + def report_json_schema_merge_warnings: () -> void + def format_deprecated_elements: (::Enumerable[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]) -> ::String + def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String + def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String + def missing_necessary_field_error_for: (Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String + def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String + def old_versions: (::Array[::Integer]) -> ::String + def files_noun_phrase: (::Array[::Integer]) -> ::String + def new_versioned_json_schema_artifact: (::Hash[::String, untyped]) -> ::ElasticGraph::SchemaDefinition::SchemaArtifact[::Hash[::String, untyped]] + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rbs new file mode 100644 index 000000000..12d81c14b --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/enum_type_extension.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + module EnumTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType + private + + def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & ScalarTypeExtension) -> void + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rbs new file mode 100644 index 000000000..bf1007001 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rbs @@ -0,0 +1,21 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + module FieldExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::Field + @json_schema_options: ::Hash[::Symbol, untyped] + @non_nullable_in_json_schema: bool + + def json_schema_options: () -> ::Hash[::Symbol, untyped] + def non_nullable_in_json_schema?: () -> bool + def json_schema: (?nullable: bool?, **untyped) -> void + def to_indexing_field_reference: () -> Indexing::FieldReference? + + private + + def type_for_json_schema: () -> (::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference & TypeReferenceExtension) + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rbs new file mode 100644 index 000000000..161725cac --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + module ObjectInterfaceExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::TypeWithSubfields + @json_schema_options: ::Hash[::Symbol, untyped] + + def json_schema_options: () -> ::Hash[::Symbol, untyped] + def json_schema: (**untyped) -> void + def to_indexing_field_type: () -> (Indexing::FieldType::Object | Indexing::FieldType::Union) + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs new file mode 100644 index 000000000..c659b4ec8 --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs @@ -0,0 +1,19 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + module ScalarTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType + @json_schema_options: ::Hash[::Symbol, untyped] + + def json_schema_options: () -> ::Hash[::Symbol, untyped] + def json_schema: (**untyped) -> void + def validate_json_schema_configuration!: () -> void + + private + + def inferred_grouping_missing_value_placeholder: () -> ::String? + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rbs new file mode 100644 index 000000000..4ff5b792b --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension.rbs @@ -0,0 +1,16 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module SchemaElements + module TypeReferenceExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference + @json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray? + def json_schema_layers: () -> ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray + + private + + def peel_json_schema_layers_once: () -> [::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray, (::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference & TypeReferenceExtension)] + end + end + end + end +end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs new file mode 100644 index 000000000..06f930cbb --- /dev/null +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/state_extension.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module JSONIngestion + module SchemaDefinition + module StateExtension: ::ElasticGraph::SchemaDefinition::State + attr_accessor json_schema_version: ::Integer? + attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? + attr_accessor enforce_json_schema_version: bool + attr_accessor allow_omitted_json_schema_fields: bool + attr_accessor allow_extra_json_schema_fields: bool + + def self.extended: (::ElasticGraph::SchemaDefinition::State & StateExtension) -> void + end + end + end +end diff --git a/elasticgraph-schema_definition/README.md b/elasticgraph-schema_definition/README.md index 5c06af483..b85f79b9e 100644 --- a/elasticgraph-schema_definition/README.md +++ b/elasticgraph-schema_definition/README.md @@ -21,9 +21,6 @@ graph LR; elasticgraph-indexer["elasticgraph-indexer"]; elasticgraph-schema_definition --> elasticgraph-indexer; class elasticgraph-indexer otherEgGemStyle; - elasticgraph-json_ingestion["elasticgraph-json_ingestion"]; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; - class elasticgraph-json_ingestion otherEgGemStyle; elasticgraph-schema_artifacts["elasticgraph-schema_artifacts"]; elasticgraph-schema_definition --> elasticgraph-schema_artifacts; class elasticgraph-schema_artifacts otherEgGemStyle; diff --git a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec index da50e4335..69b268ad5 100644 --- a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec +++ b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec @@ -43,7 +43,6 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-graphql", ElasticGraph::VERSION # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters) spec.add_dependency "elasticgraph-indexer", ElasticGraph::VERSION # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters) - spec.add_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_dependency "elasticgraph-schema_artifacts", ElasticGraph::VERSION spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION spec.add_dependency "graphql", "~> 2.6.2" @@ -52,5 +51,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-datastore_core", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + # The test suite exercises the default JSON ingestion extension, but the gem remains optional at runtime. + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb index 924f28cd6..d48299143 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb @@ -9,14 +9,15 @@ require "elastic_graph/errors" require "elastic_graph/schema_artifacts/runtime_metadata/extension" require "elastic_graph/schema_artifacts/runtime_metadata/graphql_resolver" -require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" -require "elastic_graph/schema_definition/results" -require "elastic_graph/schema_definition/state" # :nocov: -- only loaded on JRuby require "elastic_graph/schema_definition/jruby_patches" if RUBY_ENGINE == "jruby" # :nocov: +require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" +require "elastic_graph/schema_definition/results" +require "elastic_graph/schema_definition/state" + module ElasticGraph # The main entry point for schema definition from ElasticGraph applications. # @@ -294,7 +295,7 @@ def union_type(name, &block) # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end def scalar_type(name, &block) @@ -453,97 +454,32 @@ def results @results ||= @factory.new_results end - # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema - # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique - # version number. The publisher will then include this version number in published events to identify the version of the schema it - # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync. + # Records the JSON schema version of this schema definition. # - # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying to constantly - # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this - # on every JSON schema change with {#enforce_json_schema_version}. + # The default implementation is a no-op so that callers (and the test helpers) can invoke this + # uniformly regardless of whether an ingestion extension is loaded. When + # `elasticgraph-json_ingestion` is loaded, its `APIExtension` overrides this with real behavior. # - # @param version [Integer] current version number of the JSON schema artifact + # @param version [Integer] the JSON schema version # @return [void] - # @see #enforce_json_schema_version - # - # @example Set the JSON schema version to 1 - # ElasticGraph.define_schema do |schema| - # schema.json_schema_version 1 - # end def json_schema_version(version) - if !version.is_a?(Integer) || version < 1 - raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}" - end - - if @state.json_schema_version - raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}" - end - - @state.json_schema_version = version - @state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first - nil end - # Configures whether {SchemaArtifactManager} enforces the requirement that the JSON schema version is incremented every time - # dumping the JSON schemas results in a changed artifact. Defaults to `true`. - # - # @note Generally speaking, you will want this to be `true` for any ElasticGraph application that is in - # production as the versioning of JSON schemas is what supports safe schema evolution as it allows - # ElasticGraph to identify which version of the JSON schema the publishing system was operating on - # when it published an event. + # Records JSON schema version enforcement configuration. # - # It can be useful to set it to `false` before your application is in production, as you do not want - # to be forced to bump the version after every single schema change while you are building an initial - # prototype. + # The default implementation is a no-op; overridden by the JSON ingestion extension. # - # @param value [Boolean] whether to require `json_schema_version` to be incremented on changes that impact `json_schemas.yaml` + # @param value [Boolean] whether JSON schema version bumps should be enforced # @return [void] - # @see #json_schema_version - # - # @example Disable enforcement during initial prototyping - # ElasticGraph.define_schema do |schema| - # # TODO: remove this once we're past the prototyping stage - # schema.enforce_json_schema_version false - # end def enforce_json_schema_version(value) - unless value == true || value == false - raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}" - end - - @state.enforce_json_schema_version = value - nil end - # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the - # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to - # configure this behavior. - # - # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events. - # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events. - # @return [void] + # Records JSON schema strictness configuration. # - # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled - # field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher - # accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId` - # is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of - # these to `true` (or none). + # The default implementation is a no-op; overridden by the JSON ingestion extension. # - # @example Allow omitted fields and disallow extra fields - # ElasticGraph.define_schema do |schema| - # schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false - # end + # @return [void] def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true) - unless [true, false].include?(allow_omitted_fields) - raise Errors::SchemaError, "`allow_omitted_fields` must be true or false" - end - - unless [true, false].include?(allow_extra_fields) - raise Errors::SchemaError, "`allow_extra_fields` must be true or false" - end - - @state.allow_omitted_json_schema_fields = allow_omitted_fields - @state.allow_extra_json_schema_fields = allow_extra_fields - nil end # Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb index 062eb2255..0df60b76a 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb @@ -250,13 +250,12 @@ def new_object_type(name) end @@object_type_new = prevent_non_factory_instantiation_of(SchemaElements::ObjectType) - def new_object_indexing_field_type(type_name:, subfields:, mapping_options:, json_schema_options:, doc_comment:) + def new_object_indexing_field_type(type_name:, subfields:, mapping_options:, doc_comment:) @@object_indexing_field_type_new.call( schema_def_state: @state, type_name: type_name, subfields: subfields, mapping_options: mapping_options, - json_schema_options: json_schema_options, doc_comment: doc_comment ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb index 1401aa949..51dfd4a1f 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb @@ -7,7 +7,6 @@ # frozen_string_literal: true require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata" require "elastic_graph/schema_definition/indexing/list_counts_mapping" require "elastic_graph/support/hash_util" require "elastic_graph/support/memoizable_data" @@ -22,28 +21,13 @@ class Field < Support::MemoizableData.define( :name, :name_in_index, :type, - :json_schema_layers, :indexing_field_type, :accuracy_confidence, - :json_schema_customizations, :mapping_customizations, :source, :runtime_field_script, :doc_comment ) - # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema - # validation will reject values which cannot be indexed into fields of a specific mapping type. - # - # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation - # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type). - # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values. - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = { - "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1}, - "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1}, - "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, - "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} - } - # @return [Hash] the mapping for this field. The returned hash should be composed entirely # of Ruby primitives that, when converted to a JSON string, match the structure required by # [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html). @@ -63,23 +47,6 @@ def mapping end end - # @return [Hash] the JSON schema definition for this field. The returned object should - # be composed entirely of Ruby primitives that, when converted to a JSON string, match the - # requirements of [the JSON schema spec](https://json-schema.org/). - def json_schema - json_schema_layers - .reverse # resolve layers from innermost to outermost wrappings - .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) } - .merge(outer_json_schema_customizations) - .merge({"description" => doc_comment}.compact) - .then { |h| Support::HashUtil.stringify_keys(h) } - end - - # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field. - def json_schema_metadata - JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) - end - # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the # datastore does so that consistency checks between our index configuration and what's in the datastore # work properly. @@ -107,80 +74,6 @@ def self.normalized_mapping_hash_for(fields) mapping_hash end - - def nullable? - json_schema_layers.include?(:nullable) - end - - private - - def inner_json_schema - user_specified_customizations = - if user_specified_json_schema_customizations_go_on_outside? - {} # : ::Hash[::String, untyped] - else - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {} - customizations = customizations_from_mapping.merge(user_specified_customizations) - customizations = indexing_field_type.format_field_json_schema_customizations(customizations) - - ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"} - return ref if customizations.empty? - - # Combine any customizations with type ref under an "allOf" subschema: - # All of these properties must hold true for the type to be valid. - # - # Note that if we simply combine the customizations with the `$ref` - # at the same level, it will not work, because other subschema - # properties are ignored when they are in the same object as a `$ref`: - # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168 - {"allOf" => [ref, customizations]} - end - - def outer_json_schema_customizations - return {} unless user_specified_json_schema_customizations_go_on_outside? - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - # Indicates if the user-specified JSON schema customizations should go on the inside - # (where they normally go) or on the outside. They only go on the outside when it's - # an array field, because then they apply to the array itself instead of the items in the - # array. - def user_specified_json_schema_customizations_go_on_outside? - json_schema_layers.include?(:array) - end - - def process_layer(layer, schema) - case layer - when :nullable - make_nullable(schema) - when :array - make_array(schema) - else - # :nocov: - layer is only ever `:nullable` or `:array` so we never get here - schema - # :nocov: - end - end - - def make_nullable(schema) - # Here we use "anyOf" to ensure that JSON can either match the schema OR null. - # - # (Using "oneOf" would mean that if we had a schema that also allowed null, - # null would never be allowed, since "oneOf" must match exactly one subschema). - { - "anyOf" => [ - schema, - {"type" => "null"} - ] - } - end - - def make_array(schema) - {"type" => "array", "items" => schema} - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb index 070f70db3..66e4ce909 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb @@ -6,27 +6,25 @@ # # frozen_string_literal: true +require "elastic_graph/support/memoizable_data" + module ElasticGraph module SchemaDefinition module Indexing - # @!parse class FieldReference < ::Data; end - FieldReference = ::Data.define( + # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be + # resolvable yet (which is why this exists). + # + # @api private + class FieldReference < Support::MemoizableData.define( :name, :name_in_index, :type, :mapping_options, - :json_schema_options, :accuracy_confidence, :source, :runtime_field_script, :doc_comment ) - - # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be - # resolvable yet (which is why this exists). - # - # @api private - class FieldReference < ::Data # @return [Field, nil] the {Field} this reference resolves to (if it can be resolved) def resolve return nil unless (resolved_type = type.fully_unwrapped.resolved) @@ -35,10 +33,8 @@ def resolve name: name, name_in_index: name_in_index, type: type, - json_schema_layers: type.json_schema_layers, indexing_field_type: resolved_type.to_indexing_field_type, accuracy_confidence: accuracy_confidence, - json_schema_customizations: json_schema_options, mapping_customizations: mapping_options, source: source, runtime_field_script: runtime_field_script, @@ -46,7 +42,10 @@ def resolve ) end - # @dynamic initialize, with, name, name_in_index, type, mapping_options, json_schema_options, accuracy_confidence, source, runtime_field_script, doc_comment + private def after_initialize + end + + # @dynamic initialize, with, name, name_in_index, type, mapping_options, accuracy_confidence, source, runtime_field_script, doc_comment end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb index d2f72fb6f..c2a487c97 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb @@ -6,6 +6,8 @@ # # frozen_string_literal: true +require "elastic_graph/support/memoizable_data" + module ElasticGraph module SchemaDefinition module Indexing @@ -13,48 +15,19 @@ module Indexing # # @api private module FieldType - # @!parse class Enum < ::Data; end - Enum = ::Data.define(:enum_value_names) - - # Responsible for the JSON schema and mapping of a {SchemaElements::EnumType}. + # Responsible for the datastore mapping of a {SchemaElements::EnumType}. # # @!attribute [r] enum_value_names # @return [Array] list of names of values in this enum type. # # @api private - class Enum < ::Data - # @return [Hash] the JSON schema for this enum type. - def to_json_schema - {"type" => "string", "enum" => enum_value_names} - end - + class Enum < Support::MemoizableData.define(:enum_value_names) # @return [Hash] the datastore mapping for this enum type. def to_mapping {"type" => "keyword"} end - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this enum type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep - # other customizations (such as the `maxLength` field customization EG automatically applies to fields - # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway). - # - # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema - # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found - # that the presence of extra field customizations inhibits the library's ability to generate code in the way - # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to - # being wrapped in an `allOf`). - # - # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed - # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to - # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR). - customizations.slice("enum") + private def after_initialize end # @dynamic initialize, enum_value_names diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb index c01fcbd4e..c02e463b6 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb @@ -14,7 +14,7 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - # Responsible for the JSON schema and mapping of a {SchemaElements::ObjectType}. + # Responsible for the mapping of a {SchemaElements::ObjectType}. # # @!attribute [r] type_name # @return [String] name of the object type @@ -22,13 +22,11 @@ module FieldType # @return [Array] the subfields of this object type # @!attribute [r] mapping_options # @return [Hash] options to be included in the mapping - # @!attribute [r] json_schema_options - # @return [Hash] options to be included in the JSON schema # @!attribute [r] doc_comment # @return [String, nil] documentation for the type # # @api private - class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options, :json_schema_options, :doc_comment) + class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options, :doc_comment) # @return [Hash] the datastore mapping for this object type. def to_mapping @to_mapping ||= begin @@ -41,77 +39,10 @@ def to_mapping end end - # @return [Hash] the JSON schema for this object type. - def to_json_schema - @to_json_schema ||= - if json_schema_options.empty? - # Fields that are `sourced_from` an alternate type must not be included in this types JSON schema, - # since events of this type won't include them. - other_source_subfields, json_schema_candidate_subfields = subfields.partition(&:source) - validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script) - required_fields = json_schema_subfields - required_fields = required_fields.reject(&:nullable?) if schema_def_state.allow_omitted_json_schema_fields - - { - "type" => "object", - "properties" => json_schema_subfields.to_h { |f| [f.name, f.json_schema] }.merge(json_schema_typename_field), - # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present - # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want - # to require it in the context of a union type. The union's json schema requires the field. - "required" => required_fields.map(&:name).freeze, - "additionalProperties" => (false unless schema_def_state.allow_extra_json_schema_fields), - "description" => doc_comment - }.compact.freeze - else - Support::HashUtil.stringify_keys(json_schema_options) - end - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this object type. - def json_schema_field_metadata_by_field_name - subfields.to_h { |f| [f.name, f.json_schema_metadata] } - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end - - private - + # @private def after_initialize subfields.freeze end - - # Returns a __typename property which we use for union types. - # - # This must always be set to the name of the type (thus the const value). - # - # We also add a "default" value. This does not impact validation, but rather - # aids tools like our kotlin codegen to save publishers from having to set the - # property explicitly when creating events. - def json_schema_typename_field - { - "__typename" => { - "type" => "string", - "const" => type_name, - "default" => type_name - } - } - end - - def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - problem_fields = other_source_subfields.reject { |f| f.json_schema_customizations.empty? } - return if problem_fields.empty? - - field_descriptions = problem_fields.map(&:name).sort.map { |f| "`#{f}`" }.join(", ") - raise Errors::SchemaError, - "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \ - "another type and also have JSON schema customizations. Instead, put the JSON schema " \ - "customizations on the source type's field definitions." - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb index cb9c3132e..d05fa4f37 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb @@ -7,40 +7,25 @@ # frozen_string_literal: true require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" module ElasticGraph module SchemaDefinition module Indexing module FieldType - # @!parse class Scalar < ::Data; end - Scalar = ::Data.define(:scalar_type) - - # Responsible for the JSON schema and mapping of a {SchemaElements::ScalarType}. + # Responsible for the datastore mapping of a {SchemaElements::ScalarType}. # # @!attribute [r] scalar_type # @return [SchemaElements::ScalarType] the scalar type # # @api private - class Scalar < ::Data + class Scalar < Support::MemoizableData.define(:scalar_type) # @return [Hash] the datastore mapping for this scalar type. def to_mapping Support::HashUtil.stringify_keys(scalar_type.mapping_options) end - # @return [Hash] the JSON schema for this scalar type. - def to_json_schema - Support::HashUtil.stringify_keys(scalar_type.json_schema_options) - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this scalar type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations + private def after_initialize end # @dynamic initialize, scalar_type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb index e4d6e634f..b7846ac41 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb @@ -8,14 +8,14 @@ require "elastic_graph/schema_definition/indexing/field_type/object" require "elastic_graph/support/hash_util" +require "elastic_graph/support/memoizable_data" module ElasticGraph module SchemaDefinition module Indexing module FieldType - # Responsible for the JSON schema and mapping of a {SchemaElements::UnionType}. + # Responsible for the datastore mapping of a {SchemaElements::UnionType}. # - # @note In JSON schema, we model this with a `oneOf`, and a `__typename` field on each subtype. # @note Within the mapping, we have a single object type that has a set union of the properties # of the subtypes (and also a `__typename` keyword field). # @@ -23,26 +23,7 @@ module FieldType # @return [Hash] the subtypes of the union, keyed by name. # # @api private - class Union < ::Data.define(:subtypes_by_name) - # @return [Hash] the JSON schema for this union type. - def to_json_schema - subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} } - - # A union type can represent multiple subtypes, referenced by the "anyOf" clause below. - # We also add a requirement for the presence of __typename to indicate which type - # is being referenced (this property is pre-defined on the type itself as a constant). - # - # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas - # to form a union, and validate equivalently when no object can satisfy multiple of the - # subschemas (which is the case here given the __typename requirements are mutually - # exclusive), we chose to use "oneOf" here because it works better with this library: - # https://github.com/pwall567/json-kotlin-schema-codegen - { - "required" => %w[__typename], - "oneOf" => subtype_json_schemas - } - end - + class Union < Support::MemoizableData.define(:subtypes_by_name) # @return [Hash] the datastore mapping for this union type. def to_mapping mapping_subfields = subtypes_by_name.values.map(&:subfields).reduce([], :union) @@ -53,16 +34,10 @@ def to_mapping ) end - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this union type. - def json_schema_field_metadata_by_field_name - {} + private def after_initialize end - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end + # @dynamic initialize, subtypes_by_name end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index a6400db64..a0d4149f4 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -17,7 +17,7 @@ module ElasticGraph module SchemaDefinition - # Contains schema definition logic specific to indexing (such as JSON schema and mapping generation). + # Contains schema definition logic specific to indexing, such as datastore mapping generation. module Indexing # Represents an index in a datastore. Defined within an indexed type. Modeled as a separate object to facilitate # further customization of the index. @@ -65,10 +65,12 @@ def initialize(name, settings, schema_def_state, indexed_type) # us a nice efficiency boost. id_field_path = public_field_path("id", explanation: "indexed types must have an `id` field") self.routing_field_path = id_field_path - id_field_path.last_part.json_schema nullable: false end + # :nocov: -- `Factory#new_index` (and its JSON-ingestion wrapper) always pass a block; the + # else branch is here for hand-instantiation paths that the test suite doesn't exercise. yield self if block_given? + # :nocov: end # Specifies how documents in this index should sort by default, when no `orderBy` argument is provided to the GraphQL query. @@ -138,8 +140,6 @@ def rollover(frequency, timestamp_field_path_name) raise Errors::SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is a list field." end - timestamp_field_path.path_parts.each { |f| f.json_schema nullable: false } - self.rollover_config = RolloverConfig.new( frequency: frequency, timestamp_field_path: timestamp_field_path @@ -186,8 +186,6 @@ def route_with(routing_field_path_name) self.routing_field_path = routing_field_path - routing_field_path.path_parts[0..-2].each { |f| f.json_schema nullable: false } - routing_field_path.last_part.json_schema nullable: false, pattern: HAS_NON_WHITE_SPACE_REGEX indexed_type.append_to_documentation "For more performant queries on this type, please filter on `#{routing_field_path_name}` if possible." end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb index 454093ff6..b1feb8f30 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb @@ -6,27 +6,48 @@ # # frozen_string_literal: true -require "elastic_graph/support/json_schema/meta_schema_validator" - module ElasticGraph module SchemaDefinition module Mixins - # Mixin used to specify non-GraphQL type info (datastore index and JSON schema type info). + # Mixin used to specify non-GraphQL type info on schema elements. # Exists as a mixin so we can apply the same consistent API to every place we need to use this. # Currently it's used in 3 places: # - # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in JSON schema and the index. - # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in JSON schema and the index. - # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard JSON schema and the index mapping. + # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in the datastore index. + # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in the datastore index. + # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard index mapping. module HasTypeInfo # @return [Hash] datastore mapping options def mapping_options @mapping_options ||= {} end + # Returns the JSON schema options recorded for this element. + # + # The default implementation returns an empty hash; ingestion extensions (such as + # `elasticgraph-json_ingestion`) override this on the elements they extend to return real options. + # Defining it here lets call sites (and Apollo/test helpers) reference `json_schema_options` and + # call `json_schema(...)` without having to detect whether an ingestion extension is loaded. + # # @return [Hash] JSON schema options def json_schema_options - @json_schema_options ||= {} + # :nocov: -- only reached when no ingestion extension is loaded; the JSON Ingestion + # extension shadows this method on every element it extends. + {} + # :nocov: + end + + # Records JSON schema options for this element. + # + # The default implementation is a no-op; an ingestion extension overrides this on the + # elements it extends to apply real validation and storage. The default JSON ingestion extension + # provides the user-facing behavior and documentation for this API. + # + # @param options [Hash] JSON schema options + # @return [void] + def json_schema(**options) + # :nocov: -- only reached when no ingestion extension is loaded. + # :nocov: end # Set of mapping parameters that it makes sense to allow customization of, based on @@ -70,7 +91,7 @@ def json_schema_options # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end # @@ -87,13 +108,6 @@ def json_schema_options # t.field "expYear", "Int" do |f| # # Use a smaller numeric type to save space in the datastore # f.mapping type: "short" - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # # Use a smaller numeric type to save space in the datastore - # f.mapping type: "byte" - # f.json_schema minimum: 1, maximum: 12 # end # # t.index "cards" @@ -108,73 +122,6 @@ def mapping(**options) mapping_options.update(options) end - - # Defines the [JSON schema](https://json-schema.org/understanding-json-schema/) validations for this field or type. Validations - # defined here will be included in the generated `json_schemas.yaml` artifact, which is used by the ElasticGraph indexer to - # validate events before indexing their data in the datastore. In addition, the publisher may use `json_schemas.yaml` for code - # generation and to apply validation before publishing an event to ElasticGraph. - # - # Can be called multiple times; each time, the options will be merged into the existing options. - # - # This is _required_ on a {SchemaElements::ScalarType} (since we don’t know how a custom scalar type should be represented in - # JSON!). On a {SchemaElements::Field}, this is optional, but can be used to make the JSON schema validation stricter then it - # would otherwise be. For example, you could use `json_schema maxLength: 30` on a `String` field to limit the length. - # - # You can use any of the JSON schema validation keywords here. In addition, `nullable: false` is supported to configure the - # generated JSON schema to disallow `null` values for the field. Note that if you define a field with a non-nullable GraphQL type - # (e.g. `Int!`), the JSON schema will automatically disallow nulls. However, as explained in the - # {SchemaElements::TypeWithSubfields#field} documentation, we generally recommend against defining non-nullable GraphQL fields. - # `json_schema nullable: false` will disallow `null` values from being indexed, while still keeping the field nullable in the - # GraphQL schema. If you think you might want to make a field non-nullable in the GraphQL schema some day, it’s a good idea to use - # `json_schema nullable: false` now to ensure every indexed record has a non-null value for the field. - # - # @note We recommend using JSON schema validations in a limited fashion. Validations that are appropriate to apply when data is - # entering the system-of-record are often not appropriate on a secondary index like ElasticGraph. Events that violate a JSON - # schema validation will fail to index (typically they will be sent to the dead letter queue and page an oncall engineer). If an - # ElasticGraph instance is meant to contain all the data of some source system, you probably don’t want it applying stricter - # validations than the source system itself has. We recommend limiting your JSON schema validations to situations where - # violations would prevent ElasticGraph from operating correctly. - # - # @param options [Hash] JSON schema options - # @return [void] - # - # @example Define the JSON schema validations of a custom scalar type - # ElasticGraph.define_schema do |schema| - # schema.scalar_type "URL" do |t| - # t.mapping type: "keyword" - # - # # JSON schema has a built-in URI format validator: - # # https://json-schema.org/understanding-json-schema/reference/string.html#resource-identifiers - # t.json_schema type: "string", format: "uri" - # end - # end - # - # @example Define additional validations on a field - # ElasticGraph.define_schema do |schema| - # schema.object_type "Card" do |t| - # t.field "id", "ID!" - # - # t.field "expYear", "Int" do |f| - # # Use JSON schema to ensure the publisher is sending us 4 digit years, not 2 digit years. - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # f.json_schema minimum: 1, maximum: 12 - # end - # - # t.index "cards" - # end - # end - def json_schema(**options) - validatable_json_schema = Support::HashUtil.stringify_keys(options) - - if (error_msg = Support::JSONSchema.strict_meta_schema_validator.validate_with_error_message(validatable_json_schema)) - raise Errors::SchemaError, "Invalid JSON schema options set on #{self}:\n\n#{error_msg}" - end - - json_schema_options.update(options) - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb index 8ab08390e..4ae693147 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb @@ -6,8 +6,8 @@ # # frozen_string_literal: true -require "rake/tasklib" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "rake/tasklib" module ElasticGraph module SchemaDefinition @@ -40,7 +40,7 @@ class RakeTasks < ::Rake::TaskLib # @param enum_value_overrides_by_type [Hash>] overrides for the names of specific enum values for # specific enum types. For example, to rename the `DayOfWeek.MONDAY` enum to `DayOfWeek.MON`, pass `{DayOfWeek: {MONDAY: "MON"}}`. # @param extension_modules [Array] List of Ruby modules to extend onto the `SchemaDefinition::API` instance. Designed to - # support ElasticGraph extension gems (such as `elasticgraph-apollo`). + # support ElasticGraph extension gems (such as `elasticgraph-apollo` and `elasticgraph-json_ingestion`). # @param output [IO] used for printing task output # # @example Minimal setup with defaults diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb index da347058e..e228e8b5b 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb @@ -8,8 +8,6 @@ require "elastic_graph/constants" require "elastic_graph/errors" -require "elastic_graph/json_ingestion/schema_definition/indexing/event_envelope" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata" require "elastic_graph/schema_artifacts/artifacts_helper_methods" require "elastic_graph/schema_artifacts/runtime_metadata/schema" require "elastic_graph/schema_definition/indexing/relationship_resolver" @@ -46,53 +44,6 @@ def runtime_metadata @runtime_metadata ||= build_runtime_metadata end - # @param version [Integer] desired JSON schema version - # @return [Hash] the JSON schema for the requested version, if available - # @raise [Errors::NotFoundError] if the requested JSON schema version is not available - def json_schemas_for(version) - unless available_json_schema_versions.include?(version) - raise Errors::NotFoundError, "The requested json schema version (#{version}) is not available. Available versions: #{available_json_schema_versions.to_a.join(", ")}." - end - - @latest_versioned_json_schema ||= merge_field_metadata_into_json_schema(current_public_json_schema).json_schema - end - - # @return [Set] set of available JSON schema versions - def available_json_schema_versions - @available_json_schema_versions ||= Set[latest_json_schema_version] - end - - # @return [Hash] the newly generated JSON schema - def latest_json_schema_version - current_public_json_schema[JSON_SCHEMA_VERSION_KEY] - end - - # @private - def json_schema_version_setter_location - state.json_schema_version_setter_location - end - - # @private - def json_schema_field_metadata_by_type_and_field_name - @json_schema_field_metadata_by_type_and_field_name ||= json_schema_indexing_field_types_by_name - .transform_values(&:json_schema_field_metadata_by_field_name) - end - - # @private - def current_public_json_schema - @current_public_json_schema ||= build_public_json_schema - end - - # @private - def merge_field_metadata_into_json_schema(json_schema) - json_schema_with_metadata_merger.merge_metadata_into(json_schema) - end - - # @private - def unused_deprecated_elements - json_schema_with_metadata_merger.unused_deprecated_elements - end - # @private STATIC_SCRIPT_REPO = Scripting::FileSystemRepository.new(::File.join(__dir__.to_s, "scripting", "scripts")) @@ -113,10 +64,6 @@ def after_initialize state.user_definition_complete_callbacks.each(&:call) end - def json_schema_with_metadata_merger - @json_schema_with_metadata_merger ||= JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger.new(self) - end - def generate_datastore_config # We need to check this before generating our datastore configuration. # We can't generate a mapping from a recursively defined schema type. @@ -268,46 +215,6 @@ def generate_sdl [type_defs + state.sdl_parts].join("\n\n") end - def build_public_json_schema - json_schema_version = state.json_schema_version - if json_schema_version.nil? - raise Errors::SchemaError, "`json_schema_version` must be specified in the schema. To resolve, add `schema.json_schema_version 1` in a schema definition block." - end - - root_document_type_names = state.object_types_by_name.values - .select { |type| type.root_document_type? && !type.abstract? } - .reject { |type| derived_indexing_type_names.include?(type.name) } - .map(&:name) - - definitions_by_name = json_schema_indexing_field_types_by_name - .transform_values(&:to_json_schema) - .compact - - event_envelope = JSONIngestion::SchemaDefinition::Indexing::EventEnvelope - - { - "$schema" => JSON_META_SCHEMA, - JSON_SCHEMA_VERSION_KEY => json_schema_version, - "$defs" => { - "ElasticGraphEventEnvelope" => event_envelope.json_schema(root_document_type_names, json_schema_version) - }.merge(definitions_by_name) - } - end - - def json_schema_indexing_field_types_by_name - @json_schema_indexing_field_types_by_name ||= state - .types_by_name - .except("Query") - .values - .reject do |t| - derived_indexing_type_names.include?(t.name) || - # Skip graphql types (including namespace types, which are GraphQL-only). - t.graphql_only? - end - .sort_by(&:name) - .to_h { |type| [type.name, type.to_indexing_field_type] } - end - def verify_runtime_metadata(runtime_metadata) registered_resolvers = runtime_metadata.graphql_resolvers_by_name diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb index e2dac36d0..0ae2e7ff7 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb @@ -8,7 +8,6 @@ require "did_you_mean" require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/json_schema_pruner" require "elastic_graph/support/graphql_gem_loader" require "elastic_graph/support/memoizable_data" require "fileutils" @@ -36,41 +35,10 @@ def initialize(schema_definition_results:, schema_artifacts_directory:, output:, @schema_artifacts_directory = schema_artifacts_directory @output = output @max_diff_lines = max_diff_lines - - @json_schemas_artifact = new_yaml_artifact( - JSON_SCHEMAS_FILE, - JSONIngestion::SchemaDefinition::JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), - extra_comment_lines: [ - "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", - "they can perform code generation and event validation." - ] - ) end # Dumps all the schema artifacts to disk. def dump_artifacts - check_if_needs_json_schema_version_bump do |recommended_json_schema_version| - if schema_definition_results.state.enforce_json_schema_version - # @type var setter_location: ::Thread::Backtrace::Location - # We use `_ =` because while `json_schema_version_setter_location` can be nil, - # it'll never be nil if we get here and we want the type to be non-nilable. - setter_location = _ = schema_definition_results.json_schema_version_setter_location - setter_location_path = ::Pathname.new(setter_location.absolute_path.to_s).relative_path_from(::Dir.pwd) - - abort "A change has been attempted to `json_schemas.yaml`, but the `json_schema_version` has not been correspondingly incremented. Please " \ - "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \ - "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ - " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ - "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \ - "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." - else - @output.puts <<~EOS - WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. - This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`. - EOS - end - end - ::FileUtils.mkdir_p(@schema_artifacts_directory) artifacts.each { |artifact| artifact.dump(@output) } end @@ -111,18 +79,11 @@ def artifacts_from_schema_def # schema elements. graphql_schema = ::GraphQL::Schema.from_definition(schema_definition_results.graphql_schema_string).to_definition.chomp - unversioned_artifacts = [ + [ new_yaml_artifact(DATASTORE_CONFIG_FILE, schema_definition_results.datastore_config), new_yaml_artifact(RUNTIME_METADATA_FILE, pruned_runtime_metadata(graphql_schema).to_dumpable_hash), - @json_schemas_artifact, new_raw_artifact(GRAPHQL_SCHEMA_FILE, "\n" + graphql_schema) ] - - versioned_artifacts = build_desired_versioned_json_schemas(@json_schemas_artifact.desired_contents).values.map do |versioned_schema| - new_versioned_json_schema_artifact(versioned_schema) - end - - unversioned_artifacts + versioned_artifacts end def notify_about_unused_type_name_overrides @@ -172,157 +133,6 @@ def notify_about_unused_enum_value_overrides EOS end - def build_desired_versioned_json_schemas(current_public_json_schema) - versioned_parsed_yamls = ::Dir.glob(::File.join(@schema_artifacts_directory, JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v*.yaml")).map do |file| - ::YAML.safe_load_file(file) - end + [current_public_json_schema] - - results_by_json_schema_version = versioned_parsed_yamls.to_h do |parsed_yaml| - merged_schema = @schema_definition_results.merge_field_metadata_into_json_schema(parsed_yaml) - [merged_schema.json_schema_version, merged_schema] - end - - report_json_schema_merge_errors(results_by_json_schema_version.values) - report_json_schema_merge_warnings - - results_by_json_schema_version.transform_values(&:json_schema) - end - - def report_json_schema_merge_errors(merged_results) - json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] - - merged_results.each do |result| - result.missing_fields.each do |field| - json_schema_versions_by_missing_field[field] << result.json_schema_version - end - - result.missing_types.each do |type| - json_schema_versions_by_missing_type[type] << result.json_schema_version - end - - result.missing_necessary_fields.each do |missing_necessary_field| - json_schema_versions_by_missing_necessary_field[missing_necessary_field] << result.json_schema_version - end - end - - missing_field_errors = json_schema_versions_by_missing_field.map do |field, json_schema_versions| - missing_field_error_for(field, json_schema_versions) - end - - missing_type_errors = json_schema_versions_by_missing_type.map do |type, json_schema_versions| - missing_type_error_for(type, json_schema_versions) - end - - missing_necessary_field_errors = json_schema_versions_by_missing_necessary_field.map do |field, json_schema_versions| - missing_necessary_field_error_for(field, json_schema_versions) - end - - definition_conflict_errors = merged_results - .flat_map { |result| result.definition_conflicts.to_a } - .group_by(&:name) - .map do |name, deprecated_elements| - <<~EOS - The schema definition of `#{name}` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: - - #{format_deprecated_elements(deprecated_elements)} - EOS - end - - errors = missing_field_errors + missing_type_errors + missing_necessary_field_errors + definition_conflict_errors - return if errors.empty? - - abort errors.join("\n\n") - end - - def report_json_schema_merge_warnings - unused_elements = @schema_definition_results.unused_deprecated_elements - return if unused_elements.empty? - - @output.puts <<~EOS - The schema definition has #{unused_elements.size} unneeded reference(s) to deprecated schema elements. These can all be safely deleted: - - #{format_deprecated_elements(unused_elements)} - - EOS - end - - def format_deprecated_elements(deprecated_elements) - descriptions = deprecated_elements - .sort_by { |e| [e.defined_at.path, e.defined_at.lineno] } - .map(&:description) - .uniq - - descriptions.each.with_index(1).map { |desc, idx| "#{idx}. #{desc}" }.join("\n") - end - - def missing_field_error_for(qualified_field, json_schema_versions) - type, field = qualified_field.split(".") - - <<~EOS - The `#{qualified_field}` field (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this field's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{qualified_field}` field has been renamed, indicate this by calling `field.renamed_from "#{field}"` on the renamed field. - 2. If the `#{qualified_field}` field has been dropped, indicate this by calling `type.deleted_field "#{field}"` on the `#{type}` type. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_type_error_for(type, json_schema_versions) - <<~EOS - The `#{type}` type (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this type's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{type}` type has been renamed, indicate this by calling `type.renamed_from "#{type}"` on the renamed type. - 2. If the `#{type}` type has been dropped, indicate this by calling `schema.deleted_type "#{type}"` on the schema. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_necessary_field_error_for(field, json_schema_versions) - path = field.fully_qualified_path.split(".").last - # :nocov: -- we only cover one side of this ternary. - has_or_have = (json_schema_versions.size == 1) ? "has" : "have" - # :nocov: - - <<~EOS - #{describe_json_schema_versions(json_schema_versions, "and")} #{has_or_have} no field that maps to the #{field.field_type} field path of `#{field.fully_qualified_path}`. - Since the field path is required for #{field.field_type}, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: - - 1. If the `#{field.fully_qualified_path}` field has been renamed, indicate this by calling `field.renamed_from "#{path}"` on the renamed field rather than using `deleted_field`. - 2. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def describe_json_schema_versions(json_schema_versions, conjunction) - json_schema_versions = json_schema_versions.sort - - # Steep doesn't support pattern matching yet, so have to skip type checking here. - __skip__ = case json_schema_versions - in [single_version] - "JSON schema version #{single_version}" - in [version1, version2] - "JSON schema versions #{version1} #{conjunction} #{version2}" - else - *versions, last_version = json_schema_versions - "JSON schema versions #{versions.join(", ")}, #{conjunction} #{last_version}" - end - end - - def old_versions(json_schema_versions) - return "this old version" if json_schema_versions.size == 1 - "these old versions" - end - - def files_noun_phrase(json_schema_versions) - return "its file" if json_schema_versions.size == 1 - "their files" - end - def artifacts_out_of_date_error(out_of_date_artifacts) # @type var diffs: ::Array[[SchemaArtifact[untyped], ::String]] diffs = [] @@ -377,20 +187,6 @@ def new_yaml_artifact(file_name, desired_contents, extra_comment_lines: []) ) end - def new_versioned_json_schema_artifact(desired_contents) - # File name depends on the schema_version field in the json schema. - schema_version = desired_contents[JSON_SCHEMA_VERSION_KEY] - - new_yaml_artifact( - ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{schema_version}.yaml"), - desired_contents, - extra_comment_lines: [ - "This JSON schema file contains internal ElasticGraph metadata and should be considered private.", - "The unversioned JSON schema file is public and intended to be provided to publishers." - ] - ) - end - def new_raw_artifact(file_name, desired_contents) SchemaArtifact.new( ::File.join(@schema_artifacts_directory, file_name), @@ -401,17 +197,6 @@ def new_raw_artifact(file_name, desired_contents) ) end - def check_if_needs_json_schema_version_bump(&block) - if @json_schemas_artifact.out_of_date? - existing_schema_version = @json_schemas_artifact.existing_dumped_contents&.dig(JSON_SCHEMA_VERSION_KEY) || -1 - desired_schema_version = @json_schemas_artifact.desired_contents[JSON_SCHEMA_VERSION_KEY] - - if existing_schema_version >= desired_schema_version - yield existing_schema_version + 1 - end - end - end - def pruned_runtime_metadata(graphql_schema_string) schema = ::GraphQL::Schema.from_definition(graphql_schema_string) runtime_meta = schema_definition_results.runtime_metadata diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb index df513de53..90753d712 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb @@ -406,19 +406,11 @@ def register_standard_elastic_graph_types # As per the Elasticsearch docs, the field MUST come in named `lat` in Elastisearch (but we want the full name in GraphQL). t.field names.latitude, "Float", name_in_index: "lat" do |f| f.documentation "Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -90, maximum: 90, nullable: false end # As per the Elasticsearch docs, the field MUST come in named `lon` in Elastisearch (but we want the full name in GraphQL). t.field names.longitude, "Float", name_in_index: "lon" do |f| f.documentation "Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -180, maximum: 180, nullable: false end t.mapping type: "geo_point" @@ -660,17 +652,15 @@ def register_standard_elastic_graph_types # Registers the standard GraphQL scalar types. Note that the SDL for the scalar type itself isn't # included in the dumped SDL, but registering it allows us to derive a filter for each, - # which we need. In addition, this lets us define the mapping and JSON schema for each standard - # scalar type. + # which we need. In addition, this lets us define the mapping for each standard scalar type. + # Ingestion extensions can layer their own built-in configuration on top. def register_standard_graphql_scalars schema_def_api.scalar_type "Boolean" do |t| t.mapping type: "boolean" - t.json_schema type: "boolean" end schema_def_api.scalar_type "Float" do |t| t.mapping type: "double" - t.json_schema type: "number" t.customize_aggregated_values_type do |avt| # not nullable, since sum(empty_set) == 0 @@ -710,12 +700,10 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "ID" do |t| t.mapping type: "keyword" - t.json_schema type: "string" end schema_def_api.scalar_type "Int" do |t| t.mapping type: "integer" - t.json_schema type: "integer", minimum: INT_MIN, maximum: INT_MAX t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", defined_at: "elastic_graph/indexer/indexing_preparers/integer" @@ -730,7 +718,6 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "String" do |t| t.mapping type: "keyword" - t.json_schema type: "string" t.customize_filter_input_type do |fit| fit.field names.contains, schema_def_state.type_ref("StringContains").as_filter_input.name do |f| @@ -754,12 +741,11 @@ def register_standard_graphql_scalars def register_custom_elastic_graph_scalars schema_def_api.scalar_type "Cursor" do |t| - # Technically, we don't use the mapping or json_schema on this type since it's a return-only + # Technically, we don't use the mapping or ingestion config on this type since it's a return-only # type and isn't indexed. However, `scalar_type` requires them to be set (since custom scalars # defined by users will need those set) so we set them here to what they would be if we actually # used them. t.mapping type: "keyword" - t.json_schema type: "string" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/cursor" @@ -772,7 +758,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "Date" do |t| t.mapping type: "date", format: DATASTORE_DATE_FORMAT - t.json_schema type: "string", format: "date" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Date", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date" @@ -792,7 +777,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "DateTime" do |t| t.mapping type: "date", format: DATASTORE_DATE_TIME_FORMAT - t.json_schema type: "string", format: "date-time" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date_time" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::DateTime", @@ -883,8 +867,6 @@ def register_custom_elastic_graph_scalars t.mapping type: "date", format: "HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS" - t.json_schema type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN - t.customize_aggregated_values_type do |avt| define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "LocalTime") do |adjective:, full_name:| <<~EOS @@ -897,7 +879,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "TimeZone" do |t| t.mapping type: "keyword" - t.json_schema type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/time_zone" @@ -914,8 +895,6 @@ def register_custom_elastic_graph_scalars # https://github.com/json-schema-org/json-schema-spec/blob/draft-07/schema.json#L23-L29 # # ...except we are omitting `null` here; it'll be added by the nullability decorator if the field is defined as nullable. - t.json_schema type: ["array", "boolean", "integer", "number", "object", "string"] - # In the index we store this as a JSON string in a `keyword` field. t.mapping type: "keyword" @@ -940,7 +919,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "JsonSafeLong" do |t| t.mapping type: "long" - t.json_schema type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" @@ -984,7 +962,6 @@ def register_custom_elastic_graph_scalars # to do if we ingest them as strings. (The `pattern` regex to validate the range # would be *extremely* complicated). t.mapping type: "long" - t.json_schema type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb index 3297fb1ee..67b3576bc 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb @@ -145,7 +145,7 @@ def derived_graphql_types derived_scalar_types = schema_def_state.factory.new_scalar_type(name) do |t| t.mapping type: "keyword" - t.json_schema type: "string" + configure_derived_scalar_type(t) t.graphql_only graphql_only? end.derived_graphql_types @@ -156,6 +156,14 @@ def derived_graphql_types end end + # Hook for extensions to customize the scalar type derived from an enum type. + # @param scalar_type [ScalarType] the scalar type to configure + # @return [void] + # @api private + def configure_derived_scalar_type(scalar_type) + # No-op by default; extensions (e.g. JSONIngestion) override this. + end + # @return [Indexing::FieldType::Enum] indexing representation of this enum type def to_indexing_field_type schema_def_state.factory.new_enum_indexing_field_type(values_by_name.keys) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb index 3d66eeb62..e940de071 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb @@ -85,8 +85,6 @@ module SchemaElements # @private # @!attribute [rw] computation_detail # @private - # @!attribute [rw] non_nullable_in_json_schema - # @private # @!attribute [rw] as_input # @private class Field < Struct.new( @@ -95,7 +93,7 @@ class Field < Struct.new( :aggregated_values_customizations, :sort_order_enum_value_customizations, :args, :sortable, :filterable, :aggregatable, :groupable, :highlightable, :returnable, :graphql_only, :source, :runtime_field_script, :relationship, :singular_name, - :computation_detail, :non_nullable_in_json_schema, :as_input, + :computation_detail, :as_input, :name_in_index, :resolver ) include Mixins::HasDocumentation @@ -140,7 +138,6 @@ def initialize( # the `_name` suffix on the attribute for clarity. singular_name: singular, name_in_index: name_in_index, - non_nullable_in_json_schema: false, as_input: as_input, resolver: resolver ) @@ -468,22 +465,6 @@ def on_each_generated_schema_element(&customization_block) customize_sort_order_enum_values(&customization_block) end - # (see Mixins::HasTypeInfo#json_schema) - def json_schema(nullable: nil, **options) - if options.key?(:type) - raise Errors::SchemaError, "Cannot override JSON schema type of field `#{name}` with `#{options.fetch(:type)}`" - end - - case nullable - when true - raise Errors::SchemaError, "`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead." - when false - self.non_nullable_in_json_schema = true - end - - super(**options) - end - # (see Mixins::HasTypeInfo#mapping) def mapping(**options) # ElasticGraph has special handling for the nested type (e.g. we generate sub-aggregation types in the GraphQL schema for @@ -1010,9 +991,8 @@ def to_indexing_field_reference Indexing::FieldReference.new( name: name, name_in_index: name_in_index, - type: non_nullable_in_json_schema ? type.wrap_non_null : type, + type: type, mapping_options: mapping_options, - json_schema_options: json_schema_options, accuracy_confidence: accuracy_confidence, source: source, runtime_field_script: runtime_field_script, @@ -1077,11 +1057,11 @@ def index_leaf? # :medium is assigned to `Field`s that are inferred from the id fields required by a relation. # We make logical guesses about the `indexing_field_type` but if the field is also manually defined, - # it could be slightly different (e.g. additional json schema validations), so we have medium + # it could be slightly different, so we have medium # confidence of these. medium: 2, - # :low is assigned to the ElastcField inferred for the foreign key of an inbound relation. The + # :low is assigned to the field inferred for the foreign key of an inbound relation. The # nullability/cardinality of the foreign key field cannot be known from the relation metadata, # so we just guess what seems safest (`[:nullable]`). If the field is defined another way # we should prefer it, so we give these fields :low confidence. diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb index 11b961ef9..8e6e8d89b 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb @@ -25,7 +25,7 @@ module SchemaElements # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" + # t.json_schema type: "string" # end # end # @@ -60,7 +60,7 @@ class ScalarType < Struct.new( include Mixins::HasReadableToSAndInspect.new { |t| t.name } # `HasTypeInfo` provides the following methods: - # @dynamic mapping_options, json_schema_options + # @dynamic mapping_options include Mixins::HasTypeInfo # @dynamic graphql_only? @@ -78,13 +78,8 @@ def initialize(schema_def_state, name) yield self - missing = [ - ("`mapping`" if mapping_options.empty?), - ("`json_schema`" if json_schema_options.empty?) - ].compact - - if missing.any? - raise Errors::SchemaError, "Scalar types require `mapping` and `json_schema` to be configured, but `#{name}` lacks #{missing.join(" and ")}." + if mapping_options.empty? + raise Errors::SchemaError, "Scalar types require `mapping` to be configured, but `#{name}` lacks `mapping`." end if (placeholder = inferred_grouping_missing_value_placeholder) @@ -343,23 +338,8 @@ def inferred_grouping_missing_value_placeholder MISSING_STRING_PLACEHOLDER elsif FLOAT_TYPES.include?(mapping_type) MISSING_NUMERIC_PLACEHOLDER - elsif mapping_type == "long" - # It is only safe to use NaN for a long when the long's range is safe to coerce to a float - # without loss of precision. This is because using NaN as the missing value will cause - # the datastore to coerce the other bucket keys to float. - # JSON schema min/max only constrains newly indexed values, not existing data that may fall outside the range before the constraints were added. - # This is an edge case where the long range may exceed safe float precision. - # In this case, users can set grouping_missing_value_placeholder to nil. - if (json_schema_options[:minimum] || LONG_STRING_MIN) >= JSON_SAFE_LONG_MIN && - (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end - elsif mapping_type == "unsigned_long" - # Similar to the checks above for long except we only need to check the max - # (since the min is zero even if not specified) - if (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end + elsif mapping_type == "long" || mapping_type == "unsigned_long" + nil elsif INTEGER_TYPES.include?(mapping_type) # All other integer types can safely be coerced to float without loss of precision inferred_numeric_placeholder_for_integer_type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb index 483d296bf..22c574977 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb @@ -171,20 +171,6 @@ def with_reverted_override schema_def_state.type_ref(type_namer.revert_override_for(name)) end - # Returns all the JSON schema array/nullable layers of a type, from outermost to innermost. - # For example, [[Int]] will return [:nullable, :array, :nullable, :array, :nullable] - def json_schema_layers - @json_schema_layers ||= begin - layers, inner_type = peel_json_schema_layers_once - - if layers.empty? || inner_type == self - layers - else - layers + inner_type.json_schema_layers - end - end - end - # Most of ElasticGraph's derived GraphQL types have a static suffix (e.g. the full type name # is source_type + suffix). This is a map of all of these. STATIC_FORMAT_NAME_BY_CATEGORY = TypeNamer::REQUIRED_PLACEHOLDERS.filter_map do |format_name, placeholders| @@ -304,16 +290,6 @@ def after_initialize Mixins::VerifiesGraphQLName.verify_name!(unwrapped_name) end - def peel_json_schema_layers_once - if list? - return [[:array], unwrap_list] if non_null? - return [[:nullable, :array], unwrap_list] - end - - return [[], unwrap_non_null] if non_null? - [[:nullable], self] - end - def matches_format_of?(category) format_name = STATIC_FORMAT_NAME_BY_CATEGORY.fetch(category) type_namer.matches_format?(name, format_name) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb index 8f4316ccf..851694eeb 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb @@ -158,7 +158,7 @@ def name # Therefore, we recommend limiting your use of `!` to only a few situations such as defining a type’s primary key (e.g. # `t.field "id", "ID!"`) or defining a list field (e.g. `t.field "authors", "[String!]!"`) since empty lists already provide a # "no data" representation. You can still configure the ElasticGraph indexer to require a non-null value for a field using - # `f.json_schema nullable: false`. + # `f.json_schema nullable: false` (when using `elasticgraph-json_ingestion`). # # @note ElasticGraph’s understanding of datastore capabilities may override your configured # `aggregatable`/`filterable`/`groupable`/`sortable` options. For example, a field indexed as `text` for full text search will @@ -495,7 +495,6 @@ def to_indexing_field_type type_name: name, subfields: indexing_fields_by_name_in_index.values.map(&:to_indexing_field).compact, mapping_options: mapping_options, - json_schema_options: json_schema_options, doc_comment: doc_comment ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb index ccc4e76c5..3d35a0e69 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb @@ -6,14 +6,13 @@ # # frozen_string_literal: true -require "elastic_graph/constants" require "elastic_graph/errors" require "elastic_graph/schema_definition/factory" require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" require "elastic_graph/schema_definition/schema_elements/enum_value_namer" require "elastic_graph/schema_definition/schema_elements/field_path" -require "elastic_graph/schema_definition/schema_elements/type_namer" require "elastic_graph/schema_definition/schema_elements/sub_aggregation_path" +require "elastic_graph/schema_definition/schema_elements/type_namer" module ElasticGraph module SchemaDefinition @@ -41,9 +40,7 @@ class State < Struct.new( :deleted_types_by_old_name, :renamed_fields_by_type_name_and_old_field_name, :deleted_fields_by_type_name_and_old_field_name, - :json_schema_version, - :json_schema_version_setter_location, - :enforce_json_schema_version, + :reserved_type_names, :graphql_extension_modules, :graphql_resolvers_by_name, :built_in_graphql_resolvers, @@ -56,8 +53,6 @@ class State < Struct.new( :output, :type_namer, :enum_value_namer, - :allow_omitted_json_schema_fields, - :allow_extra_json_schema_fields, :indexed_types_by_index_name ) include Mixins::HasReadableToSAndInspect.new @@ -92,9 +87,7 @@ def self.with( deleted_types_by_old_name: {}, renamed_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, deleted_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, - json_schema_version_setter_location: nil, - json_schema_version: nil, - enforce_json_schema_version: true, + reserved_type_names: ::Set.new, graphql_extension_modules: [], graphql_resolvers_by_name: {}, built_in_graphql_resolvers: ::Set.new, @@ -110,8 +103,6 @@ def self.with( ), enum_value_namer: SchemaElements::EnumValueNamer.new(enum_value_overrides_by_type), output: output, - allow_omitted_json_schema_fields: false, - allow_extra_json_schema_fields: true, indexed_types_by_index_name: {} ) end @@ -119,6 +110,15 @@ def self.with( # @dynamic index_document_sizes? alias_method :index_document_sizes?, :index_document_sizes + # Returns the JSON schema version recorded on this state, or `nil` if none has been set. + # + # The default implementation returns `nil` so callers can query it uniformly regardless of whether + # an ingestion extension is active. When `elasticgraph-json_ingestion` is loaded its + # `StateExtension` overrides this with a real `attr_accessor` backed by the extension's instance state. + def json_schema_version + nil + end + def type_ref(name) # Type references are immutable and can be safely cached. Here we cache them because we've observed # it having a noticeable impact on our test suite runtime. @@ -226,12 +226,10 @@ def field_path_resolver private - RESERVED_TYPE_NAMES = [EVENT_ENVELOPE_JSON_SCHEMA_NAME].to_set - def register_type(type, additional_type_index = nil) name = (_ = type).name - if RESERVED_TYPE_NAMES.include?(name) + if reserved_type_names.include?(name) raise Errors::SchemaError, "`#{name}` cannot be used as a schema type because it is a reserved name." end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb index c4c10fb89..5e51e543d 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb @@ -75,7 +75,9 @@ def define_schema_with_schema_elements( yield api if block_given? - # Set the json_schema_version to the provided value, if needed. + # Set the json_schema_version to the provided value, if needed. When no ingestion extension is + # loaded, `State#json_schema_version` returns nil and `API#json_schema_version` is a no-op, so + # we can call through unconditionally; the JSON-ingestion extension overrides both. if !json_schema_version.nil? && api.state.json_schema_version.nil? api.json_schema_version(json_schema_version) end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs index 797db12cb..514bfab31 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs @@ -65,7 +65,6 @@ module ElasticGraph ?output: io ) -> void - def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void def raw_sdl: (::String) -> void def object_type: (::String) { (SchemaElements::ObjectType) -> void } -> void def namespace_type: (::String) ?{ (SchemaElements::NamespaceType) -> void } -> void @@ -77,12 +76,13 @@ module ElasticGraph def as_active_instance: { () -> void } -> void @results: Results? def results: () -> Results - def json_schema_version: (::Integer) -> void - def enforce_json_schema_version: (bool) -> void def register_graphql_extension: (::Module, defined_at: ::String, **untyped) -> void def register_graphql_resolver: (::Symbol, ::Class, defined_at: ::String, ?built_in: bool, **untyped) -> void def on_built_in_types: () { (SchemaElements::graphQLType) -> void } -> void def on_root_query_type: () { (SchemaElements::ObjectType) -> void } -> void + def json_schema_version: (::Integer) -> void + def enforce_json_schema_version: (bool) -> void + def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs index 0ef401056..b37970c5e 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs @@ -89,7 +89,6 @@ module ElasticGraph type_name: ::String, subfields: ::Array[Indexing::Field], mapping_options: Mixins::HasTypeInfo::optionsHash, - json_schema_options: Mixins::HasTypeInfo::optionsHash, doc_comment: ::String? ) -> Indexing::FieldType::Object @@object_indexing_field_type_new: ::Method diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs index 80f6eab04..5230505ec 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs @@ -5,10 +5,8 @@ module ElasticGraph attr_reader name: ::String attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference - attr_reader json_schema_layers: jsonSchemaLayersArray attr_reader indexing_field_type: _FieldType attr_reader accuracy_confidence: Field::accuracyConfidence - attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] attr_reader mapping_customizations: ::Hash[::Symbol, untyped] attr_reader source: SchemaElements::FieldSource? attr_accessor runtime_field_script: ::String? @@ -18,10 +16,8 @@ module ElasticGraph name: ::String, name_in_index: ::String, type: SchemaElements::TypeReference, - json_schema_layers: jsonSchemaLayersArray, indexing_field_type: _FieldType, accuracy_confidence: Field::accuracyConfidence, - json_schema_customizations: ::Hash[::Symbol, untyped], mapping_customizations: ::Hash[::Symbol, untyped], source: SchemaElements::FieldSource?, runtime_field_script: ::String?, @@ -32,10 +28,8 @@ module ElasticGraph ?name: ::String, ?name_in_index: ::String, ?type: SchemaElements::TypeReference, - ?json_schema_layers: jsonSchemaLayersArray, ?indexing_field_type: _FieldType, ?accuracy_confidence: Field::accuracyConfidence, - ?json_schema_customizations: ::Hash[::Symbol, untyped], ?mapping_customizations: ::Hash[::Symbol, untyped], ?source: SchemaElements::FieldSource?, ?runtime_field_script: ::String?, @@ -44,25 +38,11 @@ module ElasticGraph end class Field < FieldSupertype - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE: ::Hash[::String, untyped] - type accuracyConfidence = SchemaElements::Field::accuracyConfidence @mapping: ::Hash[::String, untyped]? def mapping: () -> ::Hash[::String, untyped] - def json_schema: () -> ::Hash[::String, untyped] - def json_schema_metadata: () -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata def self.normalized_mapping_hash_for: (::Array[Field]) -> ::Hash[::String, untyped] - - def inner_json_schema: () -> ::Hash[::String, untyped] - def outer_json_schema_customizations: () -> ::Hash[::String, untyped] - - def nullable?: () -> bool - - def user_specified_json_schema_customizations_go_on_outside?: () -> bool - def process_layer: (::Symbol, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_nullable: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_array: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs index 55a8f0724..d2729eedf 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs @@ -2,11 +2,12 @@ module ElasticGraph module SchemaDefinition module Indexing class FieldReference + include Support::_MemoizableDataClass + attr_reader name: ::String attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference attr_reader mapping_options: ::Hash[::Symbol, untyped] - attr_reader json_schema_options: ::Hash[::Symbol, untyped] attr_reader accuracy_confidence: Field::accuracyConfidence attr_reader source: SchemaElements::FieldSource? attr_reader runtime_field_script: ::String? @@ -17,7 +18,6 @@ module ElasticGraph name_in_index: ::String, type: SchemaElements::TypeReference, mapping_options: ::Hash[::Symbol, untyped], - json_schema_options: ::Hash[::Symbol, untyped], accuracy_confidence: Field::accuracyConfidence, source: SchemaElements::FieldSource?, runtime_field_script: ::String?, @@ -29,7 +29,6 @@ module ElasticGraph ?name_in_index: ::String, ?type: SchemaElements::TypeReference, ?mapping_options: ::Hash[::Symbol, untyped], - ?json_schema_options: ::Hash[::Symbol, untyped], ?accuracy_confidence: Field::accuracyConfidence, ?source: SchemaElements::FieldSource?, ?runtime_field_script: ::String?, diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs index bc5e08829..d01805f99 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs @@ -3,9 +3,6 @@ module ElasticGraph module Indexing interface _FieldType def to_mapping: () -> ::Hash[::String, untyped] - def to_json_schema: () -> ::Hash[::String, untyped] - def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata] - def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs index 7f8f9afd6..6c263ecbe 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs @@ -4,6 +4,8 @@ module ElasticGraph module FieldType class Enum include _FieldType + include Support::_MemoizableDataClass + attr_reader enum_value_names: ::Array[::String] def initialize: (::Array[::String]) -> void end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs index 5dd23dbaa..e1b90df44 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs @@ -3,29 +3,26 @@ module ElasticGraph module Indexing module FieldType class ObjectSuperType + attr_reader schema_def_state: State attr_reader type_name: ::String attr_reader subfields: ::Array[Field] attr_reader mapping_options: Mixins::HasTypeInfo::optionsHash - attr_reader json_schema_options: Mixins::HasTypeInfo::optionsHash attr_reader doc_comment: ::String? - attr_reader schema_def_state: State def initialize: ( + schema_def_state: State, type_name: ::String, subfields: ::Array[Field], mapping_options: Mixins::HasTypeInfo::optionsHash, - json_schema_options: Mixins::HasTypeInfo::optionsHash, - doc_comment: ::String?, - schema_def_state: State + doc_comment: ::String? ) -> void def with: ( + ?schema_def_state: State, ?type_name: ::String, ?subfields: ::Array[Field], ?mapping_options: Mixins::HasTypeInfo::optionsHash, - ?json_schema_options: Mixins::HasTypeInfo::optionsHash, - ?doc_comment: ::String?, - ?schema_def_state: State + ?doc_comment: ::String? ) -> Object end @@ -34,12 +31,6 @@ module ElasticGraph include Support::_MemoizableDataClass @to_mapping: ::Hash[::String, untyped]? - @to_json_schema: ::Hash[::String, untyped]? - - private - - def json_schema_typename_field: () -> ::Hash[::String, untyped] - def validate_sourced_fields_have_no_json_schema_overrides: (::Array[Field]) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs index 32cff29a8..9ab15148e 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs @@ -4,6 +4,8 @@ module ElasticGraph module FieldType class Scalar include _FieldType + include Support::_MemoizableDataClass + attr_reader scalar_type: SchemaElements::ScalarType def initialize: (scalar_type: SchemaElements::ScalarType) -> Scalar diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs index d100ef666..3eae92d4b 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs @@ -2,13 +2,12 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - class UnionSuperType - attr_reader subtypes_by_name: ::Hash[::String, Object] - def self.new: (::Hash[::String, Object]) -> instance - end - - class Union < UnionSuperType + class Union include _FieldType + include Support::_MemoizableDataClass + + attr_reader subtypes_by_name: ::Hash[::String, Object] + def initialize: (::Hash[::String, Object]) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs index 0dc296381..9c892787b 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs @@ -8,7 +8,12 @@ module ElasticGraph attr_reader rollover_config: RolloverConfig? attr_reader has_had_multiple_sources_flag: bool attr_reader indexed_type: indexableType + attr_reader schema_def_state: State + HAS_NON_WHITE_SPACE_REGEX: ::Regexp + + def rollover: (::Symbol, ::String) -> void + def route_with: (::String) -> void def uses_custom_routing?: () -> bool def to_index_config: () -> ::Hash[::String, untyped] def to_index_template_config: () -> ::Hash[::String, untyped] @@ -16,6 +21,7 @@ module ElasticGraph private + def public_field_path: (::String, explanation: ::String) -> SchemaElements::FieldPath def date_and_datetime_types: () -> ::Array[::String] end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs index 1f3c1391b..7ad7426ac 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs @@ -6,9 +6,9 @@ module ElasticGraph type optionsHash = ::Hash[::Symbol, untyped] attr_reader mapping_options: optionsHash - attr_reader json_schema_options: optionsHash def mapping: (**untyped) -> void + def json_schema_options: () -> optionsHash def json_schema: (**untyped) -> void end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs index ca21710f0..11a4c10ba 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs @@ -6,43 +6,30 @@ module ElasticGraph end class Results < ResultsSupertype - include _SchemaArtifacts include Support::_MemoizableDataClass - def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? - def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]] - def current_public_json_schema: () -> ::Hash[::String, untyped] - def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata - def unused_deprecated_elements: () -> ::Set[SchemaElements::DeprecatedElement] + def graphql_schema_string: () -> ::String + def datastore_config: () -> ::Hash[::String, untyped] + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def derived_indexing_type_names: () -> ::Set[::String] @graphql_schema_string: ::String? @datastore_config: ::Hash[::String, untyped] @runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema? - @current_json_schemas: ::Hash[::String, untyped]? @static_script_repo: Scripting::FileSystemRepository? - @available_json_schema_versions: ::Set[::Integer]? @no_circular_dependencies: bool? @field_path_resolver: SchemaElements::FieldPath::Resolver? - @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_FieldType]? @derived_indexing_type_names: ::Set[::String]? - @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]]? - @current_public_json_schema: ::Hash[::String, untyped]? - @latest_versioned_json_schema: ::Hash[::String, untyped]? - @json_schema_with_metadata_merger: JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger? STATIC_SCRIPT_REPO: Scripting::FileSystemRepository private - def json_schema_with_metadata_merger: () -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger def generate_datastore_config: () -> ::Hash[::String, untyped] def build_dynamic_scripts: () -> ::Array[Scripting::Script] def build_runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def identify_extra_update_targets_by_object_type_name: () -> ::Hash[::String, ::Array[SchemaArtifacts::RuntimeMetadata::UpdateTarget]] def generate_sdl: () -> ::String - def build_public_json_schema: () -> ::Hash[::String, untyped] - def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_FieldType] def verify_runtime_metadata: (SchemaArtifacts::RuntimeMetadata::Schema) -> void def strip_trailing_whitespace: (::String) -> ::String def check_for_circular_dependencies!: () -> void diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs index 389dfbfdd..713693ac3 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs @@ -20,22 +20,11 @@ module ElasticGraph @output: io @max_diff_lines: ::Integer @artifacts: ::Array[SchemaArtifact[untyped]]? - @json_schemas_artifact: SchemaArtifact[untyped] def artifacts: () -> ::Array[SchemaArtifact[untyped]] def artifacts_from_schema_def: () -> ::Array[SchemaArtifact[untyped]] def notify_about_unused_type_name_overrides: () -> void def notify_about_unused_enum_value_overrides: () -> void - def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] - def report_json_schema_merge_errors: (::Array[JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata]) -> void - def report_json_schema_merge_warnings: () -> void - def format_deprecated_elements: (::Enumerable[SchemaElements::DeprecatedElement]) -> ::String - def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_necessary_field_error_for: (JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String - def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String - def old_versions: (::Array[::Integer]) -> ::String - def files_noun_phrase: (::Array[::Integer]) -> ::String def artifacts_out_of_date_error: (::Array[SchemaArtifact[untyped]]) -> ::String def truncate_diff: (::String, ::Integer) -> [::String, ::String] @@ -45,9 +34,7 @@ module ElasticGraph ?extra_comment_lines: ::Array[::String] ) -> SchemaArtifact[::Hash[::String, untyped]] - def new_versioned_json_schema_artifact: (::Hash[::String, untyped]) -> SchemaArtifact[::Hash[::String, untyped]] def new_raw_artifact: (::String, ::String) -> SchemaArtifact[::String] - def check_if_needs_json_schema_version_bump: () { (::Integer) -> void } -> void def pruned_runtime_metadata: (::String) -> SchemaArtifacts::RuntimeMetadata::Schema end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs index 518792bcb..43b6472ee 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/enum_type.rbs @@ -26,6 +26,10 @@ module ElasticGraph def values: (*::String) -> void def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Enum::Type def as_input: () -> EnumType + + private + + def configure_derived_scalar_type: (ScalarType) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs index b32e01646..07f2dc653 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs @@ -22,7 +22,6 @@ module ElasticGraph attr_accessor computation_detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail attr_reader filter_customizations: ::Array[^(Field) -> void] attr_reader sort_order_enum_value_customizations: ::Array[^(SortOrderEnumValue) -> void] - attr_reader non_nullable_in_json_schema: bool attr_reader source: FieldSource? attr_accessor relationship: Relationship? attr_reader resolver: SchemaArtifacts::RuntimeMetadata::ConfiguredGraphQLResolver? diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs index 181b563a6..47cb75435 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs @@ -31,7 +31,6 @@ module ElasticGraph def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::ScalarType def mapping_options: () -> Mixins::HasTypeInfo::optionsHash - def json_schema_options: () -> Mixins::HasTypeInfo::optionsHash private diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs index 9d418a38f..388116f00 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs @@ -30,9 +30,6 @@ module ElasticGraph def scalar_type_needing_grouped_by_object?: () -> bool def with_reverted_override: () -> TypeReference - @json_schema_layers: jsonSchemaLayersArray? - def json_schema_layers: () -> jsonSchemaLayersArray - def to_final_form: (?as_input: bool) -> TypeReference STATIC_FORMAT_NAME_BY_CATEGORY: ::Hash[::Symbol, ::Symbol] @@ -59,8 +56,6 @@ module ElasticGraph private - def peel_json_schema_layers_once: () -> [jsonSchemaLayersArray, TypeReference] - def matches_format_of?: (::Symbol) -> bool def parent_aggregation_type: (::Array[::String]) -> ::String def renamed_with_same_wrappings: (::String) -> TypeReference diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs index bb9a73291..8431b7c3e 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs @@ -18,9 +18,7 @@ module ElasticGraph attr_reader deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] attr_reader renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] attr_reader deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] - attr_accessor json_schema_version: ::Integer? - attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? - attr_accessor enforce_json_schema_version: bool + attr_reader reserved_type_names: ::Set[::String] attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension] attr_reader graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver] attr_reader built_in_graphql_resolvers: ::Set[::Symbol] @@ -30,11 +28,9 @@ module ElasticGraph attr_accessor user_definition_complete_callbacks: ::Array[^() -> void] attr_accessor sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]] attr_accessor type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference] + attr_accessor output: io attr_reader type_namer: SchemaElements::TypeNamer attr_reader enum_value_namer: SchemaElements::EnumValueNamer - attr_accessor output: io - attr_accessor allow_omitted_json_schema_fields: bool - attr_accessor allow_extra_json_schema_fields: bool attr_accessor indexed_types_by_index_name: ::Hash[::String, indexableType] def initialize: ( @@ -55,9 +51,7 @@ module ElasticGraph deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement], renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], - json_schema_version: Integer?, - json_schema_version_setter_location: ::Thread::Backtrace::Location?, - enforce_json_schema_version: bool, + reserved_type_names: ::Set[::String], graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension], graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver], built_in_graphql_resolvers: ::Set[::Symbol], @@ -67,12 +61,10 @@ module ElasticGraph user_definition_complete_callbacks: ::Array[^() -> void], sub_aggregation_paths_by_type: ::Hash[Mixins::SupportsFilteringAndAggregation, ::Array[SchemaElements::SubAggregationPath]], type_refs_by_name: ::Hash[::String, SchemaElements::TypeReference], + output: io, type_namer: SchemaElements::TypeNamer, enum_value_namer: SchemaElements::EnumValueNamer, - output: io, - allow_omitted_json_schema_fields: bool, - allow_extra_json_schema_fields: bool, - indexed_types_by_index_name: ::Hash[::String, indexableType], + indexed_types_by_index_name: ::Hash[::String, indexableType] ) -> void end @@ -89,6 +81,7 @@ module ElasticGraph def index_document_sizes?: () -> bool def type_ref: (::String) -> SchemaElements::TypeReference + def json_schema_version: () -> ::Integer? def register_index: (::String, indexableType) -> void def register_object_interface_or_union_type: (SchemaElements::ObjectType | SchemaElements::InterfaceType | SchemaElements::UnionType) -> void @@ -119,7 +112,6 @@ module ElasticGraph private - RESERVED_TYPE_NAMES: ::Set[::String] def register_type: [T] (T & SchemaElements::graphQLType, ?::Hash[::String, T]?) -> T end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs index 211039ae3..1b92524c5 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs @@ -12,7 +12,7 @@ module ElasticGraph ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], ?output: io?, ?reload_schema_artifacts: bool, - ) ?{ (API) -> void } -> _SchemaArtifacts + ) ?{ (API) -> void } -> (Results | SchemaArtifacts::FromDisk) def define_schema_with_schema_elements: ( SchemaArtifacts::RuntimeMetadata::SchemaElementNames, @@ -24,7 +24,7 @@ module ElasticGraph ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], ?output: io?, ?reload_schema_artifacts: bool, - ) ?{ (API) -> void } -> _SchemaArtifacts + ) ?{ (API) -> void } -> (Results | SchemaArtifacts::FromDisk) DOC_COMMENTS: ::String diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb index e26a47ca4..cf465bbe4 100644 --- a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb @@ -8,6 +8,7 @@ require "bundler" require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/schema_definition/schema_elements/type_namer" require "graphql" @@ -216,7 +217,12 @@ module SchemaDefinition "`schema.json_schema_version 4`" ).and matching(json_schema_version_setter_location_regex) - write_elastic_graph_schema_def_code(component_suffix: "3", json_schema_version: 3, component_extras: "t.renamed_from 'Component'", enforce_json_schema_version: false) + write_elastic_graph_schema_def_code( + component_suffix: "3", + json_schema_version: 3, + component_extras: "t.renamed_from 'Component'", + enforce_json_schema_version: false + ) expect { output = run_rake("schema_artifacts:dump") @@ -494,9 +500,13 @@ module SchemaDefinition }) ) - # Here we add a different new field (`ordinal: Int!`), without bumping the version, and disable - # enforcement so we do not have to bump the version. - write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: "\nt.field 'ordinal', 'Int!'", enforce_json_schema_version: false) + # Here we add a different new field (`ordinal: Int!`), without bumping the version (and using `enforce_json_schema_version: false` + # to not have to bump the version)... + write_elastic_graph_schema_def_code( + json_schema_version: 2, + component_name_extras: "\nt.field 'ordinal', 'Int!'", + enforce_json_schema_version: false + ) run_rake("schema_artifacts:dump") # It should not be added to the v1 schema... @@ -822,7 +832,7 @@ module SchemaDefinition expect(runtime_meta["object_types_by_name"].keys).to exclude("DateTimeListFilterInput") end - it "successfully checks schema artifacts when the rake task is run within a bundle that only includes the `elasticgraph-schema_definition` gem" do + it "successfully checks schema artifacts when the rake task is run within a minimal schema definition bundle" do # We want to ensure that `elasticgraph-schema_definition` gem declares (in its gemspec) all the # dependencies necessary for the schema definition rake tasks. Unfortunately, it's test suite # alone can't detect this, even when run via `script/run_gem_specs`, due to transitive dependencies @@ -837,6 +847,7 @@ module SchemaDefinition source "https://rubygems.org" gem "elasticgraph-schema_definition", path: "#{CommonSpecHelpers::REPO_ROOT}/elasticgraph-schema_definition" + gem "elasticgraph-json_ingestion", path: "#{CommonSpecHelpers::REPO_ROOT}/elasticgraph-json_ingestion" register_gemspec_gems_with_path = lambda do |eg_gem_name| gemspec_contents = ::File.read("#{CommonSpecHelpers::REPO_ROOT}/\#{eg_gem_name}/\#{eg_gem_name}.gemspec") @@ -854,6 +865,7 @@ module SchemaDefinition ::File.write("Rakefile", <<~EOS) project_root = "#{CommonSpecHelpers::REPO_ROOT}" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" ElasticGraph::SchemaDefinition::RakeTasks.new( @@ -861,6 +873,7 @@ module SchemaDefinition index_document_sizes: true, path_to_schema: "\#{project_root}/config/schema.rb", schema_artifacts_directory: "\#{project_root}/config/schema/artifacts", + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] ) EOS @@ -902,7 +915,7 @@ def expect_successful_run_of(*shell_commands) /line 7 at `(\S*\/?)schema\.rb`/ end - def write_elastic_graph_schema_def_code(json_schema_version:, enforce_json_schema_version: true, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false) + def write_elastic_graph_schema_def_code(json_schema_version:, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false, enforce_json_schema_version: true) code = <<~EOS Thread.current[:eg_schema_load_count] = (Thread.current[:eg_schema_load_count] || 0) + 1 if Thread.current[:eg_schema_load_count] > 1 @@ -1088,7 +1101,7 @@ def as_active_instance index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", - extension_modules: [extension_module].compact, + extension_modules: [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, extension_module].compact, derived_type_name_formats: derived_type_name_formats, type_name_overrides: type_name_overrides, enum_value_overrides_by_type: enum_value_overrides_by_type, diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb new file mode 100644 index 000000000..03c1f21eb --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb @@ -0,0 +1,28 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" + +module ElasticGraph + module SchemaDefinition + RSpec.describe Factory do + it "can create an object type without a customization block" do + api = API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :snake_case, overrides: {}), + true, + extension_modules: [] + ) + + object_type = api.factory.new_object_type("Widget") + + expect(object_type.name).to eq("Widget") + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb index f3cd503ac..2e87e57ce 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb @@ -1054,7 +1054,11 @@ def metadata_for(json_schema, type, field) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end def have_dumped_metadata(name_in_index, type) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb index 4e101b378..450ae447d 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb @@ -85,7 +85,7 @@ module SchemaDefinition expect(json_schema_field_metadata).to all eq({}) end - it "generates no field metadata for user-defined union or interface types since the JSON schema" do + it "generates no field metadata for user-defined union or interface types since they have no direct JSON fields" do metadata_by_type_and_field_name = dump_metadata do |schema| schema.interface_type "Named" do |t| t.field "name", "String" @@ -141,7 +141,11 @@ def dump_metadata(&schema_definition) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end def field_meta_of(type, name_in_index) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb index 3f1b1df12..b0dcbdc1d 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb @@ -16,7 +16,12 @@ module SchemaDefinition include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport def define_schema(**options, &block) - super(schema_element_name_form: "snake_case", **options) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + **options, + &block + ) end end end diff --git a/elasticgraph-warehouse/README.md b/elasticgraph-warehouse/README.md index 6890bbe63..ebe55cb36 100644 --- a/elasticgraph-warehouse/README.md +++ b/elasticgraph-warehouse/README.md @@ -46,21 +46,22 @@ diff --git a/Rakefile b/Rakefile index 2943335..26633c3 100644 --- a/Rakefile +++ b/Rakefile -@@ -1,5 +1,6 @@ +@@ -1,6 +1,7 @@ project_root = File.expand_path(__dir__) -+require "elastic_graph/warehouse/schema_definition/api_extension" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" ++require "elastic_graph/warehouse/schema_definition/api_extension" require "rspec/core/rake_task" -@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new( - local_config_yaml: settings_file, - path_to_schema: "#{project_root}/config/schema.rb" - ) do |tasks| -+ tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension -+ +@@ -16,5 +17,6 @@ ElasticGraph::Local::RakeTasks.new( # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension +- ++ tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension ++ + # Customizes the names of fields generated by ElasticGraph. ``` After running `bundle exec rake schema_artifacts:dump`, a `data_warehouse.yaml` file will be diff --git a/elasticgraph-warehouse/elasticgraph-warehouse.gemspec b/elasticgraph-warehouse/elasticgraph-warehouse.gemspec index 1467bdbcc..3d03e7e20 100644 --- a/elasticgraph-warehouse/elasticgraph-warehouse.gemspec +++ b/elasticgraph-warehouse/elasticgraph-warehouse.gemspec @@ -34,6 +34,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = [">= 3.4", "< 4.1"] - spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + + spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt index 7507ebfbe..6a8f878e3 100644 --- a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt +++ b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt @@ -1,5 +1,6 @@ project_root = File.expand_path(__dir__) +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" require "rspec/core/rake_task" @@ -14,6 +15,7 @@ ElasticGraph::Local::RakeTasks.new( ) do |tasks| # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension # Customizes the names of fields generated by ElasticGraph. tasks.schema_element_name_overrides = { diff --git a/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb index 0d87095f3..a9769863a 100644 --- a/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb +++ b/spec_support/lib/elastic_graph/spec_support/enable_simplecov.rb @@ -103,6 +103,9 @@ def wait_for_other_processes # status if we're not running it's test suite. add_filter "/elasticgraph-local/" unless spec_files_to_run.any? { |f| f.include?("/elasticgraph-local/") } + # The JSON ingestion gem is being introduced by extracting implementation first and moving its tests later. + add_filter "/elasticgraph-json_ingestion/" + # This version file is loaded from our gemspecs, which can get loaded by bundler before we get here. # SimpleCov is only able to track coverage of files loaded after it starts, so we need to filter them out if # their constant is already defined. They don't contain any branching statements or anything so it's ok to diff --git a/spec_support/lib/elastic_graph/spec_support/profiling.rb b/spec_support/lib/elastic_graph/spec_support/profiling.rb index abf647781..df1079d60 100644 --- a/spec_support/lib/elastic_graph/spec_support/profiling.rb +++ b/spec_support/lib/elastic_graph/spec_support/profiling.rb @@ -43,7 +43,10 @@ def self.record_raw(id, duration, caller_frames: []) end def self.report_results - computed_results = results.map do |id, results_for_id| + computed_results = results.filter_map do |id, results_for_id| + results_for_id = results_for_id.compact + next if results_for_id.empty? + durations = results_for_id.map { |h| h.fetch(:duration) } total = durations.sum count = durations.count diff --git a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb index 0c3cf8538..f24d95a9d 100644 --- a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb +++ b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb @@ -6,18 +6,47 @@ # # frozen_string_literal: true +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/test_support" -# Combines `:capture_logs` with `ElasicGraph::SchemaDefinition::TestSupport` in order +# Combines `:capture_logs` with `ElasticGraph::SchemaDefinition::TestSupport` in order # to silence log output and fail if any tests result in logged warnings. ::RSpec.shared_context "SchemaDefinitionHelpers", :capture_logs do include ::ElasticGraph::SchemaDefinition::TestSupport + def define_schema( + schema_element_name_form:, + schema_element_name_overrides: {}, + index_document_sizes: true, + json_schema_version: 1, + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension], + derived_type_name_formats: {}, + type_name_overrides: {}, + enum_value_overrides_by_type: {}, + reload_schema_artifacts: false, + output: nil, + &block + ) + super( + schema_element_name_form: schema_element_name_form, + schema_element_name_overrides: schema_element_name_overrides, + index_document_sizes: index_document_sizes, + json_schema_version: json_schema_version, + extension_modules: extension_modules, + derived_type_name_formats: derived_type_name_formats, + type_name_overrides: type_name_overrides, + enum_value_overrides_by_type: enum_value_overrides_by_type, + reload_schema_artifacts: reload_schema_artifacts, + output: output || log_device, + &block + ) + end + def define_schema_with_schema_elements( schema_elements, index_document_sizes: true, json_schema_version: 1, - extension_modules: [], + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension], derived_type_name_formats: {}, type_name_overrides: {}, enum_value_overrides_by_type: {}, @@ -32,7 +61,7 @@ def define_schema_with_schema_elements( derived_type_name_formats: derived_type_name_formats, type_name_overrides: type_name_overrides, enum_value_overrides_by_type: enum_value_overrides_by_type, - reload_schema_artifacts: false, + reload_schema_artifacts: reload_schema_artifacts, output: output || log_device ) end diff --git a/spec_support/spec_helper.rb b/spec_support/spec_helper.rb index 26c0b7227..7bf1008ad 100644 --- a/spec_support/spec_helper.rb +++ b/spec_support/spec_helper.rb @@ -366,12 +366,22 @@ def generate_schema_artifacts( require "elastic_graph/schema_definition/test_support" require "stringio" + extension_modules = begin + require "elastic_graph/json_ingestion/schema_definition/api_extension" + [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] + rescue LoadError => e + raise unless e.path == "elastic_graph/json_ingestion/schema_definition/api_extension" + + [] + end + output = ::StringIO.new # to silence warnings. ::ElasticGraph::SchemaDefinition::TestSupport.define_schema( schema_element_name_form: schema_element_name_form, schema_element_name_overrides: schema_element_name_overrides, derived_type_name_formats: derived_type_name_formats, enum_value_overrides_by_type: enum_value_overrides_by_type, + extension_modules: extension_modules, reload_schema_artifacts: reload_schema_artifacts, output: output ) do |schema|