Wire JSON ingestion schema extension

CODEBASE_OVERVIEW.md

@@ -111,7 +111,6 @@ graph LR;
     rackup["rackup"];
     rake["rake"];
     webrick["webrick"];
-    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     graphql["graphql"];
     elasticgraph --> elasticgraph-support;
@@ -126,7 +125,6 @@ graph LR;
     elasticgraph-local --> webrick;
     elasticgraph-schema_definition --> elasticgraph-graphql;
     elasticgraph-schema_definition --> elasticgraph-indexer;
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
     elasticgraph-schema_definition --> elasticgraph-schema_artifacts;
     elasticgraph-schema_definition --> elasticgraph-support;
     elasticgraph-schema_definition --> graphql;
@@ -143,7 +141,6 @@ graph LR;
     class rackup externalGemCatStyle;
     class rake externalGemCatStyle;
     class webrick externalGemCatStyle;
-    class elasticgraph-json_ingestion otherEgGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class graphql externalGemCatStyle;
     click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank;

Gemfile.lock

@@ -198,7 +198,6 @@ PATH
     elasticgraph-schema_definition (1.2.1.pre)
       elasticgraph-graphql (= 1.2.1.pre)
       elasticgraph-indexer (= 1.2.1.pre)
-      elasticgraph-json_ingestion (= 1.2.1.pre)
       elasticgraph-schema_artifacts (= 1.2.1.pre)
       elasticgraph-support (= 1.2.1.pre)
       graphql (~> 2.6.2)

Rakefile

@@ -8,9 +8,10 @@
 
 require "delegate"
 require "elastic_graph/apollo/schema_definition/api_extension"
-require "elastic_graph/warehouse/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 require "elastic_graph/schema_definition/rake_tasks"
+require "elastic_graph/warehouse/schema_definition/api_extension"
 require "yaml"
 
 project_root = File.expand_path(__dir__)
@@ -49,6 +50,7 @@ configure_local_rake_tasks = ->(tasks) do
   tasks.index_document_sizes = true
   tasks.env_port_mapping = {test: test_port}
   tasks.output = schema_def_output
+  tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
 
   tasks.define_fake_data_batch_for(:widgets) do
     require "rspec/core" # the factories file expects RSpec to be loaded, so load it.

config/site/Rakefile

@@ -433,6 +433,7 @@ module ElasticGraph
 
         # When we are releasing gems, these dependencies aren't available.
         unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile")
+          require "elastic_graph/json_ingestion/schema_definition/api_extension"
           require "elastic_graph/local/rake_tasks"
           require "elastic_graph/query_registry/rake_tasks"
 
@@ -460,6 +461,7 @@ module ElasticGraph
                   # Here we provide a "default Rakefile" for examples that don't have any specialized needs and that aren't trying
                   # to demonstrate any `Rakefile` APIs.
                   ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks|
+                    tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
                     tasks.opensearch_versions = []
                   end
                 end

config/site/support/doctest_helper.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names"
 require "elastic_graph/schema_definition/api"
 require "elastic_graph/schema_definition/schema_artifact_manager"
@@ -51,6 +52,7 @@ module ElasticGraph
     descriptions_needing_schema_def_api_and_extension_modules = {
       "ElasticGraph.define_schema" => [],
       "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension],
+      "ElasticGraph::JSONIngestion::SchemaDefinition" => [],
       "ElasticGraph::SchemaDefinition" => [],
       "ElasticGraph::Warehouse::SchemaDefinition" => [ElasticGraph::Warehouse::SchemaDefinition::APIExtension]
     }
@@ -60,7 +62,7 @@ module ElasticGraph
         @api = SchemaDefinition::API.new(
           SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}),
           true,
-          extension_modules: extension_modules
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension] + extension_modules
         )
 
         # This is required in all schemas, but we don't want to have to put in all our examples,
@@ -90,11 +92,27 @@ module ElasticGraph
       end
     end
 
+    doctest.before "ElasticGraph::JSONIngestion::SchemaDefinition" do
+      @api = SchemaDefinition::API.new(
+        SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}),
+        true,
+        extension_modules: [JSONIngestion::SchemaDefinition::APIExtension]
+      )
+
+      @api.json_schema_version 1
+      ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = @api
+    end
+
+    doctest.after "ElasticGraph::JSONIngestion::SchemaDefinition" do
+      ::Thread.current[:ElasticGraph_SchemaDefinition_API_instance] = nil
+    end
+
     doctest.before "ElasticGraph::SchemaDefinition::API#json_schema_version" do
       ElasticGraph.define_schema do |schema|
         # `schema.json_schema_version` raises an error when the version is set more than once.
         # By default we set it above. Here we clear it to allow our example to set it.
         schema.state.json_schema_version = nil
+        schema.state.json_schema_version_setter_location = nil
       end
     end
 

elasticgraph-admin/elasticgraph-admin.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rake", "~> 13.4", ">= 13.4.2"
 
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-apollo/README.md

@@ -49,29 +49,30 @@ index 4a5ef1e..5c16c2b 100644
 
 ```
 
-Finally, update `Rakefile` so that `ElasticGraph::GraphQL::Apollo::SchemaDefinition::APIExtension` is
-used as one of the `extension_modules`:
+Finally, update `Rakefile` so that `ElasticGraph::Apollo::SchemaDefinition::APIExtension` is
+used as one of the `schema_definition_extension_modules`:
 
 ```diff
 diff --git a/Rakefile b/Rakefile
 index 2943335..26633c3 100644
 --- a/Rakefile
 +++ b/Rakefile
-@@ -1,5 +1,6 @@
+@@ -1,6 +1,7 @@
  project_root = File.expand_path(__dir__)
 
 +require "elastic_graph/apollo/schema_definition/api_extension"
+ require "elastic_graph/json_ingestion/schema_definition/api_extension"
  require "elastic_graph/local/rake_tasks"
  require "elastic_graph/query_registry/rake_tasks"
  require "rspec/core/rake_task"
-@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new(
-   local_config_yaml: settings_file,
-   path_to_schema: "#{project_root}/config/schema.rb"
- ) do |tasks|
-+  tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension
-+
+@@ -16,5 +17,6 @@ ElasticGraph::Local::RakeTasks.new(
    # Determines casing of field names. Can be either `:camelCase` or `:snake_case`.
    tasks.schema_element_name_form = :camelCase
+   tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
+-
++  tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension
++
+   # Customizes the names of fields generated by ElasticGraph.
 ```
 
 That's it!

elasticgraph-apollo/apollo_tests_implementation/Rakefile

@@ -6,8 +6,9 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/schema_definition/rake_tasks"
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
+require "elastic_graph/schema_definition/rake_tasks"
 require "pathname"
 
 project_root = Pathname.new(__dir__)
@@ -17,5 +18,8 @@ ElasticGraph::SchemaDefinition::RakeTasks.new(
   index_document_sizes: false,
   path_to_schema: project_root / "config/products_schema.rb",
   schema_artifacts_directory: project_root / "config/schema/artifacts",
-  extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
+  extension_modules: [
+    ElasticGraph::Apollo::SchemaDefinition::APIExtension,
+    ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
+  ]
 )

elasticgraph-apollo/elasticgraph-apollo.gemspec

@@ -52,6 +52,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION
 end

elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/spec_support/schema_definition_helpers"
 
 module ElasticGraph
@@ -552,7 +553,7 @@ def self.with_both_casing_forms(&block)
         end
 
         def define_schema(&block)
-          extension_modules = [SchemaDefinition::APIExtension]
+          extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, SchemaDefinition::APIExtension]
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/spec_support/runtime_metadata_support"
 require "elastic_graph/spec_support/schema_definition_helpers"
 require "graphql"
@@ -1431,7 +1432,8 @@ def expect_identifiable_type_tagging_of_token(&type_def_for)
         end
 
         def define_schema(with_apollo: true, &block)
-          extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : []
+          extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension]
+          extension_modules += [SchemaDefinition::APIExtension] if with_apollo
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-graphql/elasticgraph-graphql.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION

elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb

@@ -8,6 +8,7 @@
 
 require "elastic_graph/graphql"
 require "elastic_graph/indexer"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_definition/rake_tasks"
 require "support/graphql"
 
@@ -65,6 +66,7 @@ def dump_schema_artifacts(json_schema_version:, team_extras: "")
       run_rake "schema_artifacts:dump" do |output|
         SchemaDefinition::RakeTasks.new(
           schema_element_name_form: :snake_case,
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
           index_document_sizes: true,
           path_to_schema: path_to_schema,
           schema_artifacts_directory: "config/schema/artifacts",

elasticgraph-indexer/elasticgraph-indexer.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/indexer"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_definition/rake_tasks"
 
 module ElasticGraph
@@ -401,6 +402,7 @@ def dump_artifacts
       run_rake "schema_artifacts:dump" do |output|
         SchemaDefinition::RakeTasks.new(
           schema_element_name_form: :snake_case,
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
           index_document_sizes: true,
           path_to_schema: path_to_schema,
           schema_artifacts_directory: "config/schema/artifacts",

elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb

@@ -576,7 +576,11 @@ def build_preparer_for_old_json_schema_version(v1_def:, v2_def:)
         end
 
         def define_schema(&schema_definition)
-          super(schema_element_name_form: "snake_case", &schema_definition)
+          super(
+            schema_element_name_form: "snake_case",
+            extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
+            &schema_definition
+          )
         end
       end
 

elasticgraph-json_ingestion/README.md

@@ -2,8 +2,8 @@
 
 JSON Schema ingestion support for ElasticGraph.
 
-This gem contains the JSON Schema helper code used by schema definition to generate indexing
-event schemas and merge ElasticGraph metadata into versioned schema artifacts.
+This gem provides the schema-definition extension that generates JSON Schema artifacts for indexing
+events and validates JSON-ingestion-specific schema options.
 
 ## Dependency Diagram
 
@@ -17,7 +17,4 @@ graph LR;
     elasticgraph-support["elasticgraph-support"];
     elasticgraph-json_ingestion --> elasticgraph-support;
     class elasticgraph-support otherEgGemStyle;
-    elasticgraph-schema_definition["elasticgraph-schema_definition"];
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
-    class elasticgraph-schema_definition otherEgGemStyle;
 ```

elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec

@@ -36,8 +36,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
 
-  # This gem's schema-definition extension code references `elasticgraph-schema_definition`, but
-  # applications load it through schema-definition tasks after `elasticgraph-schema_definition` is already
-  # available. Keeping this as a development dependency avoids a runtime dependency cycle.
+  # This gem's extension code references `elasticgraph-schema_definition`, but applications load it
+  # through schema-definition tasks after `elasticgraph-schema_definition` is already available.
+  # Keeping it as a development dependency avoids pulling schema definition into runtime bundles.
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end