From cc66e8b927418b42352ecdaa8e4c012faea9c92f Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Tue, 7 Jul 2026 10:09:35 -0500 Subject: [PATCH] Scaffold the elasticgraph-proto_ingestion gem Creates the new `elasticgraph-proto_ingestion` extension gem and wires it into the repo (bundle, RBS collection, docker demo, doctest helper, docs diagrams), without any of the Protocol Buffers generation logic yet. The gem currently just defines its artifact file name constants and an `APIExtension` entry point that does not customize anything. Splitting the gem creation out this way keeps the follow-up PR that adds the actual generation logic contained to the gem's own directory. --- CODEBASE_OVERVIEW.md | 6 ++- Gemfile.lock | 8 ++++ config/docker_demo/Dockerfile | 1 + config/site/support/doctest_helper.rb | 2 + elasticgraph-proto_ingestion/.rspec | 1 + elasticgraph-proto_ingestion/.yardopts | 1 + elasticgraph-proto_ingestion/Gemfile | 1 + elasticgraph-proto_ingestion/LICENSE.txt | 21 ++++++++++ elasticgraph-proto_ingestion/README.md | 17 ++++++++ .../elasticgraph-proto_ingestion.gemspec | 40 +++++++++++++++++++ .../lib/elastic_graph/proto_ingestion.rb | 18 +++++++++ .../schema_definition/api_extension.rb | 26 ++++++++++++ .../sig/elastic_graph/proto_ingestion.rbs | 6 +++ .../schema_definition/api_extension.rbs | 8 ++++ .../spec/spec_helper.rb | 10 +++++ .../schema_definition/api_extension_spec.rb | 33 +++++++++++++++ .../elastic_graph/proto_ingestion_spec.rb | 21 ++++++++++ elasticgraph-support/README.md | 3 ++ rbs_collection.yaml | 2 + 19 files changed, 224 insertions(+), 1 deletion(-) create mode 120000 elasticgraph-proto_ingestion/.rspec create mode 120000 elasticgraph-proto_ingestion/.yardopts create mode 120000 elasticgraph-proto_ingestion/Gemfile create mode 100644 elasticgraph-proto_ingestion/LICENSE.txt create mode 100644 elasticgraph-proto_ingestion/README.md create mode 100644 elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs create mode 100644 elasticgraph-proto_ingestion/spec/spec_helper.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb diff --git a/CODEBASE_OVERVIEW.md b/CODEBASE_OVERVIEW.md index a7f4b58ea..d81fd45df 100644 --- a/CODEBASE_OVERVIEW.md +++ b/CODEBASE_OVERVIEW.md @@ -192,13 +192,14 @@ graph LR; click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" "Open on RubyGems.org" _blank; ``` -### Extensions (6 gems) +### Extensions (7 gems) These libraries extend ElasticGraph to provide optional but commonly needed functionality. * [elasticgraph-apollo](elasticgraph-apollo/README.md): Transforms an ElasticGraph project into an Apollo subgraph. * [elasticgraph-health_check](elasticgraph-health_check/README.md): Provides a health check for high availability ElasticGraph deployments. * [elasticgraph-json_ingestion](elasticgraph-json_ingestion/README.md): JSON Schema ingestion support for ElasticGraph. +* [elasticgraph-proto_ingestion](elasticgraph-proto_ingestion/README.md): Supports ingesting Protocol Buffer data into ElasticGraph. * [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): Intercepts ElasticGraph datastore queries. * [elasticgraph-query_registry](elasticgraph-query_registry/README.md): Provides a source-controlled query registry for ElasticGraph applications. * [elasticgraph-warehouse](elasticgraph-warehouse/README.md): Extends ElasticGraph to support ingestion into a data warehouse. @@ -218,6 +219,7 @@ graph LR; elasticgraph-health_check["eg-health_check"]; elasticgraph-datastore_core["eg-datastore_core"]; elasticgraph-json_ingestion["eg-json_ingestion"]; + elasticgraph-proto_ingestion["eg-proto_ingestion"]; elasticgraph-query_interceptor["eg-query_interceptor"]; elasticgraph-schema_artifacts["eg-schema_artifacts"]; elasticgraph-query_registry["eg-query_registry"]; @@ -231,6 +233,7 @@ graph LR; elasticgraph-health_check --> elasticgraph-graphql; elasticgraph-health_check --> elasticgraph-support; elasticgraph-json_ingestion --> elasticgraph-support; + elasticgraph-proto_ingestion --> elasticgraph-support; elasticgraph-query_interceptor --> elasticgraph-graphql; elasticgraph-query_interceptor --> elasticgraph-schema_artifacts; elasticgraph-query_registry --> elasticgraph-graphql; @@ -246,6 +249,7 @@ graph LR; class elasticgraph-health_check targetGemStyle; class elasticgraph-datastore_core otherEgGemStyle; class elasticgraph-json_ingestion targetGemStyle; + class elasticgraph-proto_ingestion targetGemStyle; class elasticgraph-query_interceptor targetGemStyle; class elasticgraph-schema_artifacts otherEgGemStyle; class elasticgraph-query_registry targetGemStyle; diff --git a/Gemfile.lock b/Gemfile.lock index ebd747001..0575acac3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -140,6 +140,12 @@ PATH faraday-retry (~> 2.4) opensearch-ruby (~> 3.4) +PATH + remote: elasticgraph-proto_ingestion + specs: + elasticgraph-proto_ingestion (1.2.1.pre) + elasticgraph-support (= 1.2.1.pre) + PATH remote: elasticgraph-query_interceptor specs: @@ -688,6 +694,7 @@ DEPENDENCIES elasticgraph-lambda_support (= 1.2.1.pre)! elasticgraph-local (= 1.2.1.pre)! elasticgraph-opensearch (= 1.2.1.pre)! + elasticgraph-proto_ingestion (= 1.2.1.pre)! elasticgraph-query_interceptor (= 1.2.1.pre)! elasticgraph-query_registry (= 1.2.1.pre)! elasticgraph-rack (= 1.2.1.pre)! @@ -779,6 +786,7 @@ CHECKSUMS elasticgraph-lambda_support (1.2.1.pre) elasticgraph-local (1.2.1.pre) elasticgraph-opensearch (1.2.1.pre) + elasticgraph-proto_ingestion (1.2.1.pre) elasticgraph-query_interceptor (1.2.1.pre) elasticgraph-query_registry (1.2.1.pre) elasticgraph-rack (1.2.1.pre) diff --git a/config/docker_demo/Dockerfile b/config/docker_demo/Dockerfile index 2110740fd..f40bbf37b 100644 --- a/config/docker_demo/Dockerfile +++ b/config/docker_demo/Dockerfile @@ -19,6 +19,7 @@ COPY elasticgraph-indexer elasticgraph-indexer/ COPY elasticgraph-json_ingestion elasticgraph-json_ingestion/ COPY elasticgraph-local elasticgraph-local/ COPY elasticgraph-opensearch elasticgraph-opensearch/ +COPY elasticgraph-proto_ingestion elasticgraph-proto_ingestion/ COPY elasticgraph-query_registry elasticgraph-query_registry/ COPY elasticgraph-rack elasticgraph-rack/ COPY elasticgraph-schema_artifacts elasticgraph-schema_artifacts/ diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb index ee2ec66e7..0f308bf2c 100644 --- a/config/site/support/doctest_helper.rb +++ b/config/site/support/doctest_helper.rb @@ -8,6 +8,7 @@ require "elastic_graph/apollo/schema_definition/api_extension" require "elastic_graph/json_ingestion/schema_definition/api_extension" +require "elastic_graph/proto_ingestion/schema_definition/api_extension" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" require "elastic_graph/schema_definition/api" require "elastic_graph/schema_definition/schema_artifact_manager" @@ -53,6 +54,7 @@ module ElasticGraph "ElasticGraph.define_schema" => [], "ElasticGraph::Apollo::SchemaDefinition" => [Apollo::SchemaDefinition::APIExtension], "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension], + "ElasticGraph::ProtoIngestion::SchemaDefinition" => [ProtoIngestion::SchemaDefinition::APIExtension], "ElasticGraph::SchemaDefinition" => [], "ElasticGraph::Warehouse::SchemaDefinition" => [Warehouse::SchemaDefinition::APIExtension] } diff --git a/elasticgraph-proto_ingestion/.rspec b/elasticgraph-proto_ingestion/.rspec new file mode 120000 index 000000000..67e6e21b3 --- /dev/null +++ b/elasticgraph-proto_ingestion/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/.yardopts b/elasticgraph-proto_ingestion/.yardopts new file mode 120000 index 000000000..e11a2057f --- /dev/null +++ b/elasticgraph-proto_ingestion/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/Gemfile b/elasticgraph-proto_ingestion/Gemfile new file mode 120000 index 000000000..26cb2ad91 --- /dev/null +++ b/elasticgraph-proto_ingestion/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/LICENSE.txt b/elasticgraph-proto_ingestion/LICENSE.txt new file mode 100644 index 000000000..aa18b5db8 --- /dev/null +++ b/elasticgraph-proto_ingestion/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 - 2026 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md new file mode 100644 index 000000000..ca9ad4bcb --- /dev/null +++ b/elasticgraph-proto_ingestion/README.md @@ -0,0 +1,17 @@ +# ElasticGraph::ProtoIngestion + +An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph. + +## Dependency Diagram + +```mermaid +graph LR; + classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px; + classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000; + classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9; + elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"]; + class elasticgraph-proto_ingestion targetGemStyle; + elasticgraph-support["elasticgraph-support"]; + elasticgraph-proto_ingestion --> elasticgraph-support; + class elasticgraph-support otherEgGemStyle; +``` diff --git a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec new file mode 100644 index 000000000..7ef526536 --- /dev/null +++ b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec @@ -0,0 +1,40 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../elasticgraph-support/lib/elastic_graph/version" + +Gem::Specification.new do |spec| + spec.name = "elasticgraph-proto_ingestion" + spec.version = ElasticGraph::VERSION + spec.authors = ["Josh Wilson", "Myron Marston", "Block Engineering"] + spec.email = ["joshuaw@squareup.com"] + spec.homepage = "https://block.github.io/elasticgraph/" + spec.license = "MIT" + spec.summary = "Supports ingesting Protocol Buffer data into ElasticGraph." + + spec.metadata = { + "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues", + "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}", + "documentation_uri" => "https://block.github.io/elasticgraph/api-docs/v#{ElasticGraph::VERSION}/", + "homepage_uri" => "https://block.github.io/elasticgraph/", + "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}", + "gem_category" => "extension" + } + + spec.files = Dir.chdir(File.expand_path(__dir__)) do + `git ls-files -z`.split("\x0").reject do |f| + (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)}) + end - [".rspec", "Gemfile", ".yardopts"] + end + + spec.required_ruby_version = [">= 3.4", "< 4.1"] + + spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + + spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb new file mode 100644 index 000000000..df5f14429 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb @@ -0,0 +1,18 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # Namespace for Protocol Buffers schema artifact generation extensions. + module ProtoIngestion + # The name of the generated Protocol Buffers schema file. + PROTO_SCHEMA_FILE = "schema.proto" + + # The name of the generated proto field-number mapping file. + PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml" + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb new file mode 100644 index 000000000..761fea495 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -0,0 +1,26 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" + +module ElasticGraph + module ProtoIngestion + # Namespace for all protobuf schema definition support. + # + # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module. + module SchemaDefinition + # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance + # to enable protobuf schema artifact generation. + # + # @note The protobuf schema artifact generation logic has not been implemented yet, so + # extending this module is currently a no-op. + module APIExtension + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs new file mode 100644 index 000000000..81cc74611 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs @@ -0,0 +1,6 @@ +module ElasticGraph + module ProtoIngestion + PROTO_SCHEMA_FILE: ::String + PROTO_FIELD_NUMBERS_FILE: ::String + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs new file mode 100644 index 000000000..81e79cec4 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module APIExtension + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/spec_helper.rb b/elasticgraph-proto_ingestion/spec/spec_helper.rb new file mode 100644 index 000000000..4e6c40a9a --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/spec_helper.rb @@ -0,0 +1,10 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file contains RSpec configuration for `elasticgraph-proto_ingestion`. +# It is loaded by the shared spec helper at `spec_support/spec_helper.rb`. diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb new file mode 100644 index 000000000..7f57ae738 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -0,0 +1,33 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe APIExtension do + include_context "SchemaDefinitionHelpers" + + it "can be used as a schema definition extension module, without customizing the schema definition yet" do + with_extension, without_extension = [[APIExtension], []].map do |extension_modules| + define_schema(schema_element_name_form: "snake_case", extension_modules: extension_modules) do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end.graphql_schema_string + end + + expect(with_extension).to eq(without_extension) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb new file mode 100644 index 000000000..558b42477 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb @@ -0,0 +1,21 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" + +module ElasticGraph + RSpec.describe ProtoIngestion do + it "defines the PROTO_SCHEMA_FILE constant" do + expect(ProtoIngestion::PROTO_SCHEMA_FILE).to eq("schema.proto") + end + + it "defines the PROTO_FIELD_NUMBERS_FILE constant" do + expect(ProtoIngestion::PROTO_FIELD_NUMBERS_FILE).to eq("proto_field_numbers.yaml") + end + end +end diff --git a/elasticgraph-support/README.md b/elasticgraph-support/README.md index 8464cbc88..17fc1fa9d 100644 --- a/elasticgraph-support/README.md +++ b/elasticgraph-support/README.md @@ -51,6 +51,9 @@ graph LR; elasticgraph-opensearch["elasticgraph-opensearch"]; elasticgraph-opensearch --> elasticgraph-support; class elasticgraph-opensearch otherEgGemStyle; + elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"]; + elasticgraph-proto_ingestion --> elasticgraph-support; + class elasticgraph-proto_ingestion otherEgGemStyle; elasticgraph-query_registry["elasticgraph-query_registry"]; elasticgraph-query_registry --> elasticgraph-support; class elasticgraph-query_registry otherEgGemStyle; diff --git a/rbs_collection.yaml b/rbs_collection.yaml index e19fbb28d..38c029048 100644 --- a/rbs_collection.yaml +++ b/rbs_collection.yaml @@ -82,6 +82,8 @@ gems: ignore: true - name: elasticgraph-opensearch ignore: true + - name: elasticgraph-proto_ingestion + ignore: true - name: elasticgraph-query_interceptor ignore: true - name: elasticgraph-query_registry