diff --git a/CODEBASE_OVERVIEW.md b/CODEBASE_OVERVIEW.md index a7f4b58ea..d81fd45df 100644 --- a/CODEBASE_OVERVIEW.md +++ b/CODEBASE_OVERVIEW.md @@ -192,13 +192,14 @@ graph LR; click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" "Open on RubyGems.org" _blank; ``` -### Extensions (6 gems) +### Extensions (7 gems) These libraries extend ElasticGraph to provide optional but commonly needed functionality. * [elasticgraph-apollo](elasticgraph-apollo/README.md): Transforms an ElasticGraph project into an Apollo subgraph. * [elasticgraph-health_check](elasticgraph-health_check/README.md): Provides a health check for high availability ElasticGraph deployments. * [elasticgraph-json_ingestion](elasticgraph-json_ingestion/README.md): JSON Schema ingestion support for ElasticGraph. +* [elasticgraph-proto_ingestion](elasticgraph-proto_ingestion/README.md): Supports ingesting Protocol Buffer data into ElasticGraph. * [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): Intercepts ElasticGraph datastore queries. * [elasticgraph-query_registry](elasticgraph-query_registry/README.md): Provides a source-controlled query registry for ElasticGraph applications. * [elasticgraph-warehouse](elasticgraph-warehouse/README.md): Extends ElasticGraph to support ingestion into a data warehouse. @@ -218,6 +219,7 @@ graph LR; elasticgraph-health_check["eg-health_check"]; elasticgraph-datastore_core["eg-datastore_core"]; elasticgraph-json_ingestion["eg-json_ingestion"]; + elasticgraph-proto_ingestion["eg-proto_ingestion"]; elasticgraph-query_interceptor["eg-query_interceptor"]; elasticgraph-schema_artifacts["eg-schema_artifacts"]; elasticgraph-query_registry["eg-query_registry"]; @@ -231,6 +233,7 @@ graph LR; elasticgraph-health_check --> elasticgraph-graphql; elasticgraph-health_check --> elasticgraph-support; elasticgraph-json_ingestion --> elasticgraph-support; + elasticgraph-proto_ingestion --> elasticgraph-support; elasticgraph-query_interceptor --> elasticgraph-graphql; elasticgraph-query_interceptor --> elasticgraph-schema_artifacts; elasticgraph-query_registry --> elasticgraph-graphql; @@ -246,6 +249,7 @@ graph LR; class elasticgraph-health_check targetGemStyle; class elasticgraph-datastore_core otherEgGemStyle; class elasticgraph-json_ingestion targetGemStyle; + class elasticgraph-proto_ingestion targetGemStyle; class elasticgraph-query_interceptor targetGemStyle; class elasticgraph-schema_artifacts otherEgGemStyle; class elasticgraph-query_registry targetGemStyle; diff --git a/Gemfile.lock b/Gemfile.lock index ebd747001..0575acac3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -140,6 +140,12 @@ PATH faraday-retry (~> 2.4) opensearch-ruby (~> 3.4) +PATH + remote: elasticgraph-proto_ingestion + specs: + elasticgraph-proto_ingestion (1.2.1.pre) + elasticgraph-support (= 1.2.1.pre) + PATH remote: elasticgraph-query_interceptor specs: @@ -688,6 +694,7 @@ DEPENDENCIES elasticgraph-lambda_support (= 1.2.1.pre)! elasticgraph-local (= 1.2.1.pre)! elasticgraph-opensearch (= 1.2.1.pre)! + elasticgraph-proto_ingestion (= 1.2.1.pre)! elasticgraph-query_interceptor (= 1.2.1.pre)! elasticgraph-query_registry (= 1.2.1.pre)! elasticgraph-rack (= 1.2.1.pre)! @@ -779,6 +786,7 @@ CHECKSUMS elasticgraph-lambda_support (1.2.1.pre) elasticgraph-local (1.2.1.pre) elasticgraph-opensearch (1.2.1.pre) + elasticgraph-proto_ingestion (1.2.1.pre) elasticgraph-query_interceptor (1.2.1.pre) elasticgraph-query_registry (1.2.1.pre) elasticgraph-rack (1.2.1.pre) diff --git a/config/docker_demo/Dockerfile b/config/docker_demo/Dockerfile index 2110740fd..f40bbf37b 100644 --- a/config/docker_demo/Dockerfile +++ b/config/docker_demo/Dockerfile @@ -19,6 +19,7 @@ COPY elasticgraph-indexer elasticgraph-indexer/ COPY elasticgraph-json_ingestion elasticgraph-json_ingestion/ COPY elasticgraph-local elasticgraph-local/ COPY elasticgraph-opensearch elasticgraph-opensearch/ +COPY elasticgraph-proto_ingestion elasticgraph-proto_ingestion/ COPY elasticgraph-query_registry elasticgraph-query_registry/ COPY elasticgraph-rack elasticgraph-rack/ COPY elasticgraph-schema_artifacts elasticgraph-schema_artifacts/ diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb index ee2ec66e7..0f308bf2c 100644 --- a/config/site/support/doctest_helper.rb +++ b/config/site/support/doctest_helper.rb @@ -8,6 +8,7 @@ require "elastic_graph/apollo/schema_definition/api_extension" require "elastic_graph/json_ingestion/schema_definition/api_extension" +require "elastic_graph/proto_ingestion/schema_definition/api_extension" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" require "elastic_graph/schema_definition/api" require "elastic_graph/schema_definition/schema_artifact_manager" @@ -53,6 +54,7 @@ module ElasticGraph "ElasticGraph.define_schema" => [], "ElasticGraph::Apollo::SchemaDefinition" => [Apollo::SchemaDefinition::APIExtension], "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension], + "ElasticGraph::ProtoIngestion::SchemaDefinition" => [ProtoIngestion::SchemaDefinition::APIExtension], "ElasticGraph::SchemaDefinition" => [], "ElasticGraph::Warehouse::SchemaDefinition" => [Warehouse::SchemaDefinition::APIExtension] } diff --git a/elasticgraph-proto_ingestion/.rspec b/elasticgraph-proto_ingestion/.rspec new file mode 120000 index 000000000..67e6e21b3 --- /dev/null +++ b/elasticgraph-proto_ingestion/.rspec @@ -0,0 +1 @@ +../spec_support/subdir_dot_rspec \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/.yardopts b/elasticgraph-proto_ingestion/.yardopts new file mode 120000 index 000000000..e11a2057f --- /dev/null +++ b/elasticgraph-proto_ingestion/.yardopts @@ -0,0 +1 @@ +../config/site/yardopts \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/Gemfile b/elasticgraph-proto_ingestion/Gemfile new file mode 120000 index 000000000..26cb2ad91 --- /dev/null +++ b/elasticgraph-proto_ingestion/Gemfile @@ -0,0 +1 @@ +../Gemfile \ No newline at end of file diff --git a/elasticgraph-proto_ingestion/LICENSE.txt b/elasticgraph-proto_ingestion/LICENSE.txt new file mode 100644 index 000000000..aa18b5db8 --- /dev/null +++ b/elasticgraph-proto_ingestion/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 - 2026 Block, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md new file mode 100644 index 000000000..ca9ad4bcb --- /dev/null +++ b/elasticgraph-proto_ingestion/README.md @@ -0,0 +1,17 @@ +# ElasticGraph::ProtoIngestion + +An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph. + +## Dependency Diagram + +```mermaid +graph LR; + classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px; + classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000; + classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9; + elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"]; + class elasticgraph-proto_ingestion targetGemStyle; + elasticgraph-support["elasticgraph-support"]; + elasticgraph-proto_ingestion --> elasticgraph-support; + class elasticgraph-support otherEgGemStyle; +``` diff --git a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec new file mode 100644 index 000000000..7ef526536 --- /dev/null +++ b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec @@ -0,0 +1,40 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "../elasticgraph-support/lib/elastic_graph/version" + +Gem::Specification.new do |spec| + spec.name = "elasticgraph-proto_ingestion" + spec.version = ElasticGraph::VERSION + spec.authors = ["Josh Wilson", "Myron Marston", "Block Engineering"] + spec.email = ["joshuaw@squareup.com"] + spec.homepage = "https://block.github.io/elasticgraph/" + spec.license = "MIT" + spec.summary = "Supports ingesting Protocol Buffer data into ElasticGraph." + + spec.metadata = { + "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues", + "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}", + "documentation_uri" => "https://block.github.io/elasticgraph/api-docs/v#{ElasticGraph::VERSION}/", + "homepage_uri" => "https://block.github.io/elasticgraph/", + "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}", + "gem_category" => "extension" + } + + spec.files = Dir.chdir(File.expand_path(__dir__)) do + `git ls-files -z`.split("\x0").reject do |f| + (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)}) + end - [".rspec", "Gemfile", ".yardopts"] + end + + spec.required_ruby_version = [">= 3.4", "< 4.1"] + + spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + + spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb new file mode 100644 index 000000000..df5f14429 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb @@ -0,0 +1,18 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + # Namespace for Protocol Buffers schema artifact generation extensions. + module ProtoIngestion + # The name of the generated Protocol Buffers schema file. + PROTO_SCHEMA_FILE = "schema.proto" + + # The name of the generated proto field-number mapping file. + PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml" + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb new file mode 100644 index 000000000..761fea495 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -0,0 +1,26 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" + +module ElasticGraph + module ProtoIngestion + # Namespace for all protobuf schema definition support. + # + # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module. + module SchemaDefinition + # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance + # to enable protobuf schema artifact generation. + # + # @note The protobuf schema artifact generation logic has not been implemented yet, so + # extending this module is currently a no-op. + module APIExtension + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs new file mode 100644 index 000000000..81cc74611 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs @@ -0,0 +1,6 @@ +module ElasticGraph + module ProtoIngestion + PROTO_SCHEMA_FILE: ::String + PROTO_FIELD_NUMBERS_FILE: ::String + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs new file mode 100644 index 000000000..81e79cec4 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -0,0 +1,8 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module APIExtension + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/spec_helper.rb b/elasticgraph-proto_ingestion/spec/spec_helper.rb new file mode 100644 index 000000000..4e6c40a9a --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/spec_helper.rb @@ -0,0 +1,10 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +# This file contains RSpec configuration for `elasticgraph-proto_ingestion`. +# It is loaded by the shared spec helper at `spec_support/spec_helper.rb`. diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb new file mode 100644 index 000000000..7f57ae738 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -0,0 +1,33 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/spec_support/schema_definition_helpers" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe APIExtension do + include_context "SchemaDefinitionHelpers" + + it "can be used as a schema definition extension module, without customizing the schema definition yet" do + with_extension, without_extension = [[APIExtension], []].map do |extension_modules| + define_schema(schema_element_name_form: "snake_case", extension_modules: extension_modules) do |schema| + schema.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + end.graphql_schema_string + end + + expect(with_extension).to eq(without_extension) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb new file mode 100644 index 000000000..558b42477 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb @@ -0,0 +1,21 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" + +module ElasticGraph + RSpec.describe ProtoIngestion do + it "defines the PROTO_SCHEMA_FILE constant" do + expect(ProtoIngestion::PROTO_SCHEMA_FILE).to eq("schema.proto") + end + + it "defines the PROTO_FIELD_NUMBERS_FILE constant" do + expect(ProtoIngestion::PROTO_FIELD_NUMBERS_FILE).to eq("proto_field_numbers.yaml") + end + end +end diff --git a/elasticgraph-support/README.md b/elasticgraph-support/README.md index 8464cbc88..17fc1fa9d 100644 --- a/elasticgraph-support/README.md +++ b/elasticgraph-support/README.md @@ -51,6 +51,9 @@ graph LR; elasticgraph-opensearch["elasticgraph-opensearch"]; elasticgraph-opensearch --> elasticgraph-support; class elasticgraph-opensearch otherEgGemStyle; + elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"]; + elasticgraph-proto_ingestion --> elasticgraph-support; + class elasticgraph-proto_ingestion otherEgGemStyle; elasticgraph-query_registry["elasticgraph-query_registry"]; elasticgraph-query_registry --> elasticgraph-support; class elasticgraph-query_registry otherEgGemStyle; diff --git a/rbs_collection.yaml b/rbs_collection.yaml index e19fbb28d..38c029048 100644 --- a/rbs_collection.yaml +++ b/rbs_collection.yaml @@ -82,6 +82,8 @@ gems: ignore: true - name: elasticgraph-opensearch ignore: true + - name: elasticgraph-proto_ingestion + ignore: true - name: elasticgraph-query_interceptor ignore: true - name: elasticgraph-query_registry