From d43507358c555752984c8fb79bf3c2a79a818408 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 20:57:03 -0500 Subject: [PATCH 1/8] add grpcurl integration graph traversal tests --- test/integration/v1_reflection_test.exs | 44 ++++++++ test/integration/v1alpha_reflection_test.exs | 44 ++++++++ test/support/client.ex | 104 ++++++++++++++++++- 3 files changed, 191 insertions(+), 1 deletion(-) diff --git a/test/integration/v1_reflection_test.exs b/test/integration/v1_reflection_test.exs index 569d623..3f7d5eb 100644 --- a/test/integration/v1_reflection_test.exs +++ b/test/integration/v1_reflection_test.exs @@ -436,4 +436,48 @@ defmodule GrpcReflection.V1ReflectionTest do {:list_services, ""} ] end + + test "reflection graph is traversable using grpcurl", ctx do + ops = GrpcReflection.TestClient.grpcurl_service(ctx) + + assert ops == [ + {:call, "grpc.reflection.v1.ServerReflection.ServerReflectionInfo"}, + {:call, "grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo"}, + {:call, "helloworld.Greeter.SayHello"}, + {:call, "testserviceV2.TestService.CallFunction"}, + {:call, "testserviceV3.TestService.CallFunction"}, + {:service, "grpc.reflection.v1.ServerReflection"}, + {:service, "grpc.reflection.v1alpha.ServerReflection"}, + {:service, "helloworld.Greeter"}, + {:service, "testserviceV2.TestService"}, + {:service, "testserviceV3.TestService"}, + {:type, ".google.protobuf.Any"}, + {:type, ".google.protobuf.StringValue"}, + {:type, ".google.protobuf.Timestamp"}, + {:type, ".grpc.reflection.v1.ErrorResponse"}, + {:type, ".grpc.reflection.v1.ExtensionNumberResponse"}, + {:type, ".grpc.reflection.v1.ExtensionRequest"}, + {:type, ".grpc.reflection.v1.FileDescriptorResponse"}, + {:type, ".grpc.reflection.v1.ListServiceResponse"}, + {:type, ".grpc.reflection.v1.ServerReflectionRequest"}, + {:type, ".grpc.reflection.v1.ServerReflectionResponse"}, + {:type, ".grpc.reflection.v1.ServiceResponse"}, + {:type, ".grpc.reflection.v1alpha.ErrorResponse"}, + {:type, ".grpc.reflection.v1alpha.ExtensionNumberResponse"}, + {:type, ".grpc.reflection.v1alpha.ExtensionRequest"}, + {:type, ".grpc.reflection.v1alpha.FileDescriptorResponse"}, + {:type, ".grpc.reflection.v1alpha.ListServiceResponse"}, + {:type, ".grpc.reflection.v1alpha.ServerReflectionRequest"}, + {:type, ".grpc.reflection.v1alpha.ServerReflectionResponse"}, + {:type, ".grpc.reflection.v1alpha.ServiceResponse"}, + {:type, ".helloworld.HelloReply"}, + {:type, ".helloworld.HelloRequest"}, + {:type, ".testserviceV2.Enum"}, + {:type, ".testserviceV2.TestReply"}, + {:type, ".testserviceV2.TestRequest"}, + {:type, ".testserviceV3.Enum"}, + {:type, ".testserviceV3.TestReply"}, + {:type, ".testserviceV3.TestRequest"} + ] + end end diff --git a/test/integration/v1alpha_reflection_test.exs b/test/integration/v1alpha_reflection_test.exs index 5012edf..1997e3c 100644 --- a/test/integration/v1alpha_reflection_test.exs +++ b/test/integration/v1alpha_reflection_test.exs @@ -439,4 +439,48 @@ defmodule GrpcReflection.V1alphaReflectionTest do {:list_services, ""} ] end + + test "reflection graph is traversable using grpcurl", ctx do + ops = GrpcReflection.TestClient.grpcurl_service(ctx) + + assert ops == [ + {:call, "grpc.reflection.v1.ServerReflection.ServerReflectionInfo"}, + {:call, "grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo"}, + {:call, "helloworld.Greeter.SayHello"}, + {:call, "testserviceV2.TestService.CallFunction"}, + {:call, "testserviceV3.TestService.CallFunction"}, + {:service, "grpc.reflection.v1.ServerReflection"}, + {:service, "grpc.reflection.v1alpha.ServerReflection"}, + {:service, "helloworld.Greeter"}, + {:service, "testserviceV2.TestService"}, + {:service, "testserviceV3.TestService"}, + {:type, ".google.protobuf.Any"}, + {:type, ".google.protobuf.StringValue"}, + {:type, ".google.protobuf.Timestamp"}, + {:type, ".grpc.reflection.v1.ErrorResponse"}, + {:type, ".grpc.reflection.v1.ExtensionNumberResponse"}, + {:type, ".grpc.reflection.v1.ExtensionRequest"}, + {:type, ".grpc.reflection.v1.FileDescriptorResponse"}, + {:type, ".grpc.reflection.v1.ListServiceResponse"}, + {:type, ".grpc.reflection.v1.ServerReflectionRequest"}, + {:type, ".grpc.reflection.v1.ServerReflectionResponse"}, + {:type, ".grpc.reflection.v1.ServiceResponse"}, + {:type, ".grpc.reflection.v1alpha.ErrorResponse"}, + {:type, ".grpc.reflection.v1alpha.ExtensionNumberResponse"}, + {:type, ".grpc.reflection.v1alpha.ExtensionRequest"}, + {:type, ".grpc.reflection.v1alpha.FileDescriptorResponse"}, + {:type, ".grpc.reflection.v1alpha.ListServiceResponse"}, + {:type, ".grpc.reflection.v1alpha.ServerReflectionRequest"}, + {:type, ".grpc.reflection.v1alpha.ServerReflectionResponse"}, + {:type, ".grpc.reflection.v1alpha.ServiceResponse"}, + {:type, ".helloworld.HelloReply"}, + {:type, ".helloworld.HelloRequest"}, + {:type, ".testserviceV2.Enum"}, + {:type, ".testserviceV2.TestReply"}, + {:type, ".testserviceV2.TestRequest"}, + {:type, ".testserviceV3.Enum"}, + {:type, ".testserviceV3.TestReply"}, + {:type, ".testserviceV3.TestRequest"} + ] + end end diff --git a/test/support/client.ex b/test/support/client.ex index 2fe6e76..36ea53c 100644 --- a/test/support/client.ex +++ b/test/support/client.ex @@ -28,7 +28,7 @@ defmodule GrpcReflection.TestClient do :v1alpha -> GrpcReflection.TestEndpoint.V1AlphaServer.Stub end - %{channel: channel, req: req, stub: stub, version: unquote(version)} + %{channel: channel, req: req, stub: stub, version: unquote(version), host: host} end end end @@ -148,4 +148,106 @@ defmodule GrpcReflection.TestClient do extendee_commands ++ nested_commands end) end + + def grpcurl_service(ctx) do + ctx + |> grpcurl_list_services() + |> Stream.unfold(fn + [] -> + nil + + [{:service, service} | rest] = term -> + commands = grpcurl_describe_service(ctx, service) + {term, commands ++ rest} + + [{:call, call} | rest] = term -> + commands = grpcurl_describe_call(ctx, call) + {term, commands ++ rest} + + [{:type, type} | rest] = term -> + commands = grpcurl_describe_type(ctx, type) + {term, commands ++ rest} + end) + |> Enum.to_list() + |> List.flatten() + |> Enum.uniq() + |> Enum.sort() + end + + defp grpcurl_list_services(%{host: host}) do + {result, 0} = System.cmd("grpcurl", ["-v", "-plaintext", host, "list"]) + + result + |> String.split("\n") + |> Enum.reject(&(&1 == "")) + |> Enum.map(&{:service, &1}) + end + + defp grpcurl_describe_service(%{host: host}, service) do + {result, 0} = System.cmd("grpcurl", ["-v", "-plaintext", host, "list", service]) + + result + |> String.split("\n") + |> Enum.reject(&(&1 == "")) + |> Enum.map(&{:call, &1}) + end + + defp grpcurl_describe_call(%{host: host}, call) do + {result, 0} = System.cmd("grpcurl", ["-v", "-plaintext", host, "describe", call]) + + ~r/\((?: stream)? (?[a-zA-Z0-9.]+) \)/ + |> Regex.scan(result, capture: ["symbol"]) + |> List.flatten() + |> Enum.map(&{:type, &1}) + end + + defp grpcurl_describe_type(%{host: host}, type) do + {result, 0} = System.cmd("grpcurl", ["-v", "-plaintext", host, "describe", type]) + + # we are grabbing the referenced types to we can fetch those too + # but some of them might be defined inside this file + # so we have to identify those and filter them back out + + inline_declared_symbols = + ~r/(?:message (?\w+) {)|(?})|(?{)/ + |> Regex.scan(result, capture: :all_but_first) + |> List.flatten() + |> Enum.reject(&(&1 == "")) + # replace first declaration with base type to get full names + |> then(fn [_base | rest] -> [type | rest] end) + |> remove_matched_parens() + |> Enum.reduce({[], []}, fn token, {path, inline_types} -> + case token do + "}" -> + [_ | path] = path + # End of current message — pop from path + {path, inline_types} + + name -> + path = [name | path] + name = path |> Enum.reverse() |> Enum.join(".") + {path, [name | inline_types]} + end + end) + |> elem(1) + + # now we can grab all the references, then reject the nested declarations + ~r/ (?\.[a-z]+[a-zA-Z0-9.]+) / + |> Regex.scan(result, capture: ["symbol"]) + |> List.flatten() + |> Enum.reject(&Enum.member?(inline_declared_symbols, &1)) + |> Enum.map(&{:type, &1}) + end + + defp remove_matched_parens(["{", "}" | rest]) do + remove_matched_parens(rest) + end + + defp remove_matched_parens([item | rest]) do + [item | remove_matched_parens(rest)] + end + + defp remove_matched_parens(rest) do + rest + end end From 263fd05f1dbae4bb3a4abea31b393b64b55cbf98 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:00:16 -0500 Subject: [PATCH 2/8] add grpcurl to CI setup --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 189778b..2082ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install grpcurl using snap + run: sudo snap install grpcurl --classic + - name: Set up Elixir id: beam uses: erlef/setup-beam@v1 From 86638e049b51e5d674b4fe9beaac90e792549808 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:01:55 -0500 Subject: [PATCH 3/8] seitch grpcurl from classic to edge snap --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2082ad3..d1ea321 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v4 - name: Install grpcurl using snap - run: sudo snap install grpcurl --classic + run: sudo snap install grpcurl --edge - name: Set up Elixir id: beam From c517c21db226e1c97a2da4703babdb1bbac02e26 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:06:36 -0500 Subject: [PATCH 4/8] add snap bin to path --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1ea321..586f5e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,9 @@ jobs: - uses: actions/checkout@v4 - name: Install grpcurl using snap - run: sudo snap install grpcurl --edge + run: | + sudo snap install grpcurl --edge + echo "::add-path::/snap/bin" - name: Set up Elixir id: beam From 43105c404beb391ad24a42b59cc02fb3d945e4a2 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:09:49 -0500 Subject: [PATCH 5/8] try again --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 586f5e1..7285917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,9 +43,10 @@ jobs: - uses: actions/checkout@v4 - name: Install grpcurl using snap - run: | - sudo snap install grpcurl --edge - echo "::add-path::/snap/bin" + run: sudo snap install grpcurl --edge + + - name: Add snap bin to PATH (new method) + run: echo "/snap/bin" >> $GITHUB_PATH - name: Set up Elixir id: beam From e5e7afdc0f1436179bc8a9779edcd1f2df897d71 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:18:21 -0500 Subject: [PATCH 6/8] also install grpcurl for deps unlocked step --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7285917..9853f55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Install grpcurl using snap run: sudo snap install grpcurl --edge - - name: Add snap bin to PATH (new method) + - name: Add snap bin to PATH run: echo "/snap/bin" >> $GITHUB_PATH - name: Set up Elixir @@ -140,6 +140,12 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install grpcurl using snap + run: sudo snap install grpcurl --edge + + - name: Add snap bin to PATH + run: echo "/snap/bin" >> $GITHUB_PATH + - name: Set up Elixir id: beam uses: erlef/setup-beam@v1 From 922c39798fae2508ef0242735eda96764633a4a0 Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:23:14 -0500 Subject: [PATCH 7/8] add comments --- test/support/client.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/support/client.ex b/test/support/client.ex index 36ea53c..3b5ea65 100644 --- a/test/support/client.ex +++ b/test/support/client.ex @@ -239,6 +239,9 @@ defmodule GrpcReflection.TestClient do |> Enum.map(&{:type, &1}) end + # if we only match `message XYZ {` and `}`, we get extra `}` tokens from things like the + # oneof declarations. If we also match non-message `{` we can eliminate these intermeidate + # paren-blocks, which we do here for our needs defp remove_matched_parens(["{", "}" | rest]) do remove_matched_parens(rest) end From 8b1c5a3f00923d5b1657c0cd2a98f60da69c621c Mon Sep 17 00:00:00 2001 From: mjheilmann Date: Wed, 31 Dec 2025 21:29:21 -0500 Subject: [PATCH 8/8] grab grpcurl release from github instead of using snap --- .github/workflows/ci.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9853f55..eab9ee2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,11 +42,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install grpcurl using snap - run: sudo snap install grpcurl --edge - - - name: Add snap bin to PATH - run: echo "/snap/bin" >> $GITHUB_PATH + - name: Install grpcurl + run: | + GRPCURL_VERSION="1.9.3" + curl -sSL https://github.com/fullstorydev/grpcurl/releases/download/v1.9.3/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz | tar -xvz + mv grpcurl /usr/local/bin/grpcurl + grpcurl --version - name: Set up Elixir id: beam @@ -140,11 +141,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install grpcurl using snap - run: sudo snap install grpcurl --edge - - - name: Add snap bin to PATH - run: echo "/snap/bin" >> $GITHUB_PATH + - name: Install grpcurl + run: | + GRPCURL_VERSION="1.9.3" + curl -sSL https://github.com/fullstorydev/grpcurl/releases/download/v1.9.3/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz | tar -xvz + mv grpcurl /usr/local/bin/grpcurl + grpcurl --version - name: Set up Elixir id: beam