From 2e725dbadc8e1e3f49d3247e979a8ebdf5656bdd Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:42:46 +0100 Subject: [PATCH] grpcerrors: preserve unary RPC cancellation causes Restore the local context cause when a unary RPC returns a matching bare cancellation or deadline status. Preserve the RPC status code, including when the cause is a custom error, and leave distinct remote errors intact. Handle this in the shared client interceptor so session auth, secrets, SSH checks, and gateway calls retain context from the actual RPC context. This also preserves causes attached only to a session-derived context. Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- session/auth/auth_test.go | 114 +++++++++++++++++++++++++++ util/grpcerrors/intercept.go | 13 ++- util/grpcerrors/intercept_test.go | 126 ++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 session/auth/auth_test.go create mode 100644 util/grpcerrors/intercept_test.go diff --git a/session/auth/auth_test.go b/session/auth/auth_test.go new file mode 100644 index 000000000000..40b211e5f3ea --- /dev/null +++ b/session/auth/auth_test.go @@ -0,0 +1,114 @@ +package auth + +import ( + "context" + "testing" + "time" + + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/testutil" + "github.com/moby/buildkit/util/grpcerrors" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestSessionAuthTimeout(t *testing.T) { + original := SessionAuthTimeout + SessionAuthTimeout = 100 * time.Millisecond + t.Cleanup(func() { SessionAuthTimeout = original }) + + sm, err := session.NewManager() + require.NoError(t, err) + + s, err := session.NewSession(t.Context(), "") + require.NoError(t, err) + + s.Allow(blockingAuthServer{}) + done := make(chan error, 1) + go func() { done <- s.Run(t.Context(), session.Dialer(testutil.TestStream(sm.HandleConn))) }() + t.Cleanup(func() { require.NoError(t, s.Close()); require.NoError(t, <-done) }) + + ctx, cancel := context.WithTimeoutCause(t.Context(), 10*time.Second, errors.New("waiting for auth session")) + defer cancel() + + _, err = sm.Get(ctx, s.ID(), false) + require.NoError(t, err) + + g := session.NewGroup(s.ID()) + for _, tc := range []struct { + name string + call func(context.Context) error + }{ + { + name: "resolving credentials from session", + call: func(ctx context.Context) error { + _, _, _, err := CredentialsFunc(ctx, sm, g)("registry.example.com") + return err + }, + }, + { + name: "fetching auth token from session", + call: func(ctx context.Context) error { _, err := FetchToken(ctx, &FetchTokenRequest{}, sm, g); return err }, + }, + { + name: "getting token authority from session", + call: func(ctx context.Context) error { + _, _, err := GetTokenAuthority(ctx, "registry.example.com", sm, g) + return err + }, + }, + { + name: "verifying token authority from session", + call: func(ctx context.Context) error { + _, _, err := VerifyTokenAuthority(ctx, "registry.example.com", new([32]byte), sm, g) + return err + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.call(t.Context()) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.EqualError(t, err, tc.name+": context deadline exceeded") + encoded := grpcerrors.ToGRPC(t.Context(), err) + require.Equal(t, codes.DeadlineExceeded, status.Code(encoded)) + require.Equal(t, err.Error(), status.Convert(encoded).Message()) + + parent, cancel := context.WithCancelCause(t.Context()) + cause := errors.Wrap(context.Canceled, "build canceled") + cancel(cause) + err = tc.call(parent) + require.ErrorIs(t, err, cause) + require.Equal(t, codes.Canceled, grpcerrors.Code(err)) + require.NotContains(t, err.Error(), tc.name) + }) + } +} + +type blockingAuthServer struct{ UnimplementedAuthServer } + +func (s blockingAuthServer) Register(server *grpc.Server) { + RegisterAuthServer(server, s) +} + +func (blockingAuthServer) Credentials(ctx context.Context, _ *CredentialsRequest) (*CredentialsResponse, error) { + <-ctx.Done() + return nil, context.Cause(ctx) +} + +func (blockingAuthServer) FetchToken(ctx context.Context, _ *FetchTokenRequest) (*FetchTokenResponse, error) { + <-ctx.Done() + return nil, context.Cause(ctx) +} + +func (blockingAuthServer) GetTokenAuthority(ctx context.Context, _ *GetTokenAuthorityRequest) (*GetTokenAuthorityResponse, error) { + <-ctx.Done() + return nil, context.Cause(ctx) +} + +func (blockingAuthServer) VerifyTokenAuthority(ctx context.Context, _ *VerifyTokenAuthorityRequest) (*VerifyTokenAuthorityResponse, error) { + <-ctx.Done() + return nil, context.Cause(ctx) +} diff --git a/util/grpcerrors/intercept.go b/util/grpcerrors/intercept.go index a4c774953ffa..5551611041f0 100644 --- a/util/grpcerrors/intercept.go +++ b/util/grpcerrors/intercept.go @@ -8,6 +8,7 @@ import ( "github.com/moby/buildkit/util/stack" "github.com/pkg/errors" "google.golang.org/grpc" + "google.golang.org/grpc/status" ) func UnaryServerInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { @@ -38,7 +39,17 @@ func StreamServerInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamSer } func UnaryClientInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { - err := FromGRPC(invoker(ctx, method, req, reply, cc, opts...)) + err := invoker(ctx, method, req, reply, cc, opts...) + if err != nil { + ctxErr := ctx.Err() //nolint:forbidigo // Match gRPC's status before restoring a custom cancellation cause. + if ctxErr != nil { + st, ok := status.FromError(err) + if ok && st.Code() == Code(ctxErr) && st.Message() == ctxErr.Error() && len(st.Details()) == 0 { + return errors.WithStack(WrapCode(context.Cause(ctx), st.Code())) + } + } + } + err = FromGRPC(err) if err != nil { stack.Helper() } diff --git a/util/grpcerrors/intercept_test.go b/util/grpcerrors/intercept_test.go new file mode 100644 index 000000000000..b8e3a4db3db9 --- /dev/null +++ b/util/grpcerrors/intercept_test.go @@ -0,0 +1,126 @@ +package grpcerrors_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/moby/buildkit/util/grpcerrors" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + health "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" +) + +func TestUnaryClientCancellationCause(t *testing.T) { + t.Parallel() + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer() + started := make(chan struct{}) + health.RegisterHealthServer(server, &blockingHealthServer{started: started}) + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + conn, err := grpc.NewClient("passthrough:///test", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", listener.Addr().String()) + }), + grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, conn.Close()) }) + + // Only the derived RPC context is canceled, as with session health failures. + parent := t.Context() + ctx, cancel := context.WithCancelCause(parent) + defer cancel(context.Canceled) + cause := errors.New("session healthcheck failed fatally") + result := make(chan error, 1) + go func() { _, err := health.NewHealthClient(conn).Check(ctx, &health.HealthCheckRequest{}); result <- err }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("RPC did not start") + } + cancel(cause) + select { + case err := <-result: + require.ErrorIs(t, err, cause) + require.Equal(t, codes.Canceled, grpcerrors.Code(err)) + encoded := grpcerrors.ToGRPC(parent, err) + require.Equal(t, codes.Canceled, status.Code(encoded)) + require.Contains(t, status.Convert(encoded).Message(), cause.Error()) + require.NoError(t, context.Cause(parent)) + case <-time.After(10 * time.Second): + t.Fatal("RPC did not stop") + } +} + +type blockingHealthServer struct { + health.UnimplementedHealthServer + started chan struct{} +} + +func (s *blockingHealthServer) Check(ctx context.Context, _ *health.HealthCheckRequest) (*health.HealthCheckResponse, error) { + close(s.started) + <-ctx.Done() + return nil, context.Cause(ctx) +} + +func TestUnaryClientPreservesRPCError(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancelCause(t.Context()) + cancel(errors.New("local cancellation")) + for _, tc := range []struct { + name string + err error + }{ + { + name: "success", + }, + { + name: "unrelated status", + err: status.Error(codes.PermissionDenied, "denied"), + }, + { + name: "remote cancellation", + err: status.Error(codes.Canceled, "remote operation canceled"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := grpcerrors.UnaryClientInterceptor(ctx, "/test", nil, nil, nil, func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { return tc.err }) + if tc.err == nil { + require.NoError(t, err) + return + } + require.Equal(t, status.Code(tc.err), grpcerrors.Code(err)) + require.Contains(t, err.Error(), status.Convert(tc.err).Message()) + }) + } +} + +func TestUnaryClientDeadlineCause(t *testing.T) { + t.Parallel() + cause := errors.New("frontend startup timed out") + ctx, cancel := context.WithTimeoutCause(t.Context(), -time.Second, cause) + defer cancel() + rpcErr := status.Error(codes.DeadlineExceeded, "context deadline exceeded") + invoke := func(context.Context, string, any, any, *grpc.ClientConn, ...grpc.CallOption) error { return rpcErr } + err := grpcerrors.UnaryClientInterceptor(ctx, "/test", nil, nil, nil, invoke) + require.ErrorIs(t, err, cause) + require.Equal(t, codes.DeadlineExceeded, grpcerrors.Code(err)) + encoded := grpcerrors.ToGRPC(t.Context(), err) + require.Equal(t, codes.DeadlineExceeded, status.Code(encoded)) + require.Equal(t, cause.Error(), status.Convert(encoded).Message()) + + // A remote deadline must remain intact while the local context is live. + err = grpcerrors.UnaryClientInterceptor(t.Context(), "/test", nil, nil, nil, invoke) + require.Equal(t, codes.DeadlineExceeded, grpcerrors.Code(err)) + require.NotErrorIs(t, err, cause) + require.Contains(t, err.Error(), "context deadline exceeded") +}