cmd/compile: fix type reshaping for nested instantiations

In nested instantiations, the frontend inserts a reshaping operation to
maintain the original type. However, while the CallExpr node's type was
updated, the underlying shape function used for instantiation still
retained the pointer shape in its signature.

This caused the SSA backend to use the pointer shape type where the
original reshaped type was expected.

Fix this by saving the reshape type for associated instantiations and
converting the instantiated result to the reshaped type, ensuring the
original type is properly preserved.

Fixes #78297

Change-Id: I412464f91c8e280250ca72b2505a9b9ae809e87c
Reviewed-on: https://go-review.googlesource.com/c/go/+/767161
Reviewed-by: Carlos Amedee <carlos@golang.org>
Auto-Submit: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Reviewed-by: David Chase <drchase@google.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

src/cmd/compile/internal/devirtualize/pgo.go

@@ -484,6 +484,7 @@ func condCall(curfn *ir.Func, pos src.XPos, cond ir.Node, thenCall, elseCall *ir
 	res := ir.NewInlinedCallExpr(pos, body, retvars)
 	res.SetType(thenCall.Type())
 	res.SetTypecheck(1)
+	res.Reshape = thenCall.Reshape
 	return res
 }
 

src/cmd/compile/internal/ir/expr.go

@@ -197,6 +197,7 @@ type CallExpr struct {
 	// whether it's a runtime.KeepAlive call the compiler generates to
 	// keep a variable alive. See #73137.
 	IsCompilerVarLive bool
+	Reshape           bool
 }
 
 func NewCallExpr(pos src.XPos, op Op, fun Node, args []Node) *CallExpr {
@@ -376,6 +377,7 @@ type InlinedCallExpr struct {
 	miniExpr
 	Body       Nodes
 	ReturnVars Nodes // must be side-effect free
+	Reshape    bool
 }
 
 func NewInlinedCallExpr(pos src.XPos, body, retvars []Node) *InlinedCallExpr {
@@ -391,10 +393,16 @@ func (n *InlinedCallExpr) SingleResult() Node {
 	if have := len(n.ReturnVars); have != 1 {
 		base.FatalfAt(n.Pos(), "inlined call has %v results, expected 1", have)
 	}
-	if !n.Type().HasShape() && n.ReturnVars[0].Type().HasShape() {
-		// If the type of the call is not a shape, but the type of the return value
-		// is a shape, we need to do an implicit conversion, so the real type
-		// of n is maintained.
+
+	// If the type of the call is not a shape, but the type of the return value
+	// is a shape, we need to do an implicit conversion, so the real type
+	// of n is maintained.
+	needImplicitConv := !n.Type().HasShape() && n.ReturnVars[0].Type().HasShape()
+	if n.Reshape { // or if the inlined call expr needs reshaping.
+		needImplicitConv = true
+	}
+
+	if needImplicitConv {
 		r := NewConvExpr(n.Pos(), OCONVNOP, n.Type(), n.ReturnVars[0])
 		r.SetTypecheck(1)
 		return r

src/cmd/compile/internal/noder/reader.go

@@ -982,19 +982,8 @@ func Shapify(targ *types.Type, basic bool) *types.Type {
 	// types, and discarding struct field names and tags. However, we'll
 	// need to start tracking how type parameters are actually used to
 	// implement some of these optimizations.
-	pointerShaping := basic && targ.IsPtr() && !targ.Elem().NotInHeap()
-	// The exception is when the type parameter is a pointer to a type
-	// which `Type.HasShape()` returns true, but `Type.IsShape()` returns
-	// false, like `*[]go.shape.T`. This is because the type parameter is
-	// used to instantiate a generic function inside another generic function.
-	// In this case, we want to keep the targ as-is, otherwise, we may lose the
-	// original type after `*[]go.shape.T` is shapified to `*go.shape.uint8`.
-	// See issue #54535, #71184.
-	if pointerShaping && !targ.Elem().IsShape() && targ.Elem().HasShape() {
-		return targ
-	}
 	under := targ.Underlying()
-	if pointerShaping {
+	if basic && targ.IsPtr() && !targ.Elem().NotInHeap() {
 		under = types.NewPtr(types.Types[types.TUINT8])
 	}
 
@@ -2566,6 +2555,11 @@ func (r *reader) expr() (res ir.Node) {
 		}
 
 		x.SetType(typ)
+
+		if call, ok := x.(*ir.CallExpr); ok {
+			call.Reshape = true
+		}
+
 		return x
 
 	case exprConvert:
@@ -3796,6 +3790,7 @@ func unifiedInlineCall(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlInd
 	res.SetInit(init)
 	res.SetType(call.Type())
 	res.SetTypecheck(1)
+	res.Reshape = call.Reshape
 
 	// Inlining shouldn't add any functions to todoBodies.
 	assert(len(todoBodies) == 0)

src/cmd/compile/internal/ssagen/ssa.go

@@ -5182,6 +5182,9 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool, deferExt
 		} else {
 			result = s.newValue1I(ssa.OpSelectN, fp.Type, 0, call)
 		}
+		if n.Reshape {
+			result = s.newValue1(ssa.OpCopy, n.Type(), result)
+		}
 	}
 
 	// Finish block for defers

test/fixedbugs/issue78297.go

@@ -0,0 +1,19 @@
+// compile
+
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+func G[U any]() (u U) { return }
+
+//go:noinline
+func H[U any]() (u U) { return }
+
+func F[T ~*[1]byte]() {
+	_ = G[T]()[:]
+	_ = H[T]()[:]
+}
+
+var _ = F[*[1]byte]