os: ignore O_TRUNC errors on named pipes and terminal devices on Windows

Prior to Go 1.24, os.OpenFile used to support O_TRUNC on named pipes and
terminal devices, even when the truncation was really ignored. This
behavior was consistent with Unix semantics.

CL 618836 changed the implementation of os.OpenFile on Windows and
unintentionally started returning an error when O_TRUNC was used on such
files.

Fixes #76071

Change-Id: Id10d3d8120ae9aa0548ef05423a172ff4e502ff9
Reviewed-on: https://go-review.googlesource.com/c/go/+/716420
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Damien Neil <dneil@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

src/internal/syscall/windows/at_windows.go

@@ -131,6 +131,14 @@ func Openat(dirfd syscall.Handle, name string, flag uint64, perm uint32) (_ sysc
 
 	if flag&syscall.O_TRUNC != 0 {
 		err = syscall.Ftruncate(h, 0)
+		if err == ERROR_INVALID_PARAMETER {
+			// ERROR_INVALID_PARAMETER means truncation is not supported on this file handle.
+			// Unix's O_TRUNC specification says to ignore O_TRUNC on named pipes and terminal devices.
+			// We do the same here.
+			if t, err1 := syscall.GetFileType(h); err1 == nil && (t == syscall.FILE_TYPE_PIPE || t == syscall.FILE_TYPE_CHAR) {
+				err = nil
+			}
+		}
 		if err != nil {
 			syscall.CloseHandle(h)
 			return syscall.InvalidHandle, err

src/os/os_windows_test.go

@@ -2275,3 +2275,16 @@ func TestOpenFileFlagInvalid(t *testing.T) {
 	}
 	f.Close()
 }
+
+func TestOpenFileTruncateNamedPipe(t *testing.T) {
+	t.Parallel()
+	name := pipeName()
+	pipe := newBytePipe(t, name, false)
+	defer pipe.Close()
+
+	f, err := os.OpenFile(name, os.O_TRUNC|os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+}

src/os/root_windows_test.go

@@ -228,3 +228,22 @@ func TestRootSymlinkToDirectory(t *testing.T) {
 		})
 	}
 }
+
+func TestRootOpenFileTruncateNamedPipe(t *testing.T) {
+	t.Parallel()
+	name := pipeName()
+	pipe := newBytePipe(t, name, false)
+	defer pipe.Close()
+
+	root, err := os.OpenRoot(filepath.Dir(name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer root.Close()
+
+	f, err := root.OpenFile(filepath.Base(name), os.O_TRUNC|os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.Close()
+}

src/syscall/syscall_windows.go

@@ -468,6 +468,14 @@ func Open(name string, flag int, perm uint32) (fd Handle, err error) {
 	if flag&O_TRUNC == O_TRUNC &&
 		(createmode == OPEN_EXISTING || (createmode == OPEN_ALWAYS && err == ERROR_ALREADY_EXISTS)) {
 		err = Ftruncate(h, 0)
+		if err == _ERROR_INVALID_PARAMETER {
+			// ERROR_INVALID_PARAMETER means truncation is not supported on this file handle.
+			// Unix's O_TRUNC specification says to ignore O_TRUNC on named pipes and terminal devices.
+			// We do the same here.
+			if t, err1 := GetFileType(h); err1 == nil && (t == FILE_TYPE_PIPE || t == FILE_TYPE_CHAR) {
+				err = nil
+			}
+		}
 		if err != nil {
 			CloseHandle(h)
 			return InvalidHandle, err

src/syscall/types_windows.go

@@ -34,6 +34,10 @@ const (
 	WSAECONNRESET             Errno = 10054
 )
 
+const (
+	_ERROR_INVALID_PARAMETER Errno = 87
+)
+
 const (
 	// Invented values to support what package os expects.
 	O_RDONLY       = 0x00000