Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-12-08 06:10:04 +00:00
Code Issues Projects Releases Packages Wiki Activity

cmd/go: move GOCACHEPROG protocol types to their own package

Browse source 
This is a step toward making it easy to point to them in
documentation. The other option is that we copy-paste all of these
type definitions wholesale, which seems ridiculous.

Updates #71032
Updates #59719

Change-Id: I7117e03308ae0adc721ed7a57792c33ba68ce827
Reviewed-on: https://go-review.googlesource.com/c/go/+/638995
Auto-Submit: Austin Clements <austin@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Matloob <matloob@golang.org>
This commit is contained in:
Austin Clements 2024-12-27 13:27:21 -05:00 • committed by Gopher Robot
parent 858a0e9dfd
commit 20da34c6d2
2 changed files with 155 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

163
src/cmd/go/internal/cache/prog.go vendored
View file

@ -7,6 +7,7 @@ package cache
import ( import (
"bufio" "bufio"
"cmd/go/internal/base" "cmd/go/internal/base"
"cmd/go/internal/cacheprog"
"cmd/internal/quoted" "cmd/internal/quoted"
"context" "context"
"crypto/sha256" "crypto/sha256"
@ -38,7 +39,7 @@ type ProgCache struct {
// can are the commands that the child process declared that it supports. // can are the commands that the child process declared that it supports.
// This is effectively the versioning mechanism. // This is effectively the versioning mechanism.
can map[ProgCmd]bool can map[cacheprog.ProgCmd]bool
// fuzzDirCache is another Cache implementation to use for the FuzzDir // fuzzDirCache is another Cache implementation to use for the FuzzDir
// method. In practice this is the default GOCACHE disk-based // method. In practice this is the default GOCACHE disk-based
@ -55,7 +56,7 @@ type ProgCache struct {
mu sync.Mutex // guards following fields mu sync.Mutex // guards following fields
nextID int64 nextID int64
inFlight map[int64]chan<- *ProgResponse inFlight map[int64]chan<- *cacheprog.ProgResponse
outputFile map[OutputID]string // object => abs path on disk outputFile map[OutputID]string // object => abs path on disk
// writeMu serializes writing to the child process. // writeMu serializes writing to the child process.
@ -63,134 +64,6 @@ type ProgCache struct {
writeMu sync.Mutex writeMu sync.Mutex
} }
// The following types define the protocol for a GOCACHEPROG program.
//
// By default, the go command manages a build cache stored in the file system
// itself. GOCACHEPROG can be set to the name of a command (with optional
// space-separated flags) that implements the go command build cache externally.
// This permits defining a different cache policy.
//
// The go command will start the GOCACHEPROG as a subprocess and communicate
// with it via JSON messages over stdin/stdout. The subprocess's stderr will be
// connected to the go command's stderr.
//
// The subprocess should immediately send a [ProgResponse] with its capabilities.
// After that, the go command will send a stream of [ProgRequest] messages and the
// subprocess should reply to each [ProgRequest] with a [ProgResponse] message.
// ProgCmd is a command that can be issued to a child process.
//
// If the interface needs to grow, the go command can add new commands or new
// versioned commands like "get2" in the future. The initial [ProgResponse] from
// the child process indicates which commands it supports.
type ProgCmd string
const (
// cmdPut tells the cache program to store an object in the cache.
//
// [ProgRequest.ActionID] is the cache key of this object. The cache should
// store [ProgRequest.OutputID] and [ProgRequest.Body] under this key for a
// later "get" request. It must also store the Body in a file in the local
// file system and return the path to that file in [ProgResponse.DiskPath],
// which must exist at least until a "close" request.
cmdPut = ProgCmd("put")
// cmdGet tells the cache program to retrieve an object from the cache.
//
// [ProgRequest.ActionID] specifies the key of the object to get. If the
// cache does not contain this object, it should set [ProgResponse.Miss] to
// true. Otherwise, it should populate the fields of [ProgResponse],
// including setting [ProgResponse.OutputID] to the OutputID of the original
// "put" request and [ProgResponse.DiskPath] to the path of a local file
// containing the Body of the original "put" request. That file must
// continue to exist at least until a "close" request.
cmdGet = ProgCmd("get")
// cmdClose requests that the cache program exit gracefully.
//
// The cache program should reply to this request and then exit
// (thus closing its stdout).
cmdClose = ProgCmd("close")
)
// ProgRequest is the JSON-encoded message that's sent from the go command to
// the GOCACHEPROG child process over stdin. Each JSON object is on its own
// line. A ProgRequest of Type "put" with BodySize > 0 will be followed by a
// line containing a base64-encoded JSON string literal of the body.
type ProgRequest struct {
// ID is a unique number per process across all requests.
// It must be echoed in the ProgResponse from the child.
ID int64
// Command is the type of request.
// The go command will only send commands that were declared
// as supported by the child.
Command ProgCmd
// ActionID is the cache key for "put" and "get" requests.
ActionID []byte `json:",omitempty"` // or nil if not used
// OutputID is stored with the body for "put" requests.
//
// Prior to Go 1.24, when GOCACHEPROG was still an experiment, this was
// accidentally named ObjectID. It was renamed to OutputID in Go 1.24.
OutputID []byte `json:",omitempty"` // or nil if not used
// Body is the body for "put" requests. It's sent after the JSON object
// as a base64-encoded JSON string when BodySize is non-zero.
// It's sent as a separate JSON value instead of being a struct field
// send in this JSON object so large values can be streamed in both directions.
// The base64 string body of a ProgRequest will always be written
// immediately after the JSON object and a newline.
Body io.Reader `json:"-"`
// BodySize is the number of bytes of Body. If zero, the body isn't written.
BodySize int64 `json:",omitempty"`
// ObjectID is the accidental spelling of OutputID that was used prior to Go
// 1.24.
//
// Deprecated: use OutputID. This field is only populated temporarily for
// backwards compatibility with Go 1.23 and earlier when
// GOEXPERIMENT=gocacheprog is set. It will be removed in Go 1.25.
ObjectID []byte `json:",omitempty"`
}
// ProgResponse is the JSON response from the child process to the go command.
//
// With the exception of the first protocol message that the child writes to its
// stdout with ID==0 and KnownCommands populated, these are only sent in
// response to a ProgRequest from the go command.
//
// ProgResponses can be sent in any order. The ID must match the request they're
// replying to.
type ProgResponse struct {
ID int64 // that corresponds to ProgRequest; they can be answered out of order
Err string `json:",omitempty"` // if non-empty, the error
// KnownCommands is included in the first message that cache helper program
// writes to stdout on startup (with ID==0). It includes the
// ProgRequest.Command types that are supported by the program.
//
// This lets the go command extend the protocol gracefully over time (adding
// "get2", etc), or fail gracefully when needed. It also lets the go command
// verify the program wants to be a cache helper.
KnownCommands []ProgCmd `json:",omitempty"`
// For "get" requests.
Miss bool `json:",omitempty"` // cache miss
OutputID []byte `json:",omitempty"` // the ObjectID stored with the body
Size int64 `json:",omitempty"` // body size in bytes
Time *time.Time `json:",omitempty"` // when the object was put in the cache (optional; used for cache expiration)
// For "get" and "put" requests.
// DiskPath is the absolute path on disk of the body corresponding to a
// "get" (on cache hit) or "put" request's ActionID.
DiskPath string `json:",omitempty"`
}
// startCacheProg starts the prog binary (with optional space-separated flags) // startCacheProg starts the prog binary (with optional space-separated flags)
// and returns a Cache implementation that talks to it. // and returns a Cache implementation that talks to it.
// //
@ -238,14 +111,14 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache {
stdout: out, stdout: out,
stdin: in, stdin: in,
bw: bufio.NewWriter(in), bw: bufio.NewWriter(in),
inFlight: make(map[int64]chan<- *ProgResponse), inFlight: make(map[int64]chan<- *cacheprog.ProgResponse),
outputFile: make(map[OutputID]string), outputFile: make(map[OutputID]string),
readLoopDone: make(chan struct{}), readLoopDone: make(chan struct{}),
} }
// Register our interest in the initial protocol message from the child to // Register our interest in the initial protocol message from the child to
// us, saying what it can do. // us, saying what it can do.
capResc := make(chan *ProgResponse, 1) capResc := make(chan *cacheprog.ProgResponse, 1)
pc.inFlight[0] = capResc pc.inFlight[0] = capResc
pc.jenc = json.NewEncoder(pc.bw) pc.jenc = json.NewEncoder(pc.bw)
@ -260,7 +133,7 @@ func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache {
case <-timer.C: case <-timer.C:
log.Printf("# still waiting for GOCACHEPROG %v ...", prog) log.Printf("# still waiting for GOCACHEPROG %v ...", prog)
case capRes := <-capResc: case capRes := <-capResc:
can := map[ProgCmd]bool{} can := map[cacheprog.ProgCmd]bool{}
for _, cmd := range capRes.KnownCommands { for _, cmd := range capRes.KnownCommands {
can[cmd] = true can[cmd] = true
} }
@ -277,7 +150,7 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) {
defer close(readLoopDone) defer close(readLoopDone)
jd := json.NewDecoder(c.stdout) jd := json.NewDecoder(c.stdout)
for { for {
res := new(ProgResponse) res := new(cacheprog.ProgResponse)
if err := jd.Decode(res); err != nil { if err := jd.Decode(res); err != nil {
if c.closing.Load() { if c.closing.Load() {
return // quietly return // quietly
@ -302,8 +175,8 @@ func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) {
} }
} }
func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, error) { func (c *ProgCache) send(ctx context.Context, req *cacheprog.ProgRequest) (*cacheprog.ProgResponse, error) {
resc := make(chan *ProgResponse, 1) resc := make(chan *cacheprog.ProgResponse, 1)
if err := c.writeToChild(req, resc); err != nil { if err := c.writeToChild(req, resc); err != nil {
return nil, err return nil, err
} }
@ -318,7 +191,7 @@ func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse,
} }
} }
func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (err error) { func (c *ProgCache) writeToChild(req *cacheprog.ProgRequest, resc chan<- *cacheprog.ProgResponse) (err error) {
c.mu.Lock() c.mu.Lock()
c.nextID++ c.nextID++
req.ID = c.nextID req.ID = c.nextID
@ -369,7 +242,7 @@ func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (e
} }
func (c *ProgCache) Get(a ActionID) (Entry, error) { func (c *ProgCache) Get(a ActionID) (Entry, error) {
if !c.can[cmdGet] { if !c.can[cacheprog.CmdGet] {
// They can't do a "get". Maybe they're a write-only cache. // They can't do a "get". Maybe they're a write-only cache.
// //
// TODO(bradfitz,bcmills): figure out the proper error type here. Maybe // TODO(bradfitz,bcmills): figure out the proper error type here. Maybe
@ -379,8 +252,8 @@ func (c *ProgCache) Get(a ActionID) (Entry, error) {
// error types on the Cache interface. // error types on the Cache interface.
return Entry{}, &entryNotFoundError{} return Entry{}, &entryNotFoundError{}
} }
res, err := c.send(c.ctx, &ProgRequest{ res, err := c.send(c.ctx, &cacheprog.ProgRequest{
Command: cmdGet, Command: cacheprog.CmdGet,
ActionID: a[:], ActionID: a[:],
}) })
if err != nil { if err != nil {
@ -436,7 +309,7 @@ func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64,
return OutputID{}, 0, err return OutputID{}, 0, err
} }
if !c.can[cmdPut] { if !c.can[cacheprog.CmdPut] {
// Child is a read-only cache. Do nothing. // Child is a read-only cache. Do nothing.
return out, size, nil return out, size, nil
} }
@ -448,8 +321,8 @@ func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64,
deprecatedValue = out[:] deprecatedValue = out[:]
} }
res, err := c.send(c.ctx, &ProgRequest{ res, err := c.send(c.ctx, &cacheprog.ProgRequest{
Command: cmdPut, Command: cacheprog.CmdPut,
ActionID: a[:], ActionID: a[:],
OutputID: out[:], OutputID: out[:],
ObjectID: deprecatedValue, // TODO(bradfitz): remove in Go 1.25 ObjectID: deprecatedValue, // TODO(bradfitz): remove in Go 1.25
@ -473,8 +346,8 @@ func (c *ProgCache) Close() error {
// First write a "close" message to the child so it can exit nicely // First write a "close" message to the child so it can exit nicely
// and clean up if it wants. Only after that exchange do we cancel // and clean up if it wants. Only after that exchange do we cancel
// the context that kills the process. // the context that kills the process.
if c.can[cmdClose] { if c.can[cacheprog.CmdClose] {
_, err = c.send(c.ctx, &ProgRequest{Command: cmdClose}) _, err = c.send(c.ctx, &cacheprog.ProgRequest{Command: cacheprog.CmdClose})
} }
// Cancel the context, which will close the helper's stdin. // Cancel the context, which will close the helper's stdin.
c.ctxCancel() c.ctxCancel()

137
src/cmd/go/internal/cacheprog/cacheprog.go Normal file
View file

@ -0,0 +1,137 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package cacheprog defines the protocol for a GOCACHEPROG program.
//
// By default, the go command manages a build cache stored in the file system
// itself. GOCACHEPROG can be set to the name of a command (with optional
// space-separated flags) that implements the go command build cache externally.
// This permits defining a different cache policy.
//
// The go command will start the GOCACHEPROG as a subprocess and communicate
// with it via JSON messages over stdin/stdout. The subprocess's stderr will be
// connected to the go command's stderr.
//
// The subprocess should immediately send a [ProgResponse] with its capabilities.
// After that, the go command will send a stream of [ProgRequest] messages and the
// subprocess should reply to each [ProgRequest] with a [ProgResponse] message.
package cacheprog
import (
"io"
"time"
)
// ProgCmd is a command that can be issued to a child process.
//
// If the interface needs to grow, the go command can add new commands or new
// versioned commands like "get2" in the future. The initial [ProgResponse] from
// the child process indicates which commands it supports.
type ProgCmd string
const (
// CmdPut tells the cache program to store an object in the cache.
//
// [ProgRequest.ActionID] is the cache key of this object. The cache should
// store [ProgRequest.OutputID] and [ProgRequest.Body] under this key for a
// later "get" request. It must also store the Body in a file in the local
// file system and return the path to that file in [ProgResponse.DiskPath],
// which must exist at least until a "close" request.
CmdPut = ProgCmd("put")
// CmdGet tells the cache program to retrieve an object from the cache.
//
// [ProgRequest.ActionID] specifies the key of the object to get. If the
// cache does not contain this object, it should set [ProgResponse.Miss] to
// true. Otherwise, it should populate the fields of [ProgResponse],
// including setting [ProgResponse.OutputID] to the OutputID of the original
// "put" request and [ProgResponse.DiskPath] to the path of a local file
// containing the Body of the original "put" request. That file must
// continue to exist at least until a "close" request.
CmdGet = ProgCmd("get")
// CmdClose requests that the cache program exit gracefully.
//
// The cache program should reply to this request and then exit
// (thus closing its stdout).
CmdClose = ProgCmd("close")
)
// ProgRequest is the JSON-encoded message that's sent from the go command to
// the GOCACHEPROG child process over stdin. Each JSON object is on its own
// line. A ProgRequest of Type "put" with BodySize > 0 will be followed by a
// line containing a base64-encoded JSON string literal of the body.
type ProgRequest struct {
// ID is a unique number per process across all requests.
// It must be echoed in the ProgResponse from the child.
ID int64
// Command is the type of request.
// The go command will only send commands that were declared
// as supported by the child.
Command ProgCmd
// ActionID is the cache key for "put" and "get" requests.
ActionID []byte `json:",omitempty"` // or nil if not used
// OutputID is stored with the body for "put" requests.
//
// Prior to Go 1.24, when GOCACHEPROG was still an experiment, this was
// accidentally named ObjectID. It was renamed to OutputID in Go 1.24.
OutputID []byte `json:",omitempty"` // or nil if not used
// Body is the body for "put" requests. It's sent after the JSON object
// as a base64-encoded JSON string when BodySize is non-zero.
// It's sent as a separate JSON value instead of being a struct field
// send in this JSON object so large values can be streamed in both directions.
// The base64 string body of a ProgRequest will always be written
// immediately after the JSON object and a newline.
Body io.Reader `json:"-"`
// BodySize is the number of bytes of Body. If zero, the body isn't written.
BodySize int64 `json:",omitempty"`
// ObjectID is the accidental spelling of OutputID that was used prior to Go
// 1.24.
//
// Deprecated: use OutputID. This field is only populated temporarily for
// backwards compatibility with Go 1.23 and earlier when
// GOEXPERIMENT=gocacheprog is set. It will be removed in Go 1.25.
ObjectID []byte `json:",omitempty"`
}
// ProgResponse is the JSON response from the child process to the go command.
//
// With the exception of the first protocol message that the child writes to its
// stdout with ID==0 and KnownCommands populated, these are only sent in
// response to a ProgRequest from the go command.
//
// ProgResponses can be sent in any order. The ID must match the request they're
// replying to.
type ProgResponse struct {
ID int64 // that corresponds to ProgRequest; they can be answered out of order
Err string `json:",omitempty"` // if non-empty, the error
// KnownCommands is included in the first message that cache helper program
// writes to stdout on startup (with ID==0). It includes the
// ProgRequest.Command types that are supported by the program.
//
// This lets the go command extend the protocol gracefully over time (adding
// "get2", etc), or fail gracefully when needed. It also lets the go command
// verify the program wants to be a cache helper.
KnownCommands []ProgCmd `json:",omitempty"`
// For "get" requests.
Miss bool `json:",omitempty"` // cache miss
OutputID []byte `json:",omitempty"` // the ObjectID stored with the body
Size int64 `json:",omitempty"` // body size in bytes
Time *time.Time `json:",omitempty"` // when the object was put in the cache (optional; used for cache expiration)
// For "get" and "put" requests.
// DiskPath is the absolute path on disk of the body corresponding to a
// "get" (on cache hit) or "put" request's ActionID.
DiskPath string `json:",omitempty"`
}