mirror of
https://github.com/restic/restic.git
synced 2025-12-08 06:09:56 +00:00
Update dependenciess
Exclude minio-go for now (pin to 3.x.y).
This commit is contained in:
parent
9d0f13c4c0
commit
946c8399e2
2985 changed files with 1008107 additions and 118934 deletions
75
vendor/cloud.google.com/go/bigquery/uploader.go
generated
vendored
75
vendor/cloud.google.com/go/bigquery/uploader.go
generated
vendored
|
|
@ -20,6 +20,7 @@ import (
|
|||
"reflect"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
bq "google.golang.org/api/bigquery/v2"
|
||||
)
|
||||
|
||||
// An Uploader does streaming inserts into a BigQuery table.
|
||||
|
|
@ -151,27 +152,67 @@ func toValueSaver(x interface{}) (ValueSaver, bool, error) {
|
|||
}
|
||||
|
||||
func (u *Uploader) putMulti(ctx context.Context, src []ValueSaver) error {
|
||||
var rows []*insertionRow
|
||||
for _, saver := range src {
|
||||
req, err := u.newInsertRequest(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
call := u.t.c.bqs.Tabledata.InsertAll(u.t.ProjectID, u.t.DatasetID, u.t.TableID, req)
|
||||
call = call.Context(ctx)
|
||||
setClientHeader(call.Header())
|
||||
var res *bq.TableDataInsertAllResponse
|
||||
err = runWithRetry(ctx, func() (err error) {
|
||||
res, err = call.Do()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return handleInsertErrors(res.InsertErrors, req.Rows)
|
||||
}
|
||||
|
||||
func (u *Uploader) newInsertRequest(savers []ValueSaver) (*bq.TableDataInsertAllRequest, error) {
|
||||
req := &bq.TableDataInsertAllRequest{
|
||||
TemplateSuffix: u.TableTemplateSuffix,
|
||||
IgnoreUnknownValues: u.IgnoreUnknownValues,
|
||||
SkipInvalidRows: u.SkipInvalidRows,
|
||||
}
|
||||
for _, saver := range savers {
|
||||
row, insertID, err := saver.Save()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
rows = append(rows, &insertionRow{InsertID: insertID, Row: row})
|
||||
if insertID == "" {
|
||||
insertID = randomIDFn()
|
||||
}
|
||||
m := make(map[string]bq.JsonValue)
|
||||
for k, v := range row {
|
||||
m[k] = bq.JsonValue(v)
|
||||
}
|
||||
req.Rows = append(req.Rows, &bq.TableDataInsertAllRequestRows{
|
||||
InsertId: insertID,
|
||||
Json: m,
|
||||
})
|
||||
}
|
||||
|
||||
return u.t.c.service.insertRows(ctx, u.t.ProjectID, u.t.DatasetID, u.t.TableID, rows, &insertRowsConf{
|
||||
skipInvalidRows: u.SkipInvalidRows,
|
||||
ignoreUnknownValues: u.IgnoreUnknownValues,
|
||||
templateSuffix: u.TableTemplateSuffix,
|
||||
})
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// An insertionRow represents a row of data to be inserted into a table.
|
||||
type insertionRow struct {
|
||||
// If InsertID is non-empty, BigQuery will use it to de-duplicate insertions of
|
||||
// this row on a best-effort basis.
|
||||
InsertID string
|
||||
// The data to be inserted, represented as a map from field name to Value.
|
||||
Row map[string]Value
|
||||
func handleInsertErrors(ierrs []*bq.TableDataInsertAllResponseInsertErrors, rows []*bq.TableDataInsertAllRequestRows) error {
|
||||
if len(ierrs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var errs PutMultiError
|
||||
for _, e := range ierrs {
|
||||
if int(e.Index) > len(rows) {
|
||||
return fmt.Errorf("internal error: unexpected row index: %v", e.Index)
|
||||
}
|
||||
rie := RowInsertionError{
|
||||
InsertID: rows[e.Index].InsertId,
|
||||
RowIndex: int(e.Index),
|
||||
}
|
||||
for _, errp := range e.Errors {
|
||||
rie.Errors = append(rie.Errors, bqToError(errp))
|
||||
}
|
||||
errs = append(errs, rie)
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue