feat(train): add new command to interact with aws and train models

This commit is contained in:
2021-10-17 19:15:44 +02:00
parent 5436dfebc2
commit 538cea18f2
1064 changed files with 282251 additions and 89305 deletions

35
vendor/github.com/aws/smithy-go/encoding/json/array.go generated vendored Normal file
View File

@ -0,0 +1,35 @@
package json
import (
"bytes"
)
// Array represent the encoding of a JSON Array
type Array struct {
w *bytes.Buffer
writeComma bool
scratch *[]byte
}
func newArray(w *bytes.Buffer, scratch *[]byte) *Array {
w.WriteRune(leftBracket)
return &Array{w: w, scratch: scratch}
}
// Value adds a new element to the JSON Array.
// Returns a Value type that is used to encode
// the array element.
func (a *Array) Value() Value {
if a.writeComma {
a.w.WriteRune(comma)
} else {
a.writeComma = true
}
return newValue(a.w, a.scratch)
}
// Close encodes the end of the JSON Array
func (a *Array) Close() {
a.w.WriteRune(rightBracket)
}

View File

@ -0,0 +1,15 @@
package json
const (
leftBrace = '{'
rightBrace = '}'
leftBracket = '['
rightBracket = ']'
comma = ','
quote = '"'
colon = ':'
null = "null"
)

View File

@ -0,0 +1,139 @@
package json
import (
"bytes"
"encoding/json"
"fmt"
"io"
)
// DiscardUnknownField discards unknown fields from decoder body.
// This function is useful while deserializing json body with additional
// unknown information that should be discarded.
func DiscardUnknownField(decoder *json.Decoder) error {
// This deliberately does not share logic with CollectUnknownField, even
// though it could, because if we were to delegate to that then we'd incur
// extra allocations and general memory usage.
v, err := decoder.Token()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
if _, ok := v.(json.Delim); ok {
for decoder.More() {
err = DiscardUnknownField(decoder)
}
endToken, err := decoder.Token()
if err != nil {
return err
}
if _, ok := endToken.(json.Delim); !ok {
return fmt.Errorf("invalid JSON : expected json delimiter, found %T %v",
endToken, endToken)
}
}
return nil
}
// CollectUnknownField grabs the contents of unknown fields from the decoder body
// and returns them as a byte slice. This is useful for skipping unknown fields without
// completely discarding them.
func CollectUnknownField(decoder *json.Decoder) ([]byte, error) {
result, err := collectUnknownField(decoder)
if err != nil {
return nil, err
}
buff := bytes.NewBuffer(nil)
encoder := json.NewEncoder(buff)
if err := encoder.Encode(result); err != nil {
return nil, err
}
return buff.Bytes(), nil
}
func collectUnknownField(decoder *json.Decoder) (interface{}, error) {
// Grab the initial value. This could either be a concrete value like a string or a a
// delimiter.
token, err := decoder.Token()
if err == io.EOF {
return nil, nil
}
if err != nil {
return nil, err
}
// If it's an array or object, we'll need to recurse.
delim, ok := token.(json.Delim)
if ok {
var result interface{}
if delim == '{' {
result, err = collectUnknownObject(decoder)
if err != nil {
return nil, err
}
} else {
result, err = collectUnknownArray(decoder)
if err != nil {
return nil, err
}
}
// Discard the closing token. decoder.Token handles checking for matching delimiters
if _, err := decoder.Token(); err != nil {
return nil, err
}
return result, nil
}
return token, nil
}
func collectUnknownArray(decoder *json.Decoder) ([]interface{}, error) {
// We need to create an empty array here instead of a nil array, since by getting
// into this function at all we necessarily have seen a non-nil list.
array := []interface{}{}
for decoder.More() {
value, err := collectUnknownField(decoder)
if err != nil {
return nil, err
}
array = append(array, value)
}
return array, nil
}
func collectUnknownObject(decoder *json.Decoder) (map[string]interface{}, error) {
object := make(map[string]interface{})
for decoder.More() {
key, err := collectUnknownField(decoder)
if err != nil {
return nil, err
}
// Keys have to be strings, which is particularly important as the encoder
// won't except a map with interface{} keys
stringKey, ok := key.(string)
if !ok {
return nil, fmt.Errorf("expected string key, found %T", key)
}
value, err := collectUnknownField(decoder)
if err != nil {
return nil, err
}
object[stringKey] = value
}
return object, nil
}

View File

@ -0,0 +1,30 @@
package json
import (
"bytes"
)
// Encoder is JSON encoder that supports construction of JSON values
// using methods.
type Encoder struct {
w *bytes.Buffer
Value
}
// NewEncoder returns a new JSON encoder
func NewEncoder() *Encoder {
writer := bytes.NewBuffer(nil)
scratch := make([]byte, 64)
return &Encoder{w: writer, Value: newValue(writer, &scratch)}
}
// String returns the String output of the JSON encoder
func (e Encoder) String() string {
return e.w.String()
}
// Bytes returns the []byte slice of the JSON encoder
func (e Encoder) Bytes() []byte {
return e.w.Bytes()
}

198
vendor/github.com/aws/smithy-go/encoding/json/escape.go generated vendored Normal file
View File

@ -0,0 +1,198 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Copied and modified from Go 1.8 stdlib's encoding/json/#safeSet
package json
import (
"bytes"
"unicode/utf8"
)
// safeSet holds the value true if the ASCII character with the given array
// position can be represented inside a JSON string without any further
// escaping.
//
// All values are true except for the ASCII control characters (0-31), the
// double quote ("), and the backslash character ("\").
var safeSet = [utf8.RuneSelf]bool{
' ': true,
'!': true,
'"': false,
'#': true,
'$': true,
'%': true,
'&': true,
'\'': true,
'(': true,
')': true,
'*': true,
'+': true,
',': true,
'-': true,
'.': true,
'/': true,
'0': true,
'1': true,
'2': true,
'3': true,
'4': true,
'5': true,
'6': true,
'7': true,
'8': true,
'9': true,
':': true,
';': true,
'<': true,
'=': true,
'>': true,
'?': true,
'@': true,
'A': true,
'B': true,
'C': true,
'D': true,
'E': true,
'F': true,
'G': true,
'H': true,
'I': true,
'J': true,
'K': true,
'L': true,
'M': true,
'N': true,
'O': true,
'P': true,
'Q': true,
'R': true,
'S': true,
'T': true,
'U': true,
'V': true,
'W': true,
'X': true,
'Y': true,
'Z': true,
'[': true,
'\\': false,
']': true,
'^': true,
'_': true,
'`': true,
'a': true,
'b': true,
'c': true,
'd': true,
'e': true,
'f': true,
'g': true,
'h': true,
'i': true,
'j': true,
'k': true,
'l': true,
'm': true,
'n': true,
'o': true,
'p': true,
'q': true,
'r': true,
's': true,
't': true,
'u': true,
'v': true,
'w': true,
'x': true,
'y': true,
'z': true,
'{': true,
'|': true,
'}': true,
'~': true,
'\u007f': true,
}
// copied from Go 1.8 stdlib's encoding/json/#hex
var hex = "0123456789abcdef"
// escapeStringBytes escapes and writes the passed in string bytes to the dst
// buffer
//
// Copied and modifed from Go 1.8 stdlib's encodeing/json/#encodeState.stringBytes
func escapeStringBytes(e *bytes.Buffer, s []byte) {
e.WriteByte('"')
start := 0
for i := 0; i < len(s); {
if b := s[i]; b < utf8.RuneSelf {
if safeSet[b] {
i++
continue
}
if start < i {
e.Write(s[start:i])
}
switch b {
case '\\', '"':
e.WriteByte('\\')
e.WriteByte(b)
case '\n':
e.WriteByte('\\')
e.WriteByte('n')
case '\r':
e.WriteByte('\\')
e.WriteByte('r')
case '\t':
e.WriteByte('\\')
e.WriteByte('t')
default:
// This encodes bytes < 0x20 except for \t, \n and \r.
// If escapeHTML is set, it also escapes <, >, and &
// because they can lead to security holes when
// user-controlled strings are rendered into JSON
// and served to some browsers.
e.WriteString(`\u00`)
e.WriteByte(hex[b>>4])
e.WriteByte(hex[b&0xF])
}
i++
start = i
continue
}
c, size := utf8.DecodeRune(s[i:])
if c == utf8.RuneError && size == 1 {
if start < i {
e.Write(s[start:i])
}
e.WriteString(`\ufffd`)
i += size
start = i
continue
}
// U+2028 is LINE SEPARATOR.
// U+2029 is PARAGRAPH SEPARATOR.
// They are both technically valid characters in JSON strings,
// but don't work in JSONP, which has to be evaluated as JavaScript,
// and can lead to security holes there. It is valid JSON to
// escape them, so we do so unconditionally.
// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
if c == '\u2028' || c == '\u2029' {
if start < i {
e.Write(s[start:i])
}
e.WriteString(`\u202`)
e.WriteByte(hex[c&0xF])
i += size
start = i
continue
}
i += size
}
if start < len(s) {
e.Write(s[start:])
}
e.WriteByte('"')
}

View File

@ -0,0 +1,42 @@
package json
import (
"bytes"
)
// Object represents the encoding of a JSON Object type
type Object struct {
w *bytes.Buffer
writeComma bool
scratch *[]byte
}
func newObject(w *bytes.Buffer, scratch *[]byte) *Object {
w.WriteRune(leftBrace)
return &Object{w: w, scratch: scratch}
}
func (o *Object) writeKey(key string) {
o.w.WriteRune(quote)
o.w.Write([]byte(key))
o.w.WriteRune(quote)
o.w.WriteRune(colon)
}
// Key adds the given named key to the JSON object.
// Returns a Value encoder that should be used to encode
// a JSON value type.
func (o *Object) Key(name string) Value {
if o.writeComma {
o.w.WriteRune(comma)
} else {
o.writeComma = true
}
o.writeKey(name)
return newValue(o.w, o.scratch)
}
// Close encodes the end of the JSON Object
func (o *Object) Close() {
o.w.WriteRune(rightBrace)
}

149
vendor/github.com/aws/smithy-go/encoding/json/value.go generated vendored Normal file
View File

@ -0,0 +1,149 @@
package json
import (
"bytes"
"encoding/base64"
"math/big"
"strconv"
"github.com/aws/smithy-go/encoding"
)
// Value represents a JSON Value type
// JSON Value types: Object, Array, String, Number, Boolean, and Null
type Value struct {
w *bytes.Buffer
scratch *[]byte
}
// newValue returns a new Value encoder
func newValue(w *bytes.Buffer, scratch *[]byte) Value {
return Value{w: w, scratch: scratch}
}
// String encodes v as a JSON string
func (jv Value) String(v string) {
escapeStringBytes(jv.w, []byte(v))
}
// Byte encodes v as a JSON number
func (jv Value) Byte(v int8) {
jv.Long(int64(v))
}
// Short encodes v as a JSON number
func (jv Value) Short(v int16) {
jv.Long(int64(v))
}
// Integer encodes v as a JSON number
func (jv Value) Integer(v int32) {
jv.Long(int64(v))
}
// Long encodes v as a JSON number
func (jv Value) Long(v int64) {
*jv.scratch = strconv.AppendInt((*jv.scratch)[:0], v, 10)
jv.w.Write(*jv.scratch)
}
// ULong encodes v as a JSON number
func (jv Value) ULong(v uint64) {
*jv.scratch = strconv.AppendUint((*jv.scratch)[:0], v, 10)
jv.w.Write(*jv.scratch)
}
// Float encodes v as a JSON number
func (jv Value) Float(v float32) {
jv.float(float64(v), 32)
}
// Double encodes v as a JSON number
func (jv Value) Double(v float64) {
jv.float(v, 64)
}
func (jv Value) float(v float64, bits int) {
*jv.scratch = encoding.EncodeFloat((*jv.scratch)[:0], v, bits)
jv.w.Write(*jv.scratch)
}
// Boolean encodes v as a JSON boolean
func (jv Value) Boolean(v bool) {
*jv.scratch = strconv.AppendBool((*jv.scratch)[:0], v)
jv.w.Write(*jv.scratch)
}
// Base64EncodeBytes writes v as a base64 value in JSON string
func (jv Value) Base64EncodeBytes(v []byte) {
encodeByteSlice(jv.w, (*jv.scratch)[:0], v)
}
// Write writes v directly to the JSON document
func (jv Value) Write(v []byte) {
jv.w.Write(v)
}
// Array returns a new Array encoder
func (jv Value) Array() *Array {
return newArray(jv.w, jv.scratch)
}
// Object returns a new Object encoder
func (jv Value) Object() *Object {
return newObject(jv.w, jv.scratch)
}
// Null encodes a null JSON value
func (jv Value) Null() {
jv.w.WriteString(null)
}
// BigInteger encodes v as JSON value
func (jv Value) BigInteger(v *big.Int) {
jv.w.Write([]byte(v.Text(10)))
}
// BigDecimal encodes v as JSON value
func (jv Value) BigDecimal(v *big.Float) {
if i, accuracy := v.Int64(); accuracy == big.Exact {
jv.Long(i)
return
}
// TODO: Should this try to match ES6 ToString similar to stdlib JSON?
jv.w.Write([]byte(v.Text('e', -1)))
}
// Based on encoding/json encodeByteSlice from the Go Standard Library
// https://golang.org/src/encoding/json/encode.go
func encodeByteSlice(w *bytes.Buffer, scratch []byte, v []byte) {
if v == nil {
w.WriteString(null)
return
}
w.WriteRune(quote)
encodedLen := base64.StdEncoding.EncodedLen(len(v))
if encodedLen <= len(scratch) {
// If the encoded bytes fit in e.scratch, avoid an extra
// allocation and use the cheaper Encoding.Encode.
dst := scratch[:encodedLen]
base64.StdEncoding.Encode(dst, v)
w.Write(dst)
} else if encodedLen <= 1024 {
// The encoded bytes are short enough to allocate for, and
// Encoding.Encode is still cheaper.
dst := make([]byte, encodedLen)
base64.StdEncoding.Encode(dst, v)
w.Write(dst)
} else {
// The encoded bytes are too long to cheaply allocate, and
// Encoding.Encode is no longer noticeably cheaper.
enc := base64.NewEncoder(base64.StdEncoding, w)
enc.Write(v)
enc.Close()
}
w.WriteRune(quote)
}