feat(train): add new command to interact with aws and train models

This commit is contained in:
2021-10-17 19:15:44 +02:00
parent 5436dfebc2
commit 538cea18f2
1064 changed files with 282251 additions and 89305 deletions

View File

@ -0,0 +1,58 @@
// Package ec2rolecreds provides the credentials provider implementation for
// retrieving AWS credentials from Amazon EC2 Instance Roles via Amazon EC2 IMDS.
//
// Concurrency and caching
//
// The Provider is not safe to be used concurrently, and does not provide any
// caching of credentials retrieved. You should wrap the Provider with a
// `aws.CredentialsCache` to provide concurrency safety, and caching of
// credentials.
//
// Loading credentials with the SDK's AWS Config
//
// The EC2 Instance role credentials provider will automatically be the resolved
// credential provider int he credential chain if no other credential provider is
// resolved first.
//
// To explicitly instruct the SDK's credentials resolving to use the EC2 Instance
// role for credentials, you specify a `credentials_source` property in the config
// profile the SDK will load.
//
// [default]
// credential_source = Ec2InstanceMetadata
//
// Loading credentials with the Provider directly
//
// Another way to use the EC2 Instance role credentials provider is to create it
// directly and assign it as the credentials provider for an API client.
//
// The following example creates a credentials provider for a command, and wraps
// it with the CredentialsCache before assigning the provider to the Amazon S3 API
// client's Credentials option.
//
// provider := imds.New(imds.Options{})
//
// // Create the service client value configured for credentials.
// svc := s3.New(s3.Options{
// Credentials: aws.NewCredentialsCache(provider),
// })
//
// If you need more control, you can set the configuration options on the
// credentials provider using the imds.Options type to configure the EC2 IMDS
// API Client and ExpiryWindow of the retrieved credentials.
//
// provider := imds.New(imds.Options{
// // See imds.Options type's documentation for more options available.
// Client: imds.New(Options{
// HTTPClient: customHTTPClient,
// }),
//
// // Modify how soon credentials expire prior to their original expiry time.
// ExpiryWindow: 5 * time.Minute,
// })
//
// EC2 IMDS API Client
//
// See the github.com/aws/aws-sdk-go-v2/feature/ec2/imds module for more details on
// configuring the client, and options available.
package ec2rolecreds

View File

@ -0,0 +1,174 @@
package ec2rolecreds
import (
"bufio"
"context"
"encoding/json"
"fmt"
"path"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/smithy-go"
)
// ProviderName provides a name of EC2Role provider
const ProviderName = "EC2RoleProvider"
// GetMetadataAPIClient provides the interface for an EC2 IMDS API client for the
// GetMetadata operation.
type GetMetadataAPIClient interface {
GetMetadata(context.Context, *imds.GetMetadataInput, ...func(*imds.Options)) (*imds.GetMetadataOutput, error)
}
// A Provider retrieves credentials from the EC2 service, and keeps track if
// those credentials are expired.
//
// The New function must be used to create the Provider.
//
// p := &ec2rolecreds.New(ec2rolecreds.Options{
// Client: imds.New(imds.Options{}),
//
// // Expire the credentials 10 minutes before IAM states they should.
// // Proactively refreshing the credentials.
// ExpiryWindow: 10 * time.Minute
// })
type Provider struct {
options Options
}
// Options is a list of user settable options for setting the behavior of the Provider.
type Options struct {
// The API client that will be used by the provider to make GetMetadata API
// calls to EC2 IMDS.
//
// If nil, the provider will default to the EC2 IMDS client.
Client GetMetadataAPIClient
}
// New returns an initialized Provider value configured to retrieve
// credentials from EC2 Instance Metadata service.
func New(optFns ...func(*Options)) *Provider {
options := Options{}
for _, fn := range optFns {
fn(&options)
}
if options.Client == nil {
options.Client = imds.New(imds.Options{})
}
return &Provider{
options: options,
}
}
// Retrieve retrieves credentials from the EC2 service.
// Error will be returned if the request fails, or unable to extract
// the desired credentials.
func (p *Provider) Retrieve(ctx context.Context) (aws.Credentials, error) {
credsList, err := requestCredList(ctx, p.options.Client)
if err != nil {
return aws.Credentials{Source: ProviderName}, err
}
if len(credsList) == 0 {
return aws.Credentials{Source: ProviderName},
fmt.Errorf("unexpected empty EC2 IMDS role list")
}
credsName := credsList[0]
roleCreds, err := requestCred(ctx, p.options.Client, credsName)
if err != nil {
return aws.Credentials{Source: ProviderName}, err
}
creds := aws.Credentials{
AccessKeyID: roleCreds.AccessKeyID,
SecretAccessKey: roleCreds.SecretAccessKey,
SessionToken: roleCreds.Token,
Source: ProviderName,
CanExpire: true,
Expires: roleCreds.Expiration,
}
return creds, nil
}
// A ec2RoleCredRespBody provides the shape for unmarshaling credential
// request responses.
type ec2RoleCredRespBody struct {
// Success State
Expiration time.Time
AccessKeyID string
SecretAccessKey string
Token string
// Error state
Code string
Message string
}
const iamSecurityCredsPath = "/iam/security-credentials/"
// requestCredList requests a list of credentials from the EC2 service. If
// there are no credentials, or there is an error making or receiving the
// request
func requestCredList(ctx context.Context, client GetMetadataAPIClient) ([]string, error) {
resp, err := client.GetMetadata(ctx, &imds.GetMetadataInput{
Path: iamSecurityCredsPath,
})
if err != nil {
return nil, fmt.Errorf("no EC2 IMDS role found, %w", err)
}
defer resp.Content.Close()
credsList := []string{}
s := bufio.NewScanner(resp.Content)
for s.Scan() {
credsList = append(credsList, s.Text())
}
if err := s.Err(); err != nil {
return nil, fmt.Errorf("failed to read EC2 IMDS role, %w", err)
}
return credsList, nil
}
// requestCred requests the credentials for a specific credentials from the EC2 service.
//
// If the credentials cannot be found, or there is an error reading the response
// and error will be returned.
func requestCred(ctx context.Context, client GetMetadataAPIClient, credsName string) (ec2RoleCredRespBody, error) {
resp, err := client.GetMetadata(ctx, &imds.GetMetadataInput{
Path: path.Join(iamSecurityCredsPath, credsName),
})
if err != nil {
return ec2RoleCredRespBody{},
fmt.Errorf("failed to get %s EC2 IMDS role credentials, %w",
credsName, err)
}
defer resp.Content.Close()
var respCreds ec2RoleCredRespBody
if err := json.NewDecoder(resp.Content).Decode(&respCreds); err != nil {
return ec2RoleCredRespBody{},
fmt.Errorf("failed to decode %s EC2 IMDS role credentials, %w",
credsName, err)
}
if !strings.EqualFold(respCreds.Code, "Success") {
// If an error code was returned something failed requesting the role.
return ec2RoleCredRespBody{},
fmt.Errorf("failed to get %s EC2 IMDS role credentials, %w",
credsName,
&smithy.GenericAPIError{Code: respCreds.Code, Message: respCreds.Message})
}
return respCreds, nil
}