robocar-tools/pkg/train/train.go

package train

import (
	"context"
	"fmt"
	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/sagemaker"
	"github.com/aws/aws-sdk-go-v2/service/sagemaker/types"
	"github.com/cyrilix/robocar-tools/pkg/awsutils"
	"github.com/cyrilix/robocar-tools/pkg/data"
	"github.com/cyrilix/robocar-tools/pkg/models"
	"go.uber.org/zap"
	"strconv"
	"time"
)

func New(bucketName string, ociImage, roleArn string) *Training {
	return &Training{
		config:       awsutils.MustLoadConfig(),
		bucketName:   bucketName,
		ociImage:     ociImage,
		roleArn:      roleArn,
		prefixInput:  prefixInput,
		outputBucket: fmt.Sprintf("s3://%s/output", bucketName),
	}
}

type Training struct {
	config       aws.Config
	bucketName   string
	ociImage     string
	roleArn      string
	prefixInput  string
	outputBucket string
}

func (t *Training) TrainDir(ctx context.Context, jobName, basedir string, imgWidth, imgHeight, sliceSize int, horizon int, withFlipImage bool, outputModelFile string, enableSpotTraining bool) error {
	l := zap.S()
	l.Infof("run training with data from %s", basedir)
	archive, err := data.BuildArchive(basedir, sliceSize, imgWidth, imgHeight, horizon, withFlipImage)
	if err != nil {
		return fmt.Errorf("unable to build data archive: %w", err)
	}
	l.Info("")

	err = t.UploadArchive(ctx, archive)
	if err != nil {
		return fmt.Errorf("unable to upload data arrchive: %w", err)
	}
	l.Info("")

	err = t.runTraining(
		ctx,
		jobName,
		sliceSize,
		imgHeight,
		imgWidth,
		enableSpotTraining,
	)
	if err != nil {
		return fmt.Errorf("unable to run training: %w", err)
	}

	err = t.GetTrainingOutput(ctx, jobName, outputModelFile)
	if err != nil {
		return fmt.Errorf("unable to get output model file '%s': %w", outputModelFile, err)
	}
	return nil
}

func List(bucketName string) error {
	l := zap.S()
	pfxInput := prefixInput

	// Create an Amazon S3 service client
	client := s3.NewFromConfig(awsutils.MustLoadConfig())

	// Get the first page of results for ListObjectsV2 for a bucket
	output, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
		Bucket: aws.String(bucketName),
		Prefix: &pfxInput,
	})
	if err != nil {
		l.Fatal(err)
	}

	l.Info("first page results:")
	for _, object := range output.Contents {
		if *object.Key == pfxInput {
			continue
		}
		l.Infof("key=%s size=%d", aws.ToString(object.Key), object.Size)
	}
	return nil
}

func (t *Training) runTraining(ctx context.Context, jobName string, slideSize int, imgHeight, imgWidth int, enableSpotTraining bool) error {
	l := zap.S()
	client := sagemaker.NewFromConfig(awsutils.MustLoadConfig())
	l.Infof("Start training job '%s'", jobName)

	trainingJobInput := sagemaker.CreateTrainingJobInput{
		EnableManagedSpotTraining: enableSpotTraining,
		AlgorithmSpecification: &types.AlgorithmSpecification{
			TrainingInputMode: types.TrainingInputModeFile,
			TrainingImage:     aws.String(t.ociImage),
		},
		OutputDataConfig: &types.OutputDataConfig{
			S3OutputPath: aws.String(t.outputBucket),
		},
		ResourceConfig: &types.ResourceConfig{
			InstanceCount: 1,
			//InstanceType:   types.TrainingInstanceTypeMlP2Xlarge,
			InstanceType:   types.TrainingInstanceTypeMlG4dnXlarge,
			VolumeSizeInGB: 1,
		},
		RoleArn: aws.String(t.roleArn),
		StoppingCondition: &types.StoppingCondition{
			MaxRuntimeInSeconds: 1800,
		},
		TrainingJobName: aws.String(jobName),
		HyperParameters: map[string]string{
			"sagemaker_region": "eu-west-1",
			"slide_size":       strconv.Itoa(slideSize),
			"img_height":       strconv.Itoa(imgHeight),
			"img_width":        strconv.Itoa(imgWidth),
		},
		InputDataConfig: []types.Channel{
			{
				ChannelName: aws.String("train"),
				DataSource: &types.DataSource{
					S3DataSource: &types.S3DataSource{
						S3DataType:             types.S3DataTypeS3Prefix,
						S3Uri:                  aws.String(fmt.Sprintf("s3://%s/%s", t.bucketName, t.prefixInput)),
						S3DataDistributionType: types.S3DataDistributionFullyReplicated,
					},
				},
			},
		},
	}
	if enableSpotTraining {
		trainingJobInput.StoppingCondition.MaxWaitTimeInSeconds = aws.Int32(3600)
	}

	// TODO: check train data exist
	jobOutput, err := client.CreateTrainingJob(
		ctx,
		&trainingJobInput,
	)

	if err != nil {
		return fmt.Errorf("unable to run sagemeker job: %w", err)
	}

	for {
		time.Sleep(30 * time.Second)

		status, err := client.DescribeTrainingJob(
			ctx,
			&sagemaker.DescribeTrainingJobInput{
				TrainingJobName: aws.String(jobName),
			},
		)
		if err != nil {
			l.Infof("unable to get status from ob %v: %v", jobOutput.TrainingJobArn, err)
			continue
		}
		switch status.TrainingJobStatus {
		case types.TrainingJobStatusInProgress:
			l.Infof("job in progress: %v - %v - %v", status.TrainingJobStatus, status.SecondaryStatus, *status.SecondaryStatusTransitions[len(status.SecondaryStatusTransitions)-1].StatusMessage)
			continue
		case types.TrainingJobStatusFailed:
			return fmt.Errorf("job %s finished with status %v", jobName, status.TrainingJobStatus)
		default:
			l.Infof("job %s finished with status %v", jobName, status.TrainingJobStatus)
			return nil
		}
	}
}

func (t *Training) GetTrainingOutput(ctx context.Context, jobName, outputFile string) error {
	modelPath := fmt.Sprintf("output/%s/output/model.tar.gz", jobName)
	err := models.DownloadArchiveToFile(ctx, t.bucketName, modelPath, outputFile)
	if err != nil {
		return fmt.Errorf("unable to download training model '%s' to '%s' file: %w", modelPath, outputFile, err)
	}
	return nil
}

func ListJob(ctx context.Context) error {

	client := sagemaker.NewFromConfig(awsutils.MustLoadConfig())
	jobs, err := client.ListTrainingJobs(ctx, &sagemaker.ListTrainingJobsInput{})
	if err != nil {
		return fmt.Errorf("unable to list trainings jobs: %w", err)
	}
	for _, job := range jobs.TrainingJobSummaries {
		fmt.Printf("%s\t\t%s\n", *job.TrainingJobName, job.TrainingJobStatus)
	}
	return nil
}