fix(oci): bad parse oci_ref when used with custom registry

This commit is contained in:
Cyrille Nofficial 2023-05-28 14:39:49 +02:00
parent 9fb01f7be9
commit 29ca250ba8
18 changed files with 308 additions and 66 deletions

View File

@ -27,7 +27,7 @@ var (
func main() { func main() {
var mqttBroker, username, password, clientId string var mqttBroker, username, password, clientId string
var cameraTopic, steeringTopic string var cameraTopic, steeringTopic string
var modelPath, modelsDir, ociRef string var modelPath, modelsDir, ociRegistry, ociRepository, ociTag string
var edgeVerbosity int var edgeVerbosity int
var imgWidth, imgHeight, horizon int var imgWidth, imgHeight, horizon int
@ -37,7 +37,9 @@ func main() {
cli.InitMqttFlags(DefaultClientId, &mqttBroker, &username, &password, &clientId, &mqttQos, &mqttRetain) cli.InitMqttFlags(DefaultClientId, &mqttBroker, &username, &password, &clientId, &mqttQos, &mqttRetain)
flag.StringVar(&modelPath, "model", "", "path to model file") flag.StringVar(&modelPath, "model", "", "path to model file")
flag.StringVar(&ociRef, "oci-model", "", "oci image to pull") flag.StringVar(&ociRegistry, "oci-model-registry", "", "oci registry where to fetch model")
flag.StringVar(&ociRepository, "oci-model-repository", "", "oci repository where to fetch model")
flag.StringVar(&ociTag, "oci-model-tag", "", "oci tag name for model to pull")
flag.StringVar(&modelsDir, "models-dir", "/tmp/robocar/models", "path where to store model file") flag.StringVar(&modelsDir, "models-dir", "/tmp/robocar/models", "path where to store model file")
flag.StringVar(&steeringTopic, "mqtt-topic-road", os.Getenv("MQTT_TOPIC_STEERING"), "Mqtt topic to publish road detection result, use MQTT_TOPIC_STEERING if args not set") flag.StringVar(&steeringTopic, "mqtt-topic-road", os.Getenv("MQTT_TOPIC_STEERING"), "Mqtt topic to publish road detection result, use MQTT_TOPIC_STEERING if args not set")
flag.StringVar(&cameraTopic, "mqtt-topic-camera", os.Getenv("MQTT_TOPIC_CAMERA"), "Mqtt topic that contains camera frame values, use MQTT_TOPIC_CAMERA if args not set") flag.StringVar(&cameraTopic, "mqtt-topic-camera", os.Getenv("MQTT_TOPIC_CAMERA"), "Mqtt topic that contains camera frame values, use MQTT_TOPIC_CAMERA if args not set")
@ -68,12 +70,12 @@ func main() {
cleanup := metrics.Init(context.Background()) cleanup := metrics.Init(context.Background())
defer cleanup() defer cleanup()
if modelPath == "" && ociRef == "" { if modelPath == "" && ociRepository == "" {
zap.L().Error("model path or oci image is mandatory") zap.L().Error("model path or oci image is mandatory")
flag.PrintDefaults() flag.PrintDefaults()
os.Exit(1) os.Exit(1)
} }
if modelPath != "" && ociRef != "" { if modelPath != "" && ociRepository != "" {
zap.L().Error("model path and oci image are exclusives") zap.L().Error("model path and oci image are exclusives")
flag.PrintDefaults() flag.PrintDefaults()
os.Exit(1) os.Exit(1)
@ -88,7 +90,8 @@ func main() {
zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err) zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err)
} }
} else { } else {
modelPath, modelType, width, height, horizonFromName, err = oci.PullOciImage(ociRef, modelsDir) ctx := context.Background()
modelPath, modelType, width, height, horizonFromName, err = oci.PullOciImage(ctx, ociRegistry, ociRepository, ociTag, modelsDir)
if err != nil { if err != nil {
zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err) zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err)
} }
@ -110,10 +113,10 @@ func main() {
os.Exit(1) os.Exit(1)
} }
if ociRef == "" { if ociRepository == "" {
zap.S().Infof("model path : %v", modelPath) zap.S().Infof("model path : %v", modelPath)
} else { } else {
zap.S().Infof("oci image model : %v", ociRef) zap.S().Infof("oci image model : %v/%v:%v", ociRegistry, ociRepository, ociTag)
} }
zap.S().Infof("model type : %v", modelType) zap.S().Infof("model type : %v", modelType)
zap.S().Infof("model for image width : %v", imgWidth) zap.S().Infof("model for image width : %v", imgWidth)

4
go.mod
View File

@ -14,7 +14,7 @@ require (
go.opentelemetry.io/otel/sdk/metric v0.30.0 go.opentelemetry.io/otel/sdk/metric v0.30.0
go.uber.org/zap v1.21.0 go.uber.org/zap v1.21.0
google.golang.org/protobuf v1.28.1 google.golang.org/protobuf v1.28.1
oras.land/oras-go/v2 v2.0.2 oras.land/oras-go/v2 v2.2.0
) )
require ( require (
@ -31,7 +31,7 @@ require (
go.uber.org/multierr v1.6.0 // indirect go.uber.org/multierr v1.6.0 // indirect
golang.org/x/image v0.0.0-20220601225756-64ec528b34cd // indirect golang.org/x/image v0.0.0-20220601225756-64ec528b34cd // indirect
golang.org/x/net v0.8.0 // indirect golang.org/x/net v0.8.0 // indirect
golang.org/x/sync v0.1.0 // indirect golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.6.0 // indirect golang.org/x/sys v0.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
) )

8
go.sum
View File

@ -84,8 +84,8 @@ golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -119,5 +119,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
oras.land/oras-go/v2 v2.0.2 h1:3aSQdJ7EUC0ft2e9PjJB9Jzastz5ojPA4LzZ3Q4YbUc= oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
oras.land/oras-go/v2 v2.0.2/go.mod h1:PWnWc/Kyyg7wUTUsDHshrsJkzuxXzreeMd6NrfdnFSo= oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=

View File

@ -6,26 +6,31 @@ import (
"fmt" "fmt"
"github.com/cyrilix/robocar-steering-tflite-edgetpu/pkg/tools" "github.com/cyrilix/robocar-steering-tflite-edgetpu/pkg/tools"
v1 "github.com/opencontainers/image-spec/specs-go/v1" v1 "github.com/opencontainers/image-spec/specs-go/v1"
"go.uber.org/zap"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry"
"oras.land/oras-go/v2/registry/remote"
"path" "path"
"strconv" "strconv"
"strings"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
) )
func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType tools.ModelType, imgWidth, imgHeight int, horizon int, err error) { func PullOciImage(ctx context.Context, regName, repoName, tag, modelsDir string) (modelPath string, modelType tools.ModelType, imgWidth, imgHeight int, horizon int, err error) {
repository := strings.Split(ociRef, ":")[0]
tag := strings.Split(ociRef, ":")[1]
manifest, err := fetchManifest(repository, tag) repo, err := getRepository(ctx, regName, repoName)
if err != nil { if err != nil {
err = fmt.Errorf("unable to fetch manifest '%s': %w", ociRef, err) err = fmt.Errorf("unable to fetch oci artifact from '%s/%s: %w", regName, repoName, err)
return return
} }
manifest, err := fetchManifest(ctx, repo, tag)
if err != nil {
err = fmt.Errorf("unable to fetch manifest '%s/%s:%s': %w", regName, repoName, tag, err)
return
}
zap.S().Infof("Manifest: %v", manifest)
// 0. Create a file store // 0. Create a file store
modelStore := path.Join(modelsDir, manifest.Annotations["category"]) modelStore := path.Join(modelsDir, manifest.Annotations["category"])
fs, err := file.New(modelStore) fs, err := file.New(modelStore)
@ -34,14 +39,7 @@ func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType
} }
defer fs.Close() defer fs.Close()
// 1. Connect to a remote repository // 2. Copy from the remote repoName to the file store
ctx := context.Background()
repo, err := remote.NewRepository(repository)
if err != nil {
return
}
// 2. Copy from the remote repository to the file store
_, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
if err != nil { if err != nil {
return return
@ -70,31 +68,60 @@ func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType
return return
} }
func fetchManifest(repository string, tag string) (*v1.Manifest, error) { func getRepository(ctx context.Context, registryName string, repoName string) (registry.Repository, error) {
repo, err := remote.NewRepository(repository)
reg, err := remote.NewRegistry(registryName)
if err != nil { if err != nil {
panic(err) return nil, fmt.Errorf("bad registry '%v': %w", registryName, err)
} }
ctx := context.Background() reg.RepositoryOptions.PlainHTTP = true
// For debug
//reg.Repositories(ctx, "", func(repos []string) error {
// for _, r := range repos {
// zap.S().Debugf("found repo %v", r)
// }
// return nil
//})
repo, err := reg.Repository(ctx, repoName)
if err != nil {
return nil, fmt.Errorf("unable to instanciate new repository: %w", err)
}
// For debug
/*
repo.Tags(ctx, "", func(tags []string) error {
for _, t := range tags {
zap.S().Debugf("found tag '%v'", t)
}
return nil
})
*/
return repo, nil
}
func fetchManifest(ctx context.Context, repo registry.Repository, tag string) (*v1.Manifest, error) {
descriptor, err := repo.Resolve(ctx, tag) descriptor, err := repo.Resolve(ctx, tag)
zap.S().Debugf("model descriptor: %#v", descriptor)
if err != nil { if err != nil {
panic(err) return nil, fmt.Errorf("unexpected error on tag resolving: %w", err)
} }
rc, err := repo.Fetch(ctx, descriptor) rc, err := repo.Fetch(ctx, descriptor)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to fetch manifest for image '%s:%s': %w", repository, tag, err) return nil, fmt.Errorf("unable to fetch manifest for image '%s:%s': %w", repo, tag, err)
} }
defer rc.Close() // don't forget to close defer rc.Close() // don't forget to close
pulledBlob, err := content.ReadAll(rc, descriptor) pulledBlob, err := content.ReadAll(rc, descriptor)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to read manifest content for image '%s:%s': %w", repository, tag, err) return nil, fmt.Errorf("unable to read manifest content for image '%s:%s': %w", repo, tag, err)
} }
var manifest v1.Manifest var manifest v1.Manifest
err = json.Unmarshal(pulledBlob, &manifest) err = json.Unmarshal(pulledBlob, &manifest)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to unmarsh json manifest content for image '%s:%s': %w", repository, tag, err) return nil, fmt.Errorf("unable to unmarsh json manifest content for image '%s:%s': %w", repo, tag, err)
} }
return &manifest, nil return &manifest, nil
} }

5
vendor/modules.txt vendored
View File

@ -114,7 +114,7 @@ golang.org/x/image/tiff/lzw
## explicit; go 1.17 ## explicit; go 1.17
golang.org/x/net/internal/socks golang.org/x/net/internal/socks
golang.org/x/net/proxy golang.org/x/net/proxy
# golang.org/x/sync v0.1.0 # golang.org/x/sync v0.2.0
## explicit ## explicit
golang.org/x/sync/errgroup golang.org/x/sync/errgroup
golang.org/x/sync/semaphore golang.org/x/sync/semaphore
@ -154,7 +154,7 @@ google.golang.org/protobuf/runtime/protoimpl
google.golang.org/protobuf/types/known/timestamppb google.golang.org/protobuf/types/known/timestamppb
# gopkg.in/yaml.v2 v2.4.0 # gopkg.in/yaml.v2 v2.4.0
## explicit; go 1.15 ## explicit; go 1.15
# oras.land/oras-go/v2 v2.0.2 # oras.land/oras-go/v2 v2.2.0
## explicit; go 1.19 ## explicit; go 1.19
oras.land/oras-go/v2 oras.land/oras-go/v2
oras.land/oras-go/v2/content oras.land/oras-go/v2/content
@ -173,6 +173,7 @@ oras.land/oras-go/v2/internal/platform
oras.land/oras-go/v2/internal/registryutil oras.land/oras-go/v2/internal/registryutil
oras.land/oras-go/v2/internal/resolver oras.land/oras-go/v2/internal/resolver
oras.land/oras-go/v2/internal/slices oras.land/oras-go/v2/internal/slices
oras.land/oras-go/v2/internal/spec
oras.land/oras-go/v2/internal/status oras.land/oras-go/v2/internal/status
oras.land/oras-go/v2/internal/syncutil oras.land/oras-go/v2/internal/syncutil
oras.land/oras-go/v2/registry oras.land/oras-go/v2/registry

View File

@ -38,3 +38,4 @@ dist/
*.tar.gz *.tar.gz
vendor/ vendor/
_dist/ _dist/
.cover

View File

@ -1,6 +1,8 @@
# ORAS Go library # ORAS Go library
![ORAS](https://github.com/oras-project/oras-www/raw/main/docs/assets/images/oras.png) <p align="left">
<a href="https://oras.land/"><img src="https://oras.land/img/oras.svg" alt="banner" width="100px"></a>
</p>
## Project status ## Project status
@ -45,7 +47,7 @@ to use releases with major version `2` for new features.
## Docs ## Docs
- [oras.land/client_libraries/go](https://oras.land/client_libraries/0_go/): Documentation for the ORAS Go library - [oras.land/client_libraries/go](https://oras.land/docs/Client_Libraries/go): Documentation for the ORAS Go library
- [Reviewing guide](https://github.com/oras-project/community/blob/main/REVIEWING.md): All reviewers must read the reviewing guide and agree to follow the project review guidelines. - [Reviewing guide](https://github.com/oras-project/community/blob/main/REVIEWING.md): All reviewers must read the reviewing guide and agree to follow the project review guidelines.
## Code of Conduct ## Code of Conduct

View File

@ -21,6 +21,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker" "oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
) )
// PredecessorFinder finds out the nodes directly pointing to a given node of a // PredecessorFinder finds out the nodes directly pointing to a given node of a
@ -86,13 +87,13 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
return nil, err return nil, err
} }
return index.Manifests, nil return index.Manifests, nil
case ocispec.MediaTypeArtifactManifest: case spec.MediaTypeArtifactManifest:
content, err := FetchAll(ctx, fetcher, node) content, err := FetchAll(ctx, fetcher, node)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var manifest ocispec.Artifact var manifest spec.Artifact
if err := json.Unmarshal(content, &manifest); err != nil { if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err return nil, err
} }

View File

@ -29,6 +29,7 @@ import (
"oras.land/oras-go/v2/internal/copyutil" "oras.land/oras-go/v2/internal/copyutil"
"oras.land/oras-go/v2/internal/descriptor" "oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/docker" "oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
"oras.land/oras-go/v2/internal/status" "oras.land/oras-go/v2/internal/status"
"oras.land/oras-go/v2/internal/syncutil" "oras.land/oras-go/v2/internal/syncutil"
"oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry"
@ -255,7 +256,7 @@ func (opts *ExtendedCopyGraphOptions) FilterAnnotation(key string, regex *regexp
switch p.MediaType { switch p.MediaType {
case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest, case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest,
docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex, docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest: spec.MediaTypeArtifactManifest:
annotations, err := fetchAnnotations(ctx, src, p) annotations, err := fetchAnnotations(ctx, src, p)
if err != nil { if err != nil {
return nil, err return nil, err
@ -345,7 +346,7 @@ func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) {
// if the artifact type is not present in the descriptors, // if the artifact type is not present in the descriptors,
// fetch it from the manifest content. // fetch it from the manifest content.
switch p.MediaType { switch p.MediaType {
case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: case spec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
artifactType, err := fetchArtifactType(ctx, src, p) artifactType, err := fetchArtifactType(ctx, src, p)
if err != nil { if err != nil {
return nil, err return nil, err
@ -370,8 +371,8 @@ func fetchArtifactType(ctx context.Context, src content.ReadOnlyGraphStorage, de
defer rc.Close() defer rc.Close()
switch desc.MediaType { switch desc.MediaType {
case ocispec.MediaTypeArtifactManifest: case spec.MediaTypeArtifactManifest:
var manifest ocispec.Artifact var manifest spec.Artifact
if err := json.NewDecoder(rc).Decode(&manifest); err != nil { if err := json.NewDecoder(rc).Decode(&manifest); err != nil {
return "", err return "", err
} }

View File

@ -19,6 +19,7 @@ import (
"github.com/opencontainers/go-digest" "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker" "oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
) )
// DefaultMediaType is the media type used when no media type is specified. // DefaultMediaType is the media type used when no media type is specified.
@ -70,7 +71,7 @@ func IsManifest(desc ocispec.Descriptor) bool {
docker.MediaTypeManifestList, docker.MediaTypeManifestList,
ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageManifest,
ocispec.MediaTypeImageIndex, ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest: spec.MediaTypeArtifactManifest:
return true return true
default: default:
return false return false

View File

@ -0,0 +1,49 @@
/*
Copyright The ORAS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import ocispec "github.com/opencontainers/image-spec/specs-go/v1"
// AnnotationReferrersFiltersApplied is the annotation key for the comma separated list of filters applied by the registry in the referrers listing.
const AnnotationReferrersFiltersApplied = "org.opencontainers.referrers.filtersApplied"
// MediaTypeArtifactManifest specifies the media type for a content descriptor.
const MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json"
// Artifact describes an artifact manifest.
// This structure provides `application/vnd.oci.artifact.manifest.v1+json` mediatype when marshalled to JSON.
//
// This manifest type was introduced in image-spec v1.1.0-rc1 and was removed in
// image-spec v1.1.0-rc3. It is not part of the current image-spec and is kept
// here for Go compatibility.
//
// Reference: https://github.com/opencontainers/image-spec/pull/999
type Artifact struct {
// MediaType is the media type of the object this schema refers to.
MediaType string `json:"mediaType"`
// ArtifactType is the IANA media type of the artifact this schema refers to.
ArtifactType string `json:"artifactType"`
// Blobs is a collection of blobs referenced by this manifest.
Blobs []ocispec.Descriptor `json:"blobs,omitempty"`
// Subject (reference) is an optional link from the artifact to another manifest forming an association between the artifact and the other manifest.
Subject *ocispec.Descriptor `json:"subject,omitempty"`
// Annotations contains arbitrary metadata for the artifact manifest.
Annotations map[string]string `json:"annotations,omitempty"`
}

View File

@ -27,6 +27,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/errdef" "oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/spec"
) )
const ( const (
@ -93,8 +94,8 @@ func packArtifact(ctx context.Context, pusher content.Pusher, artifactType strin
if err != nil { if err != nil {
return ocispec.Descriptor{}, err return ocispec.Descriptor{}, err
} }
manifest := ocispec.Artifact{ manifest := spec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest, MediaType: spec.MediaTypeArtifactManifest,
ArtifactType: artifactType, ArtifactType: artifactType,
Blobs: blobs, Blobs: blobs,
Subject: opts.Subject, Subject: opts.Subject,
@ -104,7 +105,7 @@ func packArtifact(ctx context.Context, pusher content.Pusher, artifactType strin
if err != nil { if err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err) return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err)
} }
manifestDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, manifestJSON) manifestDesc := content.NewDescriptorFromBytes(spec.MediaTypeArtifactManifest, manifestJSON)
// populate ArtifactType and Annotations of the manifest into manifestDesc // populate ArtifactType and Annotations of the manifest into manifestDesc
manifestDesc.ArtifactType = manifest.ArtifactType manifestDesc.ArtifactType = manifest.ArtifactType
manifestDesc.Annotations = manifest.Annotations manifestDesc.Annotations = manifest.Annotations

View File

@ -56,6 +56,12 @@ var defaultClientID = "oras-go"
// StaticCredential specifies static credentials for the given host. // StaticCredential specifies static credentials for the given host.
func StaticCredential(registry string, cred Credential) func(context.Context, string) (Credential, error) { func StaticCredential(registry string, cred Credential) func(context.Context, string) (Credential, error) {
if registry == "docker.io" {
// it is expected that traffic targeting "docker.io" will be redirected
// to "registry-1.docker.io"
// reference: https://github.com/moby/moby/blob/v24.0.0-beta.2/registry/config.go#L25-L48
registry = "registry-1.docker.io"
}
return func(_ context.Context, target string) (Credential, error) { return func(_ context.Context, target string) (Credential, error) {
if target == registry { if target == registry {
return cred, nil return cred, nil

View File

@ -20,6 +20,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker" "oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
) )
// defaultManifestMediaTypes contains the default set of manifests media types. // defaultManifestMediaTypes contains the default set of manifests media types.
@ -28,7 +29,7 @@ var defaultManifestMediaTypes = []string{
docker.MediaTypeManifestList, docker.MediaTypeManifestList,
ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageManifest,
ocispec.MediaTypeImageIndex, ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest, spec.MediaTypeArtifactManifest,
} }
// defaultManifestAcceptHeader is the default set in the `Accept` header for // defaultManifestAcceptHeader is the default set in the `Accept` header for

View File

@ -22,6 +22,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/internal/descriptor" "oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/spec"
) )
// zeroDigest represents a digest that consists of zeros. zeroDigest is used // zeroDigest represents a digest that consists of zeros. zeroDigest is used
@ -68,6 +69,38 @@ var (
errNoReferrerUpdate = errors.New("no referrer update") errNoReferrerUpdate = errors.New("no referrer update")
) )
const (
// opDeleteReferrersIndex represents the operation for deleting a
// referrers index.
opDeleteReferrersIndex = "DeleteReferrersIndex"
)
// ReferrersError records an error and the operation and the subject descriptor.
type ReferrersError struct {
// Op represents the failing operation.
Op string
// Subject is the descriptor of referenced artifact.
Subject ocispec.Descriptor
// Err is the entity of referrers error.
Err error
}
// Error returns error msg of IgnorableError.
func (e *ReferrersError) Error() string {
return e.Err.Error()
}
// Unwrap returns the inner error of IgnorableError.
func (e *ReferrersError) Unwrap() error {
return errors.Unwrap(e.Err)
}
// IsIndexDelete tells if e is kind of error related to referrers
// index deletion.
func (e *ReferrersError) IsReferrersIndexDelete() bool {
return e.Op == opDeleteReferrersIndex
}
// buildReferrersTag builds the referrers tag for the given manifest descriptor. // buildReferrersTag builds the referrers tag for the given manifest descriptor.
// Format: <algorithm>-<digest> // Format: <algorithm>-<digest>
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#unavailable-referrers-api // Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#unavailable-referrers-api
@ -80,7 +113,7 @@ func buildReferrersTag(desc ocispec.Descriptor) string {
// isReferrersFilterApplied checks annotations to see if requested is in the // isReferrersFilterApplied checks annotations to see if requested is in the
// applied filter list. // applied filter list.
func isReferrersFilterApplied(annotations map[string]string, requested string) bool { func isReferrersFilterApplied(annotations map[string]string, requested string) bool {
applied := annotations[ocispec.AnnotationReferrersFiltersApplied] applied := annotations[spec.AnnotationReferrersFiltersApplied]
if applied == "" || requested == "" { if applied == "" || requested == "" {
return false return false
} }

View File

@ -39,6 +39,7 @@ import (
"oras.land/oras-go/v2/internal/ioutil" "oras.land/oras-go/v2/internal/ioutil"
"oras.land/oras-go/v2/internal/registryutil" "oras.land/oras-go/v2/internal/registryutil"
"oras.land/oras-go/v2/internal/slices" "oras.land/oras-go/v2/internal/slices"
"oras.land/oras-go/v2/internal/spec"
"oras.land/oras-go/v2/internal/syncutil" "oras.land/oras-go/v2/internal/syncutil"
"oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry"
"oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/auth"
@ -213,6 +214,19 @@ func (r *Repository) Push(ctx context.Context, expected ocispec.Descriptor, cont
return r.blobStore(expected).Push(ctx, expected, content) return r.blobStore(expected).Push(ctx, expected, content)
} }
// Mount makes the blob with the given digest in fromRepo
// available in the repository signified by the receiver.
//
// This avoids the need to pull content down from fromRepo only to push it to r.
//
// If the registry does not implement mounting, getContent will be used to get the
// content to push. If getContent is nil, the content will be pulled from the source
// repository. If getContent returns an error, it will be wrapped inside the error
// returned from Mount.
func (r *Repository) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo string, getContent func() (io.ReadCloser, error)) error {
return r.Blobs().(registry.Mounter).Mount(ctx, desc, fromRepo, getContent)
}
// Exists returns true if the described content exists. // Exists returns true if the described content exists.
func (r *Repository) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { func (r *Repository) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) {
return r.blobStore(target).Exists(ctx, target) return r.blobStore(target).Exists(ctx, target)
@ -659,6 +673,73 @@ func (s *blobStore) Fetch(ctx context.Context, target ocispec.Descriptor) (rc io
} }
} }
// Mount mounts the given descriptor from fromRepo into s.
func (s *blobStore) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo string, getContent func() (io.ReadCloser, error)) error {
// pushing usually requires both pull and push actions.
// Reference: https://github.com/distribution/distribution/blob/v2.7.1/registry/handlers/app.go#L921-L930
ctx = registryutil.WithScopeHint(ctx, s.repo.Reference, auth.ActionPull, auth.ActionPush)
// We also need pull access to the source repo.
fromRef := s.repo.Reference
fromRef.Repository = fromRepo
ctx = registryutil.WithScopeHint(ctx, fromRef, auth.ActionPull)
url := buildRepositoryBlobMountURL(s.repo.PlainHTTP, s.repo.Reference, desc.Digest, fromRepo)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
if err != nil {
return err
}
resp, err := s.repo.client().Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusCreated {
defer resp.Body.Close()
// Check the server seems to be behaving.
return verifyContentDigest(resp, desc.Digest)
}
if resp.StatusCode != http.StatusAccepted {
defer resp.Body.Close()
return errutil.ParseErrorResponse(resp)
}
resp.Body.Close()
// From the [spec]:
//
// "If a registry does not support cross-repository mounting
// or is unable to mount the requested blob,
// it SHOULD return a 202.
// This indicates that the upload session has begun
// and that the client MAY proceed with the upload."
//
// So we need to get the content from somewhere in order to
// push it. If the caller has provided a getContent function, we
// can use that, otherwise pull the content from the source repository.
//
// [spec]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository
var r io.ReadCloser
if getContent != nil {
r, err = getContent()
} else {
r, err = s.sibling(fromRepo).Fetch(ctx, desc)
}
if err != nil {
return fmt.Errorf("cannot read source blob: %w", err)
}
defer r.Close()
return s.completePushAfterInitialPost(ctx, req, resp, desc, r)
}
// sibling returns a blob store for another repository in the same
// registry.
func (s *blobStore) sibling(otherRepoName string) *blobStore {
otherRepo := *s.repo
otherRepo.Reference.Repository = otherRepoName
return &blobStore{
repo: &otherRepo,
}
}
// Push pushes the content, matching the expected descriptor. // Push pushes the content, matching the expected descriptor.
// Existing content is not checked by Push() to minimize the number of out-going // Existing content is not checked by Push() to minimize the number of out-going
// requests. // requests.
@ -679,11 +760,8 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if err != nil { if err != nil {
return err return err
} }
reqHostname := req.URL.Hostname()
reqPort := req.URL.Port()
client := s.repo.client() resp, err := s.repo.client().Do(req)
resp, err := client.Do(req)
if err != nil { if err != nil {
return err return err
} }
@ -693,7 +771,15 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
return errutil.ParseErrorResponse(resp) return errutil.ParseErrorResponse(resp)
} }
resp.Body.Close() resp.Body.Close()
return s.completePushAfterInitialPost(ctx, req, resp, expected, content)
}
// completePushAfterInitialPost implements step 2 of the push protocol. This can be invoked either by
// Push or by Mount when the receiving repository does not implement the
// mount endpoint.
func (s *blobStore) completePushAfterInitialPost(ctx context.Context, req *http.Request, resp *http.Response, expected ocispec.Descriptor, content io.Reader) error {
reqHostname := req.URL.Hostname()
reqPort := req.URL.Port()
// monolithic upload // monolithic upload
location, err := resp.Location() location, err := resp.Location()
if err != nil { if err != nil {
@ -710,7 +796,7 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if reqPort == "443" && locationHostname == reqHostname && locationPort == "" { if reqPort == "443" && locationHostname == reqHostname && locationPort == "" {
location.Host = locationHostname + ":" + reqPort location.Host = locationHostname + ":" + reqPort
} }
url = location.String() url := location.String()
req, err = http.NewRequestWithContext(ctx, http.MethodPut, url, content) req, err = http.NewRequestWithContext(ctx, http.MethodPut, url, content)
if err != nil { if err != nil {
return err return err
@ -730,7 +816,7 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if auth := resp.Request.Header.Get("Authorization"); auth != "" { if auth := resp.Request.Header.Get("Authorization"); auth != "" {
req.Header.Set("Authorization", auth) req.Header.Set("Authorization", auth)
} }
resp, err = client.Do(req) resp, err = s.repo.client().Do(req)
if err != nil { if err != nil {
return err return err
} }
@ -946,7 +1032,7 @@ func (s *manifestStore) Delete(ctx context.Context, target ocispec.Descriptor) e
// deleteWithIndexing removes the manifest content identified by the descriptor, // deleteWithIndexing removes the manifest content identified by the descriptor,
// and indexes referrers for the manifest when needed. // and indexes referrers for the manifest when needed.
func (s *manifestStore) deleteWithIndexing(ctx context.Context, target ocispec.Descriptor) error { func (s *manifestStore) deleteWithIndexing(ctx context.Context, target ocispec.Descriptor) error {
if target.MediaType == ocispec.MediaTypeArtifactManifest || target.MediaType == ocispec.MediaTypeImageManifest { if target.MediaType == spec.MediaTypeArtifactManifest || target.MediaType == ocispec.MediaTypeImageManifest {
if state := s.repo.loadReferrersState(); state == referrersStateSupported { if state := s.repo.loadReferrersState(); state == referrersStateSupported {
// referrers API is available, no client-side indexing needed // referrers API is available, no client-side indexing needed
return s.repo.delete(ctx, target, true) return s.repo.delete(ctx, target, true)
@ -1155,7 +1241,7 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c
// and indexes referrers for the manifest when needed. // and indexes referrers for the manifest when needed.
func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.Descriptor, r io.Reader, reference string) error { func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.Descriptor, r io.Reader, reference string) error {
switch expected.MediaType { switch expected.MediaType {
case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest: case spec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
if state := s.repo.loadReferrersState(); state == referrersStateSupported { if state := s.repo.loadReferrersState(); state == referrersStateSupported {
// referrers API is available, no client-side indexing needed // referrers API is available, no client-side indexing needed
return s.push(ctx, expected, r, reference) return s.push(ctx, expected, r, reference)
@ -1183,8 +1269,8 @@ func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.D
func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error { func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error {
var subject ocispec.Descriptor var subject ocispec.Descriptor
switch desc.MediaType { switch desc.MediaType {
case ocispec.MediaTypeArtifactManifest: case spec.MediaTypeArtifactManifest:
var manifest ocispec.Artifact var manifest spec.Artifact
if err := json.Unmarshal(manifestJSON, &manifest); err != nil { if err := json.Unmarshal(manifestJSON, &manifest); err != nil {
return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err) return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err)
} }
@ -1271,7 +1357,11 @@ func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispe
// 4. delete the dangling original referrers index // 4. delete the dangling original referrers index
if !skipDelete { if !skipDelete {
if err := s.repo.delete(ctx, oldIndexDesc, true); err != nil { if err := s.repo.delete(ctx, oldIndexDesc, true); err != nil {
return fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err) return &ReferrersError{
Op: opDeleteReferrersIndex,
Err: fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err),
Subject: subject,
}
} }
} }
return nil return nil

View File

@ -20,6 +20,7 @@ import (
"net/url" "net/url"
"strings" "strings"
"github.com/opencontainers/go-digest"
"oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry"
) )
@ -87,6 +88,17 @@ func buildRepositoryBlobUploadURL(plainHTTP bool, ref registry.Reference) string
return buildRepositoryBaseURL(plainHTTP, ref) + "/blobs/uploads/" return buildRepositoryBaseURL(plainHTTP, ref) + "/blobs/uploads/"
} }
// buildRepositoryBlobMountURLbuilds the URL for cross-repository mounting.
// Format: <scheme>://<registry>/v2/<repository>/blobs/uploads/?mount=<digest>&from=<other_repository>
// Reference: https://docs.docker.com/registry/spec/api/#blob
func buildRepositoryBlobMountURL(plainHTTP bool, ref registry.Reference, d digest.Digest, fromRepo string) string {
return fmt.Sprintf("%s?mount=%s&from=%s",
buildRepositoryBlobUploadURL(plainHTTP, ref),
d,
fromRepo,
)
}
// buildReferrersURL builds the URL for querying the Referrers API. // buildReferrersURL builds the URL for querying the Referrers API.
// Format: <scheme>://<registry>/v2/<repository>/referrers/<digest>?artifactType=<artifactType> // Format: <scheme>://<registry>/v2/<repository>/referrers/<digest>?artifactType=<artifactType>
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers // Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers

View File

@ -107,6 +107,19 @@ type TagLister interface {
Tags(ctx context.Context, last string, fn func(tags []string) error) error Tags(ctx context.Context, last string, fn func(tags []string) error) error
} }
// Mounter allows cross-repository blob mounts.
// For backward compatibility reasons, this is not implemented by
// BlobStore: use a type assertion to check availability.
type Mounter interface {
// Mount makes the blob with the given descriptor in fromRepo
// available in the repository signified by the receiver.
Mount(ctx context.Context,
desc ocispec.Descriptor,
fromRepo string,
getContent func() (io.ReadCloser, error),
) error
}
// Tags lists the tags available in the repository. // Tags lists the tags available in the repository.
func Tags(ctx context.Context, repo TagLister) ([]string, error) { func Tags(ctx context.Context, repo TagLister) ([]string, error) {
var res []string var res []string