robocars/robocar-steering-tflite-edgetpu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(oci): bad parse oci_ref when used with custom registry

Browse Source
This commit is contained in:
Cyrille Nofficial 2023-05-28 14:39:49 +02:00
parent 9fb01f7be9
commit 29ca250ba8
18 changed files with 308 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/rc-steering/rc-steering.go
View File

@ -27,7 +27,7 @@ var (
func main() {
var mqttBroker, username, password, clientId string
var cameraTopic, steeringTopic string
var modelPath, modelsDir, ociRef string
var modelPath, modelsDir, ociRegistry, ociRepository, ociTag string
var edgeVerbosity int
var imgWidth, imgHeight, horizon int
@ -37,7 +37,9 @@ func main() {
cli.InitMqttFlags(DefaultClientId, &mqttBroker, &username, &password, &clientId, &mqttQos, &mqttRetain)
flag.StringVar(&modelPath, "model", "", "path to model file")
flag.StringVar(&ociRef, "oci-model", "", "oci image to pull")
flag.StringVar(&ociRegistry, "oci-model-registry", "", "oci registry where to fetch model")
flag.StringVar(&ociRepository, "oci-model-repository", "", "oci repository where to fetch model")
flag.StringVar(&ociTag, "oci-model-tag", "", "oci tag name for model to pull")
flag.StringVar(&modelsDir, "models-dir", "/tmp/robocar/models", "path where to store model file")
flag.StringVar(&steeringTopic, "mqtt-topic-road", os.Getenv("MQTT_TOPIC_STEERING"), "Mqtt topic to publish road detection result, use MQTT_TOPIC_STEERING if args not set")
flag.StringVar(&cameraTopic, "mqtt-topic-camera", os.Getenv("MQTT_TOPIC_CAMERA"), "Mqtt topic that contains camera frame values, use MQTT_TOPIC_CAMERA if args not set")
@ -68,12 +70,12 @@ func main() {
cleanup := metrics.Init(context.Background())
defer cleanup()
if modelPath == "" && ociRef == "" {
if modelPath == "" && ociRepository == "" {
zap.L().Error("model path or oci image is mandatory")
flag.PrintDefaults()
os.Exit(1)
}
if modelPath != "" && ociRef != "" {
if modelPath != "" && ociRepository != "" {
zap.L().Error("model path and oci image are exclusives")
flag.PrintDefaults()
os.Exit(1)
@ -88,7 +90,8 @@ func main() {
zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err)
}
} else {
modelPath, modelType, width, height, horizonFromName, err = oci.PullOciImage(ociRef, modelsDir)
ctx := context.Background()
modelPath, modelType, width, height, horizonFromName, err = oci.PullOciImage(ctx, ociRegistry, ociRepository, ociTag, modelsDir)
if err != nil {
zap.S().Panicf("bad model name '%v', unable to detect configuration from name pattern: %v", modelPath, err)
}
@ -110,10 +113,10 @@ func main() {
os.Exit(1)
}
if ociRef == "" {
if ociRepository == "" {
zap.S().Infof("model path : %v", modelPath)
} else {
zap.S().Infof("oci image model : %v", ociRef)
zap.S().Infof("oci image model : %v/%v:%v", ociRegistry, ociRepository, ociTag)
}
zap.S().Infof("model type : %v", modelType)
zap.S().Infof("model for image width : %v", imgWidth)

4
go.mod
View File

@ -14,7 +14,7 @@ require (
go.opentelemetry.io/otel/sdk/metric v0.30.0
go.uber.org/zap v1.21.0
google.golang.org/protobuf v1.28.1
oras.land/oras-go/v2 v2.0.2
oras.land/oras-go/v2 v2.2.0
)
require (
@ -31,7 +31,7 @@ require (
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/image v0.0.0-20220601225756-64ec528b34cd // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

8
go.sum
View File

@ -84,8 +84,8 @@ golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -119,5 +119,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
oras.land/oras-go/v2 v2.0.2 h1:3aSQdJ7EUC0ft2e9PjJB9Jzastz5ojPA4LzZ3Q4YbUc=
oras.land/oras-go/v2 v2.0.2/go.mod h1:PWnWc/Kyyg7wUTUsDHshrsJkzuxXzreeMd6NrfdnFSo=
oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=

79
pkg/oci/models.go
View File

@ -6,26 +6,31 @@ import (
"fmt"
"github.com/cyrilix/robocar-steering-tflite-edgetpu/pkg/tools"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"go.uber.org/zap"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry"
"oras.land/oras-go/v2/registry/remote"
"path"
"strconv"
"strings"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
)
func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType tools.ModelType, imgWidth, imgHeight int, horizon int, err error) {
repository := strings.Split(ociRef, ":")[0]
tag := strings.Split(ociRef, ":")[1]
func PullOciImage(ctx context.Context, regName, repoName, tag, modelsDir string) (modelPath string, modelType tools.ModelType, imgWidth, imgHeight int, horizon int, err error) {
manifest, err := fetchManifest(repository, tag)
repo, err := getRepository(ctx, regName, repoName)
if err != nil {
err = fmt.Errorf("unable to fetch manifest '%s': %w", ociRef, err)
err = fmt.Errorf("unable to fetch oci artifact from '%s/%s: %w", regName, repoName, err)
return
}
manifest, err := fetchManifest(ctx, repo, tag)
if err != nil {
err = fmt.Errorf("unable to fetch manifest '%s/%s:%s': %w", regName, repoName, tag, err)
return
}
zap.S().Infof("Manifest: %v", manifest)
// 0. Create a file store
modelStore := path.Join(modelsDir, manifest.Annotations["category"])
fs, err := file.New(modelStore)
@ -34,14 +39,7 @@ func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType
}
defer fs.Close()
// 1. Connect to a remote repository
ctx := context.Background()
repo, err := remote.NewRepository(repository)
if err != nil {
return
}
// 2. Copy from the remote repository to the file store
// 2. Copy from the remote repoName to the file store
_, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
if err != nil {
return
@ -70,31 +68,60 @@ func PullOciImage(ociRef string, modelsDir string) (modelPath string, modelType
return
}
func fetchManifest(repository string, tag string) (*v1.Manifest, error) {
repo, err := remote.NewRepository(repository)
func getRepository(ctx context.Context, registryName string, repoName string) (registry.Repository, error) {
reg, err := remote.NewRegistry(registryName)
if err != nil {
panic(err)
return nil, fmt.Errorf("bad registry '%v': %w", registryName, err)
}
ctx := context.Background()
reg.RepositoryOptions.PlainHTTP = true
// For debug
//reg.Repositories(ctx, "", func(repos []string) error {
// for _, r := range repos {
// zap.S().Debugf("found repo %v", r)
// }
// return nil
//})
repo, err := reg.Repository(ctx, repoName)
if err != nil {
return nil, fmt.Errorf("unable to instanciate new repository: %w", err)
}
// For debug
/*
repo.Tags(ctx, "", func(tags []string) error {
for _, t := range tags {
zap.S().Debugf("found tag '%v'", t)
}
return nil
})
*/
return repo, nil
}
func fetchManifest(ctx context.Context, repo registry.Repository, tag string) (*v1.Manifest, error) {
descriptor, err := repo.Resolve(ctx, tag)
zap.S().Debugf("model descriptor: %#v", descriptor)
if err != nil {
panic(err)
return nil, fmt.Errorf("unexpected error on tag resolving: %w", err)
}
rc, err := repo.Fetch(ctx, descriptor)
if err != nil {
return nil, fmt.Errorf("unable to fetch manifest for image '%s:%s': %w", repository, tag, err)
return nil, fmt.Errorf("unable to fetch manifest for image '%s:%s': %w", repo, tag, err)
}
defer rc.Close() // don't forget to close
pulledBlob, err := content.ReadAll(rc, descriptor)
if err != nil {
return nil, fmt.Errorf("unable to read manifest content for image '%s:%s': %w", repository, tag, err)
return nil, fmt.Errorf("unable to read manifest content for image '%s:%s': %w", repo, tag, err)
}
var manifest v1.Manifest
err = json.Unmarshal(pulledBlob, &manifest)
if err != nil {
return nil, fmt.Errorf("unable to unmarsh json manifest content for image '%s:%s': %w", repository, tag, err)
return nil, fmt.Errorf("unable to unmarsh json manifest content for image '%s:%s': %w", repo, tag, err)
}
return &manifest, nil
}

5
vendor/modules.txt vendored
View File

@ -114,7 +114,7 @@ golang.org/x/image/tiff/lzw
## explicit; go 1.17
golang.org/x/net/internal/socks
golang.org/x/net/proxy
# golang.org/x/sync v0.1.0
# golang.org/x/sync v0.2.0
## explicit
golang.org/x/sync/errgroup
golang.org/x/sync/semaphore
@ -154,7 +154,7 @@ google.golang.org/protobuf/runtime/protoimpl
google.golang.org/protobuf/types/known/timestamppb
# gopkg.in/yaml.v2 v2.4.0
## explicit; go 1.15
# oras.land/oras-go/v2 v2.0.2
# oras.land/oras-go/v2 v2.2.0
## explicit; go 1.19
oras.land/oras-go/v2
oras.land/oras-go/v2/content
@ -173,6 +173,7 @@ oras.land/oras-go/v2/internal/platform
oras.land/oras-go/v2/internal/registryutil
oras.land/oras-go/v2/internal/resolver
oras.land/oras-go/v2/internal/slices
oras.land/oras-go/v2/internal/spec
oras.land/oras-go/v2/internal/status
oras.land/oras-go/v2/internal/syncutil
oras.land/oras-go/v2/registry

1
vendor/oras.land/oras-go/v2/.gitignore vendored
View File

@ -38,3 +38,4 @@ dist/
*.tar.gz
vendor/
_dist/
.cover

6
vendor/oras.land/oras-go/v2/README.md vendored
View File

@ -1,6 +1,8 @@
# ORAS Go library
![ORAS](https://github.com/oras-project/oras-www/raw/main/docs/assets/images/oras.png)
<p align="left">
<a href="https://oras.land/"><img src="https://oras.land/img/oras.svg" alt="banner" width="100px"></a>
</p>
## Project status
@ -45,7 +47,7 @@ to use releases with major version `2` for new features.
## Docs
- [oras.land/client_libraries/go](https://oras.land/client_libraries/0_go/): Documentation for the ORAS Go library
- [oras.land/client_libraries/go](https://oras.land/docs/Client_Libraries/go): Documentation for the ORAS Go library
- [Reviewing guide](https://github.com/oras-project/community/blob/main/REVIEWING.md): All reviewers must read the reviewing guide and agree to follow the project review guidelines.
## Code of Conduct

5
vendor/oras.land/oras-go/v2/content/graph.go vendored
View File

@ -21,6 +21,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
)
// PredecessorFinder finds out the nodes directly pointing to a given node of a
@ -86,13 +87,13 @@ func Successors(ctx context.Context, fetcher Fetcher, node ocispec.Descriptor) (
return nil, err
}
return index.Manifests, nil
case ocispec.MediaTypeArtifactManifest:
case spec.MediaTypeArtifactManifest:
content, err := FetchAll(ctx, fetcher, node)
if err != nil {
return nil, err
}
var manifest ocispec.Artifact
var manifest spec.Artifact
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, err
}

9
vendor/oras.land/oras-go/v2/extendedcopy.go vendored
View File

@ -29,6 +29,7 @@ import (
"oras.land/oras-go/v2/internal/copyutil"
"oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
"oras.land/oras-go/v2/internal/status"
"oras.land/oras-go/v2/internal/syncutil"
"oras.land/oras-go/v2/registry"
@ -255,7 +256,7 @@ func (opts *ExtendedCopyGraphOptions) FilterAnnotation(key string, regex *regexp
switch p.MediaType {
case docker.MediaTypeManifest, ocispec.MediaTypeImageManifest,
docker.MediaTypeManifestList, ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest:
spec.MediaTypeArtifactManifest:
annotations, err := fetchAnnotations(ctx, src, p)
if err != nil {
return nil, err
@ -345,7 +346,7 @@ func (opts *ExtendedCopyGraphOptions) FilterArtifactType(regex *regexp.Regexp) {
// if the artifact type is not present in the descriptors,
// fetch it from the manifest content.
switch p.MediaType {
case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
case spec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
artifactType, err := fetchArtifactType(ctx, src, p)
if err != nil {
return nil, err
@ -370,8 +371,8 @@ func fetchArtifactType(ctx context.Context, src content.ReadOnlyGraphStorage, de
defer rc.Close()
switch desc.MediaType {
case ocispec.MediaTypeArtifactManifest:
var manifest ocispec.Artifact
case spec.MediaTypeArtifactManifest:
var manifest spec.Artifact
if err := json.NewDecoder(rc).Decode(&manifest); err != nil {
return "", err
}

3
vendor/oras.land/oras-go/v2/internal/descriptor/descriptor.go vendored
View File

@ -19,6 +19,7 @@ import (
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
)
// DefaultMediaType is the media type used when no media type is specified.
@ -70,7 +71,7 @@ func IsManifest(desc ocispec.Descriptor) bool {
docker.MediaTypeManifestList,
ocispec.MediaTypeImageManifest,
ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest:
spec.MediaTypeArtifactManifest:
return true
default:
return false

49
vendor/oras.land/oras-go/v2/internal/spec/artifact.go vendored Normal file
View File

@ -0,0 +1,49 @@
/*
Copyright The ORAS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import ocispec "github.com/opencontainers/image-spec/specs-go/v1"
// AnnotationReferrersFiltersApplied is the annotation key for the comma separated list of filters applied by the registry in the referrers listing.
const AnnotationReferrersFiltersApplied = "org.opencontainers.referrers.filtersApplied"
// MediaTypeArtifactManifest specifies the media type for a content descriptor.
const MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json"
// Artifact describes an artifact manifest.
// This structure provides `application/vnd.oci.artifact.manifest.v1+json` mediatype when marshalled to JSON.
//
// This manifest type was introduced in image-spec v1.1.0-rc1 and was removed in
// image-spec v1.1.0-rc3. It is not part of the current image-spec and is kept
// here for Go compatibility.
//
// Reference: https://github.com/opencontainers/image-spec/pull/999
type Artifact struct {
// MediaType is the media type of the object this schema refers to.
MediaType string `json:"mediaType"`
// ArtifactType is the IANA media type of the artifact this schema refers to.
ArtifactType string `json:"artifactType"`
// Blobs is a collection of blobs referenced by this manifest.
Blobs []ocispec.Descriptor `json:"blobs,omitempty"`
// Subject (reference) is an optional link from the artifact to another manifest forming an association between the artifact and the other manifest.
Subject *ocispec.Descriptor `json:"subject,omitempty"`
// Annotations contains arbitrary metadata for the artifact manifest.
Annotations map[string]string `json:"annotations,omitempty"`
}

7
vendor/oras.land/oras-go/v2/pack.go vendored
View File

@ -27,6 +27,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/errdef"
"oras.land/oras-go/v2/internal/spec"
)
const (
@ -93,8 +94,8 @@ func packArtifact(ctx context.Context, pusher content.Pusher, artifactType strin
if err != nil {
return ocispec.Descriptor{}, err
}
manifest := ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
manifest := spec.Artifact{
MediaType: spec.MediaTypeArtifactManifest,
ArtifactType: artifactType,
Blobs: blobs,
Subject: opts.Subject,
@ -104,7 +105,7 @@ func packArtifact(ctx context.Context, pusher content.Pusher, artifactType strin
if err != nil {
return ocispec.Descriptor{}, fmt.Errorf("failed to marshal manifest: %w", err)
}
manifestDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeArtifactManifest, manifestJSON)
manifestDesc := content.NewDescriptorFromBytes(spec.MediaTypeArtifactManifest, manifestJSON)
// populate ArtifactType and Annotations of the manifest into manifestDesc
manifestDesc.ArtifactType = manifest.ArtifactType
manifestDesc.Annotations = manifest.Annotations

6
vendor/oras.land/oras-go/v2/registry/remote/auth/client.go vendored
View File

@ -56,6 +56,12 @@ var defaultClientID = "oras-go"
// StaticCredential specifies static credentials for the given host.
func StaticCredential(registry string, cred Credential) func(context.Context, string) (Credential, error) {
if registry == "docker.io" {
// it is expected that traffic targeting "docker.io" will be redirected
// to "registry-1.docker.io"
// reference: https://github.com/moby/moby/blob/v24.0.0-beta.2/registry/config.go#L25-L48
registry = "registry-1.docker.io"
}
return func(_ context.Context, target string) (Credential, error) {
if target == registry {
return cred, nil

3
vendor/oras.land/oras-go/v2/registry/remote/manifest.go vendored
View File

@ -20,6 +20,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/internal/docker"
"oras.land/oras-go/v2/internal/spec"
)
// defaultManifestMediaTypes contains the default set of manifests media types.
@ -28,7 +29,7 @@ var defaultManifestMediaTypes = []string{
docker.MediaTypeManifestList,
ocispec.MediaTypeImageManifest,
ocispec.MediaTypeImageIndex,
ocispec.MediaTypeArtifactManifest,
spec.MediaTypeArtifactManifest,
}
// defaultManifestAcceptHeader is the default set in the `Accept` header for

35
vendor/oras.land/oras-go/v2/registry/remote/referrers.go vendored
View File

@ -22,6 +22,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/internal/descriptor"
"oras.land/oras-go/v2/internal/spec"
)
// zeroDigest represents a digest that consists of zeros. zeroDigest is used
@ -68,6 +69,38 @@ var (
errNoReferrerUpdate = errors.New("no referrer update")
)
const (
// opDeleteReferrersIndex represents the operation for deleting a
// referrers index.
opDeleteReferrersIndex = "DeleteReferrersIndex"
)
// ReferrersError records an error and the operation and the subject descriptor.
type ReferrersError struct {
// Op represents the failing operation.
Op string
// Subject is the descriptor of referenced artifact.
Subject ocispec.Descriptor
// Err is the entity of referrers error.
Err error
}
// Error returns error msg of IgnorableError.
func (e *ReferrersError) Error() string {
return e.Err.Error()
}
// Unwrap returns the inner error of IgnorableError.
func (e *ReferrersError) Unwrap() error {
return errors.Unwrap(e.Err)
}
// IsIndexDelete tells if e is kind of error related to referrers
// index deletion.
func (e *ReferrersError) IsReferrersIndexDelete() bool {
return e.Op == opDeleteReferrersIndex
}
// buildReferrersTag builds the referrers tag for the given manifest descriptor.
// Format: <algorithm>-<digest>
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#unavailable-referrers-api
@ -80,7 +113,7 @@ func buildReferrersTag(desc ocispec.Descriptor) string {
// isReferrersFilterApplied checks annotations to see if requested is in the
// applied filter list.
func isReferrersFilterApplied(annotations map[string]string, requested string) bool {
applied := annotations[ocispec.AnnotationReferrersFiltersApplied]
applied := annotations[spec.AnnotationReferrersFiltersApplied]
if applied == "" || requested == "" {
return false
}

112
vendor/oras.land/oras-go/v2/registry/remote/repository.go vendored
View File

@ -39,6 +39,7 @@ import (
"oras.land/oras-go/v2/internal/ioutil"
"oras.land/oras-go/v2/internal/registryutil"
"oras.land/oras-go/v2/internal/slices"
"oras.land/oras-go/v2/internal/spec"
"oras.land/oras-go/v2/internal/syncutil"
"oras.land/oras-go/v2/registry"
"oras.land/oras-go/v2/registry/remote/auth"
@ -213,6 +214,19 @@ func (r *Repository) Push(ctx context.Context, expected ocispec.Descriptor, cont
return r.blobStore(expected).Push(ctx, expected, content)
}
// Mount makes the blob with the given digest in fromRepo
// available in the repository signified by the receiver.
//
// This avoids the need to pull content down from fromRepo only to push it to r.
//
// If the registry does not implement mounting, getContent will be used to get the
// content to push. If getContent is nil, the content will be pulled from the source
// repository. If getContent returns an error, it will be wrapped inside the error
// returned from Mount.
func (r *Repository) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo string, getContent func() (io.ReadCloser, error)) error {
return r.Blobs().(registry.Mounter).Mount(ctx, desc, fromRepo, getContent)
}
// Exists returns true if the described content exists.
func (r *Repository) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) {
return r.blobStore(target).Exists(ctx, target)
@ -659,6 +673,73 @@ func (s *blobStore) Fetch(ctx context.Context, target ocispec.Descriptor) (rc io
}
}
// Mount mounts the given descriptor from fromRepo into s.
func (s *blobStore) Mount(ctx context.Context, desc ocispec.Descriptor, fromRepo string, getContent func() (io.ReadCloser, error)) error {
// pushing usually requires both pull and push actions.
// Reference: https://github.com/distribution/distribution/blob/v2.7.1/registry/handlers/app.go#L921-L930
ctx = registryutil.WithScopeHint(ctx, s.repo.Reference, auth.ActionPull, auth.ActionPush)
// We also need pull access to the source repo.
fromRef := s.repo.Reference
fromRef.Repository = fromRepo
ctx = registryutil.WithScopeHint(ctx, fromRef, auth.ActionPull)
url := buildRepositoryBlobMountURL(s.repo.PlainHTTP, s.repo.Reference, desc.Digest, fromRepo)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
if err != nil {
return err
}
resp, err := s.repo.client().Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusCreated {
defer resp.Body.Close()
// Check the server seems to be behaving.
return verifyContentDigest(resp, desc.Digest)
}
if resp.StatusCode != http.StatusAccepted {
defer resp.Body.Close()
return errutil.ParseErrorResponse(resp)
}
resp.Body.Close()
// From the [spec]:
//
// "If a registry does not support cross-repository mounting
// or is unable to mount the requested blob,
// it SHOULD return a 202.
// This indicates that the upload session has begun
// and that the client MAY proceed with the upload."
//
// So we need to get the content from somewhere in order to
// push it. If the caller has provided a getContent function, we
// can use that, otherwise pull the content from the source repository.
//
// [spec]: https://github.com/opencontainers/distribution-spec/blob/main/spec.md#mounting-a-blob-from-another-repository
var r io.ReadCloser
if getContent != nil {
r, err = getContent()
} else {
r, err = s.sibling(fromRepo).Fetch(ctx, desc)
}
if err != nil {
return fmt.Errorf("cannot read source blob: %w", err)
}
defer r.Close()
return s.completePushAfterInitialPost(ctx, req, resp, desc, r)
}
// sibling returns a blob store for another repository in the same
// registry.
func (s *blobStore) sibling(otherRepoName string) *blobStore {
otherRepo := *s.repo
otherRepo.Reference.Repository = otherRepoName
return &blobStore{
repo: &otherRepo,
}
}
// Push pushes the content, matching the expected descriptor.
// Existing content is not checked by Push() to minimize the number of out-going
// requests.
@ -679,11 +760,8 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if err != nil {
return err
}
reqHostname := req.URL.Hostname()
reqPort := req.URL.Port()
client := s.repo.client()
resp, err := client.Do(req)
resp, err := s.repo.client().Do(req)
if err != nil {
return err
}
@ -693,7 +771,15 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
return errutil.ParseErrorResponse(resp)
}
resp.Body.Close()
return s.completePushAfterInitialPost(ctx, req, resp, expected, content)
}
// completePushAfterInitialPost implements step 2 of the push protocol. This can be invoked either by
// Push or by Mount when the receiving repository does not implement the
// mount endpoint.
func (s *blobStore) completePushAfterInitialPost(ctx context.Context, req *http.Request, resp *http.Response, expected ocispec.Descriptor, content io.Reader) error {
reqHostname := req.URL.Hostname()
reqPort := req.URL.Port()
// monolithic upload
location, err := resp.Location()
if err != nil {
@ -710,7 +796,7 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if reqPort == "443" && locationHostname == reqHostname && locationPort == "" {
location.Host = locationHostname + ":" + reqPort
}
url = location.String()
url := location.String()
req, err = http.NewRequestWithContext(ctx, http.MethodPut, url, content)
if err != nil {
return err
@ -730,7 +816,7 @@ func (s *blobStore) Push(ctx context.Context, expected ocispec.Descriptor, conte
if auth := resp.Request.Header.Get("Authorization"); auth != "" {
req.Header.Set("Authorization", auth)
}
resp, err = client.Do(req)
resp, err = s.repo.client().Do(req)
if err != nil {
return err
}
@ -946,7 +1032,7 @@ func (s *manifestStore) Delete(ctx context.Context, target ocispec.Descriptor) e
// deleteWithIndexing removes the manifest content identified by the descriptor,
// and indexes referrers for the manifest when needed.
func (s *manifestStore) deleteWithIndexing(ctx context.Context, target ocispec.Descriptor) error {
if target.MediaType == ocispec.MediaTypeArtifactManifest || target.MediaType == ocispec.MediaTypeImageManifest {
if target.MediaType == spec.MediaTypeArtifactManifest || target.MediaType == ocispec.MediaTypeImageManifest {
if state := s.repo.loadReferrersState(); state == referrersStateSupported {
// referrers API is available, no client-side indexing needed
return s.repo.delete(ctx, target, true)
@ -1155,7 +1241,7 @@ func (s *manifestStore) push(ctx context.Context, expected ocispec.Descriptor, c
// and indexes referrers for the manifest when needed.
func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.Descriptor, r io.Reader, reference string) error {
switch expected.MediaType {
case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
case spec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
if state := s.repo.loadReferrersState(); state == referrersStateSupported {
// referrers API is available, no client-side indexing needed
return s.push(ctx, expected, r, reference)
@ -1183,8 +1269,8 @@ func (s *manifestStore) pushWithIndexing(ctx context.Context, expected ocispec.D
func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec.Descriptor, manifestJSON []byte) error {
var subject ocispec.Descriptor
switch desc.MediaType {
case ocispec.MediaTypeArtifactManifest:
var manifest ocispec.Artifact
case spec.MediaTypeArtifactManifest:
var manifest spec.Artifact
if err := json.Unmarshal(manifestJSON, &manifest); err != nil {
return fmt.Errorf("failed to decode manifest: %s: %s: %w", desc.Digest, desc.MediaType, err)
}
@ -1271,7 +1357,11 @@ func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispe
// 4. delete the dangling original referrers index
if !skipDelete {
if err := s.repo.delete(ctx, oldIndexDesc, true); err != nil {
return fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err)
return &ReferrersError{
Op: opDeleteReferrersIndex,
Err: fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err),
Subject: subject,
}
}
}
return nil

12
vendor/oras.land/oras-go/v2/registry/remote/url.go vendored
View File

@ -20,6 +20,7 @@ import (
"net/url"
"strings"
"github.com/opencontainers/go-digest"
"oras.land/oras-go/v2/registry"
)
@ -87,6 +88,17 @@ func buildRepositoryBlobUploadURL(plainHTTP bool, ref registry.Reference) string
return buildRepositoryBaseURL(plainHTTP, ref) + "/blobs/uploads/"
}
// buildRepositoryBlobMountURLbuilds the URL for cross-repository mounting.
// Format: <scheme>://<registry>/v2/<repository>/blobs/uploads/?mount=<digest>&from=<other_repository>
// Reference: https://docs.docker.com/registry/spec/api/#blob
func buildRepositoryBlobMountURL(plainHTTP bool, ref registry.Reference, d digest.Digest, fromRepo string) string {
return fmt.Sprintf("%s?mount=%s&from=%s",
buildRepositoryBlobUploadURL(plainHTTP, ref),
d,
fromRepo,
)
}
// buildReferrersURL builds the URL for querying the Referrers API.
// Format: <scheme>://<registry>/v2/<repository>/referrers/<digest>?artifactType=<artifactType>
// Reference: https://github.com/opencontainers/distribution-spec/blob/v1.1.0-rc1/spec.md#listing-referrers

13
vendor/oras.land/oras-go/v2/registry/repository.go vendored
View File

@ -107,6 +107,19 @@ type TagLister interface {
Tags(ctx context.Context, last string, fn func(tags []string) error) error
}
// Mounter allows cross-repository blob mounts.
// For backward compatibility reasons, this is not implemented by
// BlobStore: use a type assertion to check availability.
type Mounter interface {
// Mount makes the blob with the given descriptor in fromRepo
// available in the repository signified by the receiver.
Mount(ctx context.Context,
desc ocispec.Descriptor,
fromRepo string,
getContent func() (io.ReadCloser, error),
) error
}
// Tags lists the tags available in the repository.
func Tags(ctx context.Context, repo TagLister) ([]string, error) {
var res []string