327 lines
8.5 KiB
Go
327 lines
8.5 KiB
Go
|
/*
|
||
|
|
Copyright The containerd Authors.
|
||
|
|
|
||
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
you may not use this file except in compliance with the License.
|
||
|
|
You may obtain a copy of the License at
|
||
|
|
|
||
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
|
||
|
|
Unless required by applicable law or agreed to in writing, software
|
||
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
See the License for the specific language governing permissions and
|
||
|
|
limitations under the License.
|
||
|
|
*/
|
||
|
|
|
||
|
|
package fs
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
|
||
|
|
"golang.org/x/sync/errgroup"
|
||
|
|
|
||
|
|
"github.com/sirupsen/logrus"
|
||
|
|
)
|
||
|
|
|
||
|
|
// ChangeKind is the type of modification that
|
||
|
|
// a change is making.
|
||
|
|
type ChangeKind int
|
||
|
|
|
||
|
|
const (
|
||
|
|
// ChangeKindUnmodified represents an unmodified
|
||
|
|
// file
|
||
|
|
ChangeKindUnmodified = iota
|
||
|
|
|
||
|
|
// ChangeKindAdd represents an addition of
|
||
|
|
// a file
|
||
|
|
ChangeKindAdd
|
||
|
|
|
||
|
|
// ChangeKindModify represents a change to
|
||
|
|
// an existing file
|
||
|
|
ChangeKindModify
|
||
|
|
|
||
|
|
// ChangeKindDelete represents a delete of
|
||
|
|
// a file
|
||
|
|
ChangeKindDelete
|
||
|
|
)
|
||
|
|
|
||
|
|
func (k ChangeKind) String() string {
|
||
|
|
switch k {
|
||
|
|
case ChangeKindUnmodified:
|
||
|
|
return "unmodified"
|
||
|
|
case ChangeKindAdd:
|
||
|
|
return "add"
|
||
|
|
case ChangeKindModify:
|
||
|
|
return "modify"
|
||
|
|
case ChangeKindDelete:
|
||
|
|
return "delete"
|
||
|
|
default:
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Change represents single change between a diff and its parent.
|
||
|
|
type Change struct {
|
||
|
|
Kind ChangeKind
|
||
|
|
Path string
|
||
|
|
}
|
||
|
|
|
||
|
|
// ChangeFunc is the type of function called for each change
|
||
|
|
// computed during a directory changes calculation.
|
||
|
|
type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error
|
||
|
|
|
||
|
|
// Changes computes changes between two directories calling the
|
||
|
|
// given change function for each computed change. The first
|
||
|
|
// directory is intended to the base directory and second
|
||
|
|
// directory the changed directory.
|
||
|
|
//
|
||
|
|
// The change callback is called by the order of path names and
|
||
|
|
// should be appliable in that order.
|
||
|
|
// Due to this apply ordering, the following is true
|
||
|
|
// - Removed directory trees only create a single change for the root
|
||
|
|
// directory removed. Remaining changes are implied.
|
||
|
|
// - A directory which is modified to become a file will not have
|
||
|
|
// delete entries for sub-path items, their removal is implied
|
||
|
|
// by the removal of the parent directory.
|
||
|
|
//
|
||
|
|
// Opaque directories will not be treated specially and each file
|
||
|
|
// removed from the base directory will show up as a removal.
|
||
|
|
//
|
||
|
|
// File content comparisons will be done on files which have timestamps
|
||
|
|
// which may have been truncated. If either of the files being compared
|
||
|
|
// has a zero value nanosecond value, each byte will be compared for
|
||
|
|
// differences. If 2 files have the same seconds value but different
|
||
|
|
// nanosecond values where one of those values is zero, the files will
|
||
|
|
// be considered unchanged if the content is the same. This behavior
|
||
|
|
// is to account for timestamp truncation during archiving.
|
||
|
|
func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error {
|
||
|
|
if a == "" {
|
||
|
|
logrus.Debugf("Using single walk diff for %s", b)
|
||
|
|
return addDirChanges(ctx, changeFn, b)
|
||
|
|
} else if diffOptions := detectDirDiff(b, a); diffOptions != nil {
|
||
|
|
logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a)
|
||
|
|
return diffDirChanges(ctx, changeFn, a, diffOptions)
|
||
|
|
}
|
||
|
|
|
||
|
|
logrus.Debugf("Using double walk diff for %s from %s", b, a)
|
||
|
|
return doubleWalkDiff(ctx, changeFn, a, b)
|
||
|
|
}
|
||
|
|
|
||
|
|
func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error {
|
||
|
|
return filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
// Rebase path
|
||
|
|
path, err = filepath.Rel(root, path)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
path = filepath.Join(string(os.PathSeparator), path)
|
||
|
|
|
||
|
|
// Skip root
|
||
|
|
if path == string(os.PathSeparator) {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
return changeFn(ChangeKindAdd, path, f, nil)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// diffDirOptions is used when the diff can be directly calculated from
|
||
|
|
// a diff directory to its base, without walking both trees.
|
||
|
|
type diffDirOptions struct {
|
||
|
|
diffDir string
|
||
|
|
skipChange func(string) (bool, error)
|
||
|
|
deleteChange func(string, string, os.FileInfo) (string, error)
|
||
|
|
}
|
||
|
|
|
||
|
|
// diffDirChanges walks the diff directory and compares changes against the base.
|
||
|
|
func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error {
|
||
|
|
changedDirs := make(map[string]struct{})
|
||
|
|
return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error {
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
// Rebase path
|
||
|
|
path, err = filepath.Rel(o.diffDir, path)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
path = filepath.Join(string(os.PathSeparator), path)
|
||
|
|
|
||
|
|
// Skip root
|
||
|
|
if path == string(os.PathSeparator) {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// TODO: handle opaqueness, start new double walker at this
|
||
|
|
// location to get deletes, and skip tree in single walker
|
||
|
|
|
||
|
|
if o.skipChange != nil {
|
||
|
|
if skip, err := o.skipChange(path); skip {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
var kind ChangeKind
|
||
|
|
|
||
|
|
deletedFile, err := o.deleteChange(o.diffDir, path, f)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
// Find out what kind of modification happened
|
||
|
|
if deletedFile != "" {
|
||
|
|
path = deletedFile
|
||
|
|
kind = ChangeKindDelete
|
||
|
|
f = nil
|
||
|
|
} else {
|
||
|
|
// Otherwise, the file was added
|
||
|
|
kind = ChangeKindAdd
|
||
|
|
|
||
|
|
// ...Unless it already existed in a base, in which case, it's a modification
|
||
|
|
stat, err := os.Stat(filepath.Join(base, path))
|
||
|
|
if err != nil && !os.IsNotExist(err) {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err == nil {
|
||
|
|
// The file existed in the base, so that's a modification
|
||
|
|
|
||
|
|
// However, if it's a directory, maybe it wasn't actually modified.
|
||
|
|
// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
|
||
|
|
if stat.IsDir() && f.IsDir() {
|
||
|
|
if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
|
||
|
|
// Both directories are the same, don't record the change
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
kind = ChangeKindModify
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files.
|
||
|
|
// This block is here to ensure the change is recorded even if the
|
||
|
|
// modify time, mode and size of the parent directory in the rw and ro layers are all equal.
|
||
|
|
// Check https://github.com/docker/docker/pull/13590 for details.
|
||
|
|
if f.IsDir() {
|
||
|
|
changedDirs[path] = struct{}{}
|
||
|
|
}
|
||
|
|
if kind == ChangeKindAdd || kind == ChangeKindDelete {
|
||
|
|
parent := filepath.Dir(path)
|
||
|
|
if _, ok := changedDirs[parent]; !ok && parent != "/" {
|
||
|
|
pi, err := os.Stat(filepath.Join(o.diffDir, parent))
|
||
|
|
if err := changeFn(ChangeKindModify, parent, pi, err); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
changedDirs[parent] = struct{}{}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return changeFn(kind, path, f, nil)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// doubleWalkDiff walks both directories to create a diff
|
||
|
|
func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) {
|
||
|
|
g, ctx := errgroup.WithContext(ctx)
|
||
|
|
|
||
|
|
var (
|
||
|
|
c1 = make(chan *currentPath)
|
||
|
|
c2 = make(chan *currentPath)
|
||
|
|
|
||
|
|
f1, f2 *currentPath
|
||
|
|
rmdir string
|
||
|
|
)
|
||
|
|
g.Go(func() error {
|
||
|
|
defer close(c1)
|
||
|
|
return pathWalk(ctx, a, c1)
|
||
|
|
})
|
||
|
|
g.Go(func() error {
|
||
|
|
defer close(c2)
|
||
|
|
return pathWalk(ctx, b, c2)
|
||
|
|
})
|
||
|
|
g.Go(func() error {
|
||
|
|
for c1 != nil || c2 != nil {
|
||
|
|
if f1 == nil && c1 != nil {
|
||
|
|
f1, err = nextPath(ctx, c1)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if f1 == nil {
|
||
|
|
c1 = nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if f2 == nil && c2 != nil {
|
||
|
|
f2, err = nextPath(ctx, c2)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if f2 == nil {
|
||
|
|
c2 = nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if f1 == nil && f2 == nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
var f os.FileInfo
|
||
|
|
k, p := pathChange(f1, f2)
|
||
|
|
switch k {
|
||
|
|
case ChangeKindAdd:
|
||
|
|
if rmdir != "" {
|
||
|
|
rmdir = ""
|
||
|
|
}
|
||
|
|
f = f2.f
|
||
|
|
f2 = nil
|
||
|
|
case ChangeKindDelete:
|
||
|
|
// Check if this file is already removed by being
|
||
|
|
// under of a removed directory
|
||
|
|
if rmdir != "" && strings.HasPrefix(f1.path, rmdir) {
|
||
|
|
f1 = nil
|
||
|
|
continue
|
||
|
|
} else if f1.f.IsDir() {
|
||
|
|
rmdir = f1.path + string(os.PathSeparator)
|
||
|
|
} else if rmdir != "" {
|
||
|
|
rmdir = ""
|
||
|
|
}
|
||
|
|
f1 = nil
|
||
|
|
case ChangeKindModify:
|
||
|
|
same, err := sameFile(f1, f2)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if f1.f.IsDir() && !f2.f.IsDir() {
|
||
|
|
rmdir = f1.path + string(os.PathSeparator)
|
||
|
|
} else if rmdir != "" {
|
||
|
|
rmdir = ""
|
||
|
|
}
|
||
|
|
f = f2.f
|
||
|
|
f1 = nil
|
||
|
|
f2 = nil
|
||
|
|
if same {
|
||
|
|
if !isLinked(f) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
k = ChangeKindUnmodified
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := changeFn(k, p, f, nil); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
})
|
||
|
|
|
||
|
|
return g.Wait()
|
||
|
|
}
|