github/kairos-agent
7
0
Fork 0
mirror of https://github.com/kairos-io/kairos-agent.git synced 2025-06-03 01:44:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify rsync implementation (#61)

Browse Source
This commit is contained in:
Itxaka 2023-06-20 12:18:31 +02:00 committed by GitHub
parent 75586ad601
commit 738bfbdb69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 62 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -29,7 +29,6 @@ require (
github.com/spf13/viper v1.8.1
github.com/twpayne/go-vfs v1.7.2
github.com/urfave/cli/v2 v2.25.1
github.com/zloylos/grsync v1.7.0
golang.org/x/net v0.10.0
golang.org/x/oauth2 v0.7.0
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0

2
pkg/constants/constants.go
View File

@ -115,6 +115,8 @@ const (
Archx86 = "x86_64"
ArchArm64 = "arm64"
SignedShim = "shim.efi"
Rsync = "rsync"
)
func GetCloudInitPaths() []string {

2
pkg/elemental/elemental.go
View File

@ -369,7 +369,7 @@ func (e *Elemental) DumpSource(target string, imgSrc *v1.ImageSource) (info inte
}
} else if imgSrc.IsDir() {
excludes := []string{"/mnt", "/proc", "/sys", "/dev", "/tmp", "/host", "/run"}
err = utils.SyncData(e.config.Logger, e.config.Fs, imgSrc.Value(), target, excludes...)
err = utils.SyncData(e.config.Logger, e.config.Runner, e.config.Fs, imgSrc.Value(), target, excludes...)
if err != nil {
return nil, err
}

35
pkg/elemental/elemental_test.go
View File

@ -584,11 +584,6 @@ var _ = Describe("Elemental", Label("elemental"), func() {
_, err := el.DeployImage(img, true)
Expect(err).NotTo(BeNil())
})
It("Fails copying the image if source does not exist", func() {
img.Source = v1.NewDirSrc("/welp")
_, err := el.DeployImage(img, true)
Expect(err).NotTo(BeNil())
})
It("Fails unmounting the image after copying", func() {
mounter.ErrorOnUnmount = true
_, err := el.DeployImage(img, false)
@ -605,14 +600,23 @@ var _ = Describe("Elemental", Label("elemental"), func() {
Expect(err).ShouldNot(HaveOccurred())
})
It("Copies files from a directory source", func() {
sourceDir, err := utils.TempDir(fs, "", "elemental")
rsyncCount := 0
src := ""
dest := ""
runner.SideEffect = func(cmd string, args ...string) ([]byte, error) {
if cmd == constants.Rsync {
rsyncCount += 1
src = args[len(args)-2]
dest = args[len(args)-1]
}
return []byte{}, nil
}
_, err := e.DumpSource("/dest", v1.NewDirSrc("/source"))
Expect(err).ShouldNot(HaveOccurred())
_, err = e.DumpSource(destDir, v1.NewDirSrc(sourceDir))
Expect(err).To(BeNil())
})
It("Fails if source directory does not exist", func() {
_, err := e.DumpSource(destDir, v1.NewDirSrc("/welp"))
Expect(err).ToNot(BeNil())
Expect(rsyncCount).To(Equal(1))
Expect(src).To(HaveSuffix("/source/"))
Expect(dest).To(HaveSuffix("/dest/"))
})
It("Unpacks a docker image to target", Label("docker"), func() {
_, err := e.DumpSource(destDir, v1.NewDockerSrc("docker/image:latest"))
@ -633,15 +637,12 @@ var _ = Describe("Elemental", Label("elemental"), func() {
})
It("Copies image file to target", func() {
sourceImg := "/source.img"
destFile := filepath.Join(destDir, "active.img")
_, err := fs.Create(sourceImg)
Expect(err).To(BeNil())
destFile := filepath.Join(destDir, "active.img")
_, err = fs.Stat(destFile)
Expect(err).NotTo(BeNil())
_, err = e.DumpSource(destFile, v1.NewFileSrc(sourceImg))
Expect(err).To(BeNil())
_, err = fs.Stat(destFile)
Expect(err).To(BeNil())
Expect(runner.IncludesCmds([][]string{{constants.Rsync}}))
})
It("Fails to copy, source file is not present", func() {
_, err := e.DumpSource("whatever", v1.NewFileSrc("/source.img"))

68
pkg/utils/common.go
View File

@ -32,11 +32,9 @@ import (
"github.com/distribution/distribution/reference"
"github.com/joho/godotenv"
"github.com/twpayne/go-vfs"
"github.com/zloylos/grsync"
cnst "github.com/kairos-io/kairos/v2/pkg/constants"
v1 "github.com/kairos-io/kairos/v2/pkg/types/v1"
"github.com/twpayne/go-vfs"
)
func CommandExists(command string) bool {
@ -156,7 +154,7 @@ func CreateDirStructure(fs v1.FS, target string) error {
// SyncData rsync's source folder contents to a target folder content,
// both are expected to exist beforehand.
func SyncData(log v1.Logger, fs v1.FS, source string, target string, excludes ...string) error {
func SyncData(log v1.Logger, runner v1.Runner, fs v1.FS, source string, target string, excludes ...string) error {
if fs != nil {
if s, err := fs.RawPath(source); err == nil {
source = s
@ -174,46 +172,46 @@ func SyncData(log v1.Logger, fs v1.FS, source string, target string, excludes ..
target = fmt.Sprintf("%s/", target)
}
task := grsync.NewTask(
source,
target,
grsync.RsyncOptions{
Quiet: false,
Archive: true,
XAttrs: true,
ACLs: true,
Exclude: excludes,
},
)
log.Infof("Starting rsync...")
args := []string{"--progress", "--partial", "--human-readable", "--archive", "--xattrs", "--acls"}
for _, e := range excludes {
args = append(args, fmt.Sprintf("--exclude=%s", e))
}
args = append(args, source, target)
done := displayProgress(log, 5*time.Second, "Syncing data...")
_, err := runner.Run(cnst.Rsync, args...)
close(done)
if err != nil {
log.Errorf("rsync finished with errors: %s", err.Error())
return err
}
log.Info("Finished syncing")
return nil
}
func displayProgress(log v1.Logger, tick time.Duration, message string) chan bool {
ticker := time.NewTicker(tick)
done := make(chan bool)
quit := make(chan bool)
go func() {
for {
select {
case <-quit:
case <-done:
ticker.Stop()
return
case <-time.After(5 * time.Second):
state := task.State()
log.Debugf(
"progress rsync %s to %s: %.2f / rem. %d / tot. %d / sp. %s",
source,
target,
state.Progress,
state.Remain,
state.Total,
state.Speed,
)
case <-ticker.C:
log.Debug(message)
}
}
}()
err := task.Run()
quit <- true
if err != nil {
return fmt.Errorf("%w: %s", err, strings.Join([]string{task.Log().Stderr, task.Log().Stdout}, "\n"))
}
return nil
return done
}
// Reboot reboots the system afater the given delay (in seconds) time passed.

14
pkg/utils/utils_test.go
View File

@ -53,6 +53,7 @@ var _ = Describe("Utils", Label("utils"), func() {
var syscall *v1mock.FakeSyscall
var client *v1mock.FakeHTTPClient
var mounter *v1mock.ErrorMounter
var realRunner *v1.RealRunner
var fs vfs.FS
var cleanup func()
@ -62,6 +63,7 @@ var _ = Describe("Utils", Label("utils"), func() {
mounter = v1mock.NewErrorMounter()
client = &v1mock.FakeHTTPClient{}
logger = v1.NewNullLogger()
realRunner = &v1.RealRunner{Logger: logger}
// Ensure /tmp exists in the VFS
fs, cleanup, _ = vfst.NewTestFS(nil)
fs.Mkdir("/tmp", constants.DirPerm)
@ -455,7 +457,7 @@ var _ = Describe("Utils", Label("utils"), func() {
_, _ = utils.TempFile(fs, sourceDir, "file*")
}
Expect(utils.SyncData(logger, fs, sourceDir, destDir)).To(BeNil())
Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil())
filesDest, err := fs.ReadDir(destDir)
Expect(err).To(BeNil())
@ -486,7 +488,7 @@ var _ = Describe("Utils", Label("utils"), func() {
_, _ = utils.TempFile(fs, sourceDir, "file*")
}
Expect(utils.SyncData(logger, fs, sourceDir, destDir, "host", "run")).To(BeNil())
Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "host", "run")).To(BeNil())
filesDest, err := fs.ReadDir(destDir)
Expect(err).To(BeNil())
@ -524,7 +526,7 @@ var _ = Describe("Utils", Label("utils"), func() {
utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm)
utils.MkdirAll(fs, filepath.Join(sourceDir, "tmp", "host"), constants.DirPerm)
Expect(utils.SyncData(logger, fs, sourceDir, destDir, "/host", "/run")).To(BeNil())
Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "/host", "/run")).To(BeNil())
filesDest, err := fs.ReadDir(destDir)
Expect(err).To(BeNil())
@ -550,17 +552,17 @@ var _ = Describe("Utils", Label("utils"), func() {
Expect(err).ShouldNot(HaveOccurred())
destDir, err := utils.TempDir(fs, "", "elementaltarget")
Expect(err).ShouldNot(HaveOccurred())
Expect(utils.SyncData(logger, fs, sourceDir, destDir)).To(BeNil())
Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil())
})
It("should NOT fail if destination does not exist", func() {
sourceDir, err := os.MkdirTemp("", "elemental")
err = os.WriteFile(filepath.Join(sourceDir, "testfile"), []byte("sdjfnsdjkfjkdsanfkjsnda"), os.ModePerm)
Expect(err).ToNot(HaveOccurred())
err = utils.SyncData(logger, nil, sourceDir, "/welp")
err = utils.SyncData(logger, realRunner, nil, sourceDir, "/welp")
Expect(err).To(BeNil())
})
It("should fail if source does not exist", func() {
Expect(utils.SyncData(logger, fs, "/welp", "/walp")).NotTo(BeNil())
Expect(utils.SyncData(logger, realRunner, fs, "/welp", "/walp")).NotTo(BeNil())
})
})
Describe("IsLocalURI", Label("uri"), func() {