From a113147f8ab123276a82b258653d09dfbfb5fe37 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Wed, 7 Jun 2023 12:28:37 +0300 Subject: [PATCH] 1225 config collector elemental (#32) Co-authored-by: Mauro Morales Co-authored-by: Itxaka Co-authored-by: Dimitris Karakasilis --- go.mod | 2 +- go.sum | 2 + internal/agent/install.go | 65 +++-- internal/agent/install_test.go | 182 ++++++++++++ internal/agent/interactive_install.go | 10 +- main.go | 5 +- pkg/action/install_test.go | 5 +- pkg/cloudinit/layout_plugin.go | 20 +- pkg/cloudinit/layout_test.go | 403 ++++++++++++++++++++++++++ pkg/config/config.go | 1 + pkg/elemental/elemental.go | 2 + pkg/partitioner/disk.go | 15 +- pkg/types/v1/config.go | 1 + 13 files changed, 663 insertions(+), 50 deletions(-) create mode 100644 pkg/cloudinit/layout_test.go diff --git a/go.mod b/go.mod index d282d60..fba3faf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/itchyny/gojq v0.12.12 github.com/jaypipes/ghw v0.10.0 github.com/joho/godotenv v1.5.1 - github.com/kairos-io/kairos-sdk v0.0.6-0.20230526103201-c90740d747f8 + github.com/kairos-io/kairos-sdk v0.0.6 github.com/labstack/echo/v4 v4.10.2 github.com/mitchellh/mapstructure v1.4.2 github.com/mudler/go-nodepair v0.0.0-20221223092639-ba399a66fdfb diff --git a/go.sum b/go.sum index 5f95f6a..526b992 100644 --- a/go.sum +++ b/go.sum @@ -393,6 +393,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kairos-io/kairos-sdk v0.0.6-0.20230526103201-c90740d747f8 h1:7l/hlFjt31+L70HMQxr4RJ/Gk19nYn+whrjPxL9OTJ8= github.com/kairos-io/kairos-sdk v0.0.6-0.20230526103201-c90740d747f8/go.mod h1:DW5fUA1rfH1lBak5eobLzfYLjzDoYggaFzxNjY/BO7U= +github.com/kairos-io/kairos-sdk v0.0.6 h1:+faSwcqxAGhQP/Saiufpr5sGgLhbdy2NT+0IjBX22nU= +github.com/kairos-io/kairos-sdk v0.0.6/go.mod h1:DW5fUA1rfH1lBak5eobLzfYLjzDoYggaFzxNjY/BO7U= github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329 h1:qq2nCpSrXrmvDGRxW0ruW9BVEV1CN2a9YDOExdt+U0o= github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4= github.com/kendru/darwin/go/depgraph v0.0.0-20221105232959-877d6a81060c h1:eKb4PqwAMhlqwXw0W3atpKaYaPGlXE/Fwh+xpCEYaPk= diff --git a/internal/agent/install.go b/internal/agent/install.go index 98a04ff..6b18981 100644 --- a/internal/agent/install.go +++ b/internal/agent/install.go @@ -5,13 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/sirupsen/logrus" "net/url" "os" "strings" "syscall" "time" + "github.com/sirupsen/logrus" + events "github.com/kairos-io/kairos-sdk/bus" "github.com/kairos-io/kairos-sdk/machine" "github.com/kairos-io/kairos-sdk/utils" @@ -30,18 +31,6 @@ import ( "gopkg.in/yaml.v2" ) -func optsToArgs(options map[string]string) (res []string) { - for k, v := range options { - if k != "device" && k != "cc" && k != "reboot" && k != "poweroff" { - res = append(res, fmt.Sprintf("--%s", k)) - if v != "" { - res = append(res, v) - } - } - } - return -} - func displayInfo(agentConfig *Config) { fmt.Println("--------------------------") fmt.Println("No providers found, dropping to a shell. \n -- For instructions on how to install manually, see: https://kairos.io/docs/installation/manual/") @@ -106,7 +95,14 @@ func ManualInstall(c string, options map[string]string, strictValidations, debug } } - return RunInstall(debug, options) + // Load the installation Config from the system + installConfig, err := elementalConfig.ReadConfigRun("/etc/elemental") + if err != nil { + return err + } + installConfig.Debug = debug + + return RunInstall(installConfig, options) } func Install(debug bool, dir ...string) error { @@ -133,6 +129,13 @@ func Install(debug bool, dir ...string) error { ensureDataSourceReady() + // Load the installation Config from the system + installConfig, err := elementalConfig.ReadConfigRun("/etc/elemental") + if err != nil { + return err + } + installConfig.Debug = debug + // Reads config, and if present and offline is defined, // runs the installation cc, err := config.Scan(collector.Directories(dir...), collector.MergeBootLine, collector.NoLogs) @@ -145,7 +148,7 @@ func Install(debug bool, dir ...string) error { r["device"] = cc.Install.Device mergeOption(configStr, r) - err = RunInstall(debug, r) + err = RunInstall(installConfig, r) if err != nil { return err } @@ -239,7 +242,7 @@ func Install(debug bool, dir ...string) error { pterm.Info.Println("Starting installation") - if err := RunInstall(debug, r); err != nil { + if err := RunInstall(installConfig, r); err != nil { return err } @@ -256,20 +259,11 @@ func Install(debug bool, dir ...string) error { return nil } -func RunInstall(debug bool, options map[string]string) error { - // Load the installation Config from the system - installConfig, err := elementalConfig.ReadConfigRun("/etc/elemental") - if err != nil { - return err - } - if debug { +func RunInstall(installConfig *v1.RunConfig, options map[string]string) error { + if installConfig.Debug { installConfig.Logger.SetLevel(logrus.DebugLevel) } - // Run pre-install stage - _ = elementalUtils.RunStage(&installConfig.Config, "kairos-install.pre", installConfig.Strict, installConfig.CloudInitPaths...) - events.RunHookScript("/usr/bin/kairos-agent.install.pre.hook") //nolint:errcheck - f, _ := os.CreateTemp("", "xxxx") defer os.RemoveAll(f.Name()) @@ -292,7 +286,7 @@ func RunInstall(debug bool, options map[string]string) error { env := append(c.Install.Env, c.Env...) utils.SetEnv(env) - err = os.WriteFile(f.Name(), []byte(cloudInit), os.ModePerm) + err := os.WriteFile(f.Name(), []byte(cloudInit), os.ModePerm) if err != nil { fmt.Printf("could not write cloud init: %s\n", err.Error()) os.Exit(1) @@ -308,10 +302,10 @@ func RunInstall(debug bool, options map[string]string) error { } // Generate the installation spec - installSpec, err := elementalConfig.ReadInstallSpec(installConfig) - if err != nil { - return err - } + installSpec := elementalConfig.NewInstallSpec(installConfig.Config) + + installSpec.NoFormat = c.Install.NoFormat + // Set our cloud-init to the file we just created installSpec.CloudInit = append(installSpec.CloudInit, f.Name()) // Get the source of the installation if we are overriding it @@ -338,6 +332,13 @@ func RunInstall(debug bool, options map[string]string) error { if err != nil { return err } + + // Add user's cloud-config (to run user defined "before-install" stages) + installConfig.CloudInitPaths = append(installConfig.CloudInitPaths, installSpec.CloudInit...) + + // Run pre-install stage + _ = elementalUtils.RunStage(&installConfig.Config, "kairos-install.pre", installConfig.Strict, installConfig.CloudInitPaths...) + events.RunHookScript("/usr/bin/kairos-agent.install.pre.hook") //nolint:errcheck // Create the action installAction := action.NewInstallAction(installConfig, installSpec) // Run it diff --git a/internal/agent/install_test.go b/internal/agent/install_test.go index f9349e9..d0e18b9 100644 --- a/internal/agent/install_test.go +++ b/internal/agent/install_test.go @@ -1,16 +1,32 @@ package agent import ( + "bytes" "context" + "fmt" + "github.com/jaypipes/ghw/pkg/block" + "github.com/kairos-io/kairos/v2/pkg/constants" + conf "github.com/kairos-io/kairos/v2/pkg/elementalConfig" + "github.com/kairos-io/kairos/v2/pkg/utils" "os" + "path/filepath" "github.com/kairos-io/kairos/v2/pkg/config" + v1 "github.com/kairos-io/kairos/v2/pkg/types/v1" + v1mock "github.com/kairos-io/kairos/v2/tests/mocks" + "github.com/twpayne/go-vfs" + "github.com/twpayne/go-vfs/vfst" "gopkg.in/yaml.v3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +const printOutput = `BYT; +/dev/loop0:50593792s:loopback:512:512:gpt:Loopback device:;` +const partTmpl = ` +%d:%ss:%ss:2048s:ext4::type=83;` + var _ = Describe("prepareConfiguration", func() { path := "/foo/bar" url := "https://example.com" @@ -41,9 +57,175 @@ var _ = Describe("prepareConfiguration", func() { It("cleans up the configuration file after context is done", func() { source, err := prepareConfiguration(ctx, url) + Expect(err).ToNot(HaveOccurred()) cancel() _, err = os.Stat(source) Expect(os.IsNotExist(err)) }) }) + +var _ = Describe("RunInstall", func() { + var installConfig *v1.RunConfig + var options map[string]string + var err error + var fs vfs.FS + var cloudInit *v1mock.FakeCloudInitRunner + var cleanup func() + var memLog *bytes.Buffer + var ghwTest v1mock.GhwMock + var cmdline func() ([]byte, error) + + BeforeEach(func() { + // Default mock objects + runner := v1mock.NewFakeRunner() + syscall := &v1mock.FakeSyscall{} + mounter := v1mock.NewErrorMounter() + memLog = &bytes.Buffer{} + logger := v1.NewBufferLogger(memLog) + logger = v1.NewLogger() + extractor := v1mock.NewFakeImageExtractor(logger) + //logger.SetLevel(v1.DebugLevel()) + cloudInit = &v1mock.FakeCloudInitRunner{} + // Set default cmdline function so we dont panic :o + cmdline = func() ([]byte, error) { + return []byte{}, nil + } + + // Init test fs + var err error + fs, cleanup, err = vfst.NewTestFS(map[string]interface{}{"/proc/cmdline": ""}) + Expect(err).Should(BeNil()) + // Create tmp dir + utils.MkdirAll(fs, "/tmp", constants.DirPerm) + // Create grub confg + grubCfg := filepath.Join(constants.ActiveDir, constants.GrubConf) + err = utils.MkdirAll(fs, filepath.Dir(grubCfg), constants.DirPerm) + Expect(err).To(BeNil()) + _, err = fs.Create(grubCfg) + Expect(err).To(BeNil()) + + // Create new runconfig with all mocked objects + installConfig = conf.NewRunConfig( + conf.WithFs(fs), + conf.WithRunner(runner), + conf.WithLogger(logger), + conf.WithMounter(mounter), + conf.WithSyscall(syscall), + conf.WithCloudInitRunner(cloudInit), + conf.WithImageExtractor(extractor), + ) + + // Side effect of runners, hijack calls to commands and return our stuff + partNum := 0 + partedOut := printOutput + runner.SideEffect = func(cmd string, args ...string) ([]byte, error) { + switch cmd { + case "parted": + idx := 0 + for i, arg := range args { + if arg == "mkpart" { + idx = i + break + } + } + if idx > 0 { + partNum++ + partedOut += fmt.Sprintf(partTmpl, partNum, args[idx+3], args[idx+4]) + _, _ = fs.Create(fmt.Sprintf("/some/device%d", partNum)) + } + return []byte(partedOut), nil + case "lsblk": + return []byte(`{ +"blockdevices": + [ + {"label": "COS_ACTIVE", "type": "loop", "path": "/some/loop0"}, + {"label": "COS_OEM", "type": "part", "path": "/some/device1"}, + {"label": "COS_RECOVERY", "type": "part", "path": "/some/device2"}, + {"label": "COS_STATE", "type": "part", "path": "/some/device3"}, + {"label": "COS_PERSISTENT", "type": "part", "path": "/some/device4"} + ] +}`), nil + case "cat": + if args[0] == "/proc/cmdline" { + return cmdline() + } + return []byte{}, nil + default: + return []byte{}, nil + } + } + + device := "/some/device" + err = utils.MkdirAll(fs, filepath.Dir(device), constants.DirPerm) + Expect(err).To(BeNil()) + _, err = fs.Create(device) + Expect(err).ShouldNot(HaveOccurred()) + + userConfig := ` +#cloud-config + +install: + image: test +` + + options = map[string]string{ + "device": "/some/device", + "cc": userConfig, + } + + mainDisk := block.Disk{ + Name: "device", + Partitions: []*block.Partition{ + { + Name: "device1", + FilesystemLabel: "COS_GRUB", + Type: "ext4", + }, + { + Name: "device2", + FilesystemLabel: "COS_STATE", + Type: "ext4", + }, + { + Name: "device3", + FilesystemLabel: "COS_PERSISTENT", + Type: "ext4", + }, + { + Name: "device4", + FilesystemLabel: "COS_ACTIVE", + Type: "ext4", + }, + { + Name: "device5", + FilesystemLabel: "COS_PASSIVE", + Type: "ext4", + }, + { + Name: "device5", + FilesystemLabel: "COS_RECOVERY", + Type: "ext4", + }, + { + Name: "device6", + FilesystemLabel: "COS_OEM", + Type: "ext4", + }, + }, + } + ghwTest = v1mock.GhwMock{} + ghwTest.AddDisk(mainDisk) + ghwTest.CreateDevices() + }) + + AfterEach(func() { + cleanup() + }) + + It("runs the install", func() { + Skip("Not ready yet") + err = RunInstall(installConfig, options) + Expect(err).ToNot(HaveOccurred()) + }) +}) diff --git a/internal/agent/interactive_install.go b/internal/agent/interactive_install.go index 187a979..55daf5b 100644 --- a/internal/agent/interactive_install.go +++ b/internal/agent/interactive_install.go @@ -8,6 +8,7 @@ import ( "github.com/kairos-io/kairos/v2/internal/bus" "github.com/kairos-io/kairos/v2/internal/cmd" config "github.com/kairos-io/kairos/v2/pkg/config" + "github.com/kairos-io/kairos/v2/pkg/elementalConfig" events "github.com/kairos-io/kairos-sdk/bus" "github.com/kairos-io/kairos-sdk/unstructured" @@ -275,7 +276,14 @@ func InteractiveInstall(debug, spawnShell bool) error { pterm.Info.Println("Starting installation") pterm.Info.Println(finalCloudConfig) - err = RunInstall(debug, map[string]string{ + // Load the installation Config from the system + installConfig, err := elementalConfig.ReadConfigRun("/etc/elemental") + if err != nil { + return err + } + installConfig.Debug = debug + + err = RunInstall(installConfig, map[string]string{ "device": device, "cc": finalCloudConfig, }) diff --git a/main.go b/main.go index df1920b..945102c 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" + "runtime" + "github.com/kairos-io/kairos/v2/pkg/elementalConfig" v1 "github.com/kairos-io/kairos/v2/pkg/types/v1" "github.com/kairos-io/kairos/v2/pkg/utils" "github.com/sirupsen/logrus" - "path/filepath" "regexp" - "runtime" "os" "strings" diff --git a/pkg/action/install_test.go b/pkg/action/install_test.go index 53176f9..a82897d 100644 --- a/pkg/action/install_test.go +++ b/pkg/action/install_test.go @@ -20,6 +20,9 @@ import ( "bytes" "errors" "fmt" + "path/filepath" + "regexp" + "github.com/jaypipes/ghw/pkg/block" "github.com/kairos-io/kairos/v2/pkg/action" "github.com/kairos-io/kairos/v2/pkg/constants" @@ -31,8 +34,6 @@ import ( . "github.com/onsi/gomega" "github.com/twpayne/go-vfs" "github.com/twpayne/go-vfs/vfst" - "path/filepath" - "regexp" ) const printOutput = `BYT; diff --git a/pkg/cloudinit/layout_plugin.go b/pkg/cloudinit/layout_plugin.go index c41f1a0..83d8265 100644 --- a/pkg/cloudinit/layout_plugin.go +++ b/pkg/cloudinit/layout_plugin.go @@ -78,15 +78,6 @@ func layoutPlugin(l logger.Interface, s schema.Stage, fs vfs.FS, console plugins return errors.New("Target disk not found") } - if s.Layout.Expand != nil { - l.Infof("Extending last partition up to %d MiB", s.Layout.Expand.Size) - out, err := dev.ExpandLastPartition(s.Layout.Expand.Size) - if err != nil { - l.Error(out) - return err - } - } - for _, part := range s.Layout.Parts { _, err := utils.GetFullDeviceByLabel(runner, part.FSLabel, 1) if err == nil { @@ -111,7 +102,16 @@ func layoutPlugin(l logger.Interface, s schema.Stage, fs vfs.FS, console plugins return fmt.Errorf("Formatting partition failed: %s\nError: %w", out, err) } } - return nil } + + if s.Layout.Expand != nil { + l.Infof("Extending last partition up to %d MiB", s.Layout.Expand.Size) + out, err := dev.ExpandLastPartition(s.Layout.Expand.Size) + if err != nil { + l.Error(out) + return err + } + } + return nil } diff --git a/pkg/cloudinit/layout_test.go b/pkg/cloudinit/layout_test.go new file mode 100644 index 0000000..8d49256 --- /dev/null +++ b/pkg/cloudinit/layout_test.go @@ -0,0 +1,403 @@ +package cloudinit + +import ( + "fmt" + "strconv" + + "github.com/jaypipes/ghw/pkg/block" + "github.com/kairos-io/kairos/v2/pkg/partitioner" + v1 "github.com/kairos-io/kairos/v2/pkg/types/v1" + v1mock "github.com/kairos-io/kairos/v2/tests/mocks" + "github.com/mudler/yip/pkg/schema" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + "github.com/twpayne/go-vfs" + "github.com/twpayne/go-vfs/vfst" +) + +var _ = Describe("Layout", Label("layout"), func() { + // unit test stolen from yip + var logger v1.Logger + var stage schema.Stage + var fs vfs.FS + var console *cloudInitConsole + var runner *v1mock.FakeRunner + var ghwTest v1mock.GhwMock + var defaultSizeForTest uint + var device string + + BeforeEach(func() { + device = "/dev/device" + defaultSizeForTest = 100 + logger = v1.NewLogger() + logger.SetLevel(logrus.DebugLevel) + fs, _, _ = vfst.NewTestFS(map[string]interface{}{device: ""}) + runner = v1mock.NewFakeRunner() + console = newCloudInitConsole(logger, runner) + mainDisk := block.Disk{ + Name: "device", + Partitions: []*block.Partition{ + { + Name: "device1", + FilesystemLabel: "FAKE", + Type: "ext4", + MountPoint: "/mnt/fake", + SizeBytes: 0, + }, + { + Name: "device2", + FilesystemLabel: "FAKE", + Type: "ext4", + MountPoint: "/mnt/fake", + SizeBytes: 0, + }, + }, + } + ghwTest = v1mock.GhwMock{} + ghwTest.AddDisk(mainDisk) + ghwTest.CreateDevices() + }) + + Describe("Expand partition", Label("expand"), func() { + BeforeEach(func() { + partition := "/dev/device1" + + layout := schema.Layout{ + Device: &schema.Device{ + Label: "FAKE", + Path: device, + }, + Expand: &schema.Expand{Size: defaultSizeForTest}, + Parts: []schema.Partition{}, + } + stage = schema.Stage{ + Layout: layout, + } + + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + /* + + Getting free sectors is called by running: + `parted --script --machine -- /dev/device unit s print` + And returns the following: + BYT; + /dev/nvme0n1:7814037168s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; + 1:2048s:206847s:204800s:fat32:EFI System Partition:boot, esp, no_automount; + 2:206848s:239615s:32768s::Microsoft reserved partition:msftres, no_automount; + 3:239616s:2046941183s:2046701568s:ntfs:Basic data partition:msftdata; + 4:2046941184s:2048237567s:1296384s:ntfs::hidden, diag, no_automount; + 5:2048237568s:2050334719s:2097152s:ext4::; + 6:2050334720s:7814035455s:5763700736s:btrfs::; + + So it's the device and its total sectors and picks the last partition and its final sector. + In this case: + /dev/nvme0n1:7814037168s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; + ^device ^total sectors + 6:2050334720s:7814035455s:5763700736s:btrfs::; + ^partition ^end sector + + And you rest (total - end secor of last partition) to know how many free sectors there are. + At least 20480 sectors are needed to expand properly + */ + // Return 1.000.000 total sectors - 1000 used by the partition + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000s:0s:ext4::;` + return []byte(rtn), nil + } + // removing the first partition and creating a new one + if command == "parted" && len(args) == 13 { + if args[6] == "rm" && args[7] == "1" && args[8] == "mkpart" { + // Create the device + _, err := fs.Create(partition) + Expect(err).ToNot(HaveOccurred()) + return nil, err + } + } + return nil, nil + } + }) + + AfterEach(func() { + ghwTest.Clean() + }) + + It("Expands latest partition", func() { + err := layoutPlugin(logger, stage, fs, console) + Expect(err).ToNot(HaveOccurred()) + // This is the sector size that it's going to be passed to parted to increase the new partition size + // Remember to remove 1 last sector, don't ask me why + Sectors := partitioner.MiBToSectors(defaultSizeForTest, 512) - 1 + // Check that it tried to delete+create and check the new fs for the new partition and resize it + Expect(runner.IncludesCmds([][]string{ + {"udevadm", "settle"}, + {"parted", "--script", "--machine", "--", "/dev/device", "unit", "s", "print"}, + {"parted", "--script", "--machine", "--", "/dev/device", "unit", "s", "rm", "1", "mkpart", "part1", "", "0", strconv.Itoa(int(Sectors))}, + {"e2fsck", "-fy", "/dev/device1"}, + {"resize2fs", "/dev/device1"}, + })).ToNot(HaveOccurred()) + }) + It("Fails if there is not enough space", func() { + // Override runner side effect to return 0 sectors when asked + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000000s:0s:ext4::;` + return []byte(rtn), nil + } + return nil, nil + } + err := layoutPlugin(logger, stage, fs, console) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not enough free space")) + }) + It("Fails if device doesnt exists", func() { + // Override runner side effect to return 0 sectors when asked + _ = fs.RemoveAll("/dev/device") + err := layoutPlugin(logger, stage, fs, console) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Target disk not found")) + }) + It("Fails if new device didnt get created", func() { + // Override runner side effect to return error when partition is recreated + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000s:0s:ext4::;` + return []byte(rtn), nil + } + // removing the first partition and creating a new one + if command == "parted" && len(args) == 13 { + if args[6] == "rm" && args[7] == "1" && args[8] == "mkpart" { + // return an error + return nil, fmt.Errorf("failed") + } + } + return nil, nil + } + err := layoutPlugin(logger, stage, fs, console) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed")) + }) + It("Fails if new device didnt get created, even when command didnt return an error", func() { + // Override runner side effect to return error when partition is recreated + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000s:0s:ext4::;` + return []byte(rtn), nil + } + // removing the first partition and creating a new one + if command == "parted" && len(args) == 13 { + if args[6] == "rm" && args[7] == "1" && args[8] == "mkpart" { + // Do nothing like the command failed + return nil, nil + } + } + return nil, nil + } + err := layoutPlugin(logger, stage, fs, console) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find partition device")) + Expect(err.Error()).To(ContainSubstring("/dev/device1")) + }) + }) + + Describe("Add partitions", Label("add", "partitions"), func() { + BeforeEach(func() { + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000s:0s:ext4::;` + return []byte(rtn), nil + } + return nil, nil + } + }) + AfterEach(func() { + ghwTest.Clean() + }) + It("Adds one partition", func() { + fslabel := "jojo" + fstype := "ext3" + plabel := "dio" + + layout := schema.Layout{ + Device: &schema.Device{ + Label: "FAKE", + Path: device, + }, + Parts: []schema.Partition{ + { + Size: defaultSizeForTest, + FSLabel: fslabel, + FileSystem: fstype, + PLabel: plabel, + }, + }, + } + stage = schema.Stage{ + Layout: layout, + } + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:; +1:0s:1000s:0s:ext4::;` + return []byte(rtn), nil + } + // removing the first partition and creating a new one + if command == "parted" && len(args) == 11 { + // creating partition with our given label and fs type + if args[6] == "mkpart" && args[7] == plabel && args[8] == fstype { + logger.Info("Creating part") + //Create the device + _, err := fs.Create("/dev/device2") + Expect(err).ToNot(HaveOccurred()) + return nil, nil + } + } + return nil, nil + } + err := layoutPlugin(logger, stage, fs, console) + Expect(err).ToNot(HaveOccurred()) + // Because this is adding a new partition and according to our fake parted the first partitions occupies 1000 sectors + // We need to sum 1000 sectors to this number to calculate the sectors passed to parted + // As parted will create the new partition from sector 1001 to MBsToSectors+1001 + Sectors := partitioner.MiBToSectors(defaultSizeForTest, 512) - 1 + 1001 + // Checks that commands to create the new partition were called with the proper fs, size and labels + Expect(runner.IncludesCmds([][]string{ + {"udevadm", "settle"}, + {"parted", "--script", "--machine", "--", "/dev/device", "unit", "s", "mkpart", plabel, fstype, "1001", strconv.Itoa(int(Sectors))}, + {"mkfs.ext3", "-L", fslabel, "/dev/device2"}, + })).ToNot(HaveOccurred()) + }) + + It("Adds multiple partitions", func() { + partitions := []schema.Partition{ + { + Size: 100, + FSLabel: "fs-label-part-1", + FileSystem: "ext3", + PLabel: "label-part-1", + }, + { + Size: 120, + FSLabel: "fs-label-part-2", + FileSystem: "ext4", + PLabel: "label-part-2", + }, + } + + layout := schema.Layout{ + Device: &schema.Device{ + Label: "FAKE", + Path: device, + }, + Parts: partitions, + Expand: &schema.Expand{Size: 0}, // Expand to the end of disk + } + stage = schema.Stage{ + Layout: layout, + } + + type partitionData struct { + StartSector int + EndSector int + TotalSectors int + PartitionNumber int + Filesystem string + PLabel string + } + createdPartitions := []partitionData{} + + runner.SideEffect = func(command string, args ...string) ([]byte, error) { + if command == "parted" && args[4] == "unit" && args[5] == "s" && args[6] == "print" { + rtn := ` +BYT; +/dev/device:1000000s:nvme:512:512:gpt:KINGSTON SFYRD4000G:;` + for _, p := range createdPartitions { + rtn += fmt.Sprintf("\n%d:%ds:%ds:%ds:%s::;", p.PartitionNumber, p.StartSector, p.EndSector, p.TotalSectors, p.Filesystem) + } + + return []byte(rtn), nil + } + + // removing the first partition and creating a new one + if command == "parted" && len(args) == 11 { + // creating partition with our given label and fs type + if args[6] == "mkpart" { + endSector, err := strconv.Atoi(args[10]) + Expect(err).ToNot(HaveOccurred()) + startSector, err := strconv.Atoi(args[9]) + Expect(err).ToNot(HaveOccurred()) + + newPart := partitionData{ + StartSector: startSector, + EndSector: endSector, + TotalSectors: endSector - startSector, + PartitionNumber: len(createdPartitions) + 1, + Filesystem: args[8], + PLabel: args[7], + } + + createdPartitions = append(createdPartitions, newPart) + _, err = fs.Create(fmt.Sprintf("/dev/device%d", newPart.PartitionNumber)) + Expect(err).ToNot(HaveOccurred()) + return nil, nil + } + } + // removing the first partition and creating a new one (expand) + if command == "parted" && len(args) == 13 { + if args[6] == "rm" && args[7] == "2" && args[8] == "mkpart" { + // Create the device + _, err := fs.Create("/dev/device2") + Expect(err).ToNot(HaveOccurred()) + // Normally we would update these to match the truth: + //createdPartitions[1].EndSector = 1000000 + //createdPartitions[1].TotalSectors = createdPartitions[1].EndSector - createdPartitions[1].StartSector + // but the test below, needs the old value to check if the command + // that created the first version of the partition was run (using the old EndSector) + + return nil, err + } + } + return nil, nil + } + err := layoutPlugin(logger, stage, fs, console) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(createdPartitions)).To(Equal(len(partitions))) + // Checks that commands to create the new partition were called with the proper fs, size and labels + partedCmds := [][]string{} + for _, p := range createdPartitions { + partedCmds = append(partedCmds, []string{ + "parted", "--script", "--machine", "--", "/dev/device", "unit", "s", "mkpart", p.PLabel, p.Filesystem, strconv.Itoa(p.StartSector), strconv.Itoa(p.EndSector), + }) + partedCmds = append(partedCmds, []string{ + fmt.Sprintf("mkfs.%s", p.Filesystem), "-L", fmt.Sprintf("fs-%s", p.PLabel), fmt.Sprintf("/dev/device%d", p.PartitionNumber), + }) + } + + Expect(runner.IncludesCmds(partedCmds)).ToNot(HaveOccurred()) + + Expect(runner.IncludesCmds([][]string{ + {"parted", "--script", "--machine", "--", "/dev/device", "unit", "s", "rm", "2", "mkpart", "part2", "", strconv.Itoa(createdPartitions[1].StartSector), "100%"}, + {"e2fsck", "-fy", "/dev/device2"}, + {"resize2fs", "/dev/device2"}, + })).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/pkg/config/config.go b/pkg/config/config.go index 52f2c1a..faaefc2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -23,6 +23,7 @@ const ( type Install struct { Auto bool `yaml:"auto,omitempty"` Reboot bool `yaml:"reboot,omitempty"` + NoFormat bool `yaml:"no_format,omitempty"` Device string `yaml:"device,omitempty"` Poweroff bool `yaml:"poweroff,omitempty"` GrubOptions map[string]string `yaml:"grub_options,omitempty"` diff --git a/pkg/elemental/elemental.go b/pkg/elemental/elemental.go index 57f62b3..cef1f65 100644 --- a/pkg/elemental/elemental.go +++ b/pkg/elemental/elemental.go @@ -391,8 +391,10 @@ func (e *Elemental) DumpSource(target string, imgSrc *v1.ImageSource) (info inte // CopyCloudConfig will check if there is a cloud init in the config and store it on the target func (e *Elemental) CopyCloudConfig(cloudInit []string) (err error) { + e.config.Logger.Infof("List of cloud inits to copy: %+v\n", cloudInit) for i, ci := range cloudInit { customConfig := filepath.Join(cnst.OEMDir, fmt.Sprintf("9%d_custom.yaml", i)) + e.config.Logger.Infof("Starting copying cloud config file %s to %s", ci, customConfig) err = utils.GetSource(e.config, ci, customConfig) if err != nil { return err diff --git a/pkg/partitioner/disk.go b/pkg/partitioner/disk.go index ea9342e..0a7cae1 100644 --- a/pkg/partitioner/disk.go +++ b/pkg/partitioner/disk.go @@ -48,8 +48,19 @@ type Disk struct { logger v1.Logger } -func MiBToSectors(size uint, sectorSize uint) uint { - return size * 1048576 / sectorSize +// MiBToSectors returns the number of sectors that correspond to the given amount +// of MB. +func MiBToSectors(totalMB uint, sectorSize uint) uint { + bytes := totalMB * 1024 * 1024 + return bytes / sectorSize +} + +// SectorsToMiB returns the number of MBs that correspond to the given amount +// of sectors. +func SectorsToMiB(totalSectors uint, sectorSize uint) uint { + bytes := totalSectors * sectorSize + + return bytes / (1024 * 1024) // Mb } func NewDisk(device string, opts ...DiskOptions) *Disk { diff --git a/pkg/types/v1/config.go b/pkg/types/v1/config.go index 93c8840..488133c 100644 --- a/pkg/types/v1/config.go +++ b/pkg/types/v1/config.go @@ -123,6 +123,7 @@ func (c *Config) Sanitize() error { } type RunConfig struct { + Debug bool `yaml:"strict,omitempty" mapstructure:"debug"` Strict bool `yaml:"strict,omitempty" mapstructure:"strict"` Reboot bool `yaml:"reboot,omitempty" mapstructure:"reboot"` PowerOff bool `yaml:"poweroff,omitempty" mapstructure:"poweroff"`