From 2e86e483a7c775d0fb026c813e36ecc3f76e0515 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 7 Jul 2022 16:57:38 +0000 Subject: [PATCH] gear: Extract netboot artifacts This changeset also adds a `config_url` and `options` keyword in the c3os config. Along with that the config logic is changed so the configuration is taken also from boot commands and merged in the final installed config file. --- pkg/config/config.go | 118 +++++++++++++++++++++++++++----------- pkg/config/config_test.go | 31 +++++++++- pkg/config/options.go | 36 ++++++++++++ 3 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 pkg/config/options.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 62bbe69..9e5603a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -4,9 +4,12 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" "os" "path/filepath" + retry "github.com/avast/retry-go" + "github.com/c3os-io/c3os/internal/machine" yip "github.com/mudler/yip/pkg/schema" "gopkg.in/yaml.v2" @@ -33,25 +36,43 @@ type K3s struct { } type Config struct { - C3OS *C3OS `yaml:"c3os,omitempty"` - K3sAgent K3s `yaml:"k3s-agent,omitempty"` - K3s K3s `yaml:"k3s,omitempty"` - VPN map[string]string `yaml:"vpn,omitempty"` - cloudFileContent string - location string + C3OS *C3OS `yaml:"c3os,omitempty"` + K3sAgent K3s `yaml:"k3s-agent,omitempty"` + K3s K3s `yaml:"k3s,omitempty"` + VPN map[string]string `yaml:"vpn,omitempty"` + //cloudFileContent string + originalData map[string]interface{} + location string + ConfigURL string `yaml:"config_url,omitempty"` + Options map[string]string `yaml:"options,omitempty"` +} + +func (c Config) Data() map[string]interface{} { + return c.originalData } func (c Config) String() string { - if c.cloudFileContent == "" { + if len(c.originalData) == 0 { dat, err := yaml.Marshal(c) if err == nil { return string(dat) } } - return c.cloudFileContent + + dat, _ := yaml.Marshal(c.originalData) + return string(dat) } -func Scan(dir ...string) (c *Config, err error) { +func Scan(opts ...Option) (c *Config, err error) { + + o := &Options{} + + if err := o.Apply(opts...); err != nil { + return nil, err + } + + dir := o.ScanDir + c = &Config{} files := []string{} for _, d := range dir { @@ -69,12 +90,55 @@ func Scan(dir ...string) (c *Config, err error) { if err == nil { yaml.Unmarshal(b, c) if c.C3OS != nil || c.K3s.Enabled || c.K3sAgent.Enabled { - c.cloudFileContent = string(b) + // c.cloudFileContent = string(b) c.location = f + yaml.Unmarshal(b, &c.originalData) break } } } + + if o.MergeBootCMDLine { + d, err := machine.DotToYAML(o.BootCMDLineFile) + if err == nil { // best-effort + yaml.Unmarshal(d, c) + // Merge back to originalData only config which are part of the config structure + // This avoid garbage as unrelated bootargs to be merged in. + dat, err := yaml.Marshal(c) + if err == nil { + yaml.Unmarshal(dat, &c.originalData) + } + } + } + + if c.ConfigURL != "" { + var body []byte + + err := retry.Do( + func() error { + resp, err := http.Get(c.ConfigURL) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return nil + }, + ) + + if err != nil { + return c, fmt.Errorf("could not merge configs: %w", err) + } + + yaml.Unmarshal(body, c) + yaml.Unmarshal(body, &c.originalData) + } + return c, nil } @@ -100,6 +164,7 @@ func fileSize(f string) float64 { megabytes = (float64)(kilobytes / 1024) // cast to type float64 return megabytes } + func listFiles(dir string) ([]string, error) { content := []string{} @@ -117,29 +182,9 @@ func listFiles(dir string) ([]string, error) { } func ReplaceToken(dir []string, token string) (err error) { - c := &Config{} - files := []string{} - for _, d := range dir { - if f, err := listFiles(d); err == nil { - files = append(files, f...) - } - } - var configFile string - perms := os.ModePerm - for _, f := range files { - b, err := ioutil.ReadFile(f) - if err == nil { - yaml.Unmarshal(b, c) - if c.C3OS != nil { - configFile = f - c.cloudFileContent = string(b) - i, err := os.Stat(f) - if err == nil { - perms = i.Mode() - } - break - } - } + c, err := Scan(Directories(dir...)) + if err != nil { + return err } if c.C3OS == nil { @@ -176,7 +221,12 @@ func ReplaceToken(dir []string, token string) (err error) { return err } - return ioutil.WriteFile(configFile, d, perms) + fi, err := os.Stat(c.location) + if err != nil { + return err + } + + return ioutil.WriteFile(c.location, d, fi.Mode().Perm()) } type Stage string diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 2477947..9eeb080 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,9 +30,9 @@ var _ = Describe("Get config", func() { Context("directory", func() { It("reads config file greedly", func() { - var cc string = ` + var cc string = `baz: bar c3os: - network_token: "foo" + network_token: foo ` d, _ := ioutil.TempDir("", "xxxx") defer os.RemoveAll(d) @@ -44,7 +44,7 @@ fooz: `), os.ModePerm) Expect(err).ToNot(HaveOccurred()) - c, err := Scan(d) + c, err := Scan(Directories(d)) Expect(err).ToNot(HaveOccurred()) Expect(c).ToNot(BeNil()) Expect(c.C3OS.NetworkToken).To(Equal("foo")) @@ -86,5 +86,30 @@ fooz: "bb": map[interface{}]interface{}{"nothing": "foo"}, })) }) + + It("merges with bootargs", func() { + + var cc string = ` +c3os: + network_token: "foo" + +bb: + nothing: "foo" +` + d, _ := ioutil.TempDir("", "xxxx") + defer os.RemoveAll(d) + + err := ioutil.WriteFile(filepath.Join(d, "test"), []byte(cc), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + err = ioutil.WriteFile(filepath.Join(d, "b"), []byte(`zz.foo="baa" options.foo=bar`), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + c, err := Scan(Directories(d), MergeBootLine, WithBootCMDLineFile(filepath.Join(d, "b"))) + Expect(err).ToNot(HaveOccurred()) + Expect(c.Options["foo"]).To(Equal("bar")) + Expect(c.C3OS.NetworkToken).To(Equal("foo")) + _, exists := c.Data()["zz"] + Expect(exists).To(BeFalse()) + }) }) }) diff --git a/pkg/config/options.go b/pkg/config/options.go new file mode 100644 index 0000000..8279e77 --- /dev/null +++ b/pkg/config/options.go @@ -0,0 +1,36 @@ +package config + +type Options struct { + ScanDir []string + BootCMDLineFile string + MergeBootCMDLine bool +} + +type Option func(o *Options) error + +func (o *Options) Apply(opts ...Option) error { + for _, oo := range opts { + if err := oo(o); err != nil { + return err + } + } + return nil +} + +var MergeBootLine = func(o *Options) error { + o.MergeBootCMDLine = true + return nil +} + +func WithBootCMDLineFile(s string) Option { + return func(o *Options) error { + o.BootCMDLineFile = s + return nil + } +} +func Directories(d ...string) Option { + return func(o *Options) error { + o.ScanDir = d + return nil + } +}