diff --git a/cmd/vendor_pull.go b/cmd/vendor_pull.go index 50fe6cec1..b616d365d 100644 --- a/cmd/vendor_pull.go +++ b/cmd/vendor_pull.go @@ -32,6 +32,6 @@ func init() { vendorPullCmd.PersistentFlags().StringP("type", "t", "terraform", "atmos vendor pull --component --type=terraform|helmfile") vendorPullCmd.PersistentFlags().Bool("dry-run", false, "atmos vendor pull --component --dry-run") vendorPullCmd.PersistentFlags().String("tags", "", "Only vendor the components that have the specified tags: atmos vendor pull --tags=dev,test") - + vendorPullCmd.PersistentFlags().Bool("everything", false, "Vendor all components: atmos vendor pull --everything") vendorCmd.AddCommand(vendorPullCmd) } diff --git a/examples/demo-component-versions/atmos.yaml b/examples/demo-component-versions/atmos.yaml index e24b091f4..35ad30e31 100644 --- a/examples/demo-component-versions/atmos.yaml +++ b/examples/demo-component-versions/atmos.yaml @@ -25,4 +25,4 @@ commands: - name: "test" description: "Run all tests" steps: - - atmos vendor pull + - atmos vendor pull --everything diff --git a/examples/demo-vendoring/atmos.yaml b/examples/demo-vendoring/atmos.yaml index 20983e4f9..0f0506e81 100644 --- a/examples/demo-vendoring/atmos.yaml +++ b/examples/demo-vendoring/atmos.yaml @@ -37,4 +37,4 @@ commands: - name: "test" description: "Run all tests" steps: - - atmos vendor pull + - atmos vendor pull --everything diff --git a/go.mod b/go.mod index 640a2fabe..73e47727f 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/jwalton/go-supportscolor v1.2.0 github.com/kubescape/go-git-url v0.0.30 github.com/lrstanley/bubblezone v0.0.0-20240914071701-b48c55a5e78e + github.com/mattn/go-isatty v0.0.20 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-wordwrap v1.0.1 github.com/mitchellh/mapstructure v1.5.0 @@ -95,6 +96,7 @@ require ( github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chainguard-dev/git-urls v1.0.2 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -170,7 +172,6 @@ require ( github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect diff --git a/go.sum b/go.sum index 7031a7d37..822b319f9 100644 --- a/go.sum +++ b/go.sum @@ -399,6 +399,8 @@ github.com/charmbracelet/bubbletea v1.2.1 h1:J041h57zculJKEKf/O2pS4edXGIz+V0Yvoj github.com/charmbracelet/bubbletea v1.2.1/go.mod h1:viLoDL7hG4njLJSKU2gw7kB3LSEmWsrM80rO1dBJWBI= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= github.com/charmbracelet/x/ansi v0.4.5 h1:LqK4vwBNaXw2AyGIICa5/29Sbdq58GbGdFngSexTdRM= diff --git a/internal/exec/vendor_component_utils.go b/internal/exec/vendor_component_utils.go index ac31f3d9d..0af8edc8c 100644 --- a/internal/exec/vendor_component_utils.go +++ b/internal/exec/vendor_component_utils.go @@ -8,17 +8,16 @@ import ( "os" "path" "path/filepath" - "strconv" "strings" "text/template" - "time" "github.com/Masterminds/sprig/v3" + tea "github.com/charmbracelet/bubbletea" cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/schema" u "github.com/cloudposse/atmos/pkg/utils" "github.com/hairyhenderson/gomplate/v3" - "github.com/hashicorp/go-getter" + "github.com/mattn/go-isatty" cp "github.com/otiai10/copy" ) @@ -95,6 +94,101 @@ func ReadAndProcessComponentVendorConfigFile( // https://opencontainers.org/ // https://github.com/google/go-containerregistry // https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html + +// ExecuteStackVendorInternal executes the command to vendor an Atmos stack +// TODO: implement this +func ExecuteStackVendorInternal( + stack string, + dryRun bool, +) error { + return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") +} +func copyComponentToDestination(cliConfig schema.CliConfiguration, tempDir, componentPath string, vendorComponentSpec schema.VendorComponentSpec, sourceIsLocalFile bool, uri string) error { + // Copy from the temp folder to the destination folder and skip the excluded files + copyOptions := cp.Options{ + // Skip specifies which files should be skipped + Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if filepath.Base(src) == ".git" { + return true, nil + } + + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + + // Exclude the files that match the 'excluded_paths' patterns + // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) + // https://en.wikipedia.org/wiki/Glob_(programming) + // https://github.com/bmatcuk/doublestar#patterns + for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { + excludeMatch, err := u.PathMatch(excludePath, src) + if err != nil { + return true, err + } else if excludeMatch { + // If the file matches ANY of the 'excluded_paths' patterns, exclude the file + u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", + trimmedSrc, + excludePath, + )) + return true, nil + } + } + + // Only include the files that match the 'included_paths' patterns (if any pattern is specified) + if len(vendorComponentSpec.Source.IncludedPaths) > 0 { + anyMatches := false + for _, includePath := range vendorComponentSpec.Source.IncludedPaths { + includeMatch, err := u.PathMatch(includePath, src) + if err != nil { + return true, err + } else if includeMatch { + // If the file matches ANY of the 'included_paths' patterns, include the file + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", + trimmedSrc, + includePath, + )) + anyMatches = true + break + } + } + + if anyMatches { + return false, nil + } else { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) + return true, nil + } + } + + // If 'included_paths' is not provided, include all files that were not excluded + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) + return false, nil + }, + + // Preserve the atime and the mtime of the entries + // On linux we can preserve only up to 1 millisecond accuracy + PreserveTimes: false, + + // Preserve the uid and the gid of all entries + PreserveOwner: false, + + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + componentPath2 := componentPath + if sourceIsLocalFile { + if filepath.Ext(componentPath) == "" { + componentPath2 = path.Join(componentPath, filepath.Base(uri)) + } + } + + if err := cp.Copy(tempDir, componentPath2, copyOptions); err != nil { + return err + } + return nil +} func ExecuteComponentVendorInternal( cliConfig schema.CliConfiguration, vendorComponentSpec schema.VendorComponentSpec, @@ -102,7 +196,6 @@ func ExecuteComponentVendorInternal( componentPath string, dryRun bool, ) error { - var tempDir string var err error var t *template.Template var uri string @@ -128,7 +221,6 @@ func ExecuteComponentVendorInternal( } else { uri = vendorComponentSpec.Source.Uri } - useOciScheme := false useLocalFileSystem := false sourceIsLocalFile := false @@ -152,162 +244,27 @@ func ExecuteComponentVendorInternal( } } } - - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", - component, - uri, - componentPath, - )) - - if !dryRun { - // Create temp folder - // We are using a temp folder for the following reasons: - // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) - // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder - tempDir, err = os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - - defer removeTempDir(cliConfig, tempDir) - - // Download the source into the temp directory - if useOciScheme { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) - if err != nil { - return err - } - } else if useLocalFileSystem { - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - tempDir2 := tempDir - if sourceIsLocalFile { - tempDir2 = path.Join(tempDir, filepath.Base(uri)) - } - - if err = cp.Copy(uri, tempDir2, copyOptions); err != nil { - return err - } - } else { - // Use `go-getter` to download the sources into the temp directory - // When cloning from the root of a repo w/o using modules (sub-paths), `go-getter` does the following: - // - If the destination directory does not exist, it creates it and runs `git init` - // - If the destination directory exists, it should be an already initialized Git repository (otherwise an error will be thrown) - // For more details, refer to - // - https://github.com/hashicorp/go-getter/issues/114 - // - https://github.com/hashicorp/go-getter?tab=readme-ov-file#subdirectories - // We add the `uri` to the already created `tempDir` so it does not exist to allow `go-getter` to create - // and correctly initialize it - tempDir = path.Join(tempDir, filepath.Base(uri)) - - client := &getter.Client{ - Ctx: context.Background(), - // Define the destination where the files will be stored. This will create the directory if it doesn't exist - Dst: tempDir, - // Source - Src: uri, - Mode: getter.ClientModeAny, - } - - if err = client.Get(); err != nil { - return err - } - } - - // Copy from the temp folder to the destination folder and skip the excluded files - copyOptions := cp.Options{ - // Skip specifies which files should be skipped - Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.HasSuffix(src, ".git") { - return true, nil - } - - trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) - - // Exclude the files that match the 'excluded_paths' patterns - // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) - // https://en.wikipedia.org/wiki/Glob_(programming) - // https://github.com/bmatcuk/doublestar#patterns - for _, excludePath := range vendorComponentSpec.Source.ExcludedPaths { - excludeMatch, err := u.PathMatch(excludePath, src) - if err != nil { - return true, err - } else if excludeMatch { - // If the file matches ANY of the 'excluded_paths' patterns, exclude the file - u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", - trimmedSrc, - excludePath, - )) - return true, nil - } - } - - // Only include the files that match the 'included_paths' patterns (if any pattern is specified) - if len(vendorComponentSpec.Source.IncludedPaths) > 0 { - anyMatches := false - for _, includePath := range vendorComponentSpec.Source.IncludedPaths { - includeMatch, err := u.PathMatch(includePath, src) - if err != nil { - return true, err - } else if includeMatch { - // If the file matches ANY of the 'included_paths' patterns, include the file - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", - trimmedSrc, - includePath, - )) - anyMatches = true - break - } - } - - if anyMatches { - return false, nil - } else { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) - return true, nil - } - } - - // If 'included_paths' is not provided, include all files that were not excluded - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) - return false, nil - }, - - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - componentPath2 := componentPath - if sourceIsLocalFile { - if filepath.Ext(componentPath) == "" { - componentPath2 = path.Join(componentPath, filepath.Base(uri)) - } - } - - if err = cp.Copy(tempDir, componentPath2, copyOptions); err != nil { - return err - } + var pType pkgType + if useOciScheme { + pType = pkgTypeOci + } else if useLocalFileSystem { + pType = pkgTypeLocal + } else { + pType = pkgTypeRemote } + componentPkg := pkgComponentVendor{ + uri: uri, + name: component, + componentPath: componentPath, + sourceIsLocalFile: sourceIsLocalFile, + pkgType: pType, + version: vendorComponentSpec.Source.Version, + vendorComponentSpec: vendorComponentSpec, + IsComponent: true, + } + var packages []pkgComponentVendor + packages = append(packages, componentPkg) // Process mixins if len(vendorComponentSpec.Mixins) > 0 { for _, mixin := range vendorComponentSpec.Mixins { @@ -352,72 +309,54 @@ func ExecuteComponentVendorInternal( uri = absPath } } - - u.LogInfo(cliConfig, fmt.Sprintf( - "Pulling the mixin '%s' for the component '%s' into '%s'\n", - uri, - component, - path.Join(componentPath, mixin.Filename), - )) - - if !dryRun { - err = os.RemoveAll(tempDir) - if err != nil { - return err - } - - // Download the mixin into the temp file - if !useOciScheme { - client := &getter.Client{ - Ctx: context.Background(), - Dst: path.Join(tempDir, mixin.Filename), - Src: uri, - Mode: getter.ClientModeFile, - } - - if err = client.Get(); err != nil { - return err - } - } else { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) - if err != nil { - return err - } - } - - // Copy from the temp folder to the destination folder - copyOptions := cp.Options{ - // Preserve the atime and the mtime of the entries - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - // Prevent the error: - // symlink components/terraform/mixins/context.tf components/terraform/infra/vpc-flow-logs-bucket/context.tf: file exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, + // Check if it's a local file + if absPath, err := u.JoinAbsolutePathWithPath(componentPath, uri); err == nil { + if u.FileExists(absPath) { + pType = pkgTypeLocal + continue } + } + if useOciScheme { + pType = pkgTypeOci + } else { + pType = pkgTypeRemote + } - if err = cp.Copy(tempDir, componentPath, copyOptions); err != nil { - return err - } + pkg := pkgComponentVendor{ + uri: uri, + pkgType: pType, + name: "mixin " + uri, + sourceIsLocalFile: false, + IsMixins: true, + vendorComponentSpec: vendorComponentSpec, + version: mixin.Version, + componentPath: componentPath, + mixinFilename: mixin.Filename, } + + packages = append(packages, pkg) + } + } + // Run TUI to process packages + if len(packages) > 0 { + model, err := newModelComponentVendorInternal(packages, dryRun, cliConfig) + if err != nil { + return fmt.Errorf("error initializing model: %v", err) + } + var opts []tea.ProgramOption + // Disable TUI if no TTY support is available + if !CheckTTYSupport() { + opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} + u.LogWarning(cliConfig, "TTY is not supported. Running in non-interactive mode") + } + if _, err := tea.NewProgram(&model, opts...).Run(); err != nil { + return fmt.Errorf("running download error: %w", err) } } - return nil } -// ExecuteStackVendorInternal executes the command to vendor an Atmos stack -// TODO: implement this -func ExecuteStackVendorInternal( - stack string, - dryRun bool, -) error { - return fmt.Errorf("command 'atmos vendor pull --stack ' is not supported yet") +// CheckTTYSupport checks if stdout supports TTY for displaying the progress UI. +func CheckTTYSupport() bool { + return isatty.IsTerminal(os.Stdout.Fd()) } diff --git a/internal/exec/vendor_model.go b/internal/exec/vendor_model.go new file mode 100644 index 000000000..f72e00281 --- /dev/null +++ b/internal/exec/vendor_model.go @@ -0,0 +1,338 @@ +package exec + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/cloudposse/atmos/pkg/schema" + u "github.com/cloudposse/atmos/pkg/utils" + "github.com/hashicorp/go-getter" + cp "github.com/otiai10/copy" +) + +type pkgType int + +const ( + pkgTypeRemote pkgType = iota + pkgTypeOci + pkgTypeLocal +) + +func (p pkgType) String() string { + names := [...]string{"remote", "oci", "local"} + if p < pkgTypeRemote || p > pkgTypeLocal { + return "unknown" + } + return names[p] +} + +type pkgVendor struct { + name string + version string + atmosPackage *pkgAtmosVendor + componentPackage *pkgComponentVendor +} +type pkgAtmosVendor struct { + uri string + name string + targetPath string + sourceIsLocalFile bool + pkgType pkgType + version string + atmosVendorSource schema.AtmosVendorSource +} +type modelVendor struct { + packages []pkgVendor + index int + width int + height int + spinner spinner.Model + progress progress.Model + done bool + dryRun bool + failedPkg int + cliConfig schema.CliConfiguration + isTTY bool +} + +var ( + currentPkgNameStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("211")) + doneStyle = lipgloss.NewStyle().Margin(1, 2) + checkMark = lipgloss.NewStyle().Foreground(lipgloss.Color("42")).SetString("✓") + xMark = lipgloss.NewStyle().Foreground(lipgloss.Color("9")).SetString("x") + grayColor = lipgloss.NewStyle().Foreground(lipgloss.Color("241")) +) + +func newModelAtmosVendorInternal(pkgs []pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelVendor, error) { + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(30), + progress.WithoutPercentage(), + ) + s := spinner.New() + s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) + if len(pkgs) == 0 { + return modelVendor{done: true}, nil + } + tty := CheckTTYSupport() + var vendorPks []pkgVendor + for _, pkg := range pkgs { + p := pkgVendor{ + name: pkg.name, + version: pkg.version, + atmosPackage: &pkg, + } + vendorPks = append(vendorPks, p) + } + return modelVendor{ + packages: vendorPks, + spinner: s, + progress: p, + dryRun: dryRun, + cliConfig: cliConfig, + isTTY: tty, + }, nil +} + +func (m *modelVendor) Init() tea.Cmd { + if len(m.packages) == 0 { + m.done = true + return nil + } + return tea.Batch(ExecuteInstall(m.packages[0], m.dryRun, m.cliConfig), m.spinner.Tick) +} +func (m *modelVendor) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + if m.width > 120 { + m.width = 120 + } + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "esc", "q": + return m, tea.Quit + } + + case installedPkgMsg: + // ensure index is within bounds + if m.index >= len(m.packages) { + return m, nil + } + pkg := m.packages[m.index] + + mark := checkMark + + if msg.err != nil { + u.LogError(m.cliConfig, fmt.Errorf("Failed to vendor %s: error : %s", pkg.name, msg.err)) + mark = xMark + m.failedPkg++ + } + version := "" + if pkg.version != "" { + version = fmt.Sprintf("(%s)", pkg.version) + } + if m.index >= len(m.packages)-1 { + // Everything's been installed. We're done! + m.done = true + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + if m.dryRun { + u.LogInfo(m.cliConfig, "Done! Dry run completed. No components vendored.\n") + } + if m.failedPkg > 0 { + u.LogInfo(m.cliConfig, fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", len(m.packages)-m.failedPkg, m.failedPkg)) + } + u.LogInfo(m.cliConfig, fmt.Sprintf("Vendored %d components.\n", len(m.packages))) + } + version := grayColor.Render(version) + return m, tea.Sequence( + tea.Printf("%s %s %s", mark, pkg.name, version), + tea.Quit, + ) + } + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("%s %s %s", mark, pkg.name, version)) + } + m.index++ + // Update progress bar + progressCmd := m.progress.SetPercent(float64(m.index) / float64(len(m.packages))) + if !m.isTTY { + u.LogInfo(m.cliConfig, fmt.Sprintf("Pulling %s", m.packages[m.index].name)) + } + version = grayColor.Render(version) + return m, tea.Batch( + progressCmd, + tea.Printf("%s %s %s", mark, pkg.name, version), // print success message above our program + ExecuteInstall(m.packages[m.index], m.dryRun, m.cliConfig), // download the next package + ) + case spinner.TickMsg: + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case progress.FrameMsg: + newModel, cmd := m.progress.Update(msg) + if newModel, ok := newModel.(progress.Model); ok { + m.progress = newModel + } + return m, cmd + } + return m, nil +} + +func (m modelVendor) View() string { + + n := len(m.packages) + w := lipgloss.Width(fmt.Sprintf("%d", n)) + if m.done { + if m.dryRun { + return doneStyle.Render("Done! Dry run completed. No components vendored.\n") + } + if m.failedPkg > 0 { + return doneStyle.Render(fmt.Sprintf("Vendored %d components. Failed to vendor %d components.\n", n-m.failedPkg, m.failedPkg)) + } + return doneStyle.Render(fmt.Sprintf("Vendored %d components.\n", n)) + } + + pkgCount := fmt.Sprintf(" %*d/%*d", w, m.index, w, n) + spin := m.spinner.View() + " " + prog := m.progress.View() + cellsAvail := max(0, m.width-lipgloss.Width(spin+prog+pkgCount)) + if m.index >= len(m.packages) { + return "" + } + pkgName := currentPkgNameStyle.Render(m.packages[m.index].name) + + info := lipgloss.NewStyle().MaxWidth(cellsAvail).Render("Pulling " + pkgName) + + cellsRemaining := max(0, m.width-lipgloss.Width(spin+info+prog+pkgCount)) + gap := strings.Repeat(" ", cellsRemaining) + + return spin + info + gap + prog + pkgCount +} + +type installedPkgMsg struct { + err error + name string +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} +func downloadAndInstall(p *pkgAtmosVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + return func() tea.Msg { + if dryRun { + // Simulate the action + time.Sleep(500 * time.Millisecond) + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + + // Create temp directory + tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) + if err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to create temp directory: %w", err), + name: p.name, + } + } + // Ensure directory permissions are restricted + if err := os.Chmod(tempDir, 0700); err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to set temp directory permissions: %w", err), + name: p.name, + } + } + defer removeTempDir(cliConfig, tempDir) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + switch p.pkgType { + case pkgTypeRemote: + // Use go-getter to download remote packages + client := &getter.Client{ + Ctx: ctx, + Dst: tempDir, + Src: p.uri, + Mode: getter.ClientModeAny, + } + if err := client.Get(); err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to download package: %w", err), + name: p.name, + } + } + + case pkgTypeOci: + // Process OCI images + if err := processOciImage(cliConfig, p.uri, tempDir); err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to process OCI image: %w", err), + name: p.name, + } + } + + case pkgTypeLocal: + // Copy from local file system + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + if p.sourceIsLocalFile { + tempDir = path.Join(tempDir, filepath.Base(p.uri)) + } + if err := cp.Copy(p.uri, tempDir, copyOptions); err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to copy package: %w", err), + name: p.name, + } + } + default: + return installedPkgMsg{ + err: fmt.Errorf("unknown package type %s for package %s", p.pkgType.String(), p.name), + name: p.name, + } + + } + if err := copyToTarget(cliConfig, tempDir, p.targetPath, &p.atmosVendorSource, p.sourceIsLocalFile, p.uri); err != nil { + return installedPkgMsg{ + err: fmt.Errorf("failed to copy package: %w", err), + name: p.name, + } + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } +} +func ExecuteInstall(installer pkgVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + if installer.atmosPackage != nil { + return downloadAndInstall(installer.atmosPackage, dryRun, cliConfig) + } + if installer.componentPackage != nil { + return downloadComponentAndInstall(installer.componentPackage, dryRun, cliConfig) + } + // No valid package provided + return func() tea.Msg { + err := fmt.Errorf("no valid installer package provided for %s", installer.name) + return installedPkgMsg{ + err: err, + name: installer.name, + } + } +} diff --git a/internal/exec/vendor_model_component.go b/internal/exec/vendor_model_component.go new file mode 100644 index 000000000..bfb1b1627 --- /dev/null +++ b/internal/exec/vendor_model_component.go @@ -0,0 +1,240 @@ +package exec + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "time" + + "github.com/charmbracelet/bubbles/progress" + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/cloudposse/atmos/pkg/schema" + "github.com/hashicorp/go-getter" + cp "github.com/otiai10/copy" +) + +type pkgComponentVendor struct { + uri string + name string + sourceIsLocalFile bool + pkgType pkgType + version string + vendorComponentSpec schema.VendorComponentSpec + componentPath string + IsComponent bool + IsMixins bool + mixinFilename string +} + +func newModelComponentVendorInternal(pkgs []pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) (modelVendor, error) { + p := progress.New( + progress.WithDefaultGradient(), + progress.WithWidth(30), + progress.WithoutPercentage(), + ) + s := spinner.New() + s.Style = lipgloss.NewStyle().Foreground(lipgloss.Color("63")) + if len(pkgs) == 0 { + return modelVendor{done: true}, nil + } + vendorPks := []pkgVendor{} + for _, pkg := range pkgs { + vendorPkg := pkgVendor{ + name: pkg.name, + version: pkg.version, + componentPackage: &pkg, + } + vendorPks = append(vendorPks, vendorPkg) + + } + tty := CheckTTYSupport() + return modelVendor{ + packages: vendorPks, + spinner: s, + progress: p, + dryRun: dryRun, + cliConfig: cliConfig, + isTTY: tty, + }, nil +} + +func downloadComponentAndInstall(p *pkgComponentVendor, dryRun bool, cliConfig schema.CliConfiguration) tea.Cmd { + return func() tea.Msg { + if dryRun { + // Simulate the action + time.Sleep(100 * time.Millisecond) + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + if p.IsComponent { + err := installComponent(p, cliConfig) + if err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + if p.IsMixins { + err := installMixin(p, cliConfig) + if err != nil { + return installedPkgMsg{ + err: err, + name: p.name, + } + + } + return installedPkgMsg{ + err: nil, + name: p.name, + } + } + return installedPkgMsg{ + err: fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name), + name: p.name, + } + } +} +func installComponent(p *pkgComponentVendor, cliConfig schema.CliConfiguration) error { + + // Create temp folder + // We are using a temp folder for the following reasons: + // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) + // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder + tempDir, err := os.MkdirTemp("", fmt.Sprintf("atmos-vendor-%d-*", time.Now().Unix())) + if err != nil { + return fmt.Errorf("Failed to create temp directory %s", err) + } + // Ensure directory permissions are restricted + if err := os.Chmod(tempDir, 0700); err != nil { + return fmt.Errorf("failed to set temp directory permissions: %w", err) + } + defer removeTempDir(cliConfig, tempDir) + + switch p.pkgType { + case pkgTypeRemote: + tempDir = path.Join(tempDir, filepath.Base(p.uri)) + + client := &getter.Client{ + Ctx: context.Background(), + // Define the destination where the files will be stored. This will create the directory if it doesn't exist + Dst: tempDir, + // Source + Src: p.uri, + Mode: getter.ClientModeAny, + } + + if err = client.Get(); err != nil { + return fmt.Errorf("Failed to download package %s error %s", p.name, err) + } + + case pkgTypeOci: + // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory + err = processOciImage(cliConfig, p.uri, tempDir) + if err != nil { + return fmt.Errorf("Failed to process OCI image %s error %s", p.name, err) + } + + case pkgTypeLocal: + copyOptions := cp.Options{ + PreserveTimes: false, + PreserveOwner: false, + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + tempDir2 := tempDir + if p.sourceIsLocalFile { + tempDir2 = path.Join(tempDir, filepath.Base(p.uri)) + } + + if err = cp.Copy(p.uri, tempDir2, copyOptions); err != nil { + return fmt.Errorf("failed to copy package %s error %s", p.name, err) + } + default: + return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) + + } + if err = copyComponentToDestination(cliConfig, tempDir, p.componentPath, p.vendorComponentSpec, p.sourceIsLocalFile, p.uri); err != nil { + return fmt.Errorf("failed to copy package %s error %s", p.name, err) + } + + return nil + +} +func installMixin(p *pkgComponentVendor, cliConfig schema.CliConfiguration) error { + tempDir, err := os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) + if err != nil { + return fmt.Errorf("Failed to create temp directory %s", err) + } + + defer removeTempDir(cliConfig, tempDir) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + switch p.pkgType { + case pkgTypeRemote: + client := &getter.Client{ + Ctx: ctx, + Dst: path.Join(tempDir, p.mixinFilename), + Src: p.uri, + Mode: getter.ClientModeFile, + } + + if err = client.Get(); err != nil { + return fmt.Errorf("Failed to download package %s error %s", p.name, err) + } + case pkgTypeOci: + // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory + err = processOciImage(cliConfig, p.uri, tempDir) + if err != nil { + return fmt.Errorf("Failed to process OCI image %s error %s", p.name, err) + } + case pkgTypeLocal: + if p.uri == "" { + return fmt.Errorf("local mixin URI cannot be empty") + } + // Implement local mixin installation logic + return fmt.Errorf("local mixin installation not implemented") + + default: + return fmt.Errorf("unknown package type %s package %s", p.pkgType.String(), p.name) + + } + // Copy from the temp folder to the destination folder + copyOptions := cp.Options{ + // Preserve the atime and the mtime of the entries + PreserveTimes: false, + + // Preserve the uid and the gid of all entries + PreserveOwner: false, + + // OnSymlink specifies what to do on symlink + // Override the destination file if it already exists + // Prevent the error: + // symlink components/terraform/mixins/context.tf components/terraform/infra/vpc-flow-logs-bucket/context.tf: file exists + OnSymlink: func(src string) cp.SymlinkAction { + return cp.Deep + }, + } + + if err = cp.Copy(tempDir, p.componentPath, copyOptions); err != nil { + return fmt.Errorf("Failed to copy package %s error %s", p.name, err) + } + + return nil +} diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index d2d5cd7a6..2098720e4 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -1,17 +1,14 @@ package exec import ( - "context" "fmt" "os" "path" "path/filepath" "sort" - "strconv" "strings" - "time" - "github.com/hashicorp/go-getter" + tea "github.com/charmbracelet/bubbletea" cp "github.com/otiai10/copy" "github.com/samber/lo" "github.com/spf13/cobra" @@ -25,6 +22,7 @@ import ( // ExecuteVendorPullCommand executes `atmos vendor` commands func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { + info, err := processCommandLineArgs("terraform", cmd, args, nil) if err != nil { return err @@ -69,11 +67,27 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { } if component != "" && stack != "" { - return fmt.Errorf("either '--component' or '--stack' flag can to be provided, but not both") + return fmt.Errorf("either '--component' or '--stack' flag can be provided, but not both") } if component != "" && len(tags) > 0 { - return fmt.Errorf("either '--component' or '--tags' flag can to be provided, but not both") + return fmt.Errorf("either '--component' or '--tags' flag can be provided, but not both") + } + + // Retrieve the 'everything' flag and set default behavior if no other flags are set + everything, err := flags.GetBool("everything") + if err != nil { + return err + } + + // If neither `everything`, `component`, `stack`, nor `tags` flags are set, default to `everything = true` + if !everything && !flags.Changed("everything") && component == "" && stack == "" && len(tags) == 0 { + everything = true + } + + // Validate that only one of `--everything`, `--component`, `--stack`, or `--tags` is provided + if everything && (component != "" || stack != "" || len(tags) > 0) { + return fmt.Errorf("'--everything' flag cannot be combined with '--component', '--stack', or '--tags' flags") } if stack != "" { @@ -86,7 +100,9 @@ func ExecuteVendorPullCommand(cmd *cobra.Command, args []string) error { if err != nil { return err } - + if !vendorConfigExists && everything { + return fmt.Errorf("the '--everything' flag is set, but the vendor config file '%s' does not exist", cfg.AtmosVendorConfigFileName) + } if vendorConfigExists { // Process `vendor.yaml` return ExecuteAtmosVendorInternal(cliConfig, foundVendorConfigFile, vendorConfig.Spec, component, tags, dryRun) @@ -241,16 +257,11 @@ func ExecuteAtmosVendorInternal( dryRun bool, ) error { - var tempDir string var err error - var uri string vendorConfigFilePath := path.Dir(vendorConfigFileName) - logMessage := fmt.Sprintf("Processing vendor config file '%s'", vendorConfigFileName) - if len(tags) > 0 { - logMessage = fmt.Sprintf("%s for tags {%s}", logMessage, strings.Join(tags, ", ")) - } - u.LogInfo(cliConfig, logMessage) + logInitialMessage(cliConfig, vendorConfigFileName, tags) + if len(atmosVendorSpec.Sources) == 0 && len(atmosVendorSpec.Imports) == 0 { return fmt.Errorf("either 'spec.sources' or 'spec.imports' (or both) must be defined in the vendor config file '%s'", vendorConfigFileName) } @@ -321,33 +332,14 @@ func ExecuteAtmosVendorInternal( //} // Process sources + var packages []pkgAtmosVendor for indexSource, s := range sources { - // If `--component` is specified, and it's not equal to this component, skip this component - if component != "" && s.Component != component { + if shouldSkipSource(&s, component, tags) { continue } - // If `--tags` list is specified, and it does not contain any tags defined in this component, skip this component - // https://github.com/samber/lo?tab=readme-ov-file#intersect - if len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0 { - continue - } - - if s.File == "" { - s.File = vendorConfigFileName - } - - if s.Source == "" { - return fmt.Errorf("'source' must be specified in 'sources' in the vendor config file '%s'", - s.File, - ) - } - - if len(s.Targets) == 0 { - return fmt.Errorf("'targets' must be specified for the source '%s' in the vendor config file '%s'", - s.Source, - s.File, - ) + if err := validateSourceFields(&s, vendorConfigFileName); err != nil { + return err } tmplData := struct { @@ -356,206 +348,73 @@ func ExecuteAtmosVendorInternal( }{s.Component, s.Version} // Parse 'source' template - uri, err = ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) + uri, err := ProcessTmpl(fmt.Sprintf("source-%d", indexSource), s.Source, tmplData, false) if err != nil { return err } - - useOciScheme := false - useLocalFileSystem := false - sourceIsLocalFile := false - - // Check if `uri` uses the `oci://` scheme (to download the source from an OCI-compatible registry). - if strings.HasPrefix(uri, "oci://") { - useOciScheme = true - uri = strings.TrimPrefix(uri, "oci://") + err = validateURI(uri) + if err != nil { + return err } - - if !useOciScheme { - if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, uri); err == nil { - uri = absPath - useLocalFileSystem = true - - if u.FileExists(uri) { - sourceIsLocalFile = true - } - } + useOciScheme, useLocalFileSystem, sourceIsLocalFile := determineSourceType(uri, vendorConfigFilePath) + + // Determine package type + var pType pkgType + if useOciScheme { + pType = pkgTypeOci + } else if useLocalFileSystem { + pType = pkgTypeLocal + } else { + pType = pkgTypeRemote } - // Iterate over the targets + // Process each target within the source for indexTarget, tgt := range s.Targets { - var target string - // Parse 'target' template - target, err = ProcessTmpl(fmt.Sprintf("target-%d-%d", indexSource, indexTarget), tgt, tmplData, false) + target, err := ProcessTmpl(fmt.Sprintf("target-%d-%d", indexSource, indexTarget), tgt, tmplData, false) if err != nil { return err } - targetPath := path.Join(vendorConfigFilePath, target) - - if s.Component != "" { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources for the component '%s' from '%s' into '%s'", - s.Component, - uri, - targetPath, - )) - } else { - u.LogInfo(cliConfig, fmt.Sprintf("Pulling sources from '%s' into '%s'", - uri, - targetPath, - )) + pkgName := s.Component + if pkgName == "" { + pkgName = uri } - - if dryRun { - return nil + // Create package struct + p := pkgAtmosVendor{ + uri: uri, + name: pkgName, + targetPath: targetPath, + sourceIsLocalFile: sourceIsLocalFile, + pkgType: pType, + version: s.Version, + atmosVendorSource: s, } - // Create temp folder - // We are using a temp folder for the following reasons: - // 1. 'git' does not clone into an existing folder (and we have the existing component folder with `component.yaml` in it) - // 2. We have the option to skip some files we don't need and include only the files we need when copying from the temp folder to the destination folder - tempDir, err = os.MkdirTemp("", strconv.FormatInt(time.Now().Unix(), 10)) - if err != nil { - return err - } - - defer removeTempDir(cliConfig, tempDir) - - // Download the source into the temp directory - if useOciScheme { - // Download the Image from the OCI-compatible registry, extract the layers from the tarball, and write to the destination directory - err = processOciImage(cliConfig, uri, tempDir) - if err != nil { - return err - } - } else if useLocalFileSystem { - copyOptions := cp.Options{ - PreserveTimes: false, - PreserveOwner: false, - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } - - if sourceIsLocalFile { - tempDir = path.Join(tempDir, filepath.Base(uri)) - } + packages = append(packages, p) - if err = cp.Copy(uri, tempDir, copyOptions); err != nil { - return err - } - } else { - // Use `go-getter` to download the sources into the temp directory - // When cloning from the root of a repo w/o using modules (sub-paths), `go-getter` does the following: - // - If the destination directory does not exist, it creates it and runs `git init` - // - If the destination directory exists, it should be an already initialized Git repository (otherwise an error will be thrown) - // For more details, refer to - // - https://github.com/hashicorp/go-getter/issues/114 - // - https://github.com/hashicorp/go-getter?tab=readme-ov-file#subdirectories - // We add the `uri` to the already created `tempDir` so it does not exist to allow `go-getter` to create - // and correctly initialize it - tempDir = path.Join(tempDir, filepath.Base(uri)) - - client := &getter.Client{ - Ctx: context.Background(), - // Define the destination where the files will be stored. This will create the directory if it doesn't exist - Dst: tempDir, - // Source - Src: uri, - Mode: getter.ClientModeAny, - } - - if err = client.Get(); err != nil { - return err - } - } + // Log the action (handled in downloadAndInstall) + } + } - // Copy from the temp directory to the destination folder and skip the excluded files - copyOptions := cp.Options{ - // Skip specifies which files should be skipped - Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.HasSuffix(src, ".git") { - return true, nil - } - - trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) - - // Exclude the files that match the 'excluded_paths' patterns - // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) - // https://en.wikipedia.org/wiki/Glob_(programming) - // https://github.com/bmatcuk/doublestar#patterns - for _, excludePath := range s.ExcludedPaths { - excludeMatch, err := u.PathMatch(excludePath, src) - if err != nil { - return true, err - } else if excludeMatch { - // If the file matches ANY of the 'excluded_paths' patterns, exclude the file - u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", - trimmedSrc, - excludePath, - )) - return true, nil - } - } - - // Only include the files that match the 'included_paths' patterns (if any pattern is specified) - if len(s.IncludedPaths) > 0 { - anyMatches := false - for _, includePath := range s.IncludedPaths { - includeMatch, err := u.PathMatch(includePath, src) - if err != nil { - return true, err - } else if includeMatch { - // If the file matches ANY of the 'included_paths' patterns, include the file - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", - trimmedSrc, - includePath, - )) - anyMatches = true - break - } - } - - if anyMatches { - return false, nil - } else { - u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) - return true, nil - } - } - - // If 'included_paths' is not provided, include all files that were not excluded - u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) - return false, nil - }, - - // Preserve the atime and the mtime of the entries - // On linux we can preserve only up to 1 millisecond accuracy - PreserveTimes: false, - - // Preserve the uid and the gid of all entries - PreserveOwner: false, - - // OnSymlink specifies what to do on symlink - // Override the destination file if it already exists - OnSymlink: func(src string) cp.SymlinkAction { - return cp.Deep - }, - } + // Run TUI to process packages + if len(packages) > 0 { - if sourceIsLocalFile { - if filepath.Ext(targetPath) == "" { - targetPath = path.Join(targetPath, filepath.Base(uri)) - } - } - - if err = cp.Copy(tempDir, targetPath, copyOptions); err != nil { - return err - } + var opts []tea.ProgramOption + if !CheckTTYSupport() { + // set tea.WithInput(nil) workaround tea program not run on not TTY mod issue on non TTY mode https://github.com/charmbracelet/bubbletea/issues/761 + opts = []tea.ProgramOption{tea.WithoutRenderer(), tea.WithInput(nil)} + u.LogWarning(cliConfig, "TTY is not supported. Running in non-interactive mode") } + model, err := newModelAtmosVendorInternal(packages, dryRun, cliConfig) + if err != nil { + return fmt.Errorf("error initializing model: %v", err) + } + if _, err := tea.NewProgram(&model, opts...).Run(); err != nil { + return fmt.Errorf("running atmos vendor internal download error: %w", err) + } + } + return nil } @@ -606,3 +465,149 @@ func processVendorImports( return append(mergedSources, sources...), allImports, nil } +func logInitialMessage(cliConfig schema.CliConfiguration, vendorConfigFileName string, tags []string) { + logMessage := fmt.Sprintf("Vendoring from '%s'", vendorConfigFileName) + if len(tags) > 0 { + logMessage = fmt.Sprintf("%s for tags {%s}", logMessage, strings.Join(tags, ", ")) + } + u.LogInfo(cliConfig, logMessage) + +} +func validateSourceFields(s *schema.AtmosVendorSource, vendorConfigFileName string) error { + // Ensure necessary fields are present + if s.File == "" { + s.File = vendorConfigFileName + } + if s.Source == "" { + return fmt.Errorf("'source' must be specified in 'sources' in the vendor config file '%s'", s.File) + } + if len(s.Targets) == 0 { + return fmt.Errorf("'targets' must be specified for the source '%s' in the vendor config file '%s'", s.Source, s.File) + } + return nil +} +func shouldSkipSource(s *schema.AtmosVendorSource, component string, tags []string) bool { + // Skip if component or tags do not match + // If `--component` is specified, and it's not equal to this component, skip this component + // If `--tags` list is specified, and it does not contain any tags defined in this component, skip this component + return (component != "" && s.Component != component) || (len(tags) > 0 && len(lo.Intersect(tags, s.Tags)) == 0) +} + +func determineSourceType(uri, vendorConfigFilePath string) (bool, bool, bool) { + // Determine if the URI is an OCI scheme, a local file, or remote + useOciScheme := strings.HasPrefix(uri, "oci://") + if useOciScheme { + uri = strings.TrimPrefix(uri, "oci://") + } + + useLocalFileSystem := false + sourceIsLocalFile := false + if !useOciScheme { + if absPath, err := u.JoinAbsolutePathWithPath(vendorConfigFilePath, uri); err == nil { + uri = absPath + useLocalFileSystem = true + sourceIsLocalFile = u.FileExists(uri) + } + } + return useOciScheme, useLocalFileSystem, sourceIsLocalFile +} + +func copyToTarget(cliConfig schema.CliConfiguration, tempDir, targetPath string, s *schema.AtmosVendorSource, sourceIsLocalFile bool, uri string) error { + copyOptions := cp.Options{ + Skip: generateSkipFunction(cliConfig, tempDir, s), + PreserveTimes: false, + PreserveOwner: false, + OnSymlink: func(src string) cp.SymlinkAction { return cp.Deep }, + } + + // Adjust the target path if it's a local file with no extension + if sourceIsLocalFile && filepath.Ext(targetPath) == "" { + targetPath = path.Join(targetPath, filepath.Base(uri)) + } + + return cp.Copy(tempDir, targetPath, copyOptions) +} + +// generateSkipFunction generates a function that determines whether to skip a file or directory +// based on the 'excluded_paths' and 'included_paths' patterns in the vendor source +func generateSkipFunction(cliConfig schema.CliConfiguration, tempDir string, s *schema.AtmosVendorSource) func(os.FileInfo, string, string) (bool, error) { + return func(srcInfo os.FileInfo, src, dest string) (bool, error) { + if filepath.Base(src) == ".git" { + return true, nil + } + + trimmedSrc := u.TrimBasePathFromPath(tempDir+"/", src) + + // Exclude the files that match the 'excluded_paths' patterns + // It supports POSIX-style Globs for file names/paths (double-star `**` is supported) + // https://en.wikipedia.org/wiki/Glob_(programming) + // https://github.com/bmatcuk/doublestar#patterns + for _, excludePath := range s.ExcludedPaths { + excludeMatch, err := u.PathMatch(excludePath, src) + if err != nil { + return true, err + } else if excludeMatch { + // If the file matches ANY of the 'excluded_paths' patterns, exclude the file + u.LogTrace(cliConfig, fmt.Sprintf("Excluding the file '%s' since it matches the '%s' pattern from 'excluded_paths'\n", + trimmedSrc, + excludePath, + )) + return true, nil + } + } + + // Only include the files that match the 'included_paths' patterns (if any pattern is specified) + if len(s.IncludedPaths) > 0 { + anyMatches := false + for _, includePath := range s.IncludedPaths { + includeMatch, err := u.PathMatch(includePath, src) + if err != nil { + return true, err + } else if includeMatch { + // If the file matches ANY of the 'included_paths' patterns, include the file + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s' since it matches the '%s' pattern from 'included_paths'\n", + trimmedSrc, + includePath, + )) + anyMatches = true + break + } + } + + if anyMatches { + return false, nil + } else { + u.LogTrace(cliConfig, fmt.Sprintf("Excluding '%s' since it does not match any pattern from 'included_paths'\n", trimmedSrc)) + return true, nil + } + } + + // If 'included_paths' is not provided, include all files that were not excluded + u.LogTrace(cliConfig, fmt.Sprintf("Including '%s'\n", u.TrimBasePathFromPath(tempDir+"/", src))) + return false, nil + } +} +func validateURI(uri string) error { + if uri == "" { + return fmt.Errorf("URI cannot be empty") + } + // Add more validation as needed + // Validate URI format + if strings.Contains(uri, "..") { + return fmt.Errorf("URI cannot contain path traversal sequences") + } + if strings.Contains(uri, " ") { + return fmt.Errorf("URI cannot contain spaces") + } + // Validate characters + if strings.ContainsAny(uri, "<>|&;$") { + return fmt.Errorf("URI contains invalid characters") + } + // Validate scheme + if strings.HasPrefix(uri, "oci://") { + if !strings.Contains(uri[6:], "/") { + return fmt.Errorf("invalid OCI URI format") + } + } + return nil +} diff --git a/website/docs/cheatsheets/commands.mdx b/website/docs/cheatsheets/commands.mdx index 0128c5567..eeef99923 100644 --- a/website/docs/cheatsheets/commands.mdx +++ b/website/docs/cheatsheets/commands.mdx @@ -190,7 +190,7 @@ import CardGroup from '@site/src/components/CardGroup' ``` - atmos vendor pull + atmos vendor pull ```

Pull sources and mixins from remote repositories for Terraform and Helmfile components and other artifacts

 diff --git a/website/docs/cheatsheets/vendoring.mdx b/website/docs/cheatsheets/vendoring.mdx index fc8d845b3..6c2f56df2 100644 --- a/website/docs/cheatsheets/vendoring.mdx +++ b/website/docs/cheatsheets/vendoring.mdx @@ -67,7 +67,8 @@ import CardGroup from '@site/src/components/CardGroup' ```shell - atmos vendor pull + atmos vendor pull + atmos vendor pull --everything atmos vendor pull --component vpc-mixin-1 atmos vendor pull -c vpc-mixin-2 atmos vendor pull -c vpc-mixin-3 diff --git a/website/docs/cli/commands/vendor/vendor-pull.mdx b/website/docs/cli/commands/vendor/vendor-pull.mdx index 944b1579b..6b1e2fe08 100644 --- a/website/docs/cli/commands/vendor/vendor-pull.mdx +++ b/website/docs/cli/commands/vendor/vendor-pull.mdx @@ -29,7 +29,8 @@ With Atmos vendoring, you can copy components and other artifacts from the follo Execute the `vendor pull` command like this: ```shell -atmos vendor pull +atmos vendor pull +atmos vendor pull --everything atmos vendor pull --component [options] atmos vendor pull -c [options] atmos vendor pull --tags , [options] @@ -114,7 +115,8 @@ Run `atmos vendor pull --help` to see all the available options ## Examples ```shell -atmos vendor pull +atmos vendor pull +atmos vendor pull --everything atmos vendor pull --component vpc atmos vendor pull -c vpc-flow-logs-bucket atmos vendor pull -c echo-server --type helmfile diff --git a/website/docs/core-concepts/vendor/vendor-manifest.mdx b/website/docs/core-concepts/vendor/vendor-manifest.mdx index 55f6e319c..c41e2fec0 100644 --- a/website/docs/core-concepts/vendor/vendor-manifest.mdx +++ b/website/docs/core-concepts/vendor/vendor-manifest.mdx @@ -154,7 +154,8 @@ To vendor remote artifacts, create a `vendor.yaml` file similar to the example b With this configuration, it would be possible to run the following commands: ```shell -# atmos vendor pull +# atmos vendor pull +# atmos vendor pull --everything # atmos vendor pull --component vpc-mixin-1 # atmos vendor pull -c vpc-mixin-2 # atmos vendor pull -c vpc-mixin-3