Initial support for custom Terraform plugins

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.14.0
+
+* Initial support for custom Terraform plugins
+
 ## 0.13.1
 
 * Use mapcrafter/mapcrafter:113

VERSION

@@ -1 +1 @@
-0.13.1
+0.14.0

_examples/terraform/custom-plugins/.gitignore

@@ -0,0 +1,2 @@
+terraform.d
+.terraform

_examples/terraform/custom-plugins/.lucli.yml

@@ -0,0 +1,4 @@
+terraform:
+  plugins:
+   urls:
+   - https://github.com/Mastercard/terraform-provider-restapi/releases/download/v1.5.1/terraform-provider-restapi_v1.5.1-linux-amd64

_examples/terraform/custom-plugins/README.md

@@ -0,0 +1,24 @@
+# Custom Terraform Plugins
+
+Here's an example of using a custom Terraform plugin
+
+Plugin URLs are specified as an array in your local lucli config file, i.e.
+
+```
+terraform:
+  plugins:
+   urls:
+   - https://example.com/foo/bar
+```
+
+Plugins are downloaded to the `terraform.d/plugins/linux_amd64` directory as
+part of the init function, when running `lucli terraform init`, prior to
+starting the Docker container.
+
+As it happens, the Terraform code in this example directory doesn't work.
+
+It uses the https://github.com/Mastercard/terraform-provider-restapi, which
+makes assumptions about the format of the API you're calling, that do not hold
+true for https://whoami.lmhd.me/name
+
+But you can at least see that the custom plugin works fine.

_examples/terraform/custom-plugins/example.tf

@@ -0,0 +1,18 @@
+provider "restapi" {
+  uri                  = "https://whoami.lmhd.me"
+  debug                = true
+  write_returns_object = true
+}
+
+# This will make information about the user named "John Doe" available by finding him by first name
+data "restapi_object" "name" {
+  path         = "/name"
+  search_key   = ""
+  search_value = ""
+  results_key  = "full_name"
+  id_attribute = "preferred"
+}
+
+output "name" {
+  value = "${data.restapi_object.name.api_data.preferred}"
+}

cmd/terraform.go

@@ -1,9 +1,16 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"path"
+	"time"
 
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/docker/api/types"
 	"github.com/lmhd/lucli/creds"
+	"github.com/lmhd/lucli/lib"
 	"github.com/skybet/cali"
 )
 
@@ -36,12 +43,120 @@ Examples:
 
 	// Init function, set profile, and image version
 	task.SetInitFunc(func(t *cali.Task, args []string) {
-		t.SetImage(fmt.Sprintf("%s:%s", imageName, cli.FlagValues().GetString("terraform-version")))
-
-		// TODO: custom plugins
+		finalImageName := fmt.Sprintf("%s:%s", imageName, cli.FlagValues().GetString("terraform-version"))
+		t.SetImage(finalImageName)
 
 		task.AddEnv("AWS_PROFILE", cli.FlagValues().GetString("aws-profile"))
 		_ = creds.BindAWS(t, args)
+
+		// For terraform init only, download custom plugins, if any
+		if args[0] == "init" {
+			if cli.FlagValues().IsSet("terraform.plugins.urls") {
+				pluginURLs := cli.FlagValues().GetStringSlice("terraform.plugins.urls")
+				log.Infof("Using %v custom plugins: %v", len(pluginURLs), pluginURLs)
+
+				err := downloadTerraformPlugins(pluginURLs)
+				if err != nil {
+					log.Fatalf("Error downloading plugin: %s", err)
+				}
+
+				// Apply workaround to Terraform image
+				err = fixTerraformImageForCustomPlugins(task, finalImageName)
+				if err != nil {
+					log.Fatalf("Error hacking the Terraform image: %s", err)
+				}
+			}
+		}
 	})
 
 }
+
+// downloadTerraformPlugins downloads plugins from a slice of URLs
+func downloadTerraformPlugins(pluginURLs []string) error {
+	pluginDir := "terraform.d/plugins/linux_amd64"
+
+	// TODO: In future, could add some custom handling for GitHub releases, and download the latest/specified version
+	// this will also mean we can get the size
+
+	for _, pluginURL := range pluginURLs {
+		// TODO: get replace from flag?
+
+		// TODO: get size. Can't do that for github URLs, because
+		// they're in S3, and so you can't HEAD them to find out
+		// how big they are. But can do this for other URLs, so at
+		// least attempt it
+
+		err := lib.DownloadFile(pluginURL, pluginDir, 0, false)
+		if err != nil {
+			return fmt.Errorf("Could not download from URL: %s", err)
+		}
+
+		// chmod +x the file (well, -rwxr-xr-x, but close enough)
+		filename := pluginDir + "/" + path.Base(pluginURL)
+		os.Chmod(filename, 0755)
+		if err != nil {
+			return fmt.Errorf("Could not make plugin executable: %s", err)
+		}
+
+	}
+
+	return nil
+}
+
+// fixTerraformImageForCustomPlugins applies a workaround for
+// https://github.com/hashicorp/docker-hub-images/pull/63
+// to a specified docker image
+//
+// If possible, as a workaround, run `apk add libc6-compat` in terraform container first
+// e.g. with something like...
+// https://github.com/edupo/cali/blob/3eddce060ececa2790fe6908b9f7441aac40fc3f/docker/container.go#L109
+// run image, then overwrite local version
+func fixTerraformImageForCustomPlugins(c *cali.Task, imageName string) error {
+	log.Debugf("begin fixTerraformImageForCustomPlugins")
+
+	// Always pull the image
+	if err := c.PullImage(c.Conf.Image); err != nil {
+		return fmt.Errorf("Failed to fetch image: %s", err)
+	}
+
+	// Create a container, with modified config
+	c.Conf.Entrypoint = []string{"/sbin/apk"}
+	c.Conf.Cmd = []string{"add", "--update", "libc6-compat"}
+	resp, err := c.Cli.ContainerCreate(context.Background(), c.Conf, c.HostConf,
+		c.NetConf, "")
+	if err != nil {
+		return fmt.Errorf("Failed to create container: %s", err)
+	}
+
+	// Start the container
+	if err := c.Cli.ContainerStart(context.Background(), resp.ID,
+		types.ContainerStartOptions{}); err != nil {
+		return err
+	}
+
+	// TODO: Wait until container has finished running
+	time.Sleep(10000 * time.Millisecond)
+
+	// set container config, for saving the image
+	c.Conf.Entrypoint = []string{"/bin/terraform"}
+	c.Conf.Cmd = []string{}
+
+	// Commit (save back to image name)
+	if _, err := c.Cli.ContainerCommit(context.Background(), resp.ID,
+		types.ContainerCommitOptions{
+			Reference: imageName,
+			// TODO: config
+			Config: c.Conf,
+		},
+	); err != nil {
+		return err
+	}
+
+	// Delete the container; we no longer need it
+	if err = c.DeleteContainer(resp.ID); err != nil {
+		return fmt.Errorf("Failed to remove container: %s", err)
+	}
+
+	log.Debugf("end fixTerraformImageForCustomPlugins")
+	return nil
+}

lib/http.go

@@ -2,8 +2,15 @@ package lib
 
 import (
 	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"path"
 	"time"
+
+	log "github.com/Sirupsen/logrus"
+	pb "gopkg.in/cheggaaa/pb.v2"
 )
 
 // GetJson does a HTTP request to a specified URL, then applies data to your interface
@@ -16,3 +23,52 @@ func GetJson(url string, target interface{}) error {
 	defer r.Body.Close()
 	return json.NewDecoder(r.Body).Decode(target)
 }
+
+// DownloadFile downloads a file from a URL to a specified directory
+func DownloadFile(url, dir string, size int, replace bool) error {
+
+	filename := dir + "/" + path.Base(url)
+
+	// If we don't want to replace an existing file...
+	if !replace {
+		if _, err := os.Stat(filename); !os.IsNotExist(err) {
+			log.Infof("%v already exists", filename)
+			return nil
+		}
+	}
+
+	log.Debugf("Downloading %v to %v", url, filename)
+
+	// Create dir if it doesn't exist
+	os.MkdirAll(dir, os.ModePerm)
+
+	f, err := os.Create(filename)
+	defer f.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("Could not download file: %s", err)
+	}
+	defer resp.Body.Close()
+
+	// start new bar
+	bar := pb.New(size)
+	bar.Start()
+
+	// create proxy reader
+	body, err := ioutil.ReadAll(bar.NewProxyReader(resp.Body))
+	if err != nil {
+		return fmt.Errorf("Could not download file: %s", err)
+	}
+
+	// Write to file
+	_, err = f.Write(body)
+	if err != nil {
+		return fmt.Errorf("Could not download file: %s", err)
+	}
+
+	bar.Finish()
+	log.Debugf("Downloaded %v")
+
+	return nil
+}