Improvement on upgrade mechanism

.github/workflows/build-publish-binaries.yaml

@@ -10,6 +10,7 @@ on:
 
 env:
   DF_BIN_VER: ${{ inputs.ver }}
+  VERSION: ${{ inputs.ver }}
 
 jobs:
   docker:
@@ -35,12 +36,14 @@ jobs:
           mkdir -p /tmp/binaries/$DF_BIN_VER
           cd /tmp/binaries/$DF_BIN_VER
           id=$(docker create deepfenceio/deepfence_agent_ce:latest)
-          docker cp $id:/usr/local/bin/syft - > syft
-          docker cp $id:/home/deepfence/bin/yara-hunter/YaraHunter - > YaraHunter
-          docker cp $id:/home/deepfence/bin/secret-scanner/SecretScanner - > SecretScanner
-          docker cp $id:/usr/local/discovery/deepfence-discovery - > deepfence-discovery
-          docker cp $id:/opt/td-agent-bit/bin/fluent-bit - > fluent-bit
-          docker cp $id:/usr/local/bin/compliance_check/compliance - > compliance
+          docker cp $id:/bin/deepfenced self
+          docker cp $id:/home/deepfence/bin/package-scanner package_scanner
+          docker cp $id:/home/deepfence/bin/yara-hunter/YaraHunter malware_scanner
+          docker cp $id:/home/deepfence/bin/secret-scanner/SecretScanner secret_scanner
+          docker cp $id:/usr/local/discovery/deepfence-discovery discovery
+          docker cp $id:/opt/td-agent-bit/bin/fluent-bit fluentbit
+          docker cp $id:/usr/local/bin/syft syft
+          docker cp $id:/usr/local/bin/compliance_check/compliance compliance
           tar zcvf binaries.tar.gz ./*
           docker rm -v $id
 

deepfence_bootstrapper/Makefile

@@ -1,4 +1,4 @@
-VERSION=`git describe --tags`
+VERSION?=`git describe --tags`
 
 all: deepfence_bootstrapper
 

deepfence_bootstrapper/controls/controls.go

@@ -35,6 +35,7 @@ func SetClusterAgentControls(k8sClusterName string) {
 		func(req ctl.StartAgentUpgradeRequest) error {
 			log.Info().Msg("Start Cluster Agent Upgrade")
 			router.SetUpgrade()
+			defer router.UnsetUpgrade()
 			return StartClusterAgentUpgrade(req)
 		})
 	if err != nil {
@@ -110,6 +111,7 @@ func SetAgentControls() {
 		func(req ctl.StartAgentUpgradeRequest) error {
 			log.Info().Msg("Start Agent Upgrade")
 			router.SetUpgrade()
+			defer router.UnsetUpgrade()
 			return router.StartAgentUpgrade(req)
 		})
 	if err != nil {
@@ -119,6 +121,7 @@ func SetAgentControls() {
 		func(req ctl.EnableAgentPluginRequest) error {
 			log.Info().Msg("Start & download Agent Plugin")
 			router.SetUpgrade()
+			defer router.UnsetUpgrade()
 			err = supervisor.UpgradeProcessFromURL(req.PluginName, req.BinUrl)
 			if err != nil {
 				return err

deepfence_bootstrapper/router/openapi_client_controls.go

@@ -134,6 +134,10 @@ func SetUpgrade() {
 	upgrade.Store(true)
 }
 
+func UnsetUpgrade() {
+	upgrade.Store(false)
+}
+
 func getUpgradeWorkload() int32 {
 	if upgrade.Load() {
 		return MAX_AGENT_WORKLOAD

deepfence_bootstrapper/router/upgrade.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"syscall"
 
 	"github.com/deepfence/ThreatMapper/deepfence_bootstrapper/supervisor"
 	ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls"
@@ -57,16 +58,34 @@ func StartAgentUpgrade(req ctl.StartAgentUpgradeRequest) error {
 		return err
 	}
 
+	restart := false
 	for _, plugin := range plugins {
 		err = supervisor.UpgradeProcessFromFile(plugin.name, plugin.path)
 		if err != nil {
-			log.Error().Msg(err.Error())
+			log.Error().Msgf("plugin: %v, path: %v, err: %v", plugin.name, plugin.path, err)
+		} else if plugin.name == supervisor.Self_id {
+			restart = true
 		}
 	}
 
+	if restart {
+		log.Info().Msgf("Restart self")
+		restartSelf()
+	} else {
+		log.Info().Msgf("Not self restart")
+	}
+
 	return nil
 }
 
+func restartSelf() error {
+	argv0, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		return err
+	}
+	return syscall.Exec(argv0, os.Args, os.Environ())
+}
+
 func downloadFile(filepath string, url string) (err error) {
 
 	// Create the file

deepfence_bootstrapper/supervisor/process.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	self_id                  = "self"
+	Self_id                  = "self"
 	log_root_env             = "${DF_INSTALL_DIR}/var/log/deepfenced/"
 	EXIT_CODE_BASH_NOT_FOUND = 127
 )
@@ -288,7 +288,7 @@ func WriteTo(dst, org string) error {
 }
 
 func UpgradeProcessFromFile(name, path string) error {
-	if name == self_id {
+	if name == Self_id {
 		return selfUpgradeFromFile(path)
 	}
 
@@ -325,7 +325,7 @@ func UpgradeProcessFromFile(name, path string) error {
 }
 
 func UpgradeProcessFromURL(name, url string) error {
-	if name == self_id {
+	if name == Self_id {
 		return selfUpgradeFromUrl(url)
 	}
 

deepfence_ctl/cmd/graph.go

@@ -89,7 +89,7 @@ var graphTopologySubCmd = &cobra.Command{
 
 		root, _ := cmd.Flags().GetString("root")
 
-		var res *deepfence_server_client.ApiDocsGraphResult
+		var res *deepfence_server_client.ModelGraphResult
 		var rh *stdhttp.Response
 		switch root {
 		case "":

deepfence_ctl/go.sum

@@ -2,6 +2,7 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
@@ -14,12 +15,11 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
-github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
+github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
 github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
@@ -29,8 +29,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=

deepfence_worker/cronjobs/agent.go

@@ -131,7 +131,7 @@ func ingestAgentVersion(ctx context.Context, tags_to_url map[string]string) erro
 	if _, err = tx.Run(`
 		UNWIND $batch as row
 		MERGE (n:AgentVersion{node_id: row.tag})
-		ON CREATE SET n.url = row.url`,
+		SET n.url = row.url`,
 		map[string]interface{}{"batch": tags_to_ingest}); err != nil {
 		return err
 	}