Report whether k6 runs on CI (#4063)

* Report whether k6 runs on CI Co-authored-by: Inanc Gumus <[email protected]> * Rename and relocate the 'is_ci' usage attribute --------- Co-authored-by: Inanc Gumus <[email protected]>
grafana · Nov 18, 2024 · 4a2e38f · 4a2e38f
1 parent ddc3b0b
commit 4a2e38f
Show file tree

Hide file tree

Showing 2 changed files with 123 additions and 19 deletions.
diff --git a/cmd/report.go b/cmd/report.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"runtime"
+	"strconv"
 
 	"go.k6.io/k6/execution"
 	"go.k6.io/k6/lib/consts"
@@ -27,6 +28,7 @@ func createReport(u *usage.Usage, execScheduler *execution.Scheduler) map[string
 		executors[ec.GetType()]++
 	}
 	m["executors"] = executors
+	m["is_ci"] = isCI(execState.Test.LookupEnv)
 
 	return m
 }
@@ -52,3 +54,47 @@ func reportUsage(ctx context.Context, execScheduler *execution.Scheduler, test *
 
 	return err
 }
+
+// isCI is a helper that follows a naive approach to determine if k6 is being
+// executed within a CI system. This naive approach consists of checking if
+// the "CI" environment variable (among others) is set.
+//
+// We treat the "CI" environment variable carefully, because it's the one
+// used more often, and because we know sometimes it's explicitly set to
+// "false" to signal that k6 is not running in a CI environment.
+//
+// It is not a foolproof method, but it should work for most cases.
+func isCI(lookupEnv func(key string) (val string, ok bool)) bool {
+	if ci, ok := lookupEnv("CI"); ok {
+		ciBool, err := strconv.ParseBool(ci)
+		if err == nil {
+			return ciBool
+		}
+		// If we can't parse the "CI" value as a bool, we assume return true
+		// because we know that at least it's not set to any variant of "false",
+		// which is the most common use case, and the reasoning we apply below.
+		return true
+	}
+
+	// List of common environment variables used by different CI systems.
+	ciEnvVars := []string{
+		"BUILD_ID",               // Jenkins, Cloudbees
+		"BUILD_NUMBER",           // Jenkins, TeamCity
+		"CI",                     // Travis CI, CircleCI, Cirrus CI, Gitlab CI, Appveyor, CodeShip, dsari
+		"CI_APP_ID",              // Appflow
+		"CI_BUILD_ID",            // Appflow
+		"CI_BUILD_NUMBER",        // Appflow
+		"CI_NAME",                // Codeship and others
+		"CONTINUOUS_INTEGRATION", // Travis CI, Cirrus CI
+		"RUN_ID",                 // TaskCluster, dsari
+	}
+
+	// Check if any of these variables are set
+	for _, key := range ciEnvVars {
+		if _, ok := lookupEnv(key); ok {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/cmd/report_test.go b/cmd/report_test.go
@@ -18,32 +18,90 @@ import (
 
 func TestCreateReport(t *testing.T) {
 	t.Parallel()
+
 	logger := testutils.NewLogger(t)
 	opts, err := executor.DeriveScenariosFromShortcuts(lib.Options{
 		VUs:        null.IntFrom(10),
 		Iterations: null.IntFrom(170),
 	}, logger)
 	require.NoError(t, err)
 
-	s, err := execution.NewScheduler(&lib.TestRunState{
-		TestPreInitState: &lib.TestPreInitState{
-			Logger: logger,
-		},
-		Options: opts,
-	}, local.NewController())
-	require.NoError(t, err)
-	s.GetState().ModInitializedVUsCount(6)
-	s.GetState().AddFullIterations(uint64(opts.Iterations.Int64))
-	s.GetState().MarkStarted()
-	time.Sleep(10 * time.Millisecond)
-	s.GetState().MarkEnded()
+	initSchedulerWithEnv := func(lookupEnv func(string) (string, bool)) (*execution.Scheduler, error) {
+		return execution.NewScheduler(&lib.TestRunState{
+			TestPreInitState: &lib.TestPreInitState{
+				Logger:    logger,
+				LookupEnv: lookupEnv,
+			},
+			Options: opts,
+		}, local.NewController())
+	}
 
-	m := createReport(usage.New(), s)
-	require.NoError(t, err)
+	t.Run("default (no env)", func(t *testing.T) {
+		t.Parallel()
+
+		s, err := initSchedulerWithEnv(func(_ string) (val string, ok bool) {
+			return "", false
+		})
+		require.NoError(t, err)
+
+		s.GetState().ModInitializedVUsCount(6)
+		s.GetState().AddFullIterations(uint64(opts.Iterations.Int64))
+		s.GetState().MarkStarted()
+		time.Sleep(10 * time.Millisecond)
+		s.GetState().MarkEnded()
+
+		m := createReport(usage.New(), s)
+		require.NoError(t, err)
+
+		assert.Equal(t, consts.Version, m["k6_version"])
+		assert.EqualValues(t, map[string]int{"shared-iterations": 1}, m["executors"])
+		assert.EqualValues(t, 6, m["vus_max"])
+		assert.EqualValues(t, 170, m["iterations"])
+		assert.NotEqual(t, "0s", m["duration"])
+		assert.EqualValues(t, false, m["is_ci"])
+	})
+
+	t.Run("CI=false", func(t *testing.T) {
+		t.Parallel()
+
+		s, err := initSchedulerWithEnv(func(envVar string) (val string, ok bool) {
+			if envVar == "CI" {
+				return "false", true
+			}
+			return "", false
+		})
+		require.NoError(t, err)
+
+		m := createReport(usage.New(), s)
+		require.NoError(t, err)
+
+		assert.Equal(t, consts.Version, m["k6_version"])
+		assert.EqualValues(t, map[string]int{"shared-iterations": 1}, m["executors"])
+		assert.EqualValues(t, 0, m["vus_max"])
+		assert.EqualValues(t, 0, m["iterations"])
+		assert.Equal(t, "0s", m["duration"])
+		assert.EqualValues(t, false, m["is_ci"])
+	})
+
+	t.Run("CI=true", func(t *testing.T) {
+		t.Parallel()
+
+		s, err := initSchedulerWithEnv(func(envVar string) (val string, ok bool) {
+			if envVar == "CI" {
+				return "true", true
+			}
+			return "", false
+		})
+		require.NoError(t, err)
+
+		m := createReport(usage.New(), s)
+		require.NoError(t, err)
 
-	assert.Equal(t, consts.Version, m["k6_version"])
-	assert.EqualValues(t, map[string]int{"shared-iterations": 1}, m["executors"])
-	assert.EqualValues(t, 6, m["vus_max"])
-	assert.EqualValues(t, 170, m["iterations"])
-	assert.NotEqual(t, "0s", m["duration"])
+		assert.Equal(t, consts.Version, m["k6_version"])
+		assert.EqualValues(t, map[string]int{"shared-iterations": 1}, m["executors"])
+		assert.EqualValues(t, 0, m["vus_max"])
+		assert.EqualValues(t, 0, m["iterations"])
+		assert.Equal(t, "0s", m["duration"])
+		assert.EqualValues(t, true, m["is_ci"])
+	})
 }