Builder work end to end - main

.expeditor/config.yml

@@ -19,6 +19,8 @@ pipelines:
   - verify:
       description: Pull Request validation tests
       public: true
+  - end_to_end:
+      description: Builder end to end test
   - release_builder:
       description: Builder release process
   - post_habitat_release:

.expeditor/end_to_end.pipeline.yml

@@ -0,0 +1,22 @@
+expeditor:
+  defaults:
+    buildkite:
+      timeout_in_minutes: 30
+      env:
+        HAB_ORIGIN: "habitat"
+        HAB_BLDR_CHANNEL: "unstable"
+
+steps:
+#######################################################################
+# End to end test
+#######################################################################
+  - label: ":linux: Builder End to End"
+    command:
+      - .expeditor/scripts/end_to_end/builder_worker_test.sh
+    expeditor:
+      account:
+        - github
+      executor:
+        docker:
+          environment:
+            - GITHUB_USER

.expeditor/scripts/end_to_end/builder_worker_test.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+#set -euo pipefail
+
+source .expeditor/scripts/post_habitat_release/shared.sh
+
+export BLDR_CHANNEL=${HAB_BLDR_CHANNEL:=unstable}
+
+export HAB_BLDR_URL=https://bldr.habitat.sh
+export BLDR_ORIGIN=${HAB_BLDR_ORIGIN:=habitat}
+export HAB_LICENSE=accept
+
+curlbash_hab "${BUILD_PKG_TARGET:=x86_64-linux}"
+
+echo "--- Generating signing key"
+hab origin key generate ${BLDR_ORIGIN}
+
+#  Clean out settings
+sudo rm -rf /hab/user
+sudo rm -rf /hab/svc
+
+# Run the test
+.expeditor/scripts/end_to_end/run_e2e.sh
+
+echo "--- Terminating Habitat Supervisor"
+# Terminate supervisor and remove specs files for builder
+sudo hab sup term
+sleep 2
+
+echo "Remove builder spec files"
+sudo rm -rf /hab/sup/default/specs/builder-*

.expeditor/scripts/end_to_end/run_e2e.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -eou pipefail
+umask 0022 
+
+#  Always run from builder root directory.
+export ROOT_DIR
+ROOT_DIR=$(pwd)
+WORK_DIR=${ROOT_DIR}/test/end-to-end/worker-test
+
+source "${WORK_DIR}"/bldr-end-to-end.env
+source "${WORK_DIR}"/shared.sh
+
+export HAB_FUNC_TEST=1
+
+start_init
+
+start_builder
+
+start_jobsrv
+
+apply_db_password
+sleep 5
+
+sudo cp "${WORK_DIR}"/builder-github-app.pem /hab/svc/builder-api/files
+
+echo "start_worker"
+start_worker
+
+cat <<EOT > /tmp/builder_worker.toml
+log_level='trace'
+github.app_id = 8053
+github.webhook_secret=''
+EOT
+
+hab config apply builder-worker.default "$(date +%s)" /tmp/builder_worker.toml
+sleep 3
+echo "worker config updated"
+sudo cp "${WORK_DIR}"/builder-github-app.pem /hab/svc/builder-worker/files
+
+#  HACK  To get builder-worker HAB_FUNC_TEST changes 
+#hab svc stop habitat/builder-worker
+#sleep 5
+#cp ${WORK_DIR}/bldr-worker /hab/pkgs/habitat/builder-worker/10041/20220510143824/bin
+#
+#sleep 2
+#hab svc start habitat/builder-worker
+#  HACK  To get builder-worker HAB_FUNC_TEST changes 
+
+while hab svc status | grep --quiet down;
+do 
+  sleep 5
+done
+
+echo "Services have started - continuing with tests"
+
+#  Need to test if we need this file in this location and remove if not.
+sudo cp "${WORK_DIR}"/fixtures/neurosis-20171211220037.pub /hab/svc/builder-worker/files
+sudo cp "${WORK_DIR}"/fixtures/neurosis-20171211220037.sig.key /hab/svc/builder-worker/files
+
+sudo cp /hab/svc/builder-api/files/bldr-*.pub /hab/svc/builder-worker/files
+sudo cp /hab/svc/builder-api/files/bldr-*.box.key /hab/svc/builder-worker/files
+
+wait_for_migrations
+
+clean_test_artifacts
+
+#  Install packages
+hab pkg install core/openssl
+hab pkg install core/node --binlink
+hab pkg install core/coreutils
+
+cd test/end-to-end/worker-test
+npm install mocha
+
+if ! command -v npm >/dev/null 2>&1; then
+  hab pkg install core/node -b
+fi
+
+if ! [ -f /usr/bin/env ]; then
+  hab pkg binlink core/coreutils -d /usr/bin env
+fi
+
+if ! [ -d node_modules/mocha ]; then
+  npm install mocha
+fi
+
+if ! [ -d node_modules/chai ]; then
+  npm install chai
+fi
+
+if ! [ -d node_modules/supertest ]; then
+  npm install supertest
+fi
+
+if ! [ -d node_modules/superagent-binary-parser ]; then
+  npm install superagent-binary-parser
+fi
+
+if npm run mocha; then
+  echo "Setup tests passed"
+else
+  mocha_exit_code=$?
+  echo "Setup tests failed"
+fi
+
+echo "Exiting run script"
+exit ${mocha_exit_code:-0}

test/end-to-end/worker-test/bldr-end-to-end.env

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# Only enable and set these if you are using externally hosted PostgreSQL(RDS, Azure Database for PostgreSql etc).
+# set PG_EXT_ENABLED = true, uncomment PG_USER and update PG_PASSWORD and POSTGRES_HOST appropriatlely.
+# Migrations from local PostgreSQL to RDS are curently not supported.
+#export PG_EXT_ENABLED=false
+#export PG_USER=hab
+#export PG_PASSWORD=hab
+
+# The endpoint and port for your Postgresql(local, RDS, Azure Database for PostgreSql etc)
+# Change only if needed 
+export POSTGRES_HOST=localhost
+export POSTGRES_PORT=5432
+
+# The endpoint, key and secret for your Minio instance (see README)
+# Change these before the first install if needed
+export MINIO_ENDPOINT=http://localhost:9000
+export MINIO_BUCKET=habitat-builder-artifact-store.local
+export MINIO_ACCESS_KEY=depot
+export MINIO_SECRET_KEY=password
+
+# If you'd like to use AWS S3 instead of Minio,
+# set S3_ENABLED=true and change the other S3 associated variables appropriately.
+export S3_ENABLED=false
+export S3_REGION=us-west-2
+export S3_BUCKET=habitat-builder-artifact-store.local
+export S3_ACCESS_KEY=depotaccesskey
+export S3_SECRET_KEY=depotsecretkey
+
+export GITHUB_APP_ID=8053
+export GITHUB_API_URL="https://api.github.com"
+export GITHUB_APP_URL="https://github.com/apps/habitat-builder-dev-studio"
+
+# If you'd like to use Arifactory instead of Minio, uncomment
+# and set the following variables appropriately.
+# IMPORTANT: See the README for more info
+# export ARTIFACTORY_ENABLED=true
+# export ARTIFACTORY_API_URL=http://localhost:8081
+# export ARTIFACTORY_API_KEY=foo
+# export ARTIFACTORY_REPO=habitat-builder-artifact-store
+
+# Modify these as needed for the on-premise OAuth2 provider.
+# The variables below are configured for GitHub by default,
+# but appropriate values for Bitbucket, GitLab, Azure AD and Okta
+# are also included as comments.
+
+# Whether SSL is enabled for the on-prem depot
+export APP_SSL_ENABLED=false
+
+# The URL for this instance of the on-prem depot
+# IMPORTANT: If SSL is enabled, APP_URL should start be https
+export APP_URL=http://localhost
+
+# The OAUTH_PROVIDER value can be "github", "gitlab", "bitbucket", "azure-ad",
+# "okta" or "chef-automate"
+export OAUTH_PROVIDER=github
+# export OAUTH_PROVIDER=bitbucket
+# export OAUTH_PROVIDER=gitlab
+# export OAUTH_PROVIDER=azure-ad
+# export OAUTH_PROVIDER=okta
+# export OAUTH_PROVIDER=chef-automate
+
+# The OAUTH_USERINFO_URL is the API endpoint that will be used for user info
+export OAUTH_USERINFO_URL=https://api.github.com/user
+# export OAUTH_USERINFO_URL=https://api.bitbucket.org/1.0/user
+# export OAUTH_USERINFO_URL=https://gitlab.com/oauth/userinfo
+# export OAUTH_USERINFO_URL=https://login.microsoftonline.com/<tenant-id>/openid/userinfo
+# export OAUTH_USERINFO_URL=https://<your.okta.domain>.com/oauth2/v1/userinfo
+# export OAUTH_USERINFO_URL=https://<your.automate.domain>/session/userinfo
+
+# The OAUTH_AUTHORIZE_URL is the *fully qualified* OAuth2 authorization endpoint
+export OAUTH_AUTHORIZE_URL=https://github.com/login/oauth/authorize
+# export OAUTH_AUTHORIZE_URL=https://bitbucket.org/site/oauth2/authorize
+# export OAUTH_AUTHORIZE_URL=https://gitlab.com/oauth/authorize
+# export OAUTH_AUTHORIZE_URL=https://login.microsoftonline.com/<tenant-id>/oauth2/authorize
+# export OAUTH_AUTHORIZE_URL=https://<your.okta.domain>.com/oauth2/v1/authorize
+# export OAUTH_AUTHORIZE_URL=https://<your.automate.domain>/session/new
+
+# The OAUTH_SIGNUP_URL is the link used to register users with the OAUTH provider
+export OAUTH_SIGNUP_URL=https://github.com/join
+# export OAUTH_SIGNUP_URL=https://bitbucket.org/account/signup/
+# export OAUTH_SIGNUP_URL=https://gitlab.com/users/sign_in#register-pane
+
+# The OAUTH_TOKEN_URL is the *fully qualified* OAuth2 token endpoint
+export OAUTH_TOKEN_URL=https://github.com/login/oauth/access_token
+# export OAUTH_TOKEN_URL=https://bitbucket.org/site/oauth2/access_token
+# export OAUTH_TOKEN_URL=https://gitlab.com/oauth/token
+# export OAUTH_TOKEN_URL=https://login.microsoftonline.com/tenant-id/oauth2/token
+# export OAUTH_TOKEN_URL=https://your.okta.domain.com/oauth2/v1/token
+# export OAUTH_TOKEN_URL=https://<your.automate.domain>/session/token
+
+# The OAUTH_REDIRECT_URL is the registered OAuth2 redirect
+# IMPORTANT: If SSL is enabled, the redirect URL should be https
+# IF SCALING FRONTEND, POINT THIS AT YOUR LOAD BALANCER
+export OAUTH_REDIRECT_URL=http://localhost/
+
+# The OAUTH_CLIENT_ID is the registered OAuth2 client id
+export OAUTH_CLIENT_ID=Iv1.04b78696442708f6
+
+# The OAUTH_CLIENT_SECRET is the registerd OAuth2 client secret
+export OAUTH_CLIENT_SECRET=b4acf029896e9c7fb6613390124e8ae3c35889aa
+
+export BLDR_CHANNEL=${BLDR_CHANNEL:=unstable}
+
+export BLDR_ORIGIN=${BLDR_ORIGIN:=habitat}
+
+export HAB_BLDR_URL=${HAB_BLDR_URL:=https://bldr.habitat.sh/}
+
+# From the Automate CLI use
+# export HAB_BLDR_URL=https://MY_ON_PREM_URL/bldr/v1/
+
+# Modify if you are splitting frontend and backend on separate nodes
+# If so, this should include a --peer argument for each builder IP or hostname
+# including all frontend and backend nodes.
+# For example: export HAB_BLDR_PEER_ARG="--peer host1 --peer host2 --peer host3"
+export HAB_BLDR_PEER_ARG=""
+
+# Help us make Habitat better! Opt into analytics by changing the ANALYTICS_ENABLED
+# setting below to true, then optionally provide your company name. (Analytics is
+# disabled by default. See our privacy policy at https://www.habitat.sh/legal/privacy-policy/.)
+export ANALYTICS_ENABLED=false
+export ANALYTICS_COMPANY_NAME=""

test/end-to-end/worker-test/builder-github-app.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAsB/t16ONW0qFa5fzVTWRBFSjc2ySIMZPxhWZ9OboMC6sccyE
+ASvZZXC6r9Ogmubx7EliS6Ljwcu3L30UCAIV5KwAMt72uT3Y1EUG6/QpoWdLZD13
+NVwXgi5xKmLib16hIz5tqfk32yu44ZmXX1d9KlGkGrcxyMkqQ6XcoOBiG31fYc/b
+rJwouq0/jK2zIDaXcwXe0FtV39rXmmV5bGG2mP7pUEsCP9/trtWnE/VxDh0JbB06
+Q9Ue+H3CpIqSRJyBen2rR0/tVAkN0fvmvknUosF0W3uyOTOzG6QsebfPC0qANvo4
+FTfklPMGE3kXYKMIwF2eP8kSBQoMtwNKdlTOkQIDAQABAoIBAQCm9/4ROnoeG83U
+rJ8BHuE2C89Q85KtA5RJI91qsSVZ2EWfNNXIDnvwkShK3AQcFekvqnpHXksxRCGM
+IbXFycRbnzwo/ftErf9LbY0cT8LPww9UHnlO2tlmjwS1vD0Se9GjieGJDD0JyK9j
+IKrCtjxwzCkZMqc9xIgOzcUVtuMLTwNn/MjvtEVHZB7OiEVPfZacxMRboeR4T/y2
+9gs2uR7p0D7w4uioRwOWhaD84Yu08rYluKf4eHhkJ3Z0FgE+IABMVR+luP5Rc3XG
+gqxTVzsLEsCY1J/HUx8AIENC6kUVLpvhT4Gfj1A0+GHbqtFaz/nCudBT4OCzzHnp
+4WvuabvhAoGBANZLlKo8YIqVDYZH0Iin+98ZHzCKd7hnJOSZG7IyRVci7ODPlxkQ
+Y3ZNKgl9l8lGfddPStfV/WPW+X+3V8YHzP8KW57Z9I47oWRszOmfkZH6k5EBNNlN
+RtWS+j7CofLHXLC0WhxGnWdmwPm1lfL+0iDjVgyJad44lHvGcKA3nC19AoGBANJm
+pvqRugkdHzU1LWHhtS2CSq+wU3mhqmIYW/khScf3jZnGXvmYo9VHaQP8Ol2tNVrW
+NiHZgboB8mFrWLMvXV92dtzf1dRRlJXdXYVhnst2ApvdlJSF9VPKzq9cTHBromL0
+aJHzZyh45VK8D2hLwhpX82yja920U0s+9kWs1gGlAoGBAIOITK3Y9WDacDy1o7fd
+c+nSH6TJuQZEuQ5fM/LCjHEZjM/IW3QU7RfV8TF2F06BqAgaMVQdkrRPZUERJhz0
++MyqbOYp3cPXsg5UAcp8GPvDFARy+x9OIWZCGrue94xu9eO4L38fkMnA9j3PSUSn
+SA7LuytixMqTbAA1IzgUKhZ5AoGADV/lIWx+DPJC95jnx58Xa0srADwPMl4qfI3b
+2txY2lNMNwg7JVako3wkmtI/y5ClIZ8YVdKDwpU5GSQ/wTQzlUvshmHcL+D3twcZ
+WM85zukzhUDynvZc//SWauNNIKBW9D9hHlYAqrv0p+u4dwLJ1e7dVHyW8MLzwIbC
+FO/FstkCgYAdQ+f4ElBl871PqdQd2hbSpxt9sfUEoXTmmh/YHiS01iGc1mFis1gw
+kW64zKbmqbwcJQdF1xNKsDjKn/NFy/1lql4VEblLx4/3BNNfrGMlzEktAN1UNocx
+RUFoG5ak1BYVr9gMA+NEDG65t1gMqzTChxpFSdF8NndMPBgWsvAH6g==
+-----END RSA PRIVATE KEY-----

test/end-to-end/worker-test/fixtures/neurosis-20171211220037.pub

@@ -0,0 +1,4 @@
+SIG-PUB-1
+neurosis-20171211220037
+
+iAqYvmgFiMI6IijSWQyiLPd8hFVmqTfzRCpJdqmomFQ=

test/end-to-end/worker-test/fixtures/neurosis-20171211220037.sig.key

@@ -0,0 +1,4 @@
+SIG-SEC-1
+neurosis-20171211220037
+
+Wh4SVmUV1eAoxL/CWFQpA8YF1kawikEM0CD4pNVRQ16ICpi+aAWIwjoiKNJZDKIs93yEVWapN/NEKkl2qaiYVA==

test/end-to-end/worker-test/fixtures/testapp_original/plan.ps1

@@ -0,0 +1,23 @@
+$pkg_name="testapp"
+$pkg_origin="neurosis"
+$pkg_version="0.1.4"
+$pkg_description="This is actually just an empty package"
+$pkg_upstream_url="http://www.habitat.sh/"
+$pkg_license=@("Apache-2")
+$pkg_build_deps=@("core/dmake")
+$pkg_bin_dirs=@("bin")
+$pkg_lib_dirs=@("lib")
+
+function Invoke-SetupEnvironment {
+}
+
+function Invoke-Build {
+}
+
+
+function Invoke-Install {
+}
+
+
+function Invoke-Check {
+}

test/end-to-end/worker-test/fixtures/testapp_original/plan.sh

@@ -0,0 +1,17 @@
+pkg_name=testapp
+pkg_origin=neurosis
+pkg_version="0.1.3"
+pkg_maintainer="The Habitat Maintainers <[email protected]>"
+pkg_license=("Apache-2.0")
+pkg_deps=(core/glibc)
+pkg_description="This is a dummy app for testing builder APIs"
+pkg_upstream_url="https://habitat.sh"
+
+do_build() {
+    return 0
+}
+
+do_install() {
+    return 0
+}
+