From 9e01985a05d01a44fc565f61b146c45336be44b7 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Thu, 24 Oct 2024 11:16:52 +0100 Subject: [PATCH] Support Runtime images based on Debian Bookwork. Fixes #33 --- buildkit/internal/transform/transforms.go | 67 +++++++++++++++++-- .../internal/transform/transforms_test.go | 52 +++++++------- 2 files changed, 86 insertions(+), 33 deletions(-) diff --git a/buildkit/internal/transform/transforms.go b/buildkit/internal/transform/transforms.go index ac8fa97..8d40c34 100644 --- a/buildkit/internal/transform/transforms.go +++ b/buildkit/internal/transform/transforms.go @@ -40,11 +40,20 @@ ENV ASTRO_PYENV_{{.Name}} /home/astro/.venv/{{.Name}}/bin/python pyenvCommand = "PYENV" astroRuntimeImage = "quay.io/astronomer/astro-runtime" - defaultImageFlavour = "slim-bullseye" // Sadly for the `RUN --mount,uid=$uid` we need to use a numeric ID. defaultAstroUid = 50000 ) +var runtimeImageToFlavour = []struct { + *semver.Version + Flavour string +}{ + // Everything before Runtime 12 used bullseye + {semver.New("12.0.0"), "slim-bullseye"}, + // Everything after used bookworm. + {nil, "slim-bookworm"}, +} + var ( venvNamePattern = regexp.MustCompile(`[a-zA-Z0-9_-]+`) pythonVersionPattern = regexp.MustCompile(`[0-9]+\.[0-9]+(\.[0-9]+)?(-.*)?`) @@ -55,6 +64,9 @@ type Transformer struct { buildArgs map[string]string pythonVersions map[string]struct{} virtualEnvs map[string]struct{} + + // The version of the astro Runtime image we are based upon, if we can determine it + runtimeImageVersion string } type virtualEnv struct { @@ -185,7 +197,7 @@ func (t *Transformer) processArg(node *parser.Node) error { } func (t *Transformer) processPyenv(pyenv *parser.Node) (*parser.Node, error) { - venv, err := parsePyenvDirective(pyenv.Original) + venv, err := t.parsePyenvDirective(pyenv.Original) if err != nil { return nil, err } @@ -211,15 +223,52 @@ func (t *Transformer) processPyenv(pyenv *parser.Node) (*parser.Node, error) { return newNode, nil } -func parsePyenvDirective(s string) (*virtualEnv, error) { +func (t *Transformer) getImageFlavour() (string, error) { + // If we are ever unsure, return the latest flavour + candidate := runtimeImageToFlavour[len(runtimeImageToFlavour)-1].Flavour + + if t.runtimeImageVersion == "" { + // No runtime version we could find, return whatever the current default version is. + return candidate, nil + } + parts := strings.SplitN(t.runtimeImageVersion, "-", 1) + ver, err := semver.NewVersion(parts[0]) + if err != nil { + // If we couldn't parse the version, return a default, don't fail! + return candidate, err + } + + msg := "" + candidate = runtimeImageToFlavour[0].Flavour + // Look through all versions rules until we find one that doesn't match. At + // that point we know the previous answer we got is the one we need to use + for _, cond := range runtimeImageToFlavour { + msg += fmt.Sprintf("Comparing %#v < %#v: %v\n", ver, cond.Version, (cond.Version != nil && ver.LessThan(*cond.Version))) + if cond.Version != nil && ver.LessThan(*cond.Version) { + break + } + candidate = cond.Flavour + } + // return candidate, fmt.Errorf("%s, candidate=%s", msg, candidate) + return candidate, nil +} + +func (t *Transformer) parsePyenvDirective(s string) (*virtualEnv, error) { tokens := strings.Split(s, " ") if len(tokens) < 3 { return nil, fmt.Errorf("invalid PYENV directive: '%s', should be 'PYENV PYTHON_VERSION VENV_NAME [REQS_FILE]'", s) } + + // TODO: Work out how to show warnings to user via docker build backend! + // For now don't ever fail unless there is no flavour + flavour, err := t.getImageFlavour() + if flavour == "" && err != nil { + return nil, err + } env := &virtualEnv{ PythonVersion: tokens[1], - // For now we just hardcode this -- will add an option later - PythonFlavour: defaultImageFlavour, + // For now we just detect this -- will add an option later to let user control it + PythonFlavour: flavour, Name: tokens[2], } if len(tokens) > 3 { @@ -332,9 +381,13 @@ func (t *Transformer) ensureValidBaseImage(node *parser.Node) error { if ref.Name() == astroRuntimeImage { if tagged, ok := ref.(reference.NamedTagged); ok { - if !strings.HasSuffix(tagged.Tag(), "-base") { - ref, _ = reference.WithTag(ref, tagged.Tag()+"-base") + tag := tagged.Tag() + if !strings.HasSuffix(tag, "-base") { + t.runtimeImageVersion = tag + ref, _ = reference.WithTag(ref, tag+"-base") imgNode.Value = ref.String() + } else { + t.runtimeImageVersion = strings.TrimSuffix(tag, "-base") } } } diff --git a/buildkit/internal/transform/transforms_test.go b/buildkit/internal/transform/transforms_test.go index 22fa852..8d424ea 100644 --- a/buildkit/internal/transform/transforms_test.go +++ b/buildkit/internal/transform/transforms_test.go @@ -10,14 +10,21 @@ import ( func AssertDockerfileTransform(t *testing.T, input, expectedPreamble, expectedDockerfile string) { t.Helper() + AssertDockerfileTransformWithBuildArgs(t, input, map[string]string{}, expectedPreamble, expectedDockerfile) +} + +func AssertDockerfileTransformWithBuildArgs(t *testing.T, input string, buildArgs map[string]string, expectedPreamble, expectedDockerfile string) { + t.Helper() - preamble, body, err := Transform([]byte(input), map[string]string{}) + preamble, body, err := Transform([]byte(input), buildArgs) require.NoError(t, err) assert.NotNil(t, preamble) assert.NotNil(t, body) bodyText, err := dockerfile.Print(body) + require.NoError(t, err) assert.Equal(t, expectedDockerfile, bodyText) preambleText, err := dockerfile.Print(preamble) + require.NoError(t, err) assert.Equal(t, expectedPreamble, preambleText) } @@ -32,14 +39,14 @@ RUN mkdir /tmp/bar ` expectedPreamble := ` ARG baseimage -FROM ${baseimage} +FROM quay.io/astronomer/astro-runtime:13.0.0-base ` expectedDockerfile := `USER root -COPY --link --from=python:3.8-slim-bullseye /usr/local/bin/*3.8* /usr/local/bin/ -COPY --link --from=python:3.8-slim-bullseye /usr/local/include/python3.8* /usr/local/include/python3.8 -COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/pkgconfig/*3.8* /usr/local/lib/pkgconfig/ -COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/*3.8*.so* /usr/local/lib/ -COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/python3.8 /usr/local/lib/python3.8 +COPY --link --from=python:3.8-slim-bookworm /usr/local/bin/*3.8* /usr/local/bin/ +COPY --link --from=python:3.8-slim-bookworm /usr/local/include/python3.8* /usr/local/include/python3.8 +COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/pkgconfig/*3.8* /usr/local/lib/pkgconfig/ +COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/*3.8*.so* /usr/local/lib/ +COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/python3.8 /usr/local/lib/python3.8 RUN /sbin/ldconfig /usr/local/lib USER astro @@ -50,11 +57,11 @@ ENV ASTRO_PYENV_venv1 /home/astro/.venv/venv1/bin/python RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt COPY foo bar USER root -COPY --link --from=python:3.10-slim-bullseye /usr/local/bin/*3.10* /usr/local/bin/ -COPY --link --from=python:3.10-slim-bullseye /usr/local/include/python3.10* /usr/local/include/python3.10 -COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/pkgconfig/*3.10* /usr/local/lib/pkgconfig/ -COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/*3.10*.so* /usr/local/lib/ -COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/python3.10 /usr/local/lib/python3.10 +COPY --link --from=python:3.10-slim-bookworm /usr/local/bin/*3.10* /usr/local/bin/ +COPY --link --from=python:3.10-slim-bookworm /usr/local/include/python3.10* /usr/local/include/python3.10 +COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/pkgconfig/*3.10* /usr/local/lib/pkgconfig/ +COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/*3.10*.so* /usr/local/lib/ +COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/python3.10 /usr/local/lib/python3.10 RUN /sbin/ldconfig /usr/local/lib USER astro @@ -65,7 +72,7 @@ ENV ASTRO_PYENV_venv2 /home/astro/.venv/venv2/bin/python RUN mkdir /tmp/bar ` - AssertDockerfileTransform(t, testDockerfile, expectedPreamble, expectedDockerfile) + AssertDockerfileTransformWithBuildArgs(t, testDockerfile, map[string]string{"baseimage": "quay.io/astronomer/astro-runtime:13.0.0"}, expectedPreamble, expectedDockerfile) } func TestPython3_8(t *testing.T) { @@ -80,11 +87,11 @@ ARG baseimage FROM ${baseimage} ` expectedDockerfile := `USER root -COPY --link --from=python:3.7-slim-bullseye /usr/local/bin/*3.7* /usr/local/bin/ -COPY --link --from=python:3.7-slim-bullseye /usr/local/include/python3.7* /usr/local/include/python3.7 -COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/pkgconfig/*3.7* /usr/local/lib/pkgconfig/ -COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/*3.7*.so* /usr/local/lib/ -COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/python3.7 /usr/local/lib/python3.7 +COPY --link --from=python:3.7-slim-bookworm /usr/local/bin/*3.7* /usr/local/bin/ +COPY --link --from=python:3.7-slim-bookworm /usr/local/include/python3.7* /usr/local/include/python3.7 +COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/pkgconfig/*3.7* /usr/local/lib/pkgconfig/ +COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/*3.7*.so* /usr/local/lib/ +COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/python3.7 /usr/local/lib/python3.7 RUN /sbin/ldconfig /usr/local/lib RUN ln -s /usr/local/include/python3.7 /usr/local/include/python3.7m @@ -123,14 +130,7 @@ ARG ver=7.0.0 FROM quay.io/astronomer/astro-runtime:7.4.0-base ` - preamble, body, err := Transform([]byte(testDockerfile), map[string]string{"ver": "7.4.0"}) - require.NoError(t, err) - assert.NotNil(t, preamble) - assert.NotNil(t, body) - bodyText, err := dockerfile.Print(body) - assert.Equal(t, "", bodyText) - preambleText, err := dockerfile.Print(preamble) - assert.Equal(t, expectedPreamble, preambleText) + AssertDockerfileTransformWithBuildArgs(t, testDockerfile, map[string]string{"ver": "7.4.0"}, expectedPreamble, "") } func TestTransformsNonRuntimeLeftAlone(t *testing.T) {