Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Runtime images based on Debian Bookwork. #34

Merged
merged 1 commit into from
Oct 24, 2024
Merged

Support Runtime images based on Debian Bookwork. #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 60 additions & 7 deletions buildkit/internal/transform/transforms.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,20 @@ ENV ASTRO_PYENV_{{.Name}} /home/astro/.venv/{{.Name}}/bin/python
pyenvCommand = "PYENV"
astroRuntimeImage = "quay.io/astronomer/astro-runtime"

defaultImageFlavour = "slim-bullseye"
// Sadly for the `RUN --mount,uid=$uid` we need to use a numeric ID.
defaultAstroUid = 50000
)

var runtimeImageToFlavour = []struct {
*semver.Version
Flavour string
}{
// Everything before Runtime 12 used bullseye
{semver.New("12.0.0"), "slim-bullseye"},
// Everything after used bookworm.
{nil, "slim-bookworm"},
}

var (
venvNamePattern = regexp.MustCompile(`[a-zA-Z0-9_-]+`)
pythonVersionPattern = regexp.MustCompile(`[0-9]+\.[0-9]+(\.[0-9]+)?(-.*)?`)
Expand All @@ -55,6 +64,9 @@ type Transformer struct {
buildArgs map[string]string
pythonVersions map[string]struct{}
virtualEnvs map[string]struct{}

// The version of the astro Runtime image we are based upon, if we can determine it
runtimeImageVersion string
}

type virtualEnv struct {
Expand Down Expand Up @@ -185,7 +197,7 @@ func (t *Transformer) processArg(node *parser.Node) error {
}

func (t *Transformer) processPyenv(pyenv *parser.Node) (*parser.Node, error) {
venv, err := parsePyenvDirective(pyenv.Original)
venv, err := t.parsePyenvDirective(pyenv.Original)
if err != nil {
return nil, err
}
Expand All @@ -211,15 +223,52 @@ func (t *Transformer) processPyenv(pyenv *parser.Node) (*parser.Node, error) {
return newNode, nil
}

func parsePyenvDirective(s string) (*virtualEnv, error) {
func (t *Transformer) getImageFlavour() (string, error) {
// If we are ever unsure, return the latest flavour
candidate := runtimeImageToFlavour[len(runtimeImageToFlavour)-1].Flavour

if t.runtimeImageVersion == "" {
// No runtime version we could find, return whatever the current default version is.
return candidate, nil
}
parts := strings.SplitN(t.runtimeImageVersion, "-", 1)
ver, err := semver.NewVersion(parts[0])
if err != nil {
// If we couldn't parse the version, return a default, don't fail!
return candidate, err
}

msg := ""
candidate = runtimeImageToFlavour[0].Flavour
// Look through all versions rules until we find one that doesn't match. At
// that point we know the previous answer we got is the one we need to use
for _, cond := range runtimeImageToFlavour {
msg += fmt.Sprintf("Comparing %#v < %#v: %v\n", ver, cond.Version, (cond.Version != nil && ver.LessThan(*cond.Version)))
if cond.Version != nil && ver.LessThan(*cond.Version) {
break
}
candidate = cond.Flavour
}
// return candidate, fmt.Errorf("%s, candidate=%s", msg, candidate)
return candidate, nil
}

func (t *Transformer) parsePyenvDirective(s string) (*virtualEnv, error) {
tokens := strings.Split(s, " ")
if len(tokens) < 3 {
return nil, fmt.Errorf("invalid PYENV directive: '%s', should be 'PYENV PYTHON_VERSION VENV_NAME [REQS_FILE]'", s)
}

// TODO: Work out how to show warnings to user via docker build backend!
// For now don't ever fail unless there is no flavour
flavour, err := t.getImageFlavour()
if flavour == "" && err != nil {
return nil, err
}
env := &virtualEnv{
PythonVersion: tokens[1],
// For now we just hardcode this -- will add an option later
PythonFlavour: defaultImageFlavour,
// For now we just detect this -- will add an option later to let user control it
PythonFlavour: flavour,
Name: tokens[2],
}
if len(tokens) > 3 {
Expand Down Expand Up @@ -332,9 +381,13 @@ func (t *Transformer) ensureValidBaseImage(node *parser.Node) error {

if ref.Name() == astroRuntimeImage {
if tagged, ok := ref.(reference.NamedTagged); ok {
if !strings.HasSuffix(tagged.Tag(), "-base") {
ref, _ = reference.WithTag(ref, tagged.Tag()+"-base")
tag := tagged.Tag()
if !strings.HasSuffix(tag, "-base") {
t.runtimeImageVersion = tag
ref, _ = reference.WithTag(ref, tag+"-base")
imgNode.Value = ref.String()
} else {
t.runtimeImageVersion = strings.TrimSuffix(tag, "-base")
}
}
}
Expand Down
52 changes: 26 additions & 26 deletions buildkit/internal/transform/transforms_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,21 @@ import (

func AssertDockerfileTransform(t *testing.T, input, expectedPreamble, expectedDockerfile string) {
t.Helper()
AssertDockerfileTransformWithBuildArgs(t, input, map[string]string{}, expectedPreamble, expectedDockerfile)
}

func AssertDockerfileTransformWithBuildArgs(t *testing.T, input string, buildArgs map[string]string, expectedPreamble, expectedDockerfile string) {
t.Helper()

preamble, body, err := Transform([]byte(input), map[string]string{})
preamble, body, err := Transform([]byte(input), buildArgs)
require.NoError(t, err)
assert.NotNil(t, preamble)
assert.NotNil(t, body)
bodyText, err := dockerfile.Print(body)
require.NoError(t, err)
assert.Equal(t, expectedDockerfile, bodyText)
preambleText, err := dockerfile.Print(preamble)
require.NoError(t, err)
assert.Equal(t, expectedPreamble, preambleText)
}

Expand All @@ -32,14 +39,14 @@ RUN mkdir /tmp/bar
`
expectedPreamble := `
ARG baseimage
FROM ${baseimage}
FROM quay.io/astronomer/astro-runtime:13.0.0-base
`
expectedDockerfile := `USER root
COPY --link --from=python:3.8-slim-bullseye /usr/local/bin/*3.8* /usr/local/bin/
COPY --link --from=python:3.8-slim-bullseye /usr/local/include/python3.8* /usr/local/include/python3.8
COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/pkgconfig/*3.8* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/*3.8*.so* /usr/local/lib/
COPY --link --from=python:3.8-slim-bullseye /usr/local/lib/python3.8 /usr/local/lib/python3.8
COPY --link --from=python:3.8-slim-bookworm /usr/local/bin/*3.8* /usr/local/bin/
COPY --link --from=python:3.8-slim-bookworm /usr/local/include/python3.8* /usr/local/include/python3.8
COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/pkgconfig/*3.8* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/*3.8*.so* /usr/local/lib/
COPY --link --from=python:3.8-slim-bookworm /usr/local/lib/python3.8 /usr/local/lib/python3.8
RUN /sbin/ldconfig /usr/local/lib

USER astro
Expand All @@ -50,11 +57,11 @@ ENV ASTRO_PYENV_venv1 /home/astro/.venv/venv1/bin/python
RUN --mount=type=cache,uid=50000,gid=0,target=/home/astro/.cache/pip /home/astro/.venv/venv1/bin/pip --cache-dir=/home/astro/.cache/pip install -r /home/astro/.venv/venv1/requirements.txt
COPY foo bar
USER root
COPY --link --from=python:3.10-slim-bullseye /usr/local/bin/*3.10* /usr/local/bin/
COPY --link --from=python:3.10-slim-bullseye /usr/local/include/python3.10* /usr/local/include/python3.10
COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/pkgconfig/*3.10* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/*3.10*.so* /usr/local/lib/
COPY --link --from=python:3.10-slim-bullseye /usr/local/lib/python3.10 /usr/local/lib/python3.10
COPY --link --from=python:3.10-slim-bookworm /usr/local/bin/*3.10* /usr/local/bin/
COPY --link --from=python:3.10-slim-bookworm /usr/local/include/python3.10* /usr/local/include/python3.10
COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/pkgconfig/*3.10* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/*3.10*.so* /usr/local/lib/
COPY --link --from=python:3.10-slim-bookworm /usr/local/lib/python3.10 /usr/local/lib/python3.10
RUN /sbin/ldconfig /usr/local/lib

USER astro
Expand All @@ -65,7 +72,7 @@ ENV ASTRO_PYENV_venv2 /home/astro/.venv/venv2/bin/python
RUN mkdir /tmp/bar
`

AssertDockerfileTransform(t, testDockerfile, expectedPreamble, expectedDockerfile)
AssertDockerfileTransformWithBuildArgs(t, testDockerfile, map[string]string{"baseimage": "quay.io/astronomer/astro-runtime:13.0.0"}, expectedPreamble, expectedDockerfile)
}

func TestPython3_8(t *testing.T) {
Expand All @@ -80,11 +87,11 @@ ARG baseimage
FROM ${baseimage}
`
expectedDockerfile := `USER root
COPY --link --from=python:3.7-slim-bullseye /usr/local/bin/*3.7* /usr/local/bin/
COPY --link --from=python:3.7-slim-bullseye /usr/local/include/python3.7* /usr/local/include/python3.7
COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/pkgconfig/*3.7* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/*3.7*.so* /usr/local/lib/
COPY --link --from=python:3.7-slim-bullseye /usr/local/lib/python3.7 /usr/local/lib/python3.7
COPY --link --from=python:3.7-slim-bookworm /usr/local/bin/*3.7* /usr/local/bin/
COPY --link --from=python:3.7-slim-bookworm /usr/local/include/python3.7* /usr/local/include/python3.7
COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/pkgconfig/*3.7* /usr/local/lib/pkgconfig/
COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/*3.7*.so* /usr/local/lib/
COPY --link --from=python:3.7-slim-bookworm /usr/local/lib/python3.7 /usr/local/lib/python3.7
RUN /sbin/ldconfig /usr/local/lib
RUN ln -s /usr/local/include/python3.7 /usr/local/include/python3.7m

Expand Down Expand Up @@ -123,14 +130,7 @@ ARG ver=7.0.0
FROM quay.io/astronomer/astro-runtime:7.4.0-base
`

preamble, body, err := Transform([]byte(testDockerfile), map[string]string{"ver": "7.4.0"})
require.NoError(t, err)
assert.NotNil(t, preamble)
assert.NotNil(t, body)
bodyText, err := dockerfile.Print(body)
assert.Equal(t, "", bodyText)
preambleText, err := dockerfile.Print(preamble)
assert.Equal(t, expectedPreamble, preambleText)
AssertDockerfileTransformWithBuildArgs(t, testDockerfile, map[string]string{"ver": "7.4.0"}, expectedPreamble, "")
}

func TestTransformsNonRuntimeLeftAlone(t *testing.T) {
Expand Down
Loading