Skip to content

Commit

Permalink
Update the python convert script for multiple versions of python.
Browse files Browse the repository at this point in the history 
To do this add data and vars to use for substitution as well as a
subapckage and remove the unversioned python dep since it will be
generated correctly as part of the build.
  • Loading branch information
@justinvreeland
justinvreeland committed Sep 3, 2024
1 parent 734ded7 commit 444e7a0
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 31 deletions.
45 changes: 32 additions & 13 deletions pkg/convert/python/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ func TestGenerateManifest(t *testing.T) {
assert.EqualValues(t, got.Package.Epoch, 0)
assert.Equal(t, got.Package.Description, "Low-level, data-driven core of boto 3.")
assert.Equal(t, got.Package.Dependencies.Runtime, []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]})
assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority)

// Check Package.Copyright
assert.Equal(t, len(got.Package.Copyright), 1)
Expand All @@ -60,17 +61,40 @@ func TestGenerateManifest(t *testing.T) {
"build-base",
"busybox",
"ca-certificates-bundle",
"py3-supported-pip",
"wolfi-base",
})

// Check Pipeline
assert.Equal(t, len(got.Pipeline), 3)
assert.Equal(t, 1, len(got.Pipeline))

// Check Pipeline - fetch
assert.Equal(t, got.Pipeline[0].Uses, "fetch")

// Check Subpackages
assert.Equal(t, "py-versions", got.Subpackages[0].Range)
assert.Equal(t, "py3-${{vars.pypi-package}}", got.Subpackages[0].Dependencies.Provides[0])
assert.Equal(t, "py/pip-build-install", got.Subpackages[0].Pipeline[0].Uses)
var expectedRuntimeDeps []string = []string{
"py3.10-jmespath",
"py3.10-python-dateutil",
"py3.10-urllib3",
"python-3.10",
}
// Subpackages aren't added to this array in the same order every time causing spurious
// test failues. To fix this Just check what is seen and replace the deps with the version string.
var replacementPyVersion = got.Subpackages[0].Dependencies.Runtime[0][2:6]
for idx, dep := range expectedRuntimeDeps {
expectedRuntimeDeps[idx] = strings.Replace(dep, "3.10", replacementPyVersion, 1)
}

assert.Equal(t, expectedRuntimeDeps, got.Subpackages[0].Dependencies.Runtime)
assert.Equal(t, "${{range.value}}", got.Subpackages[0].Dependencies.ProviderPriority)
releases, ok := pythonctx.Package.Releases[pythonctx.PackageVersion]

assert.Equal(t, "python/import", got.Subpackages[0].Test.Pipeline[0].Uses)
assert.Equal(t, "${{vars.module_name}}", got.Subpackages[0].Test.Pipeline[0].With["import"])

// If the key exists
assert.True(t, ok)

Expand All @@ -89,9 +113,6 @@ func TestGenerateManifest(t *testing.T) {
"uri": strings.ReplaceAll(tempURI, pythonctx.PackageVersion, "${{package.version}}"),
})

// Check Pipeline - runs
assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel")
assert.Equal(t, got.Pipeline[2].Uses, "strip")
}
}

Expand All @@ -117,11 +138,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
assert.Equal(t, got.Package.Description,
"Backported and Experimental Type Hints for Python 3.8+",
)
assert.Equal(t, got.Package.Dependencies.Runtime,
[]string{
"python-" + versions[i],
},
)
assert.Equal(t, []string{}, got.Package.Dependencies.Runtime)
assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority)

// Check Package.Copyright
assert.Equal(t, len(got.Package.Copyright), 1)
Expand All @@ -132,11 +150,12 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
"build-base",
"busybox",
"ca-certificates-bundle",
"py3-supported-pip",
"wolfi-base",
})

// Check Pipeline
assert.Equal(t, len(got.Pipeline), 3)
assert.Equal(t, 1, len(got.Pipeline))

// Check Pipeline - fetch
assert.Equal(t, got.Pipeline[0].Uses, "fetch")
Expand All @@ -160,8 +179,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) {
"uri": "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-${{package.version}}.tar.gz",
})

// Check Pipeline - runs
assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel")
assert.Equal(t, got.Pipeline[2].Uses, "strip")
// Check Tests
assert.Equal(t, "python/import", got.Test.Pipeline[0].Uses)
assert.Equal(t, "${{vars.module_name}}", got.Test.Pipeline[0].With["import"])
}
}
100 changes: 89 additions & 11 deletions pkg/convert/python/python.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,11 @@ func (c *PythonContext) generateManifest(ctx context.Context, pack Package, vers
// Generate each field in the manifest
generated.GeneratedFromComment = pack.Info.ProjectURL
generated.Package = c.generatePackage(ctx, pack, version)
generated.Data = c.generateRange(ctx)
generated.Vars = c.generateVars(pack)
generated.Subpackages = c.generateSubpackages(ctx, pack)
generated.Environment = c.generateEnvironment(ctx, pack)
generated.Test = c.generateTest(ctx, pack)

pipelines, err := c.generatePipeline(ctx, pack, version, ghVersions)
if err != nil {
Expand Down Expand Up @@ -333,16 +337,15 @@ func (c *PythonContext) generatePackage(ctx context.Context, pack Package, versi

log.Infof("[%s] Run time Deps %v", pack.Info.Name, pack.Dependencies)

pack.Dependencies = append(pack.Dependencies, "python-"+c.PythonVersion)

pkg := config.Package{
Name: fmt.Sprintf("py%s-%s", c.PythonVersion, pack.Info.Name),
Version: version,
Epoch: 0,
Description: pack.Info.Summary,
Copyright: []config.Copyright{},
Dependencies: config.Dependencies{
Runtime: pack.Dependencies,
Runtime: pack.Dependencies,
ProviderPriority: "0",
},
}

Expand All @@ -363,6 +366,7 @@ func (c *PythonContext) generateEnvironment(ctx context.Context, pack Package) a
"build-base",
"busybox",
"ca-certificates-bundle",
"py3-supported-pip",
"wolfi-base",
}

Expand Down Expand Up @@ -455,16 +459,90 @@ func (c *PythonContext) generatePipeline(ctx context.Context, pack Package, vers
})
}

pythonBuild := config.Pipeline{
Name: "Python Build",
Uses: "python/build-wheel",
return pipeline, nil
}

// generateVars handles generated variables for multi version python generateSubpackages
func (c *PythonContext) generateRange(ctx context.Context) []config.RangeData {
return []config.RangeData{{
Name: "py-versions",
Items: map[string]string{
"3.10": "310",
"3.11": "311",
"3.12": "312",
}},
}
}

strip := config.Pipeline{
Uses: "strip",
// Generate the vars for pypi package name and pip
// Set pypi-package and module_name to the same value because it's the most common case.
// Someone else can fix it up if the build fails
func (c *PythonContext) generateVars(pack Package) map[string]string {
return map[string]string{
"pypi-package": pack.Info.Name,
"module_name": pack.Info.Name,
}
pipeline = append(pipeline, pythonBuild)
pipeline = append(pipeline, strip)
}

return pipeline, nil
// generateSubpackages handles generating suibpackages field of the melange manifest
func (c *PythonContext) generateSubpackages(ctx context.Context, pack Package) []config.Subpackage {
log := clog.FromContext(ctx)

log.Infof("[%s] Generating Subpackages", pack.Info.Name)

importTest := config.Test{
Pipeline: []config.Pipeline{config.Pipeline{
Name: "Import Test",
Uses: "python/import",
With: map[string]string{
"python": "python${{range.key}}",
"import": "${{vars.module_name}}",
},
},
},
}

pythonSubpackages := config.Subpackage{
Range: "py-versions",
Name: "py${{range.key}}-${{vars.pypi-package}}",
Dependencies: config.Dependencies{
Runtime: pack.Dependencies,
Provides: []string{"py3-${{vars.pypi-package}}"},
ProviderPriority: "${{range.value}}",
},
Pipeline: []config.Pipeline{config.Pipeline{
Name: "Python Build",
Uses: "py/pip-build-install",
With: map[string]string{
"python": "python${{range.key}}",
},
},
},
Test: &importTest,
}

return []config.Subpackage{pythonSubpackages}
}

// generate file-level package test. When building python packages for multiple
// python versions we want to ensure that we don't generate -support packages with
// contents in /bin as well as ensuring that people installing the unversioned package
// receive on and only one version of the library
func (c *PythonContext) generateTest(ctx context.Context, pack Package) *config.Test {
log := clog.FromContext(ctx)

log.Infof("[%s] Generating Tests", pack.Info.Name)

importTest := config.Test{
Pipeline: []config.Pipeline{config.Pipeline{
Name: "Import Test",
Uses: "python/import",
With: map[string]string{
"import": "${{vars.module_name}}",
},
},
},
}

return &importTest
}
17 changes: 10 additions & 7 deletions pkg/convert/python/python_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func TestFindDependencies(t *testing.T) {
}
}

// TestGeneratePackage tests when a gem has multiple licenses
// TestGeneratePackage tests when a python package has multiple licenses
func TestGeneratePackage(t *testing.T) {
for i := range versions {
pythonctxs, err := SetupContext(versions[i])
Expand All @@ -157,11 +157,12 @@ func TestGeneratePackage(t *testing.T) {
},
},
Dependencies: config.Dependencies{
Runtime: []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]},
Runtime: []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]},
ProviderPriority: "0",
},
}

assert.Equal(t, got, expected)
assert.Equal(t, expected, got)
}
}

Expand All @@ -177,7 +178,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
botocorepythonctx.PackageVersion = "1.29.78"
botocorepythonctx.PythonVersion = version

// Read the gem meta into
// Read the pypi meta into
data, err := os.ReadFile(filepath.Join(botocoreMeta, "json"))
if err != nil {
return nil, err
Expand All @@ -190,7 +191,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
}

botocorepythonctx.Package = botocorePackageMeta
botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3"}
botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3", "python-" + version}

jsonschemapythonctx, err := New("jsonschema")
if err != nil {
Expand All @@ -203,7 +204,7 @@ func SetupContext(version string) ([]*PythonContext, error) {
jsonschemapythonctx.PackageVersion = "4.17.3"
jsonschemapythonctx.PythonVersion = version

// Read the gem meta into
// Read the pypi meta into
data, err = os.ReadFile(filepath.Join(jsonschemaMeta, "json"))
if err != nil {
return nil, err
Expand Down Expand Up @@ -238,7 +239,7 @@ func SetupContextPreserveURI(version string) ([]*PythonContext, error) {
typingextctx.PythonVersion = version
typingextctx.PreserveBaseURI = true

// Read the gem meta into
// Read the pypi package meta into
data, err := os.ReadFile(filepath.Join(typingextMeta, "json"))
if err != nil {
return nil, err
Expand Down Expand Up @@ -291,6 +292,7 @@ func TestGenerateEnvironment(t *testing.T) {
"build-base",
"busybox",
"ca-certificates-bundle",
"py3-supported-pip",
"wolfi-base",
},
},
Expand All @@ -314,6 +316,7 @@ func TestGenerateEnvironment(t *testing.T) {
"build-base",
"busybox",
"ca-certificates-bundle",
"py3-supported-pip",
"wolfi-base",
},
},
Expand Down
Binary file modified pkg/sca/testdata/generated/x86_64/shbang-test-1-r1.apk
View file
Binary file not shown.

0 comments on commit 444e7a0

Please sign in to comment.