diff --git a/pkg/convert/python/e2e_test.go b/pkg/convert/python/e2e_test.go index f8143c6cc..8a9230f89 100644 --- a/pkg/convert/python/e2e_test.go +++ b/pkg/convert/python/e2e_test.go @@ -50,9 +50,10 @@ func TestGenerateManifest(t *testing.T) { assert.EqualValues(t, got.Package.Epoch, 0) assert.Equal(t, got.Package.Description, "Low-level, data-driven core of boto 3.") assert.Equal(t, got.Package.Dependencies.Runtime, []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]}) + assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority) // Check Package.Copyright - assert.Equal(t, len(got.Package.Copyright), 1) +q assert.Equal(t, len(got.Package.Copyright), 1) assert.Equal(t, got.Package.Copyright[0].License, "Apache License 2.0") // Check Environment @@ -60,17 +61,40 @@ func TestGenerateManifest(t *testing.T) { "build-base", "busybox", "ca-certificates-bundle", + "py3-supported-pip", "wolfi-base", }) // Check Pipeline - assert.Equal(t, len(got.Pipeline), 3) + assert.Equal(t, 1, len(got.Pipeline)) // Check Pipeline - fetch assert.Equal(t, got.Pipeline[0].Uses, "fetch") + // Check Subpackages + assert.Equal(t, "py-versions", got.Subpackages[0].Range) + assert.Equal(t, "py3-${{vars.pypi-package}}", got.Subpackages[0].Dependencies.Provides[0]) + assert.Equal(t, "py/pip-build-install", got.Subpackages[0].Pipeline[0].Uses) + var expectedRuntimeDeps []string = []string{ + "py3.10-jmespath", + "py3.10-python-dateutil", + "py3.10-urllib3", + "python-3.10", + } + // Subpackages aren't added to this array in the same order every time causing spurious + // test failues. To fix this Just check what is seen and replace the deps with the version string. + var replacementPyVersion = got.Subpackages[0].Dependencies.Runtime[0][2:6] + for idx, dep := range expectedRuntimeDeps { + expectedRuntimeDeps[idx] = strings.Replace(dep, "3.10", replacementPyVersion, 1) + } + + assert.Equal(t, expectedRuntimeDeps, got.Subpackages[0].Dependencies.Runtime) + assert.Equal(t, "${{range.value}}", got.Subpackages[0].Dependencies.ProviderPriority) releases, ok := pythonctx.Package.Releases[pythonctx.PackageVersion] + assert.Equal(t, "python/import", got.Subpackages[0].Test.Pipeline[0].Uses) + assert.Equal(t, "${{vars.module_name}}", got.Subpackages[0].Test.Pipeline[0].With["import"]) + // If the key exists assert.True(t, ok) @@ -89,9 +113,6 @@ func TestGenerateManifest(t *testing.T) { "uri": strings.ReplaceAll(tempURI, pythonctx.PackageVersion, "${{package.version}}"), }) - // Check Pipeline - runs - assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel") - assert.Equal(t, got.Pipeline[2].Uses, "strip") } } @@ -117,11 +138,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) { assert.Equal(t, got.Package.Description, "Backported and Experimental Type Hints for Python 3.8+", ) - assert.Equal(t, got.Package.Dependencies.Runtime, - []string{ - "python-" + versions[i], - }, - ) + assert.Equal(t, []string{}, got.Package.Dependencies.Runtime) + assert.Equal(t, "0", got.Package.Dependencies.ProviderPriority) // Check Package.Copyright assert.Equal(t, len(got.Package.Copyright), 1) @@ -132,11 +150,12 @@ func TestGenerateManifestPreserveURI(t *testing.T) { "build-base", "busybox", "ca-certificates-bundle", + "py3-supported-pip", "wolfi-base", }) // Check Pipeline - assert.Equal(t, len(got.Pipeline), 3) + assert.Equal(t, 1, len(got.Pipeline)) // Check Pipeline - fetch assert.Equal(t, got.Pipeline[0].Uses, "fetch") @@ -160,8 +179,8 @@ func TestGenerateManifestPreserveURI(t *testing.T) { "uri": "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-${{package.version}}.tar.gz", }) - // Check Pipeline - runs - assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel") - assert.Equal(t, got.Pipeline[2].Uses, "strip") + // Check Tests + assert.Equal(t, "python/import", got.Test.Pipeline[0].Uses) + assert.Equal(t, "${{vars.module_name}}", got.Test.Pipeline[0].With["import"]) } } diff --git a/pkg/convert/python/python.go b/pkg/convert/python/python.go index c73f4da46..28ba96e95 100644 --- a/pkg/convert/python/python.go +++ b/pkg/convert/python/python.go @@ -273,7 +273,11 @@ func (c *PythonContext) generateManifest(ctx context.Context, pack Package, vers // Generate each field in the manifest generated.GeneratedFromComment = pack.Info.ProjectURL generated.Package = c.generatePackage(ctx, pack, version) + generated.Data = c.generateRange(ctx) + generated.Vars = c.generateVars(pack) + generated.Subpackages = c.generateSubpackages(ctx, pack) generated.Environment = c.generateEnvironment(ctx, pack) + generated.Test = c.generateTest(ctx, pack) pipelines, err := c.generatePipeline(ctx, pack, version, ghVersions) if err != nil { @@ -333,8 +337,6 @@ func (c *PythonContext) generatePackage(ctx context.Context, pack Package, versi log.Infof("[%s] Run time Deps %v", pack.Info.Name, pack.Dependencies) - pack.Dependencies = append(pack.Dependencies, "python-"+c.PythonVersion) - pkg := config.Package{ Name: fmt.Sprintf("py%s-%s", c.PythonVersion, pack.Info.Name), Version: version, @@ -342,7 +344,8 @@ func (c *PythonContext) generatePackage(ctx context.Context, pack Package, versi Description: pack.Info.Summary, Copyright: []config.Copyright{}, Dependencies: config.Dependencies{ - Runtime: pack.Dependencies, + Runtime: pack.Dependencies, + ProviderPriority: "0", }, } @@ -363,6 +366,7 @@ func (c *PythonContext) generateEnvironment(ctx context.Context, pack Package) a "build-base", "busybox", "ca-certificates-bundle", + "py3-supported-pip", "wolfi-base", } @@ -455,16 +459,90 @@ func (c *PythonContext) generatePipeline(ctx context.Context, pack Package, vers }) } - pythonBuild := config.Pipeline{ - Name: "Python Build", - Uses: "python/build-wheel", + return pipeline, nil +} + +// generateVars handles generated variables for multi version python generateSubpackages +func (c *PythonContext) generateRange(ctx context.Context) []config.RangeData { + return []config.RangeData{{ + Name: "py-versions", + Items: map[string]string{ + "3.10": "310", + "3.11": "311", + "3.12": "312", + }}, } +} - strip := config.Pipeline{ - Uses: "strip", +// Generate the vars for pypi package name and pip +// Set pypi-package and module_name to the same value because it's the most common case. +// Someone else can fix it up if the build fails +func (c *PythonContext) generateVars(pack Package) map[string]string { + return map[string]string{ + "pypi-package": pack.Info.Name, + "module_name": pack.Info.Name, } - pipeline = append(pipeline, pythonBuild) - pipeline = append(pipeline, strip) +} - return pipeline, nil +// generateSubpackages handles generating suibpackages field of the melange manifest +func (c *PythonContext) generateSubpackages(ctx context.Context, pack Package) []config.Subpackage { + log := clog.FromContext(ctx) + + log.Infof("[%s] Generating Subpackages", pack.Info.Name) + + importTest := config.Test{ + Pipeline: []config.Pipeline{config.Pipeline{ + Name: "Import Test", + Uses: "python/import", + With: map[string]string{ + "python": "python${{range.key}}", + "import": "${{vars.module_name}}", + }, + }, + }, + } + + pythonSubpackages := config.Subpackage{ + Range: "py-versions", + Name: "py${{range.key}}-${{vars.pypi-package}}", + Dependencies: config.Dependencies{ + Runtime: pack.Dependencies, + Provides: []string{"py3-${{vars.pypi-package}}"}, + ProviderPriority: "${{range.value}}", + }, + Pipeline: []config.Pipeline{config.Pipeline{ + Name: "Python Build", + Uses: "py/pip-build-install", + With: map[string]string{ + "python": "python${{range.key}}", + }, + }, + }, + Test: &importTest, + } + + return []config.Subpackage{pythonSubpackages} +} + +// generate file-level package test. When building python packages for multiple +// python versions we want to ensure that we don't generate -support packages with +// contents in /bin as well as ensuring that people installing the unversioned package +// receive on and only one version of the library +func (c *PythonContext) generateTest(ctx context.Context, pack Package) *config.Test { + log := clog.FromContext(ctx) + + log.Infof("[%s] Generating Tests", pack.Info.Name) + + importTest := config.Test{ + Pipeline: []config.Pipeline{config.Pipeline{ + Name: "Import Test", + Uses: "python/import", + With: map[string]string{ + "import": "${{vars.module_name}}", + }, + }, + }, + } + + return &importTest } diff --git a/pkg/convert/python/python_test.go b/pkg/convert/python/python_test.go index eb12b7b51..e7b283936 100644 --- a/pkg/convert/python/python_test.go +++ b/pkg/convert/python/python_test.go @@ -136,7 +136,7 @@ func TestFindDependencies(t *testing.T) { } } -// TestGeneratePackage tests when a gem has multiple licenses +// TestGeneratePackage tests when a python package has multiple licenses func TestGeneratePackage(t *testing.T) { for i := range versions { pythonctxs, err := SetupContext(versions[i]) @@ -157,11 +157,12 @@ func TestGeneratePackage(t *testing.T) { }, }, Dependencies: config.Dependencies{ - Runtime: []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]}, + Runtime: []string{"py" + versions[i] + "-jmespath", "py" + versions[i] + "-python-dateutil", "py" + versions[i] + "-urllib3", "python-" + versions[i]}, + ProviderPriority: "0", }, } - assert.Equal(t, got, expected) + assert.Equal(t, expected, got) } } @@ -177,7 +178,7 @@ func SetupContext(version string) ([]*PythonContext, error) { botocorepythonctx.PackageVersion = "1.29.78" botocorepythonctx.PythonVersion = version - // Read the gem meta into + // Read the pypi meta into data, err := os.ReadFile(filepath.Join(botocoreMeta, "json")) if err != nil { return nil, err @@ -190,7 +191,7 @@ func SetupContext(version string) ([]*PythonContext, error) { } botocorepythonctx.Package = botocorePackageMeta - botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3"} + botocorepythonctx.Package.Dependencies = []string{"py" + version + "-jmespath", "py" + version + "-python-dateutil", "py" + version + "-urllib3", "python-" + version} jsonschemapythonctx, err := New("jsonschema") if err != nil { @@ -203,7 +204,7 @@ func SetupContext(version string) ([]*PythonContext, error) { jsonschemapythonctx.PackageVersion = "4.17.3" jsonschemapythonctx.PythonVersion = version - // Read the gem meta into + // Read the pypi meta into data, err = os.ReadFile(filepath.Join(jsonschemaMeta, "json")) if err != nil { return nil, err @@ -238,7 +239,7 @@ func SetupContextPreserveURI(version string) ([]*PythonContext, error) { typingextctx.PythonVersion = version typingextctx.PreserveBaseURI = true - // Read the gem meta into + // Read the pypi package meta into data, err := os.ReadFile(filepath.Join(typingextMeta, "json")) if err != nil { return nil, err @@ -291,6 +292,7 @@ func TestGenerateEnvironment(t *testing.T) { "build-base", "busybox", "ca-certificates-bundle", + "py3-supported-pip", "wolfi-base", }, }, @@ -314,6 +316,7 @@ func TestGenerateEnvironment(t *testing.T) { "build-base", "busybox", "ca-certificates-bundle", + "py3-supported-pip", "wolfi-base", }, },