Skip to content

Commit

Permalink
KAFKA-17735: release.py must not use home.apache.org (#17421)
Browse files Browse the repository at this point in the history
Previously, Apache Kafka was uploading release candidate (RC) artifacts
to users' home directories on home.apache.org. However, since this
resource has been decommissioned, we need to follow the standard
approach of putting release candidate artifacts into the appropriate
subversion directory, at https://dist.apache.org/repos/dist/dev/kafka/.

Reviewers: Justine Olshan <[email protected]>
  • Loading branch information
cmccabe authored Oct 8, 2024
1 parent 0a70c3a commit 8af063a
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 78 deletions.
1 change: 0 additions & 1 deletion release/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ This directory contains the tools used to publish a release.
* python 3.12
* git
* gpg 2.4
* sftp

The full instructions for producing a release are available in
https://cwiki.apache.org/confluence/display/KAFKA/Release+Process.
Expand Down
12 changes: 8 additions & 4 deletions release/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,12 @@
import gpg
import notes
import preferences
import sftp
import svn
import templates
import textfiles

from svn import SVN_DEV_URL


def get_jdk(version):
"""
Expand Down Expand Up @@ -243,7 +245,6 @@ def prereq(name, soft_check):
except Exception as e:
fail(f"Pre-requisite not met: {name}. Error: {e}")
prereq('Apache Maven CLI (mvn) in PATH', lambda: "Apache Maven" in execute("mvn -v"))
prereq('Apache sftp connection', lambda: sftp.test(apache_id))
prereq("svn CLI in PATH", lambda: "svn" in execute("svn --version"))
prereq("Verifying that you have no unstaged git changes", lambda: git.has_unstaged_changes())
prereq("Verifying that you have no staged git changes", lambda: git.has_staged_changes())
Expand Down Expand Up @@ -298,7 +299,7 @@ def delete_gitrefs():
git.create_tag(rc_tag)
git.switch_branch(starting_branch)

# Note that we don't use tempfile here because mkdtemp causes problems with sftp and being able to determine the absolute path to a file.
# Note that we don't use tempfile here because mkdtemp causes problems with being able to determine the absolute path to a file.
# Instead we rely on a fixed path
work_dir = os.path.join(repo_dir, ".release_work_dir")
clean_up_work_dir = lambda: cmd("Cleaning up work directory", f"rm -rf {work_dir}")
Expand All @@ -315,6 +316,7 @@ def delete_gitrefs():
git.create_branch(release_version, rc_tag, cwd=kafka_dir)
current_year = datetime.datetime.now().year
cmd("Verifying the correct year in NOTICE", f"grep {current_year} NOTICE", cwd=kafka_dir)
svn.checkout_svn_dev(work_dir)

print("Generating release notes")
try:
Expand Down Expand Up @@ -348,7 +350,9 @@ def delete_gitrefs():

cmd("Listing artifacts to be uploaded:", f"ls -R {artifacts_dir}")
cmd("Zipping artifacts", f"tar -czf {artifact_name}.tar.gz {artifact_name}", cwd=work_dir)
sftp.upload_artifacts(apache_id, artifacts_dir)

confirm_or_fail(f"Going to check in artifacts to svn under {SVN_DEV_URL}/{rc_tag}. OK?")
svn.commit_artifacts(rc_tag, artifacts_dir, work_dir)

confirm_or_fail("Going to build and upload mvn artifacts based on these settings:\n" + textfiles.read(global_gradle_props) + '\nOK?')
cmd("Building and uploading archives", "./gradlewAll publish", cwd=kafka_dir, env=jdk8_env, shell=True)
Expand Down
4 changes: 2 additions & 2 deletions release/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,12 @@ def cmd(action, cmd_arg, *args, **kwargs):
try:
output = execute(cmd_arg, *args, stderr=subprocess.STDOUT, **kwargs)
print(_prefix("> ", output.strip()))
return
return True
except subprocess.CalledProcessError as e:
print(e.output.decode("utf-8"))

if allow_failure:
return
return False

retry = confirm("Retry?")

Expand Down
59 changes: 0 additions & 59 deletions release/sftp.py

This file was deleted.

60 changes: 60 additions & 0 deletions release/svn.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Auxiliary functions to interact with svn(1).
"""

import os
import shutil
import subprocess

from runtime import cmd

SVN_DEV_URL="https://dist.apache.org/repos/dist/dev/kafka"

def delete_old_rc_directory_if_needed(rc_tag, src, work_dir):
svn_dev = os.path.join(work_dir, "svn_dev")
cmd_desc = f"Check if {rc_tag} exists in the subversion repository."
cmd_str = f"svn info --show-item revision {SVN_DEV_URL}/{rc_tag}"
if not cmd(cmd_desc, cmd_str, cwd = svn_dev, allow_failure = True):
print(f"Nothing under {SVN_DEV_URL}/{rc_tag}. Continuing.")
return
cmd_desc = f"Committing the deletion of {SVN_DEV_URL}/{rc_tag} from the svn repo."
cmd_str = f"svn delete -m Remove_{rc_tag} {SVN_DEV_URL}/{rc_tag}"
cmd(cmd_desc, cmd_str, cwd = svn_dev)

def commit_artifacts(rc_tag, src, work_dir):
delete_old_rc_directory_if_needed(rc_tag, src, work_dir)
svn_dev = os.path.join(work_dir, "svn_dev")
dst = os.path.join(svn_dev, rc_tag)
print(f"Copying {src} to {dst}")
shutil.copytree(src, dst)
cmd_desc = f"Adding {SVN_DEV_URL}/{rc_tag} to the svn repo."
cmd_str = f"svn add ./{rc_tag}"
cmd(cmd_desc, cmd_str, cwd = svn_dev)
cmd_desc = f"Committing the addition of {SVN_DEV_URL}/{rc_tag} to the svn repo. Please wait, this may take a while."
cmd_str = f"svn commit -m Add_{rc_tag}"
cmd(cmd_desc, cmd_str, cwd = svn_dev)

def checkout_svn_dev(work_dir):
svn_dev = os.path.join(work_dir, "svn_dev")
if os.path.exists(svn_dev):
shutil.rmtree(svn_dev)
cmd_desc = f"Checking out {SVN_DEV_URL} at {svn_dev}"
cmd_str = f"svn checkout --depth empty {SVN_DEV_URL}/ {svn_dev}"
cmd(cmd_desc, cmd_str)
24 changes: 12 additions & 12 deletions release/templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,23 +167,23 @@ def sanity_check_instructions(release_version, rc_tag, apache_id):
Some suggested steps:
* Grab the source archive and make sure it compiles: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz
* Grab one of the binary distros and run the quickstarts against them: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}.tgz
* Extract and verify one of the site docs jars: https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
* Grab the source archive and make sure it compiles: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz
* Grab one of the binary distros and run the quickstarts against them: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz
* Extract and verify one of the site docs jars: https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}-site-docs.tgz
* Build a sample against jars in the staging repo: (TODO: Can we get a temporary URL before "closing" the staged artifacts?)
* Validate GPG signatures on at least one file:
wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz &&
wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.asc &&
wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.md5 &&
wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha1 &&
wget https://home.apache.org/~{apache_id}/kafka-{rc_tag}/kafka-{release_version}-src.tgz.sha512 &&
wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz &&
wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.asc &&
wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.md5 &&
wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha1 &&
wget https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka-{release_version}-src.tgz.sha512 &&
gpg --verify kafka-{release_version}-src.tgz.asc kafka-{release_version}-src.tgz &&
gpg --print-md md5 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.md5 &&
gpg --print-md sha1 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha1 &&
gpg --print-md sha512 kafka-{release_version}-src.tgz | diff - kafka-{release_version}-src.tgz.sha512 &&
rm kafka-{release_version}-src.tgz* &&
echo "OK" || echo "Failed"
* Validate the javadocs look ok. They are at https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
* Validate the javadocs look ok. They are at https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
*******************************************************************************************************************************************************
"""
Expand All @@ -201,7 +201,7 @@ def rc_vote_email_text(release_version, rc, rc_tag, dev_branch, docs_version, ap
<DESCRIPTION OF MAJOR CHANGES, INCLUDE INDICATION OF MAJOR/MINOR RELEASE>
Release notes for the {release_version} release:
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/RELEASE_NOTES.html
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/RELEASE_NOTES.html
*** Please download, test and vote by <VOTING DEADLINE, e.g. Monday, March 28, 9am PT>
<THE RELEASE POLICY (https://www.apache.org/legal/release-policy.html#release-approval) REQUIRES VOTES TO BE OPEN FOR MINIMUM OF 3 DAYS THEREFORE VOTING DEADLINE SHOULD BE AT LEAST 72 HOURS FROM THE TIME THIS EMAIL IS SENT.>
Expand All @@ -210,7 +210,7 @@ def rc_vote_email_text(release_version, rc, rc_tag, dev_branch, docs_version, ap
https://kafka.apache.org/KEYS
* Release artifacts to be voted upon (source and binary):
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/
<USE docker/README.md FOR STEPS TO CREATE RELEASE CANDIDATE DOCKER IMAGE>
* Docker release artifacts to be voted upon:
Expand All @@ -221,7 +221,7 @@ def rc_vote_email_text(release_version, rc, rc_tag, dev_branch, docs_version, ap
https://repository.apache.org/content/groups/staging/org/apache/kafka/
* Javadoc:
https://home.apache.org/~{apache_id}/kafka-{rc_tag}/javadoc/
https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/javadoc/
* Tag to be voted upon (off {dev_branch} branch) is the {release_version} tag:
https://github.com/apache/kafka/releases/tag/{rc_tag}
Expand Down

0 comments on commit 8af063a

Please sign in to comment.