diff --git a/README.md b/README.md index 35537d4..d3307f7 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,14 @@ Trac - GitHub integration Features -------- -This Trac plugin performs four functions: +This Trac plugin performs five functions: 1. update the local git mirror used by Trac after each push to GitHub, and notify the new changesets to Trac; 2. authenticate users with their GitHub account; -3. direct changeset TracLinks to GitHub's repository browser. -4. sync GitHub teams to Trac permission groups +3. direct changeset TracLinks to GitHub's repository browser; +4. sync GitHub teams to Trac permission groups; +5. sync any new GitHub issues and pull requests into Trac tickets. The notification of new changesets is strictly equivalent to the command described in Trac's setup guide: @@ -301,6 +302,74 @@ If you do not want to store the API secrets for `access_token` and `webhook_secret` in trac.ini, you can use the same alternatives as for `client_id` and `client_secret` documented [above](#authentication). +### Syncing Issues and Pull Requests + +**`tracext.github.GitHubIssueHook`** implements a a few GitHub hooks called +when a new issue or pull request is opened, commented on, or changed. + +It will open a new Trac ticket with the corresponding issue or pull request +title and description, setting the reporter to "username (GitHub)". It will use +all default ticket fields for everything else. It will also automatically +attach a patch file for any pull requests, along with any new patches when +someone pushes new commits to any existing pull request. + +Additionally, if any comments are left on the issue or pull request (including +inline patch comments on pulls) on GitHub, they will be posted to the Trac +ticket as well. + +Unlike the post-commit hook used for syncing any GitHub repos, these hooks are +required to configure a hook "secret" used to verify that the hooks were sent +from GitHub. This is to prevent spam tickets and comments. So first, you should +generate a random secret to be used with this hook. Ideally, it should be a +random string about 40 characters long containing only `[0-9a-f]` characters. +You can generate this on the command line by running this if you prefer: + + $ ruby -rsecurerandom -e 'puts SecureRandom.hex(20)' + +It should look like this (don't just use this string though!): +`cc6f7dddec47e4e10a423dcfbab5c102f506f72d` + +Save that somewhere safe, you will use this as your `hook_secret`. + +Edit your `trac.ini` as follows to configure syncing: + + [components] + tracext.github.GitHubIssueHook = enabled + + [github] + hook_secret = + +Reload the web server, browse to the home page of your project in Trac and +append `/github-issues` to the URL. You should see the following message: + + Endpoint is ready to accept GitHub notifications. + +This is the URL of the issues endpoint we'll use for the hook. + +If you get a Trac error page saying "No handler matched request to +/github-issues" instead, the plugin isn't installed properly. Make sure you've +followed the installation instructions correctly and search Trac's logs for +errors. + +This hook supports creating tickets for multiple GitHub repos if you want it to +watch issues and pull requests from several of them at the same time. So it's +possible to create an "organization hook", which will fire for all organization +repositories, or just create any number of "repository hooks" for the repos you +want it to watch. The configuration for either is exactly the same. + +Go to your organization's or repository's settings page on GitHub. In the +"Webhooks & Services" tab, click "Add webhook". Put the URL of the endpoint in +the "Payload URL" field, leave the "Content type" as "application/json", and set +the "Secret" to the 40 character `hook_secret` you generated earlier. Now select +the "Let me select individual events" radio option, and check the following +hooks: "Issue Comments", "Issues", "Pull Request", and "Pull Request review +comment". Then click "Add webhook", and you're done. + +If you click on the webhook you just created, at the bottom of the page, you +should see that a "ping" payload was successufully delivered to Trac. + +If you already have existing issues or pull requests, they will not be synced +to Trac. Only new issues and pull requests will be synced. Advanced setup -------------- @@ -518,6 +587,7 @@ Changelog * Add configuration option for path prefix of login and logout. (#127) * Add `GitHubPolicy` permission policy to make `[timeline]` `changeset_show_file` option work correctly. (#126) +* Add support for syncing issues and pull requests. ### 2.3 diff --git a/tracext/github.py b/tracext/github.py new file mode 100644 index 0000000..110a640 --- /dev/null +++ b/tracext/github.py @@ -0,0 +1,488 @@ +import fnmatch +from hashlib import sha1 +import hmac +import json +import os +import re +import urllib2 + +from genshi.builder import tag + +from trac.attachment import Attachment +from trac.db import with_transaction +from trac.config import ListOption, Option +from trac.core import Component, implements +from trac.ticket.model import Ticket +from trac.util.translation import _ +from trac.versioncontrol.api import is_default, NoSuchChangeset, RepositoryManager +from trac.versioncontrol.web_ui.changeset import ChangesetModule +from trac.web.api import IRequestHandler +from trac.web.auth import LoginModule + + +class GitHubMixin(object): + + def get_gh_repo(self, reponame): + key = 'repository' if is_default(reponame) else '%s.repository' % reponame + return self.config.get('github', key) + + def get_branches(self, reponame): + key = 'branches' if is_default(reponame) else '%s.branches' % reponame + return self.config.getlist('github', key, sep=' ') + + def _config(self, key): + assert key in ('client_id', 'client_secret', 'hook_secret') + value = self.config.get('github', key) + if re.match('[0-9a-f]+', value): + return value + elif value.isupper(): + return os.environ.get(value, '') + else: + with open(value) as f: + return f.read.strip() + + def verify_signature(self, req, body): + full_signature = req.get_header('X-Hub-Signature') + if not full_signature or not full_signature.find('='): + return False + sha_name, signature = full_signature.split('=') + if sha_name != 'sha1': + return False + hook_secret = str(self._config('hook_secret')) + mac = hmac.new(hook_secret, msg = str(body), digestmod = sha1) + return hmac.compare_digest(mac.hexdigest(), signature) + + +class GitHubLoginModule(GitHubMixin, LoginModule): + + # INavigationContributor methods + + def get_active_navigation_item(self, req): + return 'github_login' + + def get_navigation_items(self, req): + if req.authname and req.authname != 'anonymous': + # Use the same names as LoginModule to avoid duplicates. + yield ('metanav', 'login', _('logged in as %(user)s', + user=req.authname)) + yield ('metanav', 'logout', + tag.a(_('Logout'), href=req.href.github('logout'))) + else: + # Use a different name from LoginModule to allow both in parallel. + yield ('metanav', 'github_login', + tag.a(_('GitHub Login'), href=req.href.github('login'))) + + # IRequestHandler methods + + def match_request(self, req): + return re.match('/github/(login|oauth|logout)/?$', req.path_info) + + def process_request(self, req): + if req.path_info.startswith('/github/login'): + self._do_login(req) + elif req.path_info.startswith('/github/oauth'): + self._do_oauth(req) + elif req.path_info.startswith('/github/logout'): + self._do_logout(req) + self._redirect_back(req) + + # Internal methods + + def _do_login(self, req): + oauth = self._oauth_session(req) + authorization_url, state = oauth.authorization_url( + 'https://github.com/login/oauth/authorize') + req.session['oauth_state'] = state + req.redirect(authorization_url) + + def _do_oauth(self, req): + oauth = self._oauth_session(req) + authorization_response = req.abs_href(req.path_info) + '?' + req.query_string + client_secret = self._config('client_secret') + oauth.fetch_token( + 'https://github.com/login/oauth/access_token', + authorization_response=authorization_response, + client_secret=client_secret) + + user = oauth.get('https://api.github.com/user').json() + # Small hack to pass the username to _do_login. + req.environ['REMOTE_USER'] = user['login'] + # Save other available values in the session. + req.session.setdefault('name', user.get('name') or '') + req.session.setdefault('email', user.get('email') or '') + + return super(GitHubLoginModule, self)._do_login(req) + + def _oauth_session(self, req): + client_id = self._config('client_id') + redirect_uri = req.abs_href.github('oauth') + # Inner import to avoid a hard dependency on requests-oauthlib. + from requests_oauthlib import OAuth2Session + return OAuth2Session(client_id, redirect_uri=redirect_uri, scope=[]) + + +class GitHubBrowser(GitHubMixin, ChangesetModule): + + repository = Option('github', 'repository', '', + doc="Repository name on GitHub (/)") + + # IRequestHandler methods + + def match_request(self, req): + match = self._request_re.match(req.path_info) + if match: + rev, path = match.groups() + req.args['rev'] = rev + req.args['path'] = path or '/' + return True + + def process_request(self, req): + rev = req.args.get('rev') + path = req.args.get('path') + + rm = RepositoryManager(self.env) + reponame, repos, path = rm.get_repository_by_path(path) + gh_repo = self.get_gh_repo(reponame) + + rev = repos.normalize_rev(rev) + + if path and path != '/': + path = path.lstrip('/') + # GitHub will s/blob/tree/ if the path is a directory + url = 'https://github.com/%s/blob/%s/%s' % (gh_repo, rev, path) + else: + url = 'https://github.com/%s/commit/%s' % (gh_repo, rev) + req.redirect(url) + + # ITimelineEventProvider methods + + def get_timeline_events(self, req, start, stop, filters): + for event in super(GitHubBrowser, self).get_timeline_events(req, start, stop, filters): + assert event[0] == 'changeset' + viewable_changesets, show_location, show_files = event[3] + filtered_changesets = [] + for cset, cset_resource, (reponame,) in viewable_changesets: + branches = self.get_branches(reponame) + if rev_in_branches(cset, branches): + filtered_changesets.append((cset, cset_resource, [reponame])) + if filtered_changesets: + cset = filtered_changesets[-1][0] + yield ('changeset', cset.date, cset.author, + (filtered_changesets, show_location, show_files)) + + +class GitHubPostCommitHook(GitHubMixin, Component): + implements(IRequestHandler) + + branches = ListOption('github', 'branches', sep=' ', + doc="Notify only commits on these branches to Trac") + + # IRequestHandler methods + + _request_re = re.compile(r"/github(/.*)?$") + + def match_request(self, req): + match = self._request_re.match(req.path_info) + if match: + req.args['path'] = match.group(1) or '/' + return True + + def process_request(self, req): + path = req.args['path'] + + rm = RepositoryManager(self.env) + reponame, repos, path = rm.get_repository_by_path(path) + + if repos is None or path != '/': + msg = u'No such repository (%s)\n' % path + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 400) + + if req.method != 'POST': + msg = u'Endpoint is ready to accept GitHub notifications.\n' + self.log.warning(u'Method not allowed (%s)' % req.method) + req.send(msg.encode('utf-8'), 'text/plain', 405) + + event = req.get_header('X-GitHub-Event') + if event == 'ping': + payload = json.loads(req.read()) + req.send(payload['zen'].encode('utf-8'), 'text/plain', 200) + elif event != 'push': + msg = u'Only ping and push are supported\n' + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 400) + + output = u'Running hook on %s\n' % (reponame or '(default)') + + output += u'* Updating clone\n' + try: + git = repos.git.repo # GitRepository + except AttributeError: + git = repos.repos.git.repo # GitCachedRepository + git.remote('update', '--prune') + + # Ensure that repos.get_changeset can find the new changesets. + output += u'* Synchronizing with clone\n' + repos.sync() + + try: + payload = json.loads(req.read()) + revs = [commit['id'] + for commit in payload['commits'] if commit['distinct']] + except (ValueError, KeyError): + msg = u'Invalid payload\n' + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 400) + + branches = self.get_branches(reponame) + added, skipped, unknown = classify_commits(revs, repos, branches) + + if added: + output += u'* Adding %s\n' % describe_commits(added) + # This is where Trac gets notified of the commits in the changeset + rm.notify('changeset_added', reponame, added) + + if skipped: + output += u'* Skipping %s\n' % describe_commits(skipped) + + if unknown: + output += u'* Unknown %s\n' % describe_commits(unknown) + self.log.error(u'Payload contains unknown %s', + describe_commits(unknown)) + + for line in output.splitlines(): + self.log.debug(line) + + req.send(output.encode('utf-8'), 'text/plain', 200 if output else 204) + + +def classify_commits(revs, repos, branches): + added, skipped, unknown = [], [], [] + for rev in revs: + try: + cset = repos.get_changeset(rev) + except NoSuchChangeset: + unknown.append(rev) + else: + if rev_in_branches(cset, branches): + added.append(rev) + else: + skipped.append(rev) + return added, skipped, unknown + + +def rev_in_branches(changeset, branches): + if not branches: # no branches filter configured + return True + return any(fnmatch.fnmatchcase(cset_branch, branch) + for cset_branch, _ in changeset.get_branches() for branch in branches) + + +def describe_commits(revs): + if len(revs) == 1: + return u'commit %s' % revs[0] + else: + return u'commits %s' % u', '.join(revs) + + +class GitHubIssueHook(GitHubMixin, Component): + implements(IRequestHandler) + + _request_re = re.compile(r"/github-issues/?$") + + # IRequestHandler method + def match_request(self, req): + match = self._request_re.match(req.path_info) + if match: + return True + + # IRequestHandler method + def process_request(self, req): + if req.method == 'GET': + req.send('Endpoint is ready to accept GitHub notifications.', 'text/plain', 200) + return + if req.method != 'POST': + msg = u'Method not allowed (%s)\n' % req.method + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 405) + return + + body = req.read() + + if not self.verify_signature(req, body): + msg = u'Invalid hook signature from %s, ignoring request.\n' % req.remote_addr + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 400) + return + + event = req.get_header('X-GitHub-Event') + if event == 'ping': + payload = json.loads(body) + req.send(payload['zen'].encode('utf-8'), 'text/plain', 200) + return + if event not in ['issue_comment', 'issues', 'pull_request', 'pull_request_review_comment']: + msg = u'Unsupported event recieved (%s), ignoring request.\n' % event + self.log.warning(msg.rstrip('\n')) + req.send(msg.encode('utf-8'), 'text/plain', 400) + return + + event_method = getattr(self, '_event_' + event) + event_method(req, json.loads(body)) + + def _event_issue_comment(self, req, data): + comment = data['comment'] + author = comment['user']['login'] + ' (GitHub)' + issue = data['repository']['full_name'] + '#' + str(data['issue']['number']) + issue = issue.encode('utf-8') + + ticket_id = self.find_ticket(issue) + + if data['action'] == 'created': + ticket = Ticket(self.env, ticket_id) + ticket.save_changes(author, comment['body']) + req.send('Comment added to ticket #%d.' % ticket_id, 'text/plain', 200) + + else: + req.send('No action taken for this hook.', 'text/plain', 200) + + def _event_issues(self, req, data): + author = data['sender']['login'] + ' (GitHub)' + issue = data['repository']['full_name'] + '#' + str(data['issue']['number']) + issue = issue.encode('utf-8') + + if data['action'] == 'opened': + ticket = Ticket(self.env) + ticket['reporter'] = author + ticket['summary'] = data['issue']['title'] + ticket['description'] = data['issue']['body'] + ticket['description'] += "\n\n!GitHub Issue: %s" % data['issue']['html_url'] + ticket['status'] = 'new' + ticket_id = ticket.insert() + + self.mark_github_issue(ticket_id, issue) + + req.send('Synced to new ticket #%d' % ticket_id, 'text/plain', 200) + + elif data['action'] == 'closed': + pass + + elif data['action'] == 'reopened': + pass + + else: + req.send('No action taken for this hook.', 'text/plain', 200) + + def _event_pull_request(self, req, data): + pull = data['pull_request'] + author = data['sender']['login'] + ' (GitHub)' + issue = data['repository']['full_name'] + '#' + str(pull['number']) + issue = issue.encode('utf-8') + + if data['action'] == 'opened': + ticket = Ticket(self.env) + ticket['reporter'] = author + ticket['summary'] = pull['title'] + ticket['description'] = pull['body'] + ticket['description'] += "\n\nPull Request: %s" % pull['html_url'] + ticket['status'] = 'new' + ticket_id = ticket.insert() + + self.mark_github_issue(ticket_id, issue) + + response = urllib2.urlopen(pull['patch_url']) + self.create_attachment(ticket_id, pull['number'], response.read(), author) + + req.send('Synced to new ticket #%d' % ticket_id, 'text/plain', 200) + + elif data['action'] == 'synchronize': + ticket_id = self.find_ticket(issue) + + if ticket_id: + response = urllib2.urlopen(pull['patch_url']) + self.create_attachment(ticket_id, pull['number'], response.read(), author) + req.send('Attached new patch to ticket #%d.' % ticket_id, 'text/plain', 200) + + # This happens if sync is turned on after existing PRs were opened. + req.send('No ticket to sync patch for issue %s.' % issue, 'text/plain', 200) + + elif data['action'] == 'closed': + pass + + elif data['action'] == 'reopened': + pass + + else: + req.send('No action taken for this hook.', 'text/plain', 200) + + def _event_pull_request_review_comment(self, req, data): + comment = data['comment'] + pull = data['pull_request'] + author = comment['user']['login'] + ' (GitHub)' + issue = data['repository']['full_name'] + '#' + str(pull['number']) + issue = issue.encode('utf-8') + + ticket_id = self.find_ticket(issue) + + if data['action'] == 'created': + ticket = Ticket(self.env, ticket_id) + ticket.save_changes(author, + "%s\n\n[%s See Inline Patch Context]" % + (comment['body'], comment['html_url']) + ) + req.send('Comment added to ticket #%d.' % ticket_id, 'text/plain', 200) + + else: + req.send('No action taken for this hook.', 'text/plain', 200) + + def mark_github_issue(self, ticket_id, github_issue): + """ + Manually save a custom field on the ticket (github_issue) with the + canonical GitHub address for the issue. + + We manually do this instead of instructing users to manually setup a + custom field since it's only necessary internally. End users can still + configure it if they want though. Hopefully, this also prevents ticket + split/copy plugins from duplicating the github_issue field since this + only supports syncing with one ticket. + + Several GitHub repos could be saving here, so don't only use number. + """ + + @with_transaction(self.env) + def sql_transaction(db): + cursor = db.cursor() + cursor.execute( + "DELETE FROM ticket_custom WHERE ticket = %d AND name = %%s" % + ticket_id, ['github_issue'] + ) + cursor.execute( + "INSERT INTO ticket_custom VALUES (%d, %%s, %%s)" % + ticket_id, ['github_issue', github_issue] + ) + + def find_ticket(self, github_issue): + """ + Return the ticket_id for the given GitHub issue if it exists. + """ + + rows = self.env.db_query( + "SELECT ticket FROM ticket_custom WHERE name = %s AND value = %s", + ['github_issue', github_issue] + ) + return int(rows[0][0]) if rows else None + + def create_attachment(self, ticket_id, pull_id, patch, author = None): + if len(patch) > self.env.config.get('attachment', 'max_size'): + self.log.warning('GitHub patch (#%d) too big to attach to ticket #%d' % (pull_id, ticket_id)) + return + + # Create a temp file object for Trac attachments. + temp_fd = os.tmpfile() + temp_fd.write(patch) + temp_fd.seek(0) + + attachment = Attachment(self.env, 'ticket', ticket_id) + if author: + attachment.author = author + filename = 'github-pull-%d.patch' % pull_id + attachment.insert(filename, temp_fd, len(patch))