From 687004cda669a538b3acac37ebbc345da52197df Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Wed, 19 Sep 2018 11:07:36 +0200 Subject: [PATCH] Include pull request reviews in database This extends github-sync to also store pull request reviews in the database. Review comments are being stored in the same table as regular issue comments. This change comes with support for both, sync and async Github synchronisation. Retrieval and storage of review specific comments has been folded into the github-sync/issue-comments phase - with both, regular comments and review comments in the same table, this makes timestamp handling easier. --- db/lib/issueStoreLibrary.rb | 19 ++++++++++++++++++- github-sync-tng/issue_comment_command.rb | 3 +++ github-sync/db_issues/sync-issues.rb | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/db/lib/issueStoreLibrary.rb b/db/lib/issueStoreLibrary.rb index febf055..540e63a 100755 --- a/db/lib/issueStoreLibrary.rb +++ b/db/lib/issueStoreLibrary.rb @@ -61,7 +61,24 @@ def db_insert_comments(db, comments, org, repo) end end - def db_getMaxCommentTimestampForRepo(db, org, repo) + # Inserts new comments. If any exist already, it replaces them. + def db_insert_pr_reviews(db, comments, org, repo) + comments.each do |comment| + db["DELETE FROM item_comments WHERE id=?", comment.id].delete + # eg: https://github.com/amzn/oss-dashboard/pull/1#discussion_r207199796 + itemNumber=comment.html_url.sub(/^.*\/([0-9]*)#discussion_r[0-9]*$/, '\1') + user=comment.user ? comment.user.login : nil + db[ + "INSERT INTO item_comments ( + id, org, repo, item_number, user_login, body, created_at, updated_at + ) + VALUES ( ?, ?, ?, ?, ?, ?, ?, ? )", + comment.id, org, repo, itemNumber, user, comment.body, gh_to_db_timestamp(comment.created_at), + gh_to_db_timestamp(comment.updated_at)].insert + end + end + + def db_getMaxCommentTimestampForRepo(db, org, repo) # Normally '2015-04-18 14:17:02 UTC' # Need '2015-04-18T14:17:02Z' db["select max(updated_at) from item_comments where org='#{org}' and repo='#{repo}'"].each do |row| diff --git a/github-sync-tng/issue_comment_command.rb b/github-sync-tng/issue_comment_command.rb index 249495c..c54e8af 100644 --- a/github-sync-tng/issue_comment_command.rb +++ b/github-sync-tng/issue_comment_command.rb @@ -60,10 +60,13 @@ def sync_item_comments(context, issue_db, org, repo) # Increment the timestamp by a second to avoid getting repeats ts=DateTime.strptime(maxTimestamp, '%Y-%m-%dT%H:%M:%S') + Rational(1, 60 * 60 * 24) comments=context.client.issues_comments(orgrepo, { 'since' => ts } ) + pr_reviews=context.client.pull_requests_comments(orgrepo, { 'since' => ts } ) else comments=context.client.issues_comments(orgrepo) + pr_reviews=context.client.pull_requests_comments(orgrepo) end db_insert_comments(issue_db, comments, org, repo) + db_insert_pr_reviews(issue_db, pr_reviews, org, repo) end end diff --git a/github-sync/db_issues/sync-issues.rb b/github-sync/db_issues/sync-issues.rb index 4132070..7ca8e29 100755 --- a/github-sync/db_issues/sync-issues.rb +++ b/github-sync/db_issues/sync-issues.rb @@ -147,10 +147,13 @@ def getLatestIssueComments(context, issue_db, org, repos) # Increment the timestamp by a second to avoid getting repeats ts=DateTime.strptime(maxTimestamp, '%Y-%m-%dT%H:%M:%S') + Rational(1, 60 * 60 * 24) comments=context.client.issues_comments(repo_obj.full_name, { 'since' => ts } ) + pr_reviews=context.client.pull_requests_comments(repo_obj.full_name, { 'since' => ts } ) else comments=context.client.issues_comments(repo_obj.full_name) + pr_reviews=context.client.pull_requests_comments(repo_obj.full_name) end db_insert_comments(issue_db, comments, org, repo_obj.name) + db_insert_pr_reviews(issue_db, pr_reviews, org, repo_obj.name) end context.feedback.print '.' end @@ -171,3 +174,4 @@ def sync_issue_comments(context, sync_db) end end +