Include pull request reviews in database

This extends github-sync to also store pull request reviews in the database.

Author: MaineC

db/lib/issueStoreLibrary.rb

@@ -61,7 +61,24 @@ def db_insert_comments(db, comments, org, repo)
     end
   end
 
-  def db_getMaxCommentTimestampForRepo(db, org, repo)
+  # Inserts new comments. If any exist already, it replaces them.
+  def db_insert_pr_reviews(db, comments, org, repo)
+    comments.each do |comment|
+        db["DELETE FROM item_comments WHERE id=?", comment.id].delete
+        # eg: https://github.com/amzn/oss-dashboard/pull/1#discussion_r207199796
+        itemNumber=comment.html_url.sub(/^.*\/([0-9]*)#discussion_r[0-9]*$/, '\1')
+        user=comment.user ? comment.user.login : nil
+        db[
+         "INSERT INTO item_comments (
+               id, org, repo, item_number, user_login, body, created_at, updated_at
+          )
+          VALUES ( ?, ?, ?, ?, ?, ?, ?, ? )",
+          comment.id, org, repo, itemNumber, user, comment.body, gh_to_db_timestamp(comment.created_at),
+          gh_to_db_timestamp(comment.updated_at)].insert
+    end
+ end
+
+ def db_getMaxCommentTimestampForRepo(db, org, repo)
     # Normally '2015-04-18 14:17:02 UTC'
     # Need '2015-04-18T14:17:02Z'
     db["select max(updated_at) from item_comments where org='#{org}' and repo='#{repo}'"].each do |row|

docs/Running.md

@@ -35,6 +35,7 @@ Available phases are:
 |  github-sync/events | Syncs only the event stream |
 |  github-sync/issues | Syncs the issue data - note that this is typically the heaviest initial load |
 |  github-sync/issue-comments | Syncs the issue comments |
+|  github-sync/pr-reviews | Syncs the pr reviews |
 |  github-sync/releases | Syncs the release data |
 |  github-sync/user-mapping | Loads your user-mapping file into the database |
 |  github-sync/reporting | Runs the configured DB Reports |

github-sync/db_issues/sync-issues.rb

@@ -157,6 +157,27 @@ def getLatestIssueComments(context, issue_db, org, repos)
   end
 end
 
+def getLatestPRReviews(context, issue_db, org, repos)
+  repos.each do |repo_obj|
+    begin
+      issue_db.transaction do
+        # Get the current max timestamp in the db
+        maxTimestamp=db_getMaxCommentTimestampForRepo(issue_db, org, repo_obj.name)
+        if(maxTimestamp)
+          # Increment the timestamp by a second to avoid getting repeats
+          ts=DateTime.strptime(maxTimestamp, '%Y-%m-%dT%H:%M:%S') + Rational(1, 60 * 60 * 24)
+          comments=context.client.pull_requests_comments(repo_obj.full_name, { 'since' => ts } )
+        else
+          comments=context.client.pull_requests_comments(repo_obj.full_name)
+        end
+        db_insert_pr_reviews(issue_db, comments, org, repo_obj.name)
+      end
+      context.feedback.print '.'
+    end
+  end
+end
+
+
 def sync_issue_comments(context, sync_db)
 
   owners = context.dashboard_config['organizations+logins']
@@ -171,3 +192,17 @@ def sync_issue_comments(context, sync_db)
   end
 
 end
+
+def sync_pr_reviews(context, sync_db)
+  owners = context.dashboard_config['organizations+logins']
+  context.feedback.puts " pull-request-reviews"
+
+  owners.each do |org|
+    repos=context.repositories(org)
+
+    context.feedback.print "  #{org} "
+    getLatestPRReviews(context, sync_db, org, repos)
+    context.feedback.print "\n"
+  end
+
+end

github-sync/sync.rb

@@ -69,6 +69,10 @@ def github_sync(context, run_one)
     # TODO: Move to being based on the async code
     sync_issue_comments(context, sync_db)
   end
+  if(not(run_one) or run_one=='github-sync/pr-reviews')
+    # TODO: Implement async code
+    sync_pr_reviews(context, sync_db)
+  end
   if(not(run_one) or run_one=='github-sync/releases')
     # TODO: Move to being based on the async code
     sync_releases(context, sync_db)

refresh-dashboard.rb

@@ -29,8 +29,8 @@
 
   def xml2html(context, xmlFiles, xsltFile, outputDirectory)
     Dir.glob(xmlFiles).each do |inputFile|
-      stylesheet = LibXSLT::XSLT::Stylesheet.new( LibXML::XML::Document.file(xsltFile))
       xml_doc = LibXML::XML::Document.file(inputFile)
+      stylesheet = LibXSLT::XSLT::Stylesheet.new( LibXML::XML::Document.file(xsltFile))
       html = stylesheet.apply(xml_doc)
 
       outputFile=File.basename(inputFile, ".xml")
@@ -136,7 +136,7 @@ def xml2html(context, xmlFiles, xsltFile, outputDirectory)
 
 # TODO: Implement github-sync and generate-dashboard as aliases?
 allPhases=['init-database', 'github-sync', 'pull-source', 'review-source', 'generate-dashboard']
-legitPhases=['init-database', 'github-sync/metadata', 'github-sync/commits', 'github-sync/events', 'github-sync/issues', 'github-sync/issue-comments', 'github-sync/releases', 'github-sync/traffic', 'github-sync/user-mapping', 'github-sync/reporting', 'pull-source', 'review-source', 'generate-dashboard/xml', 'generate-dashboard/json-data', 'generate-dashboard/dashboards', 'generate-dashboard/xslt', 'github-sync', 'generate-dashboard']
+legitPhases=['init-database', 'github-sync/metadata', 'github-sync/commits', 'github-sync/events', 'github-sync/issues', 'github-sync/issue-comments', 'github-sync/pr-reviews', 'github-sync/releases', 'github-sync/traffic', 'github-sync/user-mapping', 'github-sync/reporting', 'pull-source', 'review-source', 'generate-dashboard/xml', 'generate-dashboard/json-data', 'generate-dashboard/dashboards', 'generate-dashboard/xslt', 'github-sync', 'generate-dashboard']
 
 if(ARGV[1])
   run_list=ARGV[1..-1]