make a working gem

.github/workflows/acceptance.yml

@@ -24,4 +24,11 @@ jobs:
         with:
           bundler-cache: true
 
-      # TODO
+      - name: bootstrap
+        run: script/bootstrap
+
+      - name: acceptance
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ENV: acceptance
+        run: script/acceptance

.ruby-version

@@ -1 +1 @@
-3.3.0
+3.3.3

README.md

@@ -6,4 +6,4 @@
 [![acceptance](https://github.com/runwaylab/issue-db/actions/workflows/acceptance.yml/badge.svg)](https://github.com/runwaylab/issue-db/actions/workflows/acceptance.yml)
 [![release](https://github.com/runwaylab/issue-db/actions/workflows/release.yml/badge.svg)](https://github.com/runwaylab/issue-db/actions/workflows/release.yml)
 
-Use GitHub Issues as a NoSQL JSON document db
+A Ruby Gem to use GitHub Issues as a NoSQL JSON document db

issue-db.gemspec

@@ -4,7 +4,7 @@ require_relative "lib/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "issue-db"
-  spec.version       = IssueDB::VERSION
+  spec.version       = Version::VERSION
   spec.authors       = ["runwaylab", "GrantBirki"]
   spec.license       = "MIT"
 

lib/issue_db.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "redacting_logger"
+
+require_relative "version"
+require_relative "issue_db/utils/retry"
+require_relative "issue_db/utils/init"
+require_relative "issue_db/authentication"
+require_relative "issue_db/models/repository"
+require_relative "issue_db/database"
+
+class IssueDB
+  include Version
+  include Authentication
+  include Init
+
+  attr_reader :log
+  attr_reader :version
+
+  # Create a new IssueDB object
+  # :param repo: The GitHub repository to use as the datastore (org/repo format) [required]
+  # :param log: An optional logger - created for you by default
+  # :param octokit_client: An optional pre-hydrated Octokit::Client object
+  # :param label: The label to use for issues managed in the datastore by this library
+  # :param cache_expiry: The number of seconds to cache issues in memory (default: 60)
+  # :param init: Whether or not to initialize the database on object creation (default: true) - idempotent
+  # :return: A new IssueDB object
+  def initialize(repo, log: nil, octokit_client: nil, label: nil, cache_expiry: nil, init: true)
+    @log = log || RedactingLogger.new($stdout, level: ENV.fetch("LOG_LEVEL", "INFO").upcase)
+    Retry.setup!(log: @log)
+    @version = VERSION
+    @client = Authentication.login(octokit_client)
+    @repo = Repository.new(repo)
+    @label = label || ENV.fetch("ISSUE_DB_LABEL", "issue-db")
+    @cache_expiry = cache_expiry || ENV.fetch("ISSUE_DB_CACHE_EXPIRY", 60).to_i
+    init! if init
+  end
+
+  def create(key, data, options = {})
+    db.create(key, data, options)
+  end
+
+  def read(key, options = {})
+    db.read(key)
+  end
+
+  def update(key, data, options = {})
+    db.update(key, data, options)
+  end
+
+  def delete(key, options = {})
+    db.delete(key, options)
+  end
+
+  def list(options = {})
+    db.list(options)
+  end
+
+  def list_keys(options = {})
+    db.list_keys(options)
+  end
+
+  def refresh!
+    db.refresh!
+  end
+
+  protected
+
+  def db
+    @db ||= Database.new(@log, @client, @repo, @label, @cache_expiry)
+  end
+end

lib/issue_db/authentication.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "octokit"
+
+class AuthenticationError < StandardError; end
+
+module Authentication
+  def self.login(client = nil)
+    # if the client is not nil, use the pre-provided client
+    return client unless client.nil?
+
+    # if the client is nil, check for GitHub App env vars
+    # TODO
+
+    # if the client is nil and no GitHub App env vars were found, check for the GITHUB_TOKEN
+    token = ENV.fetch("GITHUB_TOKEN", nil)
+    if token
+      octokit = Octokit::Client.new(access_token: token, page_size: 100)
+      octokit.auto_paginate = true
+      return octokit
+    end
+
+    # if we make it here, no valid auth method succeeded
+    raise AuthenticationError, "No valid GitHub authentication method was provided"
+  end
+end

lib/issue_db/cache.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Cache
+  # A helper method to update all issues in the cache
+  # :return: The updated issue cache as a list of issues
+  def update_issue_cache!
+    @log.debug("updating issue cache")
+
+    # find all issues in the repo that were created by this library
+    query = "repo:#{@repo.full_name} label:#{@label}"
+
+    search_response = nil
+    begin
+      Retryable.with_context(:default) do
+        wait_for_rate_limit!(:search) # specifically wait for the search rate limit as it is much lower
+
+        begin
+          # issues structure: { "total_count": 0, "incomplete_results": false, "items": [<issues>] }
+          search_response = @client.search_issues(query)
+        rescue StandardError => e
+          # re-raise the error but if its a secondary rate limit error, just sleep for minute (oof)
+          sleep(60) if e.message.include?("exceeded a secondary rate limit")
+          raise e
+        end
+      end
+    rescue StandardError => e
+      retry_err_msg = "error search_issues() call: #{e.message} - ran out of retries"
+      @log.error(retry_err_msg)
+      raise retry_err_msg
+    end
+
+    @log.debug("issue cache updated - cached #{search_response.total_count} issues")
+    @issues = search_response.items
+    @issues_last_updated = Time.now
+    return @issues
+  end
+end

lib/issue_db/database.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require_relative "cache"
+require_relative "utils/throttle"
+require_relative "models/record"
+require_relative "utils/generate"
+
+class RecordNotFound < StandardError; end
+
+class Database
+  include Cache
+  include Throttle
+  include Generate
+
+  # :param: log [Logger] a logger object to use for logging
+  # :param: client [Octokit::Client] an Octokit::Client object to use for interacting with the GitHub API
+  # :param: repo [Repository] a Repository object that represents the GitHub repository to use as the datastore
+  # :param: label [String] the label to use for issues managed in the datastore by this library
+  # :param: cache_expiry [Integer] the number of seconds to cache issues in memory (default: 60)
+  # :return: A new Database object
+  def initialize(log, client, repo, label, cache_expiry)
+    @log = log
+    @client = client
+    @repo = repo
+    @label = label
+    @cache_expiry = cache_expiry
+    @rate_limit_all = nil
+    @issues = nil
+    @issues_last_updated = nil
+  end
+
+  # Create a new issue/record in the database
+  # This will return the newly created issue as a Record object (parsed)
+  # :param: key [String] the key (issue title) to create
+  # :param: data [Hash] the data to use for the issue body
+  # :param: options [Hash] a hash of options containing extra data such as body_before and body_after
+  # :return: The newly created issue as a Record object
+  # usage example:
+  # data = { color: "blue", cool: true, popularity: 100, tags: ["tag1", "tag2"] }
+  # options = { body_before: "some text before the data", body_after: "some text after the data", include_closed: true }
+  # db.create("event123", {cool: true, data: "here"}, options)
+  def create(key, data, options = {})
+    @log.debug("attempting to create: #{key}")
+    issue = find_issue_by_key(key, options, create_mode: true)
+    if issue
+      @log.warn("skipping issue creation and returning existing issue - an issue already exists with the key: #{key}")
+      return Record.new(issue)
+    end
+
+    # if we make it here, no existing issues were found so we can safely create one
+
+    body = generate(data, body_before: options[:body_before], body_after: options[:body_after])
+
+    # if we make it here, no existing issues were found so we can safely create one
+    issue = Retryable.with_context(:default) do
+      wait_for_rate_limit!
+      @client.create_issue(@repo.full_name, key, body, { labels: @label })
+    end
+
+    # append the newly created issue to the issues cache
+    @issues << issue
+
+    @log.debug("issue created: #{key}")
+    return Record.new(issue)
+  end
+
+  # Read an issue/record from the database
+  # This will return the issue as a Record object (parsed)
+  # :param: key [String] the key (issue title) to read
+  # :param: options [Hash] a hash of options to pass through to the search method
+  # :return: The issue as a Record object
+  def read(key, options = {})
+    @log.debug("attempting to read: #{key}")
+    issue = find_issue_by_key(key, options)
+    @log.debug("issue found: #{key}")
+    return Record.new(issue)
+  end
+
+  # Update an issue/record in the database
+  # This will return the updated issue as a Record object (parsed)
+  # :param: key [String] the key (issue title) to update
+  # :param: data [Hash] the data to use for the issue body
+  # :param: options [Hash] a hash of options containing extra data such as body_before and body_after
+  # :return: The updated issue as a Record object
+  # usage example:
+  # data = { color: "blue", cool: true, popularity: 100, tags: ["tag1", "tag2"] }
+  # options = { body_before: "some text before the data", body_after: "some text after the data", include_closed: true }
+  # db.update("event123", {cool: true, data: "here"}, options)
+  def update(key, data, options = {})
+    @log.debug("attempting to update: #{key}")
+    issue = find_issue_by_key(key, options)
+
+    body = generate(data, body_before: options[:body_before], body_after: options[:body_after])
+
+    updated_issue = Retryable.with_context(:default) do
+      wait_for_rate_limit!
+      @client.update_issue(@repo.full_name, issue.number, key, body)
+    end
+
+    # update the issue in the cache using the reference we have
+    @issues[@issues.index(issue)] = updated_issue
+
+    @log.debug("issue updated: #{key}")
+    return Record.new(updated_issue)
+  end
+
+  # Delete an issue/record from the database - in this context, "delete" means to close the issue as "completed"
+  # :param: key [String] the key (issue title) to delete
+  # :param: options [Hash] a hash of options to pass through to the search method
+  # :return: The deleted issue as a Record object (parsed) - it may contain useful data
+  def delete(key, options = {})
+    @log.debug("attempting to delete: #{key}")
+    issue = find_issue_by_key(key, options)
+
+    deleted_issue = Retryable.with_context(:default) do
+      wait_for_rate_limit!
+      @client.close_issue(@repo.full_name, issue.number)
+    end
+
+    # remove the issue from the cache
+    @issues.delete(issue)
+
+    # return the deleted issue as a Record object as it may contain useful data
+    return Record.new(deleted_issue)
+  end
+
+  # List all keys in the database
+  # This will return an array of strings that represent the issue titles that are "keys" in the database
+  # :param: options [Hash] a hash of options to pass through to the search method
+  # :return: An array of strings that represent the issue titles that are "keys" in the database
+  # usage example:
+  # options = {include_closed: true}
+  # keys = db.list_keys(options)
+  def list_keys(options = {})
+    keys = issues.select do |issue|
+      options[:include_closed] || issue[:state] == "open"
+    end.map do |issue|
+      issue[:title]
+    end
+
+    return keys
+  end
+
+  # List all issues/record in the database as Record objects (parsed)
+  # This will return an array of Record objects that represent the issues in the database
+  # :param: options [Hash] a hash of options to pass through to the search method
+  # :return: An array of Record objects that represent the issues in the database
+  # usage example:
+  # options = {include_closed: true}
+  # records = db.list(options)
+  def list(options = {})
+    records = issues.select do |issue|
+      options[:include_closed] || issue[:state] == "open"
+    end.map do |issue|
+      Record.new(issue)
+    end
+
+    return records
+  end
+
+  # Force a refresh of the issues cache
+  # This will update the issues cache with the latest issues from the repo
+  # :return: The updated issue cache as a list of issues (Hash objects not parsed)
+  def refresh!
+    update_issue_cache!
+  end
+
+  protected
+
+  def not_found!(key)
+    raise RecordNotFound, "no record found for key: #{key}"
+  end
+
+  # A helper method to search through the issues cache and return the first issue that matches the given key
+  # :param: key [String] the key (issue title) to search for
+  # :param: options [Hash] a hash of options to pass through to the search method
+  # :param: create_mode [Boolean] a flag to indicate whether or not we are in create mode
+  # :return: A direct reference to the issue as a Hash object if found, otherwise throws a RecordNotFound error
+  # ... unless create_mode is true, in which case it returns nil as a signal to proceed with creating the issue
+  def find_issue_by_key(key, options = {}, create_mode: false)
+    issue = issues.find do |issue|
+      issue[:title] == key && (options[:include_closed] || issue[:state] == "open")
+    end
+
+    if issue.nil?
+      @log.debug("no issue found in cache for: #{key}")
+      return nil if create_mode
+
+      not_found!(key)
+    end
+
+    @log.debug("issue found in cache for: #{key}")
+    return issue
+  end
+
+  # A helper method to fetch all issues from the repo and update the issues cache
+  # It is cache aware
+  def issues
+    # update the issues cache if it is nil
+    update_issue_cache! if @issues.nil?
+
+    # update the cache if it has expired
+    issues_cache_expired = (Time.now - @issues_last_updated) > @cache_expiry
+    if issues_cache_expired
+      @log.debug("issue cache expired - last updated: #{@issues_last_updated} - refreshing now")
+      update_issue_cache!
+    end
+
+    return @issues
+  end
+end