Thread parse script

.github/workflows/main.yml

@@ -7,6 +7,12 @@ on:
     branches:
       - main
 
+# env settings for github releases
+# docker image push
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
   dependencies:
     runs-on: ubuntu-latest
@@ -162,44 +168,83 @@ jobs:
           AWS_REGION: ${{ secrets.AWS_REGION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  release:
-    needs: [test, static_code_analysis]
-    name: Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3.6.0
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v4.0.0
-        with:
-          branches: |
-            [
-              'main',
-              {
-                name: 'prerelease',
-                prerelease: true
-              },
-            ]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  elixir_docs:
-    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/elixir-docs' }}
-    name: Generate project documentation
+  # release:
+  #   needs: [test, static_code_analysis]
+  #   name: Release
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3.6.0
+  #     - name: Semantic Release
+  #       uses: cycjimmy/semantic-release-action@v4.0.0
+  #       with:
+  #         branches: |
+  #           [
+  #             'main',
+  #             {
+  #               name: 'prerelease',
+  #               prerelease: true
+  #             },
+  #           ]
+  #       env:
+  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # elixir_docs:
+  #   if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/elixir-docs' }}
+  #   name: Generate project documentation
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3.6.0
+  #     - name: Sets up an Erlang/OTP environment
+  #       uses: erlef/setup-beam@v1
+  #       with:
+  #         version-file: .tool-versions
+  #         version-type: strict
+  #     - name: Build docs
+  #       uses: lee-dohm/generate-elixir-docs@v1.0.1
+  #     - name: Publish to Pages
+  #       uses: peaceiris/actions-gh-pages@v3.9.3
+  #       with:
+  #         deploy_key: ${{ secrets.DOCS_DEPLOY_KEY }}
+  #         external_repository: epochtalk/server.epochtalk.github.io
+  #         publish_dir: ./doc
+  #         publish_branch: gh-pages
+
+  # build and push image to github container registry
+  build-and-push-image:
+    needs: [static_code_analysis, test]
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3.6.0
-      - name: Sets up an Erlang/OTP environment
-        uses: erlef/setup-beam@v1
-        with:
-          version-file: .tool-versions
-          version-type: strict
-      - name: Build docs
-        uses: lee-dohm/generate-elixir-docs@v1.0.1
-      - name: Publish to Pages
-        uses: peaceiris/actions-gh-pages@v3.9.3
-        with:
-          deploy_key: ${{ secrets.DOCS_DEPLOY_KEY }}
-          external_repository: epochtalk/server.epochtalk.github.io
-          publish_dir: ./doc
-          publish_branch: gh-pages
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.1
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3.3.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5.5.1
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v6.9.0
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+      # https://docs.github.com/en/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1.4.3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.gitignore

@@ -27,3 +27,6 @@ priv/repo/seeds/permissions.json
 
 # Ignore secret config files
 config/*.secret.exs
+
+#Ignore env file
+/.env

.tool-versions

@@ -1,2 +1,3 @@
 elixir 1.17.3-otp-27
 erlang 27.1.2
+php 8.3.7

Dockerfile

@@ -1,14 +1,21 @@
-FROM elixir:1.14.0
+FROM elixir:1.17.3
+# install php
+RUN curl -sSL https://packages.sury.org/php/README.txt | bash -x
+RUN apt update
+RUN apt install -y php8.3
+
 # work in /app instead of /
 RUN mkdir -p /app
 WORKDIR /app
 RUN mix local.hex --force
 RUN mix local.rebar --force
-ADD . .
-RUN mix deps.get
 
 # compile for production
 ENV MIX_ENV=prod
+COPY mix.exs .
+COPY mix.lock .
+RUN mix deps.get
+COPY . .
 RUN mix compile
 
 CMD until mix ecto.setup; do sleep 1; done

bbcode.php

@@ -0,0 +1,4 @@
+<?php
+    var_dump($argc); //number of arguments passed 
+    var_dump($argv); //the arguments passed
+?>

config/config.exs

@@ -10,6 +10,9 @@ import Config
 # Set ecto repos
 config :epochtalk_server, ecto_repos: [EpochtalkServer.Repo]
 
+# Configure Porcelain
+config :porcelain, driver: Porcelain.Driver.Basic
+
 # Set Guardian.DB to GuardianRedis
 config :guardian, Guardian.DB, repo: GuardianRedis.Repo
 

config/runtime.exs

@@ -404,3 +404,124 @@ if config_env() == :prod do
       port: get_env_cast_integer_with_default.("EMAILER_SMTP_PORT", "465")
   end
 end
+
+##### PROXY REPO CONFIGURATIONS #####
+
+bbc_parser_poolboy_config = [
+  name: {:local, :bbc_parser},
+  worker_module: EpochtalkServer.BBCParser,
+  size: 50,
+  max_overflow: 2,
+  strategy: :fifo
+]
+
+config :epochtalk_server, bbc_parser_poolboy_config: bbc_parser_poolboy_config
+
+# conditionally show debug logs in prod
+if config_env() == :prod do
+  logger_level =
+    System.get_env("LOGGER_LEVEL")
+    |> case do
+      "DEBUG" -> :debug
+      _ -> :info
+    end
+
+  config :logger, level: logger_level
+end
+
+# Configure proxy sequences and boards blacklist
+proxy_config =
+  case config_env() do
+    :prod ->
+      id_board_blacklist =
+        get_env_or_raise_with_message.(
+          "ID_BOARD_BLACKLIST",
+          "environment variable ID_BOARD_BLACKLIST is missing."
+        )
+
+      id_board_blacklist =
+        id_board_blacklist
+        |> String.split()
+        |> Enum.map(&String.to_integer(&1))
+
+      %{
+        threads_seq: System.get_env("THREADS_SEQ") || "6000000",
+        boards_seq: System.get_env("BOARDS_SEQ") || "500",
+        id_board_blacklist: id_board_blacklist
+      }
+
+    :dev ->
+      %{
+        threads_seq: "6000000",
+        boards_seq: "500",
+        id_board_blacklist: []
+      }
+
+    # default thread/board seq and empty blacklist
+    _ ->
+      %{
+        threads_seq: "-10",
+        boards_seq: "-10",
+        id_board_blacklist: []
+      }
+  end
+
+config :epochtalk_server, proxy_config: proxy_config
+
+# Configure SmfRepo for proxy
+# - do not configure for testing
+if config_env() != :test do
+  smf_repo_username =
+    get_env_or_raise_with_message.(
+      "SMF_REPO_USERNAME",
+      "environment variable SMF_REPO_USERNAME is missing."
+    )
+
+  smf_repo_password =
+    get_env_or_raise_with_message.(
+      "SMF_REPO_PASSWORD",
+      "environment variable SMF_REPO_PASSWORD is missing."
+    )
+
+  smf_repo_hostname =
+    get_env_or_raise_with_message.(
+      "SMF_REPO_HOSTNAME",
+      "environment variable SMF_REPO_HOSTNAME is missing."
+    )
+
+  smf_repo_database =
+    get_env_or_raise_with_message.(
+      "SMF_REPO_DATABASE",
+      "environment variable SMF_REPO_DATABASE is missing."
+    )
+
+  proxy_repo_config =
+    case config_env() do
+      :prod ->
+        [
+          username: smf_repo_username,
+          password: smf_repo_password,
+          hostname: smf_repo_hostname,
+          database: smf_repo_database,
+          port: get_env_cast_integer_with_default.("SMF_REPO_PORT", "3306"),
+          stacktrace: get_env_cast_bool_with_default.("SMF_REPO_STACKTRACE", "TRUE"),
+          show_sensitive_data_on_connection_error:
+            get_env_cast_bool_with_default.("SMF_REPO_SENSITIVE_DATA_ON_ERROR", "FALSE"),
+          pool_size: get_env_cast_integer_with_default.("SMF_REPO_POOL_SIZE", "10")
+        ]
+
+      _ ->
+        [
+          username: smf_repo_username,
+          password: smf_repo_password,
+          hostname: smf_repo_hostname,
+          database: smf_repo_database,
+          port: get_env_cast_integer_with_default.("SMF_REPO_PORT", "3306"),
+          stacktrace: true,
+          show_sensitive_data_on_connection_error: true,
+          pool_size: get_env_cast_integer_with_default.("SMF_REPO_POOL_SIZE", "10")
+        ]
+    end
+
+  config :epochtalk_server, EpochtalkServer.SmfRepo, proxy_repo_config
+end

lib/epochtalk_server/application.ex

@@ -19,6 +19,10 @@ defmodule EpochtalkServer.Application do
       {Redix, host: redix_config()[:host], name: redix_config()[:name]},
       # Start the Ecto repository
       EpochtalkServer.Repo,
+      # Start the Smf repository
+      EpochtalkServer.SmfRepo,
+      # Start the BBC Parser
+      :poolboy.child_spec(:bbc_parser, bbc_parser_poolboy_config()),
       # Start Role Cache
       EpochtalkServer.Cache.Role,
       # Warm frontend_config variable (referenced by api controllers)
@@ -38,15 +42,17 @@ defmodule EpochtalkServer.Application do
       # {EpochtalkServer.Worker, arg}
     ]
 
-    # don't run config_warmer during tests
+    # adjust supervised processes for testing
     children =
-      if Application.get_env(:epochtalk_server, :env) == :test,
-        do:
-          List.delete(
-            children,
-            {Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0}
-          ),
-        else: children
+      if Application.get_env(:epochtalk_server, :env) == :test do
+        children
+        # don't run config warmer during tests
+        |> List.delete({Task, &EpochtalkServer.Models.Configuration.warm_frontend_config/0})
+        # don't run SmfRepo during tests
+        |> List.delete(EpochtalkServer.SmfRepo)
+      else
+        children
+      end
 
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
@@ -64,4 +70,7 @@ defmodule EpochtalkServer.Application do
 
   # fetch redix config
   defp redix_config(), do: Application.get_env(:epochtalk_server, :redix)
+
+  defp bbc_parser_poolboy_config,
+    do: Application.get_env(:epochtalk_server, :bbc_parser_poolboy_config)
 end