feat: adding workflow file to run notebooks #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Haystack Cookbooks | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: '0 0 */14 * *' # Every 14 days at midnight UTC | |
| workflow_dispatch: | |
| jobs: | |
| generate-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.generator.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - id: generator | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Get cookbooks | |
| pip install requests | |
| NOTEBOOKS=$(python ./scripts/generate_matrix.py --include-main) | |
| echo "matrix={\"include\":$NOTEBOOKS}" >> "$GITHUB_OUTPUT" | |
| run-notebooks: | |
| runs-on: ubuntu-latest | |
| needs: generate-matrix | |
| container: deepset/haystack:${{ matrix.haystack_version }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 3 | |
| matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} | |
| env: | |
| HAYSTACK_TELEMETRY_ENABLED: "False" | |
| # HF_API_TOKEN: ${{ secrets.HF_API_KEY }} | |
| # Note: HF_API_TOKEN needs to be unset for notebooks using sentence-transformers | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Dump matrix content | |
| run: echo '${{ toJSON(matrix) }}' | |
| - name: Install common dependencies | |
| run: | | |
| apt-get update && apt-get install -y \ | |
| build-essential \ | |
| gcc \ | |
| libsndfile1 \ | |
| ffmpeg | |
| pip install nbconvert ipython | |
| - name: Install dependencies | |
| if: toJSON(matrix.dependencies) != '[]' | |
| run: | | |
| pip install "${{ join(matrix.dependencies, '" "')}}" | |
| - name: Convert notebook to Python | |
| run: | | |
| jupyter nbconvert --to python --RegexRemovePreprocessor.patterns '%%bash' ./notebooks/${{ matrix.notebook }}.ipynb | |
| - name: Run the converted notebook | |
| run: | | |
| python ./notebooks/${{ matrix.notebook }}.py |