[CI] Update loss_check (#11090) #291
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Disabled unittest CPU workflow - manually enabled when needed | ||
| name: Unittest CPU CI (DISABLED) | ||
| #on: [pull_request] | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | ||
| cancel-in-progress: true | ||
| env: | ||
| PR_ID: ${{ github.event.pull_request.number }} | ||
| COMMIT_ID: ${{ github.event.pull_request.head.sha }} | ||
| BRANCH: ${{ github.event.pull_request.base.ref }} | ||
| TASK: PaddleNLP-CI-${{ github.event.pull_request.number }}-unittest-cpu | ||
| HF_ENDPOINT: https://hf-mirror.com | ||
| STUDIO_GIT_HOST: http://git.prod.idc-to-cloud.aistudio.baidu-int.com | ||
| PPNLP_HOME: /home/disk1/cache | ||
| HF_DATASETS_CACHE: /home/disk1/cache/huggingface/datasets | ||
| TRANSFORMERS_CACHE: /home/disk1/cache/huggingface | ||
| #jobs: | ||
| # unittest-cpu-ci: | ||
| # name: unittest-cpu-ci (DISABLED) | ||
| runs-on: [self-hosted, ernie-cpu] | ||
| permissions: | ||
| pull-requests: write | ||
| contents: read | ||
| id-token: write | ||
| steps: | ||
| - name: Run Container | ||
| env: | ||
| work_dir: ${{ github.workspace }} | ||
| python_version: "3.10" | ||
| run: | | ||
| container_name=${TASK}-$(date +%Y%m%d-%H%M%S) | ||
| echo "container_name=${container_name}" >> ${{ github.env }} | ||
| docker_image="iregistry.baidu-int.com/paddlecloud/base-images:paddlecloud-ubuntu20.04-gcc12.2-cuda12.3-cudnn9.0-nccl2.20.3.1-openmpi4.1.5-latest" | ||
| docker run -d -t --name ${container_name} --net=host -v /dev/shm:/dev/shm --shm-size=32G \ | ||
| -v $work_dir/../../..:$work_dir/../../.. \ | ||
| -v $work_dir:/workspace \ | ||
| -v /home/.cache/pip:/home/.cache/pip \ | ||
| -v /home/disk1/cache:/home/disk1/cache \ | ||
| -e BRANCH \ | ||
| -e PR_ID \ | ||
| -e COMMIT_ID \ | ||
| -e work_dir \ | ||
| -e no_proxy \ | ||
| -e python_version \ | ||
| -e HF_ENDPOINT \ | ||
| -e STUDIO_GIT_HOST \ | ||
| -e PPNLP_HOME \ | ||
| -e HF_DATASETS_CACHE \ | ||
| -e TRANSFORMERS_CACHE \ | ||
| -w /workspace ${docker_image} | ||
| - name: Download Code | ||
| env: | ||
| work_dir: ${{ github.workspace }} | ||
| run: | | ||
| docker exec -t ${container_name} /bin/bash -c ' | ||
| rm -rf * .[^.]* | ||
| echo "Downloading PaddleNLP.tar" | ||
| wget -q --no-proxy https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddleNLP.tar --no-check-certificate | ||
| echo "Extracting PaddleNLP.tar" | ||
| tar xf PaddleNLP.tar && rm -rf PaddleNLP.tar | ||
| source $work_dir/../../../proxy | ||
| cd PaddleNLP | ||
| git config --global user.name "PaddleCI" | ||
| git config --global user.email "paddle_ci@example.com" | ||
| git pull | ||
| git submodule update --init --recursive --force | ||
| if [ -n "${PR_ID}" ]; then | ||
| git fetch origin pull/${PR_ID}/head | ||
| git checkout -b PR_${PR_ID} FETCH_HEAD | ||
| git remote add upstream https://github.com/PaddlePaddle/PaddleNLP.git | ||
| git fetch upstream ${BRANCH} | ||
| git merge ${BRANCH} --no-edit | ||
| git diff --numstat ${BRANCH} -- | awk "{print \$NF}" | ||
| else | ||
| echo "Not in a pull_request event. Skipping PR-specific operations." | ||
| fi | ||
| git log --pretty=oneline -10 | ||
| ' | ||
| - name: Setup Environment | ||
| run: | | ||
| docker exec -t $container_name /bin/bash -c ' | ||
| unlink /usr/bin/python3 | ||
| ln -sf $(which python${python_version}) /usr/bin/python3 | ||
| set -e | ||
| python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | ||
| python -m pip config set global.cache-dir "/home/.cache/pip" | ||
| source $work_dir/../../../proxy | ||
| python -m pip install --upgrade pip | ||
| cd /workspace/PaddleNLP && git config --global --add safe.directory $PWD | ||
| pip install -r tests/requirements.txt | ||
| make install | ||
| ' | ||
| - name: Test | ||
| run: | | ||
| docker exec -t $container_name /bin/bash -c ' | ||
| source $work_dir/../../../proxy | ||
| cd /workspace/PaddleNLP | ||
| set -e | ||
| make test | ||
| ' | ||
| # - name: Upload Coverage To Codecov | ||
| # uses: codecov/codecov-action@v3 | ||
| # env: | ||
| # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | ||
| # with: | ||
| # token: ${{ secrets.CODECOV_TOKEN }} | ||
| # files: ./PaddleNLP/coverage.xml | ||
| - name: Terminate And Delete the Container | ||
| if: always() | ||
| run: | | ||
| docker rm -f $container_name 2>/dev/null || true | ||
