From dca7c3f86ff70a6a410fc86ad9961aa21512702a Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Mon, 6 May 2024 17:24:53 +1000 Subject: [PATCH] Reworked CI builds for `master`/`develop`. (#23182) --- .github/workflows/ci_build_major_branch.yml | 122 ++++++++++++ .../ci_build_major_branch_keymap.yml | 181 ++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 .github/workflows/ci_build_major_branch.yml create mode 100644 .github/workflows/ci_build_major_branch_keymap.yml diff --git a/.github/workflows/ci_build_major_branch.yml b/.github/workflows/ci_build_major_branch.yml new file mode 100644 index 0000000000..608e266ce4 --- /dev/null +++ b/.github/workflows/ci_build_major_branch.yml @@ -0,0 +1,122 @@ +name: CI Build Major Branch + +permissions: + contents: read + actions: write + +on: + push: + branches: [master, develop] + workflow_dispatch: + inputs: + branch: + type: choice + description: "Branch to build" + options: [master, develop] + +env: + # https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits + # We've decreased it from 20 to 15 to allow for other GHA to run unimpeded + CONCURRENT_JOBS: 15 + +# Ensure we only have one build running at a time, cancelling any active builds if a new commit is pushed to the respective branch +concurrency: + group: ci_build-${{ github.event.inputs.branch || github.ref_name }} + cancel-in-progress: true + +jobs: + determine_concurrency: + name: "Determine concurrency" + if: github.repository == 'qmk/qmk_firmware' + runs-on: ubuntu-latest + container: ghcr.io/qmk/qmk_cli + + outputs: + slice_length: ${{ steps.generate_slice_length.outputs.slice_length }} + + steps: + - name: Install prerequisites + run: | + apt-get update + apt-get install -y jq + + - name: Disable safe.directory check + run: | + git config --global --add safe.directory '*' + + - name: Checkout QMK Firmware + uses: actions/checkout@v4 + + - name: Determine concurrency + id: generate_slice_length + run: | + target_count=$( { + qmk find -km default 2>/dev/null + qmk find -km via 2>/dev/null + } | sort | uniq | wc -l) + slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution as we're splitting default and via + echo "slice_length=$slice_length" >> $GITHUB_OUTPUT + + build_targets: + name: "Compile keymap ${{ matrix.keymap }}" + needs: determine_concurrency + strategy: + fail-fast: false + matrix: + keymap: [default, via] + uses: ./.github/workflows/ci_build_major_branch_keymap.yml + with: + branch: ${{ inputs.branch || github.ref_name }} + keymap: ${{ matrix.keymap }} + slice_length: ${{ needs.determine_concurrency.outputs.slice_length }} + + rollup_tasks: + name: "Housekeeping" + needs: build_targets + runs-on: ubuntu-latest + + steps: + - name: Download firmwares + uses: actions/download-artifact@v4 + with: + pattern: firmware-* + path: firmwares + merge-multiple: true + + - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }} + uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks --delete + env: + AWS_S3_BUCKET: qmk-ci + AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }} + AWS_REGION: nyc3 + AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com + SOURCE_DIR: firmwares + DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }} + + - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest + uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks --delete + env: + AWS_S3_BUCKET: qmk-ci + AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }} + AWS_REGION: nyc3 + AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com + SOURCE_DIR: firmwares + DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest + + - name: Check if failure marker file exists + id: check_failure_marker + uses: andstor/file-existence-action@v3 + with: + files: firmwares/.failed + + - name: Fail build if needed + if: steps.check_failure_marker.outputs.exists == 'true' + run: | + # Exit with failure if the compilation stage failed + exit 1 diff --git a/.github/workflows/ci_build_major_branch_keymap.yml b/.github/workflows/ci_build_major_branch_keymap.yml new file mode 100644 index 0000000000..f722f9f106 --- /dev/null +++ b/.github/workflows/ci_build_major_branch_keymap.yml @@ -0,0 +1,181 @@ +name: CI Build Major Branch Keymap + +permissions: + contents: read + actions: write + +on: + workflow_call: + inputs: + branch: + type: string + required: true + keymap: + type: string + required: true + slice_length: + type: string + required: true + +jobs: + generate_targets: + name: "Generate targets (${{ inputs.keymap }})" + runs-on: ubuntu-latest + container: ghcr.io/qmk/qmk_cli + + outputs: + targets: ${{ steps.generate_targets.outputs.targets }} + + steps: + - name: Install prerequisites + run: | + apt-get update + apt-get install -y jq + + - name: Disable safe.directory check + run: | + git config --global --add safe.directory '*' + + - name: Checkout QMK Firmware + uses: actions/checkout@v4 + + - name: Generate build targets + id: generate_targets + run: | + { # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer + counter=0 + echo -n '{' + qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do + if [ $counter -gt 0 ]; then + echo -n ',' + fi + counter=$((counter+1)) + printf "\"group %02d\":{" $counter + echo -n '"targets":"' + echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n + echo -n '"}' + done + echo -n '}' + } | sed -e 's@\n@@g' > targets.json + + # Output the target keys as a variable + echo "targets=$(jq -c 'keys' targets.json)" >> $GITHUB_OUTPUT + + - name: Upload targets json + uses: actions/upload-artifact@v4 + with: + name: targets-${{ inputs.keymap }} + path: targets.json + + build_targets: + name: "Compile ${{ matrix.target }} (${{ inputs.keymap }})" + needs: generate_targets + runs-on: ubuntu-latest + container: ghcr.io/qmk/qmk_cli + continue-on-error: true + + strategy: + matrix: + target: ${{ fromJson(needs.generate_targets.outputs.targets) }} + + steps: + - name: Install prerequisites + run: | + apt-get update + apt-get install -y jq + + - name: Disable safe.directory check + run: | + git config --global --add safe.directory '*' + + - name: Checkout QMK Firmware + uses: actions/checkout@v4 + + - name: Get target definitions + uses: actions/download-artifact@v4 + with: + name: targets-${{ inputs.keymap }} + path: . + + - name: Deploy submodules + run: | + qmk git-submodule -f + + - name: Dump targets + run: | + jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort + + - name: Build targets + continue-on-error: true + run: | + export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null) + qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $(jq -r '.["${{ matrix.target }}"].targets' targets.json) || touch .failed + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: firmware-${{ inputs.keymap }}-${{ matrix.target }} + if-no-files-found: ignore + path: | + *.bin + *.hex + *.uf2 + .build/failed.* + .failed + + - name: Fail build if any group failed + run: | + # Exit with failure if the compilation stage failed + [ ! -f .failed ] || exit 1 + + repack_firmware: + if: always() + name: "Repack artifacts" + needs: build_targets + runs-on: ubuntu-latest + + steps: + - name: Checkout QMK Firmware + uses: actions/checkout@v4 + + - name: Download firmwares + uses: actions/download-artifact@v4 + with: + pattern: firmware-${{ inputs.keymap }}-* + path: . + merge-multiple: true + + - name: Upload all firmwares + uses: actions/upload-artifact@v4 + with: + name: firmware-${{ inputs.keymap }} + if-no-files-found: ignore + path: | + *.bin + *.hex + *.uf2 + .build/failed.* + .failed + + - name: Generate output logs + run: | + # Generate the step summary markdown + ./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true + # Truncate to a maximum of 1MB to deal with GitHub workflow limit + truncate --size='<960K' $GITHUB_STEP_SUMMARY || true + + - name: Delete temporary build artifacts + uses: geekyeggo/delete-artifact@v4 + with: + name: | + firmware-${{ inputs.keymap }}-* + targets-${{ inputs.keymap }} + + - name: 'CI Discord Notification' + if: always() + working-directory: util/ci/ + env: + DISCORD_WEBHOOK: ${{ secrets.CI_DISCORD_WEBHOOK }} + run: | + python3 -m pip install -r requirements.txt + python3 ./discord-results.py --branch ${{ inputs.branch || github.ref_name }} --keymap ${{ inputs.keymap }} --url ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}