Reworked CI builds for `master`/`develop`. (#23182)
Nick Brassel
GitHub
parent
7fc552632e
commit
dca7c3f86f
|
|
@ -0,0 +1,122 @@
|
||||||
|
name: CI Build Major Branch
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
actions: write
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [master, develop]
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
branch:
|
||||||
|
type: choice
|
||||||
|
description: "Branch to build"
|
||||||
|
options: [master, develop]
|
||||||
|
|
||||||
|
env:
|
||||||
|
# https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits
|
||||||
|
# We've decreased it from 20 to 15 to allow for other GHA to run unimpeded
|
||||||
|
CONCURRENT_JOBS: 15
|
||||||
|
|
||||||
|
# Ensure we only have one build running at a time, cancelling any active builds if a new commit is pushed to the respective branch
|
||||||
|
concurrency:
|
||||||
|
group: ci_build-${{ github.event.inputs.branch || github.ref_name }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
determine_concurrency:
|
||||||
|
name: "Determine concurrency"
|
||||||
|
if: github.repository == 'qmk/qmk_firmware'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: ghcr.io/qmk/qmk_cli
|
||||||
|
|
||||||
|
outputs:
|
||||||
|
slice_length: ${{ steps.generate_slice_length.outputs.slice_length }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Install prerequisites
|
||||||
|
run: |
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y jq
|
||||||
|
|
||||||
|
- name: Disable safe.directory check
|
||||||
|
run: |
|
||||||
|
git config --global --add safe.directory '*'
|
||||||
|
|
||||||
|
- name: Checkout QMK Firmware
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Determine concurrency
|
||||||
|
id: generate_slice_length
|
||||||
|
run: |
|
||||||
|
target_count=$( {
|
||||||
|
qmk find -km default 2>/dev/null
|
||||||
|
qmk find -km via 2>/dev/null
|
||||||
|
} | sort | uniq | wc -l)
|
||||||
|
slice_length=$((target_count / ($CONCURRENT_JOBS - 1))) # Err on the side of caution as we're splitting default and via
|
||||||
|
echo "slice_length=$slice_length" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
build_targets:
|
||||||
|
name: "Compile keymap ${{ matrix.keymap }}"
|
||||||
|
needs: determine_concurrency
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
keymap: [default, via]
|
||||||
|
uses: ./.github/workflows/ci_build_major_branch_keymap.yml
|
||||||
|
with:
|
||||||
|
branch: ${{ inputs.branch || github.ref_name }}
|
||||||
|
keymap: ${{ matrix.keymap }}
|
||||||
|
slice_length: ${{ needs.determine_concurrency.outputs.slice_length }}
|
||||||
|
|
||||||
|
rollup_tasks:
|
||||||
|
name: "Housekeeping"
|
||||||
|
needs: build_targets
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Download firmwares
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
pattern: firmware-*
|
||||||
|
path: firmwares
|
||||||
|
merge-multiple: true
|
||||||
|
|
||||||
|
- name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }}
|
||||||
|
uses: jakejarvis/s3-sync-action@master
|
||||||
|
with:
|
||||||
|
args: --acl public-read --follow-symlinks --delete
|
||||||
|
env:
|
||||||
|
AWS_S3_BUCKET: qmk-ci
|
||||||
|
AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
|
||||||
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
|
||||||
|
AWS_REGION: nyc3
|
||||||
|
AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
|
||||||
|
SOURCE_DIR: firmwares
|
||||||
|
DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }}
|
||||||
|
|
||||||
|
- name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest
|
||||||
|
uses: jakejarvis/s3-sync-action@master
|
||||||
|
with:
|
||||||
|
args: --acl public-read --follow-symlinks --delete
|
||||||
|
env:
|
||||||
|
AWS_S3_BUCKET: qmk-ci
|
||||||
|
AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
|
||||||
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
|
||||||
|
AWS_REGION: nyc3
|
||||||
|
AWS_S3_ENDPOINT: nyc3.digitaloceanspaces.com
|
||||||
|
SOURCE_DIR: firmwares
|
||||||
|
DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest
|
||||||
|
|
||||||
|
- name: Check if failure marker file exists
|
||||||
|
id: check_failure_marker
|
||||||
|
uses: andstor/file-existence-action@v3
|
||||||
|
with:
|
||||||
|
files: firmwares/.failed
|
||||||
|
|
||||||
|
- name: Fail build if needed
|
||||||
|
if: steps.check_failure_marker.outputs.exists == 'true'
|
||||||
|
run: |
|
||||||
|
# Exit with failure if the compilation stage failed
|
||||||
|
exit 1
|
||||||
|
|
@ -0,0 +1,181 @@
|
||||||
|
name: CI Build Major Branch Keymap
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
actions: write
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
branch:
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
keymap:
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
slice_length:
|
||||||
|
type: string
|
||||||
|
required: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
generate_targets:
|
||||||
|
name: "Generate targets (${{ inputs.keymap }})"
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: ghcr.io/qmk/qmk_cli
|
||||||
|
|
||||||
|
outputs:
|
||||||
|
targets: ${{ steps.generate_targets.outputs.targets }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Install prerequisites
|
||||||
|
run: |
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y jq
|
||||||
|
|
||||||
|
- name: Disable safe.directory check
|
||||||
|
run: |
|
||||||
|
git config --global --add safe.directory '*'
|
||||||
|
|
||||||
|
- name: Checkout QMK Firmware
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Generate build targets
|
||||||
|
id: generate_targets
|
||||||
|
run: |
|
||||||
|
{ # Intentionally use `shuf` here so that we share manufacturers across all build groups -- some have a lot of ARM-based boards which inherently take longer
|
||||||
|
counter=0
|
||||||
|
echo -n '{'
|
||||||
|
qmk find -km ${{ inputs.keymap }} 2>/dev/null | sort | uniq | shuf | xargs -L${{ inputs.slice_length }} | while IFS=$'\n' read target ; do
|
||||||
|
if [ $counter -gt 0 ]; then
|
||||||
|
echo -n ','
|
||||||
|
fi
|
||||||
|
counter=$((counter+1))
|
||||||
|
printf "\"group %02d\":{" $counter
|
||||||
|
echo -n '"targets":"'
|
||||||
|
echo $target | tr ' ' '\n' | sort | uniq | xargs echo -n
|
||||||
|
echo -n '"}'
|
||||||
|
done
|
||||||
|
echo -n '}'
|
||||||
|
} | sed -e 's@\n@@g' > targets.json
|
||||||
|
|
||||||
|
# Output the target keys as a variable
|
||||||
|
echo "targets=$(jq -c 'keys' targets.json)" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Upload targets json
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: targets-${{ inputs.keymap }}
|
||||||
|
path: targets.json
|
||||||
|
|
||||||
|
build_targets:
|
||||||
|
name: "Compile ${{ matrix.target }} (${{ inputs.keymap }})"
|
||||||
|
needs: generate_targets
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: ghcr.io/qmk/qmk_cli
|
||||||
|
continue-on-error: true
|
||||||
|
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
target: ${{ fromJson(needs.generate_targets.outputs.targets) }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Install prerequisites
|
||||||
|
run: |
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y jq
|
||||||
|
|
||||||
|
- name: Disable safe.directory check
|
||||||
|
run: |
|
||||||
|
git config --global --add safe.directory '*'
|
||||||
|
|
||||||
|
- name: Checkout QMK Firmware
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Get target definitions
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: targets-${{ inputs.keymap }}
|
||||||
|
path: .
|
||||||
|
|
||||||
|
- name: Deploy submodules
|
||||||
|
run: |
|
||||||
|
qmk git-submodule -f
|
||||||
|
|
||||||
|
- name: Dump targets
|
||||||
|
run: |
|
||||||
|
jq -r '.["${{ matrix.target }}"].targets' targets.json | tr ' ' '\n' | sort
|
||||||
|
|
||||||
|
- name: Build targets
|
||||||
|
continue-on-error: true
|
||||||
|
run: |
|
||||||
|
export NCPUS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null)
|
||||||
|
qmk mass-compile -t -j $NCPUS -e DUMP_CI_METADATA=yes $(jq -r '.["${{ matrix.target }}"].targets' targets.json) || touch .failed
|
||||||
|
|
||||||
|
- name: Upload binaries
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: firmware-${{ inputs.keymap }}-${{ matrix.target }}
|
||||||
|
if-no-files-found: ignore
|
||||||
|
path: |
|
||||||
|
*.bin
|
||||||
|
*.hex
|
||||||
|
*.uf2
|
||||||
|
.build/failed.*
|
||||||
|
.failed
|
||||||
|
|
||||||
|
- name: Fail build if any group failed
|
||||||
|
run: |
|
||||||
|
# Exit with failure if the compilation stage failed
|
||||||
|
[ ! -f .failed ] || exit 1
|
||||||
|
|
||||||
|
repack_firmware:
|
||||||
|
if: always()
|
||||||
|
name: "Repack artifacts"
|
||||||
|
needs: build_targets
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout QMK Firmware
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Download firmwares
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
pattern: firmware-${{ inputs.keymap }}-*
|
||||||
|
path: .
|
||||||
|
merge-multiple: true
|
||||||
|
|
||||||
|
- name: Upload all firmwares
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: firmware-${{ inputs.keymap }}
|
||||||
|
if-no-files-found: ignore
|
||||||
|
path: |
|
||||||
|
*.bin
|
||||||
|
*.hex
|
||||||
|
*.uf2
|
||||||
|
.build/failed.*
|
||||||
|
.failed
|
||||||
|
|
||||||
|
- name: Generate output logs
|
||||||
|
run: |
|
||||||
|
# Generate the step summary markdown
|
||||||
|
./util/ci/generate_failure_markdown.sh > $GITHUB_STEP_SUMMARY || true
|
||||||
|
# Truncate to a maximum of 1MB to deal with GitHub workflow limit
|
||||||
|
truncate --size='<960K' $GITHUB_STEP_SUMMARY || true
|
||||||
|
|
||||||
|
- name: Delete temporary build artifacts
|
||||||
|
uses: geekyeggo/delete-artifact@v4
|
||||||
|
with:
|
||||||
|
name: |
|
||||||
|
firmware-${{ inputs.keymap }}-*
|
||||||
|
targets-${{ inputs.keymap }}
|
||||||
|
|
||||||
|
- name: 'CI Discord Notification'
|
||||||
|
if: always()
|
||||||
|
working-directory: util/ci/
|
||||||
|
env:
|
||||||
|
DISCORD_WEBHOOK: ${{ secrets.CI_DISCORD_WEBHOOK }}
|
||||||
|
run: |
|
||||||
|
python3 -m pip install -r requirements.txt
|
||||||
|
python3 ./discord-results.py --branch ${{ inputs.branch || github.ref_name }} --keymap ${{ inputs.keymap }} --url ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||||
Loading…
Reference in New Issue