From 794f97b62845ae0a23641b044b4fbd3272bb2535 Mon Sep 17 00:00:00 2001 From: kjeld Schouten-Lebbing Date: Wed, 30 Mar 2022 10:46:56 +0200 Subject: [PATCH] chore(ci): Include charts-only CI in this repo --- .github/actions/collect-changes/action.yaml | 45 ++++++ .github/actions/label-from-status/action.yaml | 48 +++++++ .github/scripts/check-releasenotes.sh | 49 +++++++ .github/scripts/gen-helm-docs.sh | 47 ++++++ .github/scripts/renovate-releasenotes.sh | 52 +++++++ .github/workflows/charts-changelog.yaml | 89 ++++++++++++ .github/workflows/charts-lint.yaml | 54 +++++++ .github/workflows/charts-test.yaml | 135 ++++++++++++++++++ .github/workflows/pr-metadata.yaml | 60 ++++++++ .github/workflows/pr-validate.yaml | 10 +- 10 files changed, 584 insertions(+), 5 deletions(-) create mode 100644 .github/actions/collect-changes/action.yaml create mode 100644 .github/actions/label-from-status/action.yaml create mode 100644 .github/scripts/check-releasenotes.sh create mode 100644 .github/scripts/gen-helm-docs.sh create mode 100644 .github/scripts/renovate-releasenotes.sh create mode 100644 .github/workflows/charts-changelog.yaml create mode 100644 .github/workflows/charts-lint.yaml create mode 100644 .github/workflows/charts-test.yaml create mode 100644 .github/workflows/pr-metadata.yaml diff --git a/.github/actions/collect-changes/action.yaml b/.github/actions/collect-changes/action.yaml new file mode 100644 index 00000000000..7f53c79d130 --- /dev/null +++ b/.github/actions/collect-changes/action.yaml @@ -0,0 +1,45 @@ +name: "Collect changes" +description: "Collects and stores changed files/charts" + +outputs: + changesDetected: + description: "Whether or not changes to charts have been detected" + value: ${{ steps.filter.outputs.addedOrModified }} + addedOrModifiedFiles: + description: "A list of the files changed" + value: ${{ steps.filter.outputs.addedOrModified_files }} + addedOrModifiedCharts: + description: "A list of the charts changed" + value: ${{ steps.filter-charts.outputs.addedOrModified }} + +runs: + using: "composite" + steps: + - name: Collect changed files + uses: dorny/paths-filter@v2 + id: filter + with: + list-files: shell + filters: | + addedOrModified: + - added|modified: 'charts/*/**' + + - name: Collect changed charts + if: | + steps.filter.outputs.addedOrModified == 'true' + id: filter-charts + shell: bash + run: | + CHARTS=() + PATHS=(${{ steps.filter.outputs.addedOrModified_files }}) + # Get only the chart paths + for CHARTPATH in "${PATHS[@]}" + do + IFS='/' read -r -a path_parts <<< "${CHARTPATH}" + CHARTS+=("${path_parts[1]}/${path_parts[2]}") + done + + # Remove duplicates + CHARTS=( `printf "%s\n" "${CHARTS[@]}" | sort -u` ) + # Set output to changed charts + printf "::set-output name=addedOrModified::%s\n" "${CHARTS[*]}" diff --git a/.github/actions/label-from-status/action.yaml b/.github/actions/label-from-status/action.yaml new file mode 100644 index 00000000000..d972591d355 --- /dev/null +++ b/.github/actions/label-from-status/action.yaml @@ -0,0 +1,48 @@ +name: "Set issue labels based on status" +description: "Sets / removes issue labels based on CI job status" +inputs: + token: + required: true + description: "The Github API token to use" + issue-number: + required: true + description: "The issue to label" + prefix: + required: true + description: "The label prefix (e.g. lint, install)" + job-status: + required: true + description: "The status of the CI job" + remove-on-skipped: + required: false + default: false + description: "Remove the label if the job was skipped" + +runs: + using: "composite" + steps: + - name: Label success + uses: andymckay/labeler@1.0.4 + if: ${{ inputs.job-status == 'success' }} + with: + repo-token: ${{ inputs.token }} + issue-number: ${{ inputs.issue-number }} + add-labels: "${{ inputs.prefix }}:ok" + remove-labels: "${{ inputs.prefix }}:failed" + + - name: Label failure + uses: andymckay/labeler@1.0.4 + if: ${{ inputs.job-status == 'failure' }} + with: + repo-token: ${{ inputs.token }} + issue-number: ${{ inputs.issue-number }} + add-labels: "${{ inputs.prefix }}:failed" + remove-labels: "${{ inputs.prefix }}:ok" + + - name: Remove label + uses: andymckay/labeler@1.0.4 + if: ${{ (inputs.job-status == 'skipped') && (inputs.remove-on-skipped == 'true') }} + with: + repo-token: ${{ inputs.token }} + issue-number: ${{ inputs.issue-number }} + remove-labels: "${{ inputs.prefix }}:ok, ${{ inputs.prefix }}:failed" diff --git a/.github/scripts/check-releasenotes.sh b/.github/scripts/check-releasenotes.sh new file mode 100644 index 00000000000..089a1699fc4 --- /dev/null +++ b/.github/scripts/check-releasenotes.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +# Check if release notes have been changed +# Usage ./check-releasenotes.sh path + +# require yq +command -v yq >/dev/null 2>&1 || { + printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting." + exit 1 +} + +# Absolute path of repository +repository=$(git rev-parse --show-toplevel) + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ]; then + root="$1" + chart_file="${1}/Chart.yaml" + if [ ! -f "$chart_file" ]; then + printf >&2 "File %s\n does not exist.\n" "${chart_file}" + exit 1 + fi + + cd $root + + if [ -z "$DEFAULT_BRANCH" ]; then + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + fi + + CURRENT=$(cat Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "" ] || [ "$CURRENT" == "null" ]; then + printf >&2 "Changelog annotation has not been set in %s!\n" "$chart_file" + exit 1 + fi + + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + ORIGINAL=$(git show origin/$DEFAULT_BRANCH:./Chart.yaml | yq e '.annotations."artifacthub.io/changes"' -P -) + + if [ "$CURRENT" == "$ORIGINAL" ]; then + printf >&2 "Changelog annotation has not been updated in %s!\n" "$chart_file" + exit 1 + fi +else + printf >&2 "%s\n" "No chart folder has been specified." + exit 1 +fi diff --git a/.github/scripts/gen-helm-docs.sh b/.github/scripts/gen-helm-docs.sh new file mode 100644 index 00000000000..fca06603ebe --- /dev/null +++ b/.github/scripts/gen-helm-docs.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eu + +# Generate helm-docs for Helm charts +# Usage ./gen-helm-docs.sh [stable/incubator] [chart] + +# require helm-docs +command -v helm-docs >/dev/null 2>&1 || { + echo >&2 "helm-docs (https://github.com/k8s-at-home/helm-docs) is not installed. Aborting." + exit 1 +} + +# Absolute path of repository +repository=$(git rev-parse --show-toplevel) + +# Templates to copy into each chart directory +readme_template="${repository}/hack/templates/README.md.gotmpl" +readme_config_template="${repository}/hack/templates/README_CONFIG.md.gotmpl" + +# Gather all charts using the common library, excluding common-test +charts=$(find "${repository}" -name "Chart.yaml") + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ] && [ -n "$2" ]; then + charts="${repository}/charts/$1/$2/Chart.yaml" + root="$(dirname "${charts}")" + if [ ! -f "$charts" ]; then + echo "File ${charts} does not exist." + exit 1 + fi +else + root="${repository}/charts/stable" +fi + +for chart in ${charts}; do + chart_directory="$(dirname "${chart}")" + echo "-] Copying templates to ${chart_directory}" + # Copy CONFIG template to each Chart directory, do not overwrite if exists + cp -n "${readme_config_template}" "${chart_directory}" || true +done + +# Run helm-docs for charts using the common library and the common library itself +helm-docs \ + --ignore-file="${repository}/.helmdocsignore" \ + --template-files="${readme_template}" \ + --template-files="$(basename "${readme_config_template}")" \ + --chart-search-root="${root}" diff --git a/.github/scripts/renovate-releasenotes.sh b/.github/scripts/renovate-releasenotes.sh new file mode 100644 index 00000000000..0c293fd2a3b --- /dev/null +++ b/.github/scripts/renovate-releasenotes.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -e + +# Check if release notes have been changed +# Usage ./check-releasenotes.sh path + +# require yq +command -v yq >/dev/null 2>&1 || { + printf >&2 "%s\n" "yq (https://github.com/mikefarah/yq) is not installed. Aborting." + exit 1 +} + +# Allow for a specific chart to be passed in as a argument +if [ $# -ge 1 ] && [ -n "$1" ]; then + root="$1" + chart_file="${1}/Chart.yaml" + if [ ! -f "$chart_file" ]; then + printf >&2 "File %s does not exist.\n" "${chart_file}" + exit 1 + fi + cd "${root}" + + if [ -z "$DEFAULT_BRANCH" ]; then + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + fi + + printf "Updating changelog annotation for chart %s\n" "$root" + + # Loop over all dependencies in current chart version + NEW_DEPENDENCIES=() + while IFS='' read -r line; do NEW_DEPENDENCIES+=("$line"); done < <(yq e '.dependencies[].name' -P Chart.yaml | LC_ALL=C sort) + OLD_DEPENDENCIES=$(git show "origin/$DEFAULT_BRANCH:./Chart.yaml" | yq e '.dependencies[].name' -P - | LC_ALL=C sort) + + tmpfile=$(mktemp) + trap 'rm -f "$tmpfile"' EXIT + + for DEP_NAME in "${NEW_DEPENDENCIES[@]}" + do + NEW_VERSION=$(yq e ".dependencies[] | select(.name == \"$DEP_NAME\") | .version" -P Chart.yaml) + OLD_VERSION=$(git show "origin/$DEFAULT_BRANCH:./Chart.yaml" | yq e ".dependencies[] | select(.name == \"$DEP_NAME\") | .version" -P -) + if [ "${NEW_VERSION}" != "${OLD_VERSION}" ]; then + printf "%s\n" "- kind: changed" >> "${tmpfile}" + printf " description: Upgraded \`%s\` chart dependency to version \`%s\`.\n" "${DEP_NAME}" "${NEW_VERSION}" >> "${tmpfile}" + fi + done + + yq eval-all --inplace 'select(fileIndex == 0).annotations."artifacthub.io/changes" = (select(fileIndex == 1) | to_yaml) | select(fileIndex==0)' Chart.yaml "${tmpfile}" +else + printf >&2 "%s\n" "No chart folder has been specified." + exit 1 +fi diff --git a/.github/workflows/charts-changelog.yaml b/.github/workflows/charts-changelog.yaml new file mode 100644 index 00000000000..75ca2df0a44 --- /dev/null +++ b/.github/workflows/charts-changelog.yaml @@ -0,0 +1,89 @@ +name: "Charts: Update README" + +on: + workflow_call: + inputs: + modifiedCharts: + required: true + type: string + isRenovatePR: + required: true + type: string + outputs: + commitHash: + description: "The most recent commit hash at the end of this workflow" + value: ${{ jobs.generate-changelog.outputs.commitHash }} + +jobs: + validate-changelog: + name: Validate changelog + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Check changelog annotations + if: inputs.isRenovatePR != 'true' + run: | + CHARTS=(${{ inputs.modifiedCharts }}) + for i in "${CHARTS[@]}" + do + IFS='/' read -r -a chart_parts <<< "$i" + ./.github/scripts/check-releasenotes.sh "charts/${chart_parts[0]}/${chart_parts[1]}" + echo "" + done + + generate-changelog: + name: Generate changelog annotations + runs-on: ubuntu-latest + needs: + - validate-changelog + outputs: + commitHash: ${{ steps.save-commit-hash.outputs.commit_hash }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install Kubernetes tools + if: inputs.isRenovatePR == 'true' + uses: yokawasa/action-setup-kube-tools@v0.8.0 + with: + setup-tools: | + yq + yq: "4.20.1" + + - name: Annotate Charts.yaml for Renovate PR's + if: inputs.isRenovatePR == 'true' + env: + DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}" + run: | + CHARTS=(${{ inputs.modifiedCharts }}) + for i in "${CHARTS[@]}" + do + IFS='/' read -r -a chart_parts <<< "$i" + ./.github/scripts/renovate-releasenotes.sh "charts/${chart_parts[0]}/${chart_parts[1]}" + echo "" + done + + - name: Create commit + id: create-commit + if: inputs.isRenovatePR == 'true' + uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: charts/**/ + commit_message: "chore: Auto-update chart metadata [skip ci]" + commit_user_name: ${{ github.actor }} + commit_user_email: ${{ github.actor }}@users.noreply.github.com + + - name: Save commit hash + id: save-commit-hash + run: | + if [ "${{ steps.create-commit.outputs.changes_detected || 'unknown' }}" == "true" ]; then + echo '::set-output name=commit_hash::${{ steps.create-commit.outputs.commit_hash }}' + else + echo "::set-output name=commit_hash::${GITHUB_SHA}" + fi diff --git a/.github/workflows/charts-lint.yaml b/.github/workflows/charts-lint.yaml new file mode 100644 index 00000000000..f7a4a2c1d86 --- /dev/null +++ b/.github/workflows/charts-lint.yaml @@ -0,0 +1,54 @@ +name: "Charts: Lint" + +on: + workflow_call: + inputs: + checkoutCommit: + required: true + type: string + chartChangesDetected: + required: true + type: string + +jobs: + lint: + name: Lint charts + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.0 + with: + setup-tools: | + helmv3 + helm: "3.8.0" + + - uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.2.1 + + - name: Collect changes + id: list-changed + if: inputs.chartChangesDetected == 'true' + run: | + EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml) + CHARTS=$(ct list-changed --config .github/ct-lint.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded') + echo ::set-output name=charts::${OUTPUT_JSON} + if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then + echo "::set-output name=detected::true" + fi + + - name: Run chart-testing (lint) + id: lint + if: steps.list-changed.outputs.detected == 'true' + run: ct lint --config .github/ct-lint.yaml diff --git a/.github/workflows/charts-test.yaml b/.github/workflows/charts-test.yaml new file mode 100644 index 00000000000..72ae99e7c8d --- /dev/null +++ b/.github/workflows/charts-test.yaml @@ -0,0 +1,135 @@ +name: "Charts: Test" + +on: + workflow_call: + inputs: + checkoutCommit: + required: true + type: string + chartChangesDetected: + required: true + type: string + +jobs: + unit-test: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.0 + with: + setup-tools: | + helmv3 + helm: "3.8.0" + + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + + - name: Install dependencies + env: + RUBYJQ_USE_SYSTEM_LIBRARIES: 1 + run: | + sudo apt-get update + sudo apt-get install libjq-dev + bundle install + + - name: Run tests + run: | + bundle exec m -r ./test/ + + generate-install-matrix: + name: Generate matrix for install + runs-on: ubuntu-latest + outputs: + matrix: | + { + "chart": ${{ steps.list-changed.outputs.charts }} + } + detected: ${{ steps.list-changed.outputs.detected }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.2.1 + + - name: Run chart-testing (list-changed) + id: list-changed + if: inputs.chartChangesDetected == 'true' + run: | + EXCLUDED=$(yq eval -o=json '.excluded-charts // []' .github/ct-install.yaml) + CHARTS=$(ct list-changed --config .github/ct-install.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq -R -s -c 'split("\n")[:-1]') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED}, \"all\": ${CHARTS_JSON}}" | jq -c '.all-.excluded') + echo ::set-output name=charts::${OUTPUT_JSON} + if [[ $(echo ${OUTPUT_JSON} | jq -c '. | length') -gt 0 ]]; then + echo "::set-output name=detected::true" + fi + + install-charts: + needs: + - generate-install-matrix + if: needs.generate-install-matrix.outputs.detected == 'true' + name: Install charts + strategy: + matrix: ${{ fromJson(needs.generate-install-matrix.outputs.matrix) }} + fail-fast: true + max-parallel: 15 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.checkoutCommit }} + + - name: Install Kubernetes tools + uses: yokawasa/action-setup-kube-tools@v0.8.0 + with: + setup-tools: | + helmv3 + helm: "3.6.3" + + - uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.2.1 + + - name: Create k3d cluster + uses: nolar/setup-k3d-k3s@v1 + with: + version: v1.19 + + - name: Remove node taints + run: | + kubectl taint --all=true nodes node.cloudprovider.kubernetes.io/uninitialized- || true + + - name: Run chart-testing (install) + run: ct install --config .github/ct-install.yaml --charts ${{ matrix.chart }} + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + install_success: + needs: + - generate-install-matrix + - install-charts + if: | + always() + name: Install successful + runs-on: ubuntu-latest + steps: + - name: Check install matrix status + if: ${{ (needs.generate-install-matrix.outputs.detected == 'true') && (needs.install-charts.result != 'success') }} + run: exit 1 diff --git a/.github/workflows/pr-metadata.yaml b/.github/workflows/pr-metadata.yaml new file mode 100644 index 00000000000..d92b3a9edfc --- /dev/null +++ b/.github/workflows/pr-metadata.yaml @@ -0,0 +1,60 @@ +name: "Pull Request: Get metadata" + +on: + workflow_call: + outputs: + isRenovatePR: + description: "Is the PR coming from Renovate?" + value: ${{ jobs.pr-metadata.outputs.isRenovatePR }} + isFork: + description: "Is the PR coming from a forked repo?" + value: ${{ jobs.pr-metadata.outputs.isFork }} + addedOrModified: + description: "Does the PR contain any changes?" + value: ${{ jobs.pr-changes.outputs.addedOrModified }} + addedOrModifiedFiles: + description: "A list of the files changed in this PR" + value: ${{ jobs.pr-changes.outputs.addedOrModifiedFiles }} + addedOrModifiedCharts: + description: "A list of the charts changed in this PR" + value: ${{ jobs.pr-changes.outputs.addedOrModifiedCharts }} + +jobs: + pr-metadata: + name: Collect PR metadata + runs-on: ubuntu-latest + outputs: + isRenovatePR: ${{ startsWith(steps.branch-name.outputs.current_branch, 'renovate/') }} + isFork: ${{ github.event.pull_request.head.repo.full_name != github.repository }} + steps: + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.2 + + - name: Save PR data to file + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + + - name: Store pr data in artifact + uses: actions/upload-artifact@v3 + with: + name: pr_metadata + path: ./pr_number.txt + retention-days: 5 + + pr-changes: + name: Collect PR changes + runs-on: ubuntu-latest + outputs: + addedOrModified: ${{ steps.collect-changes.outputs.changesDetected }} + addedOrModifiedFiles: ${{ steps.collect-changes.outputs.addedOrModifiedFiles }} + addedOrModifiedCharts: ${{ steps.collect-changes.outputs.addedOrModifiedCharts }} + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Collect changes + id: collect-changes + uses: ./.github/actions/collect-changes diff --git a/.github/workflows/pr-validate.yaml b/.github/workflows/pr-validate.yaml index 64689204010..8a62215ecdb 100644 --- a/.github/workflows/pr-validate.yaml +++ b/.github/workflows/pr-validate.yaml @@ -9,17 +9,17 @@ concurrency: jobs: pr-metadata: - uses: truecharts/.github/workflows/pr-metadata.yaml@master + uses: truecharts/apps/.github/workflows/pr-metadata.yaml@master pre-commit-check: - uses: truecharts/.github/workflows/pre-commit-check.yaml@master + uses: truecharts/.github/.github/workflows/pre-commit-check.yaml@master needs: - pr-metadata with: modifiedFiles: ${{ needs.pr-metadata.outputs.addedOrModifiedFiles }} #charts-changelog: - # uses: truecharts/.github/workflows/charts-changelog.yaml@master + # uses: truecharts/apps/.github/workflows/charts-changelog.yaml@master # needs: # - pr-metadata # - pre-commit-check @@ -28,7 +28,7 @@ jobs: # modifiedCharts: ${{ needs.pr-metadata.outputs.addedOrModifiedCharts }} charts-lint: - uses: truecharts/.github/workflows/charts-lint.yaml@master + uses: truecharts/apps/.github/workflows/charts-lint.yaml@master needs: - pr-metadata # - charts-changelog @@ -37,7 +37,7 @@ jobs: chartChangesDetected: ${{ needs.pr-metadata.outputs.addedOrModified }} charts-test: - uses: truecharts/.github/workflows/charts-test.yaml@master + uses: truecharts/apps/.github/workflows/charts-test.yaml@master needs: - pr-metadata # - charts-changelog