From 96042481e70e8c3e57ff89c41e96a20ee9077cb5 Mon Sep 17 00:00:00 2001 From: Kjeld Schouten Date: Wed, 21 Feb 2024 01:22:20 +0100 Subject: [PATCH] feat(ci): move catalog generation to charttool (#18451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description** This changes does a dirty implementation of charttool for catalog building **โš™๏ธ Type of change** - [x] โš™๏ธ Feature/App addition - [ ] ๐Ÿช› Bugfix - [ ] โš ๏ธ Breaking change (fix or feature that would cause existing functionality to not work as expected) - [x] ๐Ÿ”ƒ Refactor of current code **๐Ÿงช How Has This Been Tested?** **๐Ÿ“ƒ Notes:** **โœ”๏ธ Checklist:** - [ ] โš–๏ธ My code follows the style guidelines of this project - [ ] ๐Ÿ‘€ I have performed a self-review of my own code - [ ] #๏ธโƒฃ I have commented my code, particularly in hard-to-understand areas - [ ] ๐Ÿ“„ I have made corresponding changes to the documentation - [ ] โš ๏ธ My changes generate no new warnings - [ ] ๐Ÿงช I have added tests to this description that prove my fix is effective or that my feature works - [ ] โฌ†๏ธ I increased versions for any altered app according to semantic versioning **โž• App addition** If this PR is an app addition please make sure you have done the following. - [ ] ๐Ÿ–ผ๏ธ I have added an icon in the Chart's root directory called `icon.png` --- _Please don't blindly check all the boxes. Read them and only check those that apply. Those checkboxes are there for the reviewer to see what is this all about and the status of this PR with a quick glance._ --- .github/scripts/build-catalog.sh | 171 ++--------- .github/workflows/catalog-test.yaml | 68 ++++- .github/workflows/charts-release.yaml | 1 + .github/workflows/pr-validate.yaml | 1 + charts/stable/jackett/Chart.yaml | 2 +- tools/build-release.sh | 410 -------------------------- tools/gen-cat.sh | 407 ------------------------- tools/process.sh | 30 -- 8 files changed, 88 insertions(+), 1002 deletions(-) delete mode 100755 tools/build-release.sh delete mode 100755 tools/gen-cat.sh delete mode 100755 tools/process.sh diff --git a/.github/scripts/build-catalog.sh b/.github/scripts/build-catalog.sh index dd343272699..36006f4ae58 100755 --- a/.github/scripts/build-catalog.sh +++ b/.github/scripts/build-catalog.sh @@ -3,155 +3,26 @@ set -o errexit set -o nounset set -o pipefail -include_questions() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} -} -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - -} -export -f clean_catalog - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - cp -rf ${target}/SCALE/migrations/* ${target}/migrations 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Remove content that is not required for the App to install - rm -rf ${target}/security.md - rm -rf ${target}/helm-values.md - rm -rf ${target}/CONFIG.md - rm -rf ${target}/docs - rm -rf ${target}/icon.png - rm -rf ${target}/ci - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - echo "- $category" >> catalog/${train}/${chartname}/item.yaml - - # Generate screenshots - screenshots="" - if [[ -d "${target}/screenshots" ]]; then - screenshots=$(ls ${target}/screenshots) - fi - if [[ -n $screenshots ]]; then - echo "screenshots:" >> catalog/${train}/${chartname}/item.yaml - for screenshot in $screenshots; do - echo " - https://truecharts.org/img/hotlink-ok/chart-screenshots/${chartname}/${screenshot}" >> catalog/${train}/${chartname}/item.yaml - done - else - echo "screenshots: []" >> catalog/${train}/${chartname}/item.yaml - fi - rm -rf ${target}/screenshots - # Copy changelog from website - if [[ ! -f "website/docs/charts/${train}/${chartname}/CHANGELOG.md" ]]; then - touch "website/docs/charts/${train}/${chartname}/CHANGELOG.md" - fi - cp -rf "website/docs/charts/${train}/${chartname}/CHANGELOG.md" "${target}/CHANGELOG.md" 2>/dev/null || : - # If changelog is empty, add a notice - if ! grep -q "for the complete changelog, please refer to the website" "${target}/CHANGELOG.md"; then - echo "Adding changelog notice for: ${chartname}" - # Count the frontmatter lines - line_count=$(sed -n '/^---$/,/^---$/p' "${target}/CHANGELOG.md" | wc -l) - # Increase the line count by 2 - line_count=$((line_count + 2)) - # Add a line to the changelog - sed -i "${line_count}s/^/*for the complete changelog, please refer to the website*\n\n/" "${target}/CHANGELOG.md" - sed -i "${line_count}s/^/**Important:**\n/" "${target}/CHANGELOG.md" - fi - echo "Truncating changelog for: ${chartname}" - # Truncate changelog to only contain the last 100 lines - sed -i '100,$ d' "${target}/CHANGELOG.md" || : - # Generate SCALE App description file - cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This App is supplied by TrueCharts, for more information visit the manual: [https://truecharts.org/charts/${train}/${chartname}](https://truecharts.org/charts/${train}/${chartname})" >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "---" >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "TrueCharts can only exist due to the incredible effort of our staff." >> ${target}/app-readme.md - echo "Please consider making a [donation](https://truecharts.org/sponsor) or contributing back to the project any way you can!" >> ${target}/app-readme.md - echo "app-readme generated" -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying App to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -rm -rf charts/unstable -if [[ -d "charts/${1}" ]]; then - echo "Start processing charts/${1} ..." - chartversion=$(cat charts/${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename charts/${1}) - train=$(basename $(dirname "charts/${1}")) - SCALESUPPORT=$(cat charts/${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - include_questions "charts/${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "charts/${1}" "${chartname}" "$train" "${chartversion}" - else - echo "Skipping chart charts/${1}, no correct SCALE compatibility layer detected" - fi -else - echo "Chart 'charts/${1}' no longer exists in repo. Skipping it..." + +# Copy changelog from website +if [[ ! -f "website/docs/charts/${train}/${chartname}/CHANGELOG.md" ]]; then + touch "website/docs/charts/${train}/${chartname}/CHANGELOG.md" fi -echo "Done processing charts/${1} ..." +cp -rf "website/docs/charts/${train}/${chartname}/CHANGELOG.md" "${target}/CHANGELOG.md" 2>/dev/null || : +# If changelog is empty, add a notice +if ! grep -q "for the complete changelog, please refer to the website" "${target}/CHANGELOG.md"; then + echo "Adding changelog notice for: ${chartname}" + # Count the frontmatter lines + line_count=$(sed -n '/^---$/,/^---$/p' "${target}/CHANGELOG.md" | wc -l) + # Increase the line count by 2 + line_count=$((line_count + 2)) + # Add a line to the changelog + sed -i "${line_count}s/^/*for the complete changelog, please refer to the website*\n\n/" "${target}/CHANGELOG.md" + sed -i "${line_count}s/^/**Important:**\n/" "${target}/CHANGELOG.md" +fi +echo "Truncating changelog for: ${chartname}" +# Truncate changelog to only contain the last 100 lines +sed -i '100,$ d' "${target}/CHANGELOG.md" || : + + diff --git a/.github/workflows/catalog-test.yaml b/.github/workflows/catalog-test.yaml index 18d3ef18160..e11923ca986 100644 --- a/.github/workflows/catalog-test.yaml +++ b/.github/workflows/catalog-test.yaml @@ -9,13 +9,14 @@ on: chartChangesDetected: required: true type: string + modifiedCharts: + required: true + type: string jobs: catalog-tests: name: Test SCALE Catalog runs-on: ubuntu-latest - container: - image: tccr.io/tccr/devcontainer:v4.0.1 steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Checkout @@ -25,6 +26,64 @@ jobs: - name: Setting repo parent dir as safe safe.directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3 + name: Checkout-validate + with: + fetch-depth: 1 + repository: truecharts/catalog_validation + path: .catalog_validation + + - name: install catalog_validation + shell: bash + run: | + cd .catalog_validation + pip install -r requirements.txt + pip install -U --no-cache-dir . + cd - + + - name: Install go-yq + run: | + mkdir -p $HOME/.local/bin + wget https://github.com/mikefarah/yq/releases/download/v4.26.1/yq_linux_amd64 -O $HOME/.local/bin/go-yq && \ + chmod +x $HOME/.local/bin/go-yq + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Collect changes (branch-based) + id: list-changed + if: inputs.chartChangesDetected == 'true' + shell: bash + run: | + CHARTS="${{ inputs.modifiedCharts }}" + echo "Modified Charts: ${CHARTS}" + + EXCLUDED_JSON=$(go-yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq --raw-input '.' | jq --compact-output --slurp '.') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED_JSON}, \"all\": ${CHARTS_JSON}}" | jq --compact-output '.all-.excluded') + + echo CHANGED_CHARTS=${OUTPUT_JSON} >> "$GITHUB_OUTPUT" + if [[ $(echo ${OUTPUT_JSON} | jq --compact-output '. | length') -gt 0 ]]; then + echo "detected=true" >> "$GITHUB_OUTPUT" + fi + + - name: Fetch and Verify dependencies + shell: bash + if: steps.list-changed.outputs.detected == 'true' + env: + charts_path: "./" + run: | + CHANGED=$(echo '${{ steps.list-changed.outputs.CHANGED_CHARTS }}' | jq --raw-output '.[]') + ./charttool deps ${CHANGED} + + - name: Build Catalog + shell: bash + if: steps.list-changed.outputs.detected == 'true' + env: + charts_path: "./" + run: | + CHANGED=$(echo '${{ steps.list-changed.outputs.CHANGED_CHARTS }}' | jq --raw-output '.[]') + ./charttool buildSCALE ${CHANGED} + ## TODO: Only run on the existing catalog when CI or template changes are actually detected # - name: Checkout @@ -33,6 +92,7 @@ jobs: # repository: truecharts/catalog # path: catalog - - name: build-and-run + - name: Test Catalog run: | - tools/build-release.sh + echo "Starting Catalog Validation" + python3 .catalog_validation/catalog_validation/scripts/catalog_validate.py validate --path "${PWD}/catalog" --ignore-catalog-json diff --git a/.github/workflows/charts-release.yaml b/.github/workflows/charts-release.yaml index 73b0811e72d..134520a2b88 100644 --- a/.github/workflows/charts-release.yaml +++ b/.github/workflows/charts-release.yaml @@ -278,6 +278,7 @@ jobs: run: | CHARTS=(${{ steps.collect-changes.outputs.modifiedChartsAfterTag }}) ./charttool deps + ./charttool buildSCALE pip install yq parthreads=$(($(nproc) * 2)) parallel -j ${parthreads} .github/scripts/build-catalog.sh '2>&1' ::: ${CHARTS[@]} diff --git a/.github/workflows/pr-validate.yaml b/.github/workflows/pr-validate.yaml index 345ec6c2e00..fd53f7e2853 100644 --- a/.github/workflows/pr-validate.yaml +++ b/.github/workflows/pr-validate.yaml @@ -49,6 +49,7 @@ jobs: with: checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }} chartChangesDetected: ${{ needs.pr-changes.outputs.addedOrModified }} + modifiedCharts: ${{ needs.pr-changes.outputs.addedOrModifiedCharts }} print_head_msg: name: print commit message diff --git a/charts/stable/jackett/Chart.yaml b/charts/stable/jackett/Chart.yaml index 2942b42a00a..e49ae2aeba4 100644 --- a/charts/stable/jackett/Chart.yaml +++ b/charts/stable/jackett/Chart.yaml @@ -35,4 +35,4 @@ sources: - https://github.com/truecharts/charts/tree/master/charts/stable/jackett - https://ghcr.io/linuxserver/jackett type: application -version: 18.2.0 +version: 18.2.1 diff --git a/tools/build-release.sh b/tools/build-release.sh deleted file mode 100755 index 17091dbad96..00000000000 --- a/tools/build-release.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env bash -set -o errexit -set -o nounset -set -o pipefail - -DEFAULT_CHART_RELEASER_VERSION=v1.2.1 - -show_help() { -cat << EOF -Usage: $(basename "$0") - -h, --help Display help - -v, --version The chart-releaser version to use (default: $DEFAULT_CHART_RELEASER_VERSION)" - --config The path to the chart-releaser config file - -d, --charts-dir The charts directory (default: charts) - -u, --charts-repo-url The GitHub Pages URL to the charts repo (default: https://.github.io/) - -o, --owner The repo owner - -r, --repo The repo name - -p, --production Enables uploading releases to github releases - -s, --standalone Disables Chart Releaser and Catalog Validation, for local generation -EOF -} - -main() { - local version="$DEFAULT_CHART_RELEASER_VERSION" - local config= - local standalone= - local charts_dir=charts/** - local owner= - local repo= - local production= - local charts_repo_url= - local token=${CR_TOKEN:-false} - local parthreads=$(($(nproc) * 2)) - - parse_command_line "$@" - if [ "${token}" == "false" ]; then - echo "env #CR_TOKEN not found, defaulting to production=false" - production="false" - fi - - local repo_root - repo_root=$(git rev-parse --show-toplevel) - pushd "$repo_root" > /dev/null - - echo 'Looking up latest tag...' - local latest_tag - latest_tag=$(lookup_latest_tag) - - echo "Discovering changed charts since '$latest_tag'..." - local changed_charts=() - readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")" - # copy_general_docs - if [[ -n "${changed_charts[*]}" ]]; then - - prep_helm - - parallel -j ${parthreads} chart_runner '2>&1' ::: ${changed_charts[@]} - echo "Starting post-processing" - # pre_commit - validate_catalog - if [ "${production}" == "true" ]; then - gen_dh_cat - upload_catalog - upload_dhcatalog - fi - else - echo "Nothing to do. No chart changes detected." - fi - - popd > /dev/null -} - -chart_runner(){ - if [[ -d "${1}" ]]; then - echo "Start processing ${1} ..." - chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${1}) - train=$(basename $(dirname "${1}")) - SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - helm dependency build "${1}" --skip-refresh || (sleep 10 && helm dependency build "${1}" --skip-refresh) || (sleep 10 && helm dependency build "${1}" --skip-refresh) - if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "${1}" "${chartname}" "$train" "${chartversion}" - include_questions "${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "${1}" "${chartname}" "$train" "${chartversion}" - else - echo "Skipping chart ${1}, no correct SCALE compatibility layer detected" - fi - else - echo "Chart '${1}' no longer exists in repo. Skipping it..." - fi - echo "Done processing ${1} ..." -} -export -f chart_runner - -include_questions(){ - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} - } -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - - } -export -f clean_catalog - -pre_commit() { - if [[ -z "$standalone" ]]; then - echo "Running pre-commit test-and-cleanup..." - pre-commit run --all ||: - # Fix sh files to always be executable - find . -name '*.sh' | xargs chmod +x - fi - } - export -f pre_commit - -create_changelog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local prevversion="$(git tag -l "${chartname}-*" --sort=-v:refname | head -n 1)" - if [[ -z "$standalone" ]]; then - echo "Generating changelogs for: ${chartname}" - # SCALE "Changelog" containing only last change - git-chglog --next-tag ${chartname}-${chartversion} --tag-filter-pattern ${chartname} --path ${chart} -o ${chart}/app-changelog.md ${chartname}-${chartversion} - # Append SCALE changelog to actual changelog - - if [[ -f "${chart}/CHANGELOG.md" ]]; then - true - else - touch ${chart}/CHANGELOG.md - fi - sed -i '1d' ${chart}/CHANGELOG.md - cat ${chart}/app-changelog.md | cat - ${chart}/CHANGELOG.md > temp && mv temp ${chart}/CHANGELOG.md - sed -i '1s/^/# Changelog
\n\n/' ${chart}/CHANGELOG.md - rm ${chart}/app-changelog.md || echo "changelog not found..." - fi - } - export -f create_changelog - -copy_general_docs() { - yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : - yes | cp -rf .github/README.md docs/index.md 2>/dev/null || : - sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md - sed -i 's~~~g' docs/index.md - yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || : - yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || : - yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || : - yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md - yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : - sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md - } - export -f copy_general_docs - - -copy_docs() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying docs for: ${chart}" - mkdir -p docs/charts/${train}/${chartname} || echo "chart path already exists, continuing..." - yes | cp -rf ${chart}/CHANGELOG.md docs/charts/${train}/${chartname}/CHANGELOG.md 2>/dev/null || : - yes | cp -rf ${chart}/security.md docs/charts/${train}/${chartname}/security.md 2>/dev/null || : - yes | cp -rf ${chart}/CONFIG.md docs/charts/${train}/${chartname}/CONFIG.md 2>/dev/null || : - yes | cp -rf ${chart}/helm-values.md docs/charts/${train}/${chartname}/helm-values.md 2>/dev/null || : - rm docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - yes | cp -rf ${chart}/LICENSE docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - } - export -f copy_docs - -prep_helm() { - if [[ -z "$standalone" ]]; then - echo "helm repo update..." - # helm repo update - fi - } - export -f prep_helm - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - sed -i '100,$ d' ${target}/CHANGELOG.md || : - mv ${target}/app-changelog.md ${target}/CHANGELOG.md 2>/dev/null || : - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - echo "- $category" >> catalog/${train}/${chartname}/item.yaml - # Generate SCALE App description file - cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This Chart is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying Chart to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -validate_catalog() { - if [[ -z "$standalone" ]]; then - echo "Starting Catalog Validation" - /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" - fi -} -export -f validate_catalog - -upload_catalog() { - echo "Uploading Catalog..." - cd catalog - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - git add --all - git commit -sm "Commit new Chart releases for TrueCharts" || exit 0 - git push - cd - - rm -rf catalog -} -export -f upload_catalog - -parse_command_line() { - while :; do - case "${1:-}" in - -h|--help) - show_help - exit - ;; - --config) - if [[ -n "${2:-}" ]]; then - config="$2" - shift - else - config=".github/cr.yaml" - shift - fi - ;; - -v|--version) - if [[ -n "${2:-}" ]]; then - version="$2" - shift - else - echo "ERROR: '-v|--version' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -d|--charts-dir) - if [[ -n "${2:-}" ]]; then - charts_dir="$2" - shift - else - echo "ERROR: '-d|--charts-dir' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -u|--charts-repo-url) - if [[ -n "${2:-}" ]]; then - charts_repo_url="$2" - shift - else - echo "ERROR: '-u|--charts-repo-url' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -o|--owner) - if [[ -n "${2:-}" ]]; then - owner="$2" - shift - else - echo "ERROR: '--owner' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -r|--repo) - if [[ -n "${2:-}" ]]; then - repo="$2" - shift - else - echo "ERROR: '--repo' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -p|--production) - production="true" - ;; - -s|--standalone) - standalone="true" - ;; - *) - break - ;; - esac - - shift - done - - if [[ -z "$owner" ]]; then - echo "No owner configured, defaulting to truecharts" >&2 - owner="truecharts" - fi - - if [[ -z "$repo" ]]; then - echo "No repo configured, defaulting to charts" >&2 - repo="charts" - fi - - if [[ -z "$charts_repo_url" ]]; then - charts_repo_url="https://$owner.github.io/$repo" - fi -} -export -f parse_command_line - -lookup_latest_tag() { - git fetch --tags > /dev/null 2>&1 - - if ! git describe --tags --abbrev=0 2> /dev/null; then - git rev-list --max-parents=0 --first-parent HEAD - fi -} -export -f lookup_latest_tag - -filter_charts() { - while read -r chart; do - [[ ! -d "$chart" ]] && continue - if [[ $(git diff $latest_tag $chart/Chart.yaml | grep "+version") ]]; then - echo "$chart" - else - echo "Version not bumped. Skipping." 1>&2 - fi - done -} -export -f filter_charts - -lookup_changed_charts() { - local commit="$1" - - local changed_files - changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml") - - local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 )) - local fields="1-${depth}" - - cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts -} -export -f lookup_changed_charts - -main "$@" diff --git a/tools/gen-cat.sh b/tools/gen-cat.sh deleted file mode 100755 index e5e38f05274..00000000000 --- a/tools/gen-cat.sh +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env bash -set -o errexit -set -o nounset -set -o pipefail - -DEFAULT_CHART_RELEASER_VERSION=v1.2.1 - -show_help() { -cat << EOF -Usage: $(basename "$0") - -h, --help Display help - -v, --version The chart-releaser version to use (default: $DEFAULT_CHART_RELEASER_VERSION)" - --config The path to the chart-releaser config file - -d, --charts-dir The charts directory (default: charts) - -u, --charts-repo-url The GitHub Pages URL to the charts repo (default: https://.github.io/) - -o, --owner The repo owner - -r, --repo The repo name - -p, --production Enables uploading releases to github releases - -s, --standalone Disables Chart Releaser and Catalog Validation, for local generation -EOF -} - -main() { - local version="$DEFAULT_CHART_RELEASER_VERSION" - local config= - local standalone= - local charts_dir=charts/** - local owner= - local repo= - local production= - local charts_repo_url= - local token=${CR_TOKEN:-false} - local parthreads=$(($(nproc) * 2)) - - parse_command_line "$@" - if [ "${token}" == "false" ]; then - echo "env #CR_TOKEN not found, defaulting to production=false" - production="false" - fi - - local repo_root - repo_root=$(git rev-parse --show-toplevel) - pushd "$repo_root" > /dev/null - - - # readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")" - # copy_general_docs - #if [[ -n "${changed_charts[*]}" ]]; then - - prep_helm - - chart_runner ${1} - echo "Starting post-processing" - # pre_commit - # validate_catalog - # if [ "${production}" == "true" ]; then - # gen_dh_cat - # upload_catalog - # upload_dhcatalog - # fi - #else - # echo "Nothing to do. No chart changes detected." - #fi - - popd > /dev/null -} - -chart_runner(){ - if [[ -d "${1}" ]]; then - echo "Start processing ${1} ..." - chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${1}) - train=$(basename $(dirname "${1}")) - # SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - helm dependency build "${1}" --skip-refresh || (sleep 10 && helm dependency build "${1}" --skip-refresh) || (sleep 10 && helm dependency build "${1}" --skip-refresh) - # if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "${1}" "${chartname}" "$train" "${chartversion}" - include_questions "${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "${1}" "${chartname}" "$train" "${chartversion}" - # else - # echo "Skipping chart ${1}, no correct SCALE compatibility layer detected" - # fi - else - echo "Chart '${1}' no longer exists in repo. Skipping it..." - fi - echo "Done processing ${1} ..." -} -export -f chart_runner - -include_questions(){ - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} - } -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - - } -export -f clean_catalog - -pre_commit() { - if [[ -z "$standalone" ]]; then - echo "Running pre-commit test-and-cleanup..." - pre-commit run --all ||: - # Fix sh files to always be executable - find . -name '*.sh' | xargs chmod +x - fi - } - export -f pre_commit - -create_changelog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local prevversion="$(git tag -l "${chartname}-*" --sort=-v:refname | head -n 1)" - if [[ -z "$standalone" ]]; then - echo "Generating changelogs for: ${chartname}" - # SCALE "Changelog" containing only last change - git-chglog --next-tag ${chartname}-${chartversion} --tag-filter-pattern ${chartname} --path ${chart} -o ${chart}/app-changelog.md ${chartname}-${chartversion} - # Append SCALE changelog to actual changelog - - if [[ -f "${chart}/CHANGELOG.md" ]]; then - true - else - touch ${chart}/CHANGELOG.md - fi - sed -i '1d' ${chart}/CHANGELOG.md - cat ${chart}/app-changelog.md | cat - ${chart}/CHANGELOG.md > temp && mv temp ${chart}/CHANGELOG.md - sed -i '1s/^/# Changelog
\n\n/' ${chart}/CHANGELOG.md - rm ${chart}/app-changelog.md || echo "changelog not found..." - fi - } - export -f create_changelog - -copy_general_docs() { - yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : - yes | cp -rf .github/README.md docs/index.md 2>/dev/null || : - sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md - sed -i 's~~~g' docs/index.md - yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || : - yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || : - yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || : - yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md - yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : - sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md - } - export -f copy_general_docs - - -copy_docs() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying docs for: ${chart}" - mkdir -p docs/charts/${train}/${chartname} || echo "chart path already exists, continuing..." - yes | cp -rf ${chart}/CHANGELOG.md docs/charts/${train}/${chartname}/CHANGELOG.md 2>/dev/null || : - yes | cp -rf ${chart}/security.md docs/charts/${train}/${chartname}/security.md 2>/dev/null || : - yes | cp -rf ${chart}/CONFIG.md docs/charts/${train}/${chartname}/CONFIG.md 2>/dev/null || : - yes | cp -rf ${chart}/helm-values.md docs/charts/${train}/${chartname}/helm-values.md 2>/dev/null || : - rm docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - yes | cp -rf ${chart}/LICENSE docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - } - export -f copy_docs - -prep_helm() { - if [[ -z "$standalone" ]]; then - helm repo add truecharts https://charts.truecharts.org - helm repo add truecharts-deps https://deps.truecharts.org - helm repo add truecharts-library https://library-charts.truecharts.org - helm repo update - fi - } - export -f prep_helm - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - sed -i '100,$ d' ${target}/CHANGELOG.md || : - mv ${target}/app-changelog.md ${target}/CHANGELOG.md 2>/dev/null || : - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - # category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - # echo "- $category" >> catalog/${train}/${chartname}/item.yaml - # Generate SCALE App description file - # cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This Chart is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying Chart to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -validate_catalog() { - if [[ -z "$standalone" ]]; then - echo "Starting Catalog Validation" - /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" - fi -} -export -f validate_catalog - -upload_catalog() { - echo "Uploading Catalog..." - cd catalog - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - git add --all - git commit -sm "Commit new Chart releases for TrueCharts" || exit 0 - git push - cd - - rm -rf catalog -} -export -f upload_catalog - -parse_command_line() { - while :; do - case "${1:-}" in - -h|--help) - show_help - exit - ;; - --config) - if [[ -n "${2:-}" ]]; then - config="$2" - shift - else - config=".github/cr.yaml" - shift - fi - ;; - -v|--version) - if [[ -n "${2:-}" ]]; then - version="$2" - shift - else - echo "ERROR: '-v|--version' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -d|--charts-dir) - if [[ -n "${2:-}" ]]; then - charts_dir="$2" - shift - else - echo "ERROR: '-d|--charts-dir' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -u|--charts-repo-url) - if [[ -n "${2:-}" ]]; then - charts_repo_url="$2" - shift - else - echo "ERROR: '-u|--charts-repo-url' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -o|--owner) - if [[ -n "${2:-}" ]]; then - owner="$2" - shift - else - echo "ERROR: '--owner' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -r|--repo) - if [[ -n "${2:-}" ]]; then - repo="$2" - shift - else - echo "ERROR: '--repo' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -p|--production) - production="true" - ;; - -s|--standalone) - standalone="true" - ;; - *) - break - ;; - esac - - shift - done - - if [[ -z "$owner" ]]; then - echo "No owner configured, defaulting to truecharts" >&2 - owner="truecharts" - fi - - if [[ -z "$repo" ]]; then - echo "No repo configured, defaulting to charts" >&2 - repo="charts" - fi - - if [[ -z "$charts_repo_url" ]]; then - charts_repo_url="https://$owner.github.io/$repo" - fi -} -export -f parse_command_line - -lookup_latest_tag() { - git fetch --tags > /dev/null 2>&1 - - if ! git describe --tags --abbrev=0 2> /dev/null; then - git rev-list --max-parents=0 --first-parent HEAD - fi -} -export -f lookup_latest_tag - -filter_charts() { - while read -r chart; do - [[ ! -d "$chart" ]] && continue - if [[ $(git diff $latest_tag $chart/Chart.yaml | grep "+version") ]]; then - echo "$chart" - else - echo "Version not bumped. Skipping." 1>&2 - fi - done -} -export -f filter_charts - -lookup_changed_charts() { - local commit="$1" - - local changed_files - changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml") - - local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 )) - local fields="1-${depth}" - - cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts -} -export -f lookup_changed_charts - -main "$@" diff --git a/tools/process.sh b/tools/process.sh deleted file mode 100755 index 06b224d4354..00000000000 --- a/tools/process.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -eu - -# This script will do some basic processing from k8s-at-home to TrueCharts -nodeport=36150 -for chart in input/*; do - if [ -d "${chart}" ]; then - nodeport=$((nodeport+1)) - basename=$(basename ${chart}) - echo "processing ${chart} using nodeport $nodeport" - mkdir -p output/${basename} - cp -rf ../templates/app/SCALE output/${basename}/ - cp -rf ${chart}/ci output/${basename}/ || echo "no CI folder detected, continuing..." - cp -rf ${chart}/templates output/${basename}/ - cp -rf ${chart}/Chart.yaml output/${basename}/Chart.yaml - cp -rf ${chart}/.helmignore output/${basename}/.helmignore || echo "helmignore not detected, continuing..." - cp -rf ${chart}/values.yaml output/${basename}/values.yaml - iconurl=$(cat ${chart}/Chart.yaml | grep -m 1 icon: ) && iconurl=${iconurl/icon: /} - sed -i "s|PLACEHOLDERICON|${iconurl}|g" output/${basename}/SCALE/item.yaml - repo=$(cat ${chart}/values.yaml | grep -m 1 repository: ) && repo=${repo/ repository: /} - tag=$(cat ${chart}/values.yaml | grep -m 1 tag: ) && tag=${tag/ tag: /} - sed -i "s|PLACEHOLDERREPO|${repo}|g" output/${basename}/SCALE/ix_values.yaml - sed -i "s|PLACEHOLDERTAG|${tag}|g" output/${basename}/SCALE/ix_values.yaml - port=$(cat ${chart}/values.yaml | grep -m 1 port: ) && port=${port/ port: /} - sed -i "s|PLACEHOLDERPORT|${port}|g" output/${basename}/SCALE/questions.yaml - sed -i "s|PLACEHOLDERNODEPORT|${nodeport}|g" output/${basename}/SCALE/questions.yaml - - - fi -done