diff --git a/.github/scripts/build-catalog.sh b/.github/scripts/build-catalog.sh index dd343272699..36006f4ae58 100755 --- a/.github/scripts/build-catalog.sh +++ b/.github/scripts/build-catalog.sh @@ -3,155 +3,26 @@ set -o errexit set -o nounset set -o pipefail -include_questions() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} -} -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - -} -export -f clean_catalog - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - cp -rf ${target}/SCALE/migrations/* ${target}/migrations 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Remove content that is not required for the App to install - rm -rf ${target}/security.md - rm -rf ${target}/helm-values.md - rm -rf ${target}/CONFIG.md - rm -rf ${target}/docs - rm -rf ${target}/icon.png - rm -rf ${target}/ci - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - echo "- $category" >> catalog/${train}/${chartname}/item.yaml - - # Generate screenshots - screenshots="" - if [[ -d "${target}/screenshots" ]]; then - screenshots=$(ls ${target}/screenshots) - fi - if [[ -n $screenshots ]]; then - echo "screenshots:" >> catalog/${train}/${chartname}/item.yaml - for screenshot in $screenshots; do - echo " - https://truecharts.org/img/hotlink-ok/chart-screenshots/${chartname}/${screenshot}" >> catalog/${train}/${chartname}/item.yaml - done - else - echo "screenshots: []" >> catalog/${train}/${chartname}/item.yaml - fi - rm -rf ${target}/screenshots - # Copy changelog from website - if [[ ! -f "website/docs/charts/${train}/${chartname}/CHANGELOG.md" ]]; then - touch "website/docs/charts/${train}/${chartname}/CHANGELOG.md" - fi - cp -rf "website/docs/charts/${train}/${chartname}/CHANGELOG.md" "${target}/CHANGELOG.md" 2>/dev/null || : - # If changelog is empty, add a notice - if ! grep -q "for the complete changelog, please refer to the website" "${target}/CHANGELOG.md"; then - echo "Adding changelog notice for: ${chartname}" - # Count the frontmatter lines - line_count=$(sed -n '/^---$/,/^---$/p' "${target}/CHANGELOG.md" | wc -l) - # Increase the line count by 2 - line_count=$((line_count + 2)) - # Add a line to the changelog - sed -i "${line_count}s/^/*for the complete changelog, please refer to the website*\n\n/" "${target}/CHANGELOG.md" - sed -i "${line_count}s/^/**Important:**\n/" "${target}/CHANGELOG.md" - fi - echo "Truncating changelog for: ${chartname}" - # Truncate changelog to only contain the last 100 lines - sed -i '100,$ d' "${target}/CHANGELOG.md" || : - # Generate SCALE App description file - cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This App is supplied by TrueCharts, for more information visit the manual: [https://truecharts.org/charts/${train}/${chartname}](https://truecharts.org/charts/${train}/${chartname})" >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "---" >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "TrueCharts can only exist due to the incredible effort of our staff." >> ${target}/app-readme.md - echo "Please consider making a [donation](https://truecharts.org/sponsor) or contributing back to the project any way you can!" >> ${target}/app-readme.md - echo "app-readme generated" -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying App to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -rm -rf charts/unstable -if [[ -d "charts/${1}" ]]; then - echo "Start processing charts/${1} ..." - chartversion=$(cat charts/${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename charts/${1}) - train=$(basename $(dirname "charts/${1}")) - SCALESUPPORT=$(cat charts/${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "charts/${1}" "${chartname}" "$train" "${chartversion}" - include_questions "charts/${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "charts/${1}" "${chartname}" "$train" "${chartversion}" - else - echo "Skipping chart charts/${1}, no correct SCALE compatibility layer detected" - fi -else - echo "Chart 'charts/${1}' no longer exists in repo. Skipping it..." + +# Copy changelog from website +if [[ ! -f "website/docs/charts/${train}/${chartname}/CHANGELOG.md" ]]; then + touch "website/docs/charts/${train}/${chartname}/CHANGELOG.md" fi -echo "Done processing charts/${1} ..." +cp -rf "website/docs/charts/${train}/${chartname}/CHANGELOG.md" "${target}/CHANGELOG.md" 2>/dev/null || : +# If changelog is empty, add a notice +if ! grep -q "for the complete changelog, please refer to the website" "${target}/CHANGELOG.md"; then + echo "Adding changelog notice for: ${chartname}" + # Count the frontmatter lines + line_count=$(sed -n '/^---$/,/^---$/p' "${target}/CHANGELOG.md" | wc -l) + # Increase the line count by 2 + line_count=$((line_count + 2)) + # Add a line to the changelog + sed -i "${line_count}s/^/*for the complete changelog, please refer to the website*\n\n/" "${target}/CHANGELOG.md" + sed -i "${line_count}s/^/**Important:**\n/" "${target}/CHANGELOG.md" +fi +echo "Truncating changelog for: ${chartname}" +# Truncate changelog to only contain the last 100 lines +sed -i '100,$ d' "${target}/CHANGELOG.md" || : + + diff --git a/.github/workflows/catalog-test.yaml b/.github/workflows/catalog-test.yaml index 18d3ef18160..e11923ca986 100644 --- a/.github/workflows/catalog-test.yaml +++ b/.github/workflows/catalog-test.yaml @@ -9,13 +9,14 @@ on: chartChangesDetected: required: true type: string + modifiedCharts: + required: true + type: string jobs: catalog-tests: name: Test SCALE Catalog runs-on: ubuntu-latest - container: - image: tccr.io/tccr/devcontainer:v4.0.1 steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Checkout @@ -25,6 +26,64 @@ jobs: - name: Setting repo parent dir as safe safe.directory run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3 + name: Checkout-validate + with: + fetch-depth: 1 + repository: truecharts/catalog_validation + path: .catalog_validation + + - name: install catalog_validation + shell: bash + run: | + cd .catalog_validation + pip install -r requirements.txt + pip install -U --no-cache-dir . + cd - + + - name: Install go-yq + run: | + mkdir -p $HOME/.local/bin + wget https://github.com/mikefarah/yq/releases/download/v4.26.1/yq_linux_amd64 -O $HOME/.local/bin/go-yq && \ + chmod +x $HOME/.local/bin/go-yq + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Collect changes (branch-based) + id: list-changed + if: inputs.chartChangesDetected == 'true' + shell: bash + run: | + CHARTS="${{ inputs.modifiedCharts }}" + echo "Modified Charts: ${CHARTS}" + + EXCLUDED_JSON=$(go-yq eval -o=json '.excluded-charts // []' .github/ct-lint.yaml) + CHARTS_JSON=$(echo "${CHARTS}" | jq --raw-input '.' | jq --compact-output --slurp '.') + OUTPUT_JSON=$(echo "{\"excluded\": ${EXCLUDED_JSON}, \"all\": ${CHARTS_JSON}}" | jq --compact-output '.all-.excluded') + + echo CHANGED_CHARTS=${OUTPUT_JSON} >> "$GITHUB_OUTPUT" + if [[ $(echo ${OUTPUT_JSON} | jq --compact-output '. | length') -gt 0 ]]; then + echo "detected=true" >> "$GITHUB_OUTPUT" + fi + + - name: Fetch and Verify dependencies + shell: bash + if: steps.list-changed.outputs.detected == 'true' + env: + charts_path: "./" + run: | + CHANGED=$(echo '${{ steps.list-changed.outputs.CHANGED_CHARTS }}' | jq --raw-output '.[]') + ./charttool deps ${CHANGED} + + - name: Build Catalog + shell: bash + if: steps.list-changed.outputs.detected == 'true' + env: + charts_path: "./" + run: | + CHANGED=$(echo '${{ steps.list-changed.outputs.CHANGED_CHARTS }}' | jq --raw-output '.[]') + ./charttool buildSCALE ${CHANGED} + ## TODO: Only run on the existing catalog when CI or template changes are actually detected # - name: Checkout @@ -33,6 +92,7 @@ jobs: # repository: truecharts/catalog # path: catalog - - name: build-and-run + - name: Test Catalog run: | - tools/build-release.sh + echo "Starting Catalog Validation" + python3 .catalog_validation/catalog_validation/scripts/catalog_validate.py validate --path "${PWD}/catalog" --ignore-catalog-json diff --git a/.github/workflows/charts-release.yaml b/.github/workflows/charts-release.yaml index 73b0811e72d..134520a2b88 100644 --- a/.github/workflows/charts-release.yaml +++ b/.github/workflows/charts-release.yaml @@ -278,6 +278,7 @@ jobs: run: | CHARTS=(${{ steps.collect-changes.outputs.modifiedChartsAfterTag }}) ./charttool deps + ./charttool buildSCALE pip install yq parthreads=$(($(nproc) * 2)) parallel -j ${parthreads} .github/scripts/build-catalog.sh '2>&1' ::: ${CHARTS[@]} diff --git a/.github/workflows/pr-validate.yaml b/.github/workflows/pr-validate.yaml index 345ec6c2e00..fd53f7e2853 100644 --- a/.github/workflows/pr-validate.yaml +++ b/.github/workflows/pr-validate.yaml @@ -49,6 +49,7 @@ jobs: with: checkoutCommit: ${{ needs.charts-changelog.outputs.commitHash }} chartChangesDetected: ${{ needs.pr-changes.outputs.addedOrModified }} + modifiedCharts: ${{ needs.pr-changes.outputs.addedOrModifiedCharts }} print_head_msg: name: print commit message diff --git a/charts/stable/jackett/Chart.yaml b/charts/stable/jackett/Chart.yaml index 2942b42a00a..e49ae2aeba4 100644 --- a/charts/stable/jackett/Chart.yaml +++ b/charts/stable/jackett/Chart.yaml @@ -35,4 +35,4 @@ sources: - https://github.com/truecharts/charts/tree/master/charts/stable/jackett - https://ghcr.io/linuxserver/jackett type: application -version: 18.2.0 +version: 18.2.1 diff --git a/tools/build-release.sh b/tools/build-release.sh deleted file mode 100755 index 17091dbad96..00000000000 --- a/tools/build-release.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env bash -set -o errexit -set -o nounset -set -o pipefail - -DEFAULT_CHART_RELEASER_VERSION=v1.2.1 - -show_help() { -cat << EOF -Usage: $(basename "$0") - -h, --help Display help - -v, --version The chart-releaser version to use (default: $DEFAULT_CHART_RELEASER_VERSION)" - --config The path to the chart-releaser config file - -d, --charts-dir The charts directory (default: charts) - -u, --charts-repo-url The GitHub Pages URL to the charts repo (default: https://.github.io/) - -o, --owner The repo owner - -r, --repo The repo name - -p, --production Enables uploading releases to github releases - -s, --standalone Disables Chart Releaser and Catalog Validation, for local generation -EOF -} - -main() { - local version="$DEFAULT_CHART_RELEASER_VERSION" - local config= - local standalone= - local charts_dir=charts/** - local owner= - local repo= - local production= - local charts_repo_url= - local token=${CR_TOKEN:-false} - local parthreads=$(($(nproc) * 2)) - - parse_command_line "$@" - if [ "${token}" == "false" ]; then - echo "env #CR_TOKEN not found, defaulting to production=false" - production="false" - fi - - local repo_root - repo_root=$(git rev-parse --show-toplevel) - pushd "$repo_root" > /dev/null - - echo 'Looking up latest tag...' - local latest_tag - latest_tag=$(lookup_latest_tag) - - echo "Discovering changed charts since '$latest_tag'..." - local changed_charts=() - readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")" - # copy_general_docs - if [[ -n "${changed_charts[*]}" ]]; then - - prep_helm - - parallel -j ${parthreads} chart_runner '2>&1' ::: ${changed_charts[@]} - echo "Starting post-processing" - # pre_commit - validate_catalog - if [ "${production}" == "true" ]; then - gen_dh_cat - upload_catalog - upload_dhcatalog - fi - else - echo "Nothing to do. No chart changes detected." - fi - - popd > /dev/null -} - -chart_runner(){ - if [[ -d "${1}" ]]; then - echo "Start processing ${1} ..." - chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${1}) - train=$(basename $(dirname "${1}")) - SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - helm dependency build "${1}" --skip-refresh || (sleep 10 && helm dependency build "${1}" --skip-refresh) || (sleep 10 && helm dependency build "${1}" --skip-refresh) - if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "${1}" "${chartname}" "$train" "${chartversion}" - include_questions "${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "${1}" "${chartname}" "$train" "${chartversion}" - else - echo "Skipping chart ${1}, no correct SCALE compatibility layer detected" - fi - else - echo "Chart '${1}' no longer exists in repo. Skipping it..." - fi - echo "Done processing ${1} ..." -} -export -f chart_runner - -include_questions(){ - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} - } -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - - } -export -f clean_catalog - -pre_commit() { - if [[ -z "$standalone" ]]; then - echo "Running pre-commit test-and-cleanup..." - pre-commit run --all ||: - # Fix sh files to always be executable - find . -name '*.sh' | xargs chmod +x - fi - } - export -f pre_commit - -create_changelog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local prevversion="$(git tag -l "${chartname}-*" --sort=-v:refname | head -n 1)" - if [[ -z "$standalone" ]]; then - echo "Generating changelogs for: ${chartname}" - # SCALE "Changelog" containing only last change - git-chglog --next-tag ${chartname}-${chartversion} --tag-filter-pattern ${chartname} --path ${chart} -o ${chart}/app-changelog.md ${chartname}-${chartversion} - # Append SCALE changelog to actual changelog - - if [[ -f "${chart}/CHANGELOG.md" ]]; then - true - else - touch ${chart}/CHANGELOG.md - fi - sed -i '1d' ${chart}/CHANGELOG.md - cat ${chart}/app-changelog.md | cat - ${chart}/CHANGELOG.md > temp && mv temp ${chart}/CHANGELOG.md - sed -i '1s/^/# Changelog
\n\n/' ${chart}/CHANGELOG.md - rm ${chart}/app-changelog.md || echo "changelog not found..." - fi - } - export -f create_changelog - -copy_general_docs() { - yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : - yes | cp -rf .github/README.md docs/index.md 2>/dev/null || : - sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md - sed -i 's~~~g' docs/index.md - yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || : - yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || : - yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || : - yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md - yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : - sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md - } - export -f copy_general_docs - - -copy_docs() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying docs for: ${chart}" - mkdir -p docs/charts/${train}/${chartname} || echo "chart path already exists, continuing..." - yes | cp -rf ${chart}/CHANGELOG.md docs/charts/${train}/${chartname}/CHANGELOG.md 2>/dev/null || : - yes | cp -rf ${chart}/security.md docs/charts/${train}/${chartname}/security.md 2>/dev/null || : - yes | cp -rf ${chart}/CONFIG.md docs/charts/${train}/${chartname}/CONFIG.md 2>/dev/null || : - yes | cp -rf ${chart}/helm-values.md docs/charts/${train}/${chartname}/helm-values.md 2>/dev/null || : - rm docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - yes | cp -rf ${chart}/LICENSE docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - } - export -f copy_docs - -prep_helm() { - if [[ -z "$standalone" ]]; then - echo "helm repo update..." - # helm repo update - fi - } - export -f prep_helm - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - sed -i '100,$ d' ${target}/CHANGELOG.md || : - mv ${target}/app-changelog.md ${target}/CHANGELOG.md 2>/dev/null || : - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - echo "- $category" >> catalog/${train}/${chartname}/item.yaml - # Generate SCALE App description file - cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This Chart is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying Chart to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -validate_catalog() { - if [[ -z "$standalone" ]]; then - echo "Starting Catalog Validation" - /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" - fi -} -export -f validate_catalog - -upload_catalog() { - echo "Uploading Catalog..." - cd catalog - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - git add --all - git commit -sm "Commit new Chart releases for TrueCharts" || exit 0 - git push - cd - - rm -rf catalog -} -export -f upload_catalog - -parse_command_line() { - while :; do - case "${1:-}" in - -h|--help) - show_help - exit - ;; - --config) - if [[ -n "${2:-}" ]]; then - config="$2" - shift - else - config=".github/cr.yaml" - shift - fi - ;; - -v|--version) - if [[ -n "${2:-}" ]]; then - version="$2" - shift - else - echo "ERROR: '-v|--version' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -d|--charts-dir) - if [[ -n "${2:-}" ]]; then - charts_dir="$2" - shift - else - echo "ERROR: '-d|--charts-dir' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -u|--charts-repo-url) - if [[ -n "${2:-}" ]]; then - charts_repo_url="$2" - shift - else - echo "ERROR: '-u|--charts-repo-url' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -o|--owner) - if [[ -n "${2:-}" ]]; then - owner="$2" - shift - else - echo "ERROR: '--owner' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -r|--repo) - if [[ -n "${2:-}" ]]; then - repo="$2" - shift - else - echo "ERROR: '--repo' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -p|--production) - production="true" - ;; - -s|--standalone) - standalone="true" - ;; - *) - break - ;; - esac - - shift - done - - if [[ -z "$owner" ]]; then - echo "No owner configured, defaulting to truecharts" >&2 - owner="truecharts" - fi - - if [[ -z "$repo" ]]; then - echo "No repo configured, defaulting to charts" >&2 - repo="charts" - fi - - if [[ -z "$charts_repo_url" ]]; then - charts_repo_url="https://$owner.github.io/$repo" - fi -} -export -f parse_command_line - -lookup_latest_tag() { - git fetch --tags > /dev/null 2>&1 - - if ! git describe --tags --abbrev=0 2> /dev/null; then - git rev-list --max-parents=0 --first-parent HEAD - fi -} -export -f lookup_latest_tag - -filter_charts() { - while read -r chart; do - [[ ! -d "$chart" ]] && continue - if [[ $(git diff $latest_tag $chart/Chart.yaml | grep "+version") ]]; then - echo "$chart" - else - echo "Version not bumped. Skipping." 1>&2 - fi - done -} -export -f filter_charts - -lookup_changed_charts() { - local commit="$1" - - local changed_files - changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml") - - local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 )) - local fields="1-${depth}" - - cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts -} -export -f lookup_changed_charts - -main "$@" diff --git a/tools/gen-cat.sh b/tools/gen-cat.sh deleted file mode 100755 index e5e38f05274..00000000000 --- a/tools/gen-cat.sh +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env bash -set -o errexit -set -o nounset -set -o pipefail - -DEFAULT_CHART_RELEASER_VERSION=v1.2.1 - -show_help() { -cat << EOF -Usage: $(basename "$0") - -h, --help Display help - -v, --version The chart-releaser version to use (default: $DEFAULT_CHART_RELEASER_VERSION)" - --config The path to the chart-releaser config file - -d, --charts-dir The charts directory (default: charts) - -u, --charts-repo-url The GitHub Pages URL to the charts repo (default: https://.github.io/) - -o, --owner The repo owner - -r, --repo The repo name - -p, --production Enables uploading releases to github releases - -s, --standalone Disables Chart Releaser and Catalog Validation, for local generation -EOF -} - -main() { - local version="$DEFAULT_CHART_RELEASER_VERSION" - local config= - local standalone= - local charts_dir=charts/** - local owner= - local repo= - local production= - local charts_repo_url= - local token=${CR_TOKEN:-false} - local parthreads=$(($(nproc) * 2)) - - parse_command_line "$@" - if [ "${token}" == "false" ]; then - echo "env #CR_TOKEN not found, defaulting to production=false" - production="false" - fi - - local repo_root - repo_root=$(git rev-parse --show-toplevel) - pushd "$repo_root" > /dev/null - - - # readarray -t changed_charts <<< "$(lookup_changed_charts "$latest_tag")" - # copy_general_docs - #if [[ -n "${changed_charts[*]}" ]]; then - - prep_helm - - chart_runner ${1} - echo "Starting post-processing" - # pre_commit - # validate_catalog - # if [ "${production}" == "true" ]; then - # gen_dh_cat - # upload_catalog - # upload_dhcatalog - # fi - #else - # echo "Nothing to do. No chart changes detected." - #fi - - popd > /dev/null -} - -chart_runner(){ - if [[ -d "${1}" ]]; then - echo "Start processing ${1} ..." - chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }') - chartname=$(basename ${1}) - train=$(basename $(dirname "${1}")) - # SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r) - helm dependency build "${1}" --skip-refresh || (sleep 10 && helm dependency build "${1}" --skip-refresh) || (sleep 10 && helm dependency build "${1}" --skip-refresh) - # if [[ "${SCALESUPPORT}" == "true" ]]; then - clean_apps "${1}" "${chartname}" "$train" "${chartversion}" - copy_apps "${1}" "${chartname}" "$train" "${chartversion}" - patch_apps "${1}" "${chartname}" "$train" "${chartversion}" - include_questions "${1}" "${chartname}" "$train" "${chartversion}" - clean_catalog "${1}" "${chartname}" "$train" "${chartversion}" - # else - # echo "Skipping chart ${1}, no correct SCALE compatibility layer detected" - # fi - else - echo "Chart '${1}' no longer exists in repo. Skipping it..." - fi - echo "Done processing ${1} ..." -} -export -f chart_runner - -include_questions(){ - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - - local source="charts/${train}/${chartname}/questions.yaml" - local target="catalog/${train}/${chartname}/${chartversion}/questions.yaml" - - echo "Making copy of $source to $target" - cp ${source} ${target} - - echo "Including standardised questions.yaml includes for: ${chartname}" - sed -i -E 's:^.*# Include\{(.*)\}.*$:cat templates/questions/**/\1.yaml:e' ${target} - } -export -f include_questions - -clean_catalog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - cd catalog/${train}/${chartname} - local majorversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( ! -iname ".*" \) | sed 's|^\./||g') | sort -Vr | cut -c1 | uniq) - echo "Removing old versions for: ${chartname}" - for majorversion in ${majorversions}; do - local maxofmajor=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) | sed 's|^\./||g') | sort -Vr | head -n1 ) - local rmversions=$( (find . -mindepth 1 -maxdepth 1 -type d \( -iname "${majorversion}.*" \) \( ! -iname "${maxofmajor}" \) | sed 's|^\./||g') | sort -Vr ) - rm -Rf ${rmversions} - done - cd - - } -export -f clean_catalog - -pre_commit() { - if [[ -z "$standalone" ]]; then - echo "Running pre-commit test-and-cleanup..." - pre-commit run --all ||: - # Fix sh files to always be executable - find . -name '*.sh' | xargs chmod +x - fi - } - export -f pre_commit - -create_changelog() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local prevversion="$(git tag -l "${chartname}-*" --sort=-v:refname | head -n 1)" - if [[ -z "$standalone" ]]; then - echo "Generating changelogs for: ${chartname}" - # SCALE "Changelog" containing only last change - git-chglog --next-tag ${chartname}-${chartversion} --tag-filter-pattern ${chartname} --path ${chart} -o ${chart}/app-changelog.md ${chartname}-${chartversion} - # Append SCALE changelog to actual changelog - - if [[ -f "${chart}/CHANGELOG.md" ]]; then - true - else - touch ${chart}/CHANGELOG.md - fi - sed -i '1d' ${chart}/CHANGELOG.md - cat ${chart}/app-changelog.md | cat - ${chart}/CHANGELOG.md > temp && mv temp ${chart}/CHANGELOG.md - sed -i '1s/^/# Changelog
\n\n/' ${chart}/CHANGELOG.md - rm ${chart}/app-changelog.md || echo "changelog not found..." - fi - } - export -f create_changelog - -copy_general_docs() { - yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || : - yes | cp -rf .github/README.md docs/index.md 2>/dev/null || : - sed -i '1s/^/---\nhide:\n - navigation\n - toc\n---\n/' docs/index.md - sed -i 's~~~g' docs/index.md - yes | cp -rf .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md 2>/dev/null || : - yes | cp -rf .github/CONTRIBUTING docs/development/contributing.md 2>/dev/null || : - yes | cp -rf .github/SUPPORT.md docs/manual/SUPPORT.md 2>/dev/null || : - yes | cp -rf LICENSE docs/about/legal/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/about/legal/LICENSE.md - yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || : - sed -i '1s/^/# NOTICE
\n\n/' docs/about/legal/NOTICE.md - } - export -f copy_general_docs - - -copy_docs() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying docs for: ${chart}" - mkdir -p docs/charts/${train}/${chartname} || echo "chart path already exists, continuing..." - yes | cp -rf ${chart}/CHANGELOG.md docs/charts/${train}/${chartname}/CHANGELOG.md 2>/dev/null || : - yes | cp -rf ${chart}/security.md docs/charts/${train}/${chartname}/security.md 2>/dev/null || : - yes | cp -rf ${chart}/CONFIG.md docs/charts/${train}/${chartname}/CONFIG.md 2>/dev/null || : - yes | cp -rf ${chart}/helm-values.md docs/charts/${train}/${chartname}/helm-values.md 2>/dev/null || : - rm docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - yes | cp -rf ${chart}/LICENSE docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - sed -i '1s/^/# License
\n\n/' docs/charts/${train}/${chartname}/LICENSE.md 2>/dev/null || : - } - export -f copy_docs - -prep_helm() { - if [[ -z "$standalone" ]]; then - helm repo add truecharts https://charts.truecharts.org - helm repo add truecharts-deps https://deps.truecharts.org - helm repo add truecharts-library https://library-charts.truecharts.org - helm repo update - fi - } - export -f prep_helm - -clean_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Cleaning SCALE catalog for Chart: ${chartname}" - rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || : - rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || : -} -export -f clean_apps - -patch_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - local target="catalog/${train}/${chartname}/${chartversion}" - echo "Applying SCALE patches for Chart: ${chartname}" - sed -i '100,$ d' ${target}/CHANGELOG.md || : - mv ${target}/app-changelog.md ${target}/CHANGELOG.md 2>/dev/null || : - # Temporary fix to prevent the UI from bugging out on 21.08 - mv ${target}/values.yaml ${target}/ix_values.yaml 2>/dev/null || : - touch ${target}/values.yaml - # mv ${target}/SCALE/ix_values.yaml ${target}/ 2>/dev/null || : - cp -rf ${target}/SCALE/templates/* ${target}/templates 2>/dev/null || : - rm -rf ${target}/SCALE 2>/dev/null || : - touch ${target}/values.yaml - # Generate item.yaml - cat ${target}/Chart.yaml | grep "icon" >> catalog/${train}/${chartname}/item.yaml - sed -i "s|^icon:|icon_url:|g" catalog/${train}/${chartname}/item.yaml - echo "categories:" >> catalog/${train}/${chartname}/item.yaml - # category=$(cat ${target}/Chart.yaml | yq '.annotations."truecharts.org/category"' -r) - # echo "- $category" >> catalog/${train}/${chartname}/item.yaml - # Generate SCALE App description file - # cat ${target}/Chart.yaml | yq .description -r >> ${target}/app-readme.md - echo "" >> ${target}/app-readme.md - echo "This Chart is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md -} -export -f patch_apps - -copy_apps() { - local chart="$1" - local chartname="$2" - local train="$3" - local chartversion="$4" - echo "Copying Chart to Catalog: ${2}" - mkdir -p catalog/${train}/${chartname}/${chartversion} - cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/ - -} -export -f copy_apps - -validate_catalog() { - if [[ -z "$standalone" ]]; then - echo "Starting Catalog Validation" - /usr/local/bin/catalog_validate validate --path "${PWD}/catalog" - fi -} -export -f validate_catalog - -upload_catalog() { - echo "Uploading Catalog..." - cd catalog - git config user.name "TrueCharts-Bot" - git config user.email "bot@truecharts.org" - git add --all - git commit -sm "Commit new Chart releases for TrueCharts" || exit 0 - git push - cd - - rm -rf catalog -} -export -f upload_catalog - -parse_command_line() { - while :; do - case "${1:-}" in - -h|--help) - show_help - exit - ;; - --config) - if [[ -n "${2:-}" ]]; then - config="$2" - shift - else - config=".github/cr.yaml" - shift - fi - ;; - -v|--version) - if [[ -n "${2:-}" ]]; then - version="$2" - shift - else - echo "ERROR: '-v|--version' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -d|--charts-dir) - if [[ -n "${2:-}" ]]; then - charts_dir="$2" - shift - else - echo "ERROR: '-d|--charts-dir' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -u|--charts-repo-url) - if [[ -n "${2:-}" ]]; then - charts_repo_url="$2" - shift - else - echo "ERROR: '-u|--charts-repo-url' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -o|--owner) - if [[ -n "${2:-}" ]]; then - owner="$2" - shift - else - echo "ERROR: '--owner' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -r|--repo) - if [[ -n "${2:-}" ]]; then - repo="$2" - shift - else - echo "ERROR: '--repo' cannot be empty." >&2 - show_help - exit 1 - fi - ;; - -p|--production) - production="true" - ;; - -s|--standalone) - standalone="true" - ;; - *) - break - ;; - esac - - shift - done - - if [[ -z "$owner" ]]; then - echo "No owner configured, defaulting to truecharts" >&2 - owner="truecharts" - fi - - if [[ -z "$repo" ]]; then - echo "No repo configured, defaulting to charts" >&2 - repo="charts" - fi - - if [[ -z "$charts_repo_url" ]]; then - charts_repo_url="https://$owner.github.io/$repo" - fi -} -export -f parse_command_line - -lookup_latest_tag() { - git fetch --tags > /dev/null 2>&1 - - if ! git describe --tags --abbrev=0 2> /dev/null; then - git rev-list --max-parents=0 --first-parent HEAD - fi -} -export -f lookup_latest_tag - -filter_charts() { - while read -r chart; do - [[ ! -d "$chart" ]] && continue - if [[ $(git diff $latest_tag $chart/Chart.yaml | grep "+version") ]]; then - echo "$chart" - else - echo "Version not bumped. Skipping." 1>&2 - fi - done -} -export -f filter_charts - -lookup_changed_charts() { - local commit="$1" - - local changed_files - changed_files=$(git diff --find-renames --name-only "$commit" -- "$charts_dir" | grep "Chart.yaml") - - local depth=$(( $(tr "/" "\n" <<< "$charts_dir" | sed '/^\(\.\)*$/d' | wc -l) + 1 )) - local fields="1-${depth}" - - cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts -} -export -f lookup_changed_charts - -main "$@" diff --git a/tools/process.sh b/tools/process.sh deleted file mode 100755 index 06b224d4354..00000000000 --- a/tools/process.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -eu - -# This script will do some basic processing from k8s-at-home to TrueCharts -nodeport=36150 -for chart in input/*; do - if [ -d "${chart}" ]; then - nodeport=$((nodeport+1)) - basename=$(basename ${chart}) - echo "processing ${chart} using nodeport $nodeport" - mkdir -p output/${basename} - cp -rf ../templates/app/SCALE output/${basename}/ - cp -rf ${chart}/ci output/${basename}/ || echo "no CI folder detected, continuing..." - cp -rf ${chart}/templates output/${basename}/ - cp -rf ${chart}/Chart.yaml output/${basename}/Chart.yaml - cp -rf ${chart}/.helmignore output/${basename}/.helmignore || echo "helmignore not detected, continuing..." - cp -rf ${chart}/values.yaml output/${basename}/values.yaml - iconurl=$(cat ${chart}/Chart.yaml | grep -m 1 icon: ) && iconurl=${iconurl/icon: /} - sed -i "s|PLACEHOLDERICON|${iconurl}|g" output/${basename}/SCALE/item.yaml - repo=$(cat ${chart}/values.yaml | grep -m 1 repository: ) && repo=${repo/ repository: /} - tag=$(cat ${chart}/values.yaml | grep -m 1 tag: ) && tag=${tag/ tag: /} - sed -i "s|PLACEHOLDERREPO|${repo}|g" output/${basename}/SCALE/ix_values.yaml - sed -i "s|PLACEHOLDERTAG|${tag}|g" output/${basename}/SCALE/ix_values.yaml - port=$(cat ${chart}/values.yaml | grep -m 1 port: ) && port=${port/ port: /} - sed -i "s|PLACEHOLDERPORT|${port}|g" output/${basename}/SCALE/questions.yaml - sed -i "s|PLACEHOLDERNODEPORT|${nodeport}|g" output/${basename}/SCALE/questions.yaml - - - fi -done