diff --git a/.github/scripts/fetch_helm_deps.sh b/.github/scripts/fetch_helm_deps.sh deleted file mode 100755 index 20d065838e1..00000000000 --- a/.github/scripts/fetch_helm_deps.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/bin/bash - -# require go-yq -command -v go-yq >/dev/null 2>&1 || { - printf >&2 "%s\n" "❌ go-yq (https://github.com/mikefarah/yq) is not installed. Aborting." - exit 1 -} - -# define defaults -cache_path=${cache_path:-./tgz_cache} -charts_path=${charts_path:-./charts} -# Do NOT persist this directory, in order to always have the latest index for this run. -index_cache=${index_cache:-./index_cache} - -mkdir -p "$cache_path" - -trains=( - "enterprise" - "stable" - "incubator" - "dependency" - "operators" -) - -load_gpg_key() { -echo "" -echo "⏬ Downloading and Loading TrueCharts pgp Public Key" -gpg_dir=.cr-gpg -mkdir -p "$gpg_dir" -curl --silent https://keybase.io/truecharts/pgp_keys.asc | gpg --dearmor > $gpg_dir/pubring.gpg || echo "❌ Couldn't load Public Key." -curl --silent https://cert-manager.io/public-keys/cert-manager-keyring-2021-09-20-1020CF3C033D4F35BAE1C19E1226061C665DF13E.gpg > $gpg_dir/certman.gpg || echo "❌ Couldn't load certman Public Key." -echo "✅ Public Key loaded successfully..." -echo "" -} -export -f load_gpg_key - -download_deps() { -local train_chart="$1" - -# Extract dependencies for the Chart -deps=$(go-yq '.dependencies' "$charts_path/$train_chart/Chart.yaml") -# Find how many deps exist, so we can loop through them -length=$(echo "$deps" | go-yq '. | length') - -echo "🤖🔨 Processing <$charts_path/$train_chart>... Dependencies: $length" -echo "" - -for idx in $(eval echo "{0..$length}"); do - # Retrieve info for the dep in the current index.. - curr_dep=$(echo "$deps" | pos="$idx" go-yq '.[env(pos)]') - - if [ ! "$curr_dep" == null ]; then - name=$(echo "$curr_dep" | go-yq '.name') - version=$(echo "$curr_dep" | go-yq '.version') - repo=$(echo "$curr_dep" | go-yq '.repository') - - # Remove http:// or https:// from url to create a dir name - repo_dir="${repo#http://}" - repo_dir="${repo#https://}" - - echo "**********" - echo "🔗 Dependency: $name" - echo "🆚 Version: $version" - echo "🏠 Repository: $repo" - echo "" - - if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then - echo "✅ Dependency exists in cache..." - else - echo "🤷‍♂️ Dependency does not exists in cache..." - - repo_url="$repo/index.yaml" - if [ -f "$index_cache/$repo_dir/index.yaml" ]; then - echo "✅ Index for <$repo> exists!" - else - echo "⏬ Index for <$repo> is missing. Downloading from <$repo_url>..." - - mkdir -p $index_cache/$repo_dir - wget --quiet "$repo_url" -O "$index_cache/$repo_dir/index.yaml" - if [ ! $? ]; then - echo "❌ wget encountered an error..." - exit 1 - fi - - if [ -f "$index_cache/$repo_dir/index.yaml" ]; then - echo "✅ Downloaded index for <$repo>!" - else - echo "❌ Failed to download index for <$repo> from <$repo_url>" - exit 1 - fi - fi - - # At the time of writing this, only 1 url existed (.urls[0]) pointing to the actual tgz. - # Extract url from repo_url. It's under .entries.DEP_NAME.urls. We filter the specific version first (.version) - dep_url=$(v="$version" n="$name" go-yq '.entries.[env(n)].[] | select (.version == env(v)) | .urls.[0]' "$index_cache/$repo_dir/index.yaml") - - if [[ ! "$dep_url" == "https"* ]]; then - # If the chart uses relative path, drop the "/index.yaml" - # from the repo_url and append the relative path - repo=$(echo "$repo_url" | sed 's/\/index\.yaml//') - dep_url="${repo}/${dep_url}" - fi - - if [[ -z "$dep_url" ]]; then - echo "❌ Dependency URL is empty." - echo "Make sure the version is correct and the dependency exists in the index." - echo "Aborting..." - exit 1 - fi - - echo "🔗 Dependency URL: $dep_url" - - echo "" - echo "⏬ Downloading dependency $name-$version from $dep_url..." - mkdir -p "$cache_path/$repo_dir" - wget --quiet "$dep_url" -P "$cache_path/$repo_dir" - wget --quiet "$dep_url.prov" -P "$cache_path/$repo_dir" || echo "failed downloading .prov" - - if [ ! $? ]; then - echo "❌ wget encountered an error..." - if [[ "$train_chart" =~ incubator\/.* ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "velero" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "metallb" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "cloudnative-pg" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "kube-prometheus-stack" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "openebs" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "csi-driver-smb" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "csi-driver-nfs" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "cert-manager" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/certman.gpg || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/certman.gpg || exit 1 - else - helm dependency build "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/pubring.gpg || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/pubring.gpg || exit 1 - fi - fi - - if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then - echo "✅ Dependency Downloaded!" - if [[ "$name" =~ "cert-manager" ]]; then - helm verify $cache_path/$repo_dir/$name-$version.tgz --keyring $gpg_dir/certman.gpg || \ - helm verify $cache_path/$repo_dir/$name-$version.tgz --keyring $gpg_dir/certman.gpg || exit 1 - elif [[ "$name" =~ "velero" ]]; then - echo "Velero is not signed..." - elif [[ "$name" =~ "metallb" ]]; then - echo "metallb is not signed..." - elif [[ "$name" =~ "cloudnative-pg" ]]; then - echo "cloudnative-pg is not signed..." - elif [[ "$name" =~ "kube-prometheus-stack" ]]; then - echo "kube-prometheus-stack is not signed..." - elif [[ "$name" =~ "openebs" ]]; then - echo "openebs is not signed..." - elif [[ "$name" =~ "csi-driver-smb" ]]; then - echo "csi-driver-smb is not signed..." - elif [[ "$name" =~ "csi-driver-nfs" ]]; then - echo "csi-driver-nfs is not signed..." - elif [[ ! "$train_chart" =~ incubator\/.* ]]; then - echo "Validating dependency signature..." - helm verify $cache_path/$repo_dir/$name-$version.tgz --keyring $gpg_dir/pubring.gpg || \ - helm verify $cache_path/$repo_dir/$name-$version.tgz --keyring $gpg_dir/pubring.gpg || exit 1 - else - echo "Skipping dependency signature verification for $train_chart..." - fi - else - echo "❌ Failed to download dependency" - # Try helm dependency build/update or otherwise fail fast if a dep fails to download... - if [[ "$train_chart" =~ incubator\/.* ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "velero" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "metallb" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "cloudnative-pg" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "kube-prometheus-stack" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "openebs" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "csi-driver-smb" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "csi-driver-nfs" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml"|| exit 1 - elif [[ "$name" =~ "cert-manager" ]]; then - helm dependency build "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/certman.gpg || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/certman.gpg || exit 1 - else - helm dependency build "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/pubring.gpg || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml" --verify --keyring $gpg_dir/pubring.gpg || exit 1 - fi - fi - fi - echo "" - - mkdir -p "$charts_path/$train_chart/charts" - echo "📝 Copying dependency <$name-$version.tgz> to <$charts_path/$train_chart/charts>..." - cp "$cache_path/$repo_dir/$name-$version.tgz" "$charts_path/$train_chart/charts" - - if [ -f "$charts_path/$train_chart/charts/$name-$version.tgz" ]; then - echo "✅ Dependency copied!" - echo "" - else - echo "❌ Failed to copy dependency" - # Try helm dependency build/update or otherwise fail fast if a dep fails to copy... - helm dependency build "$charts_path/$train_chart/Chart.yaml" || \ - helm dependency update "$charts_path/$train_chart/Chart.yaml" || exit 1 - fi - fi -done -} -export -f download_deps - -load_gpg_key - -if [ -z "$1" ]; then - for train in "${trains[@]}"; do - for chart in $(ls "$charts_path/$train"); do - download_deps "${train}/${chart}" - done - done -else - download_deps "$1" -fi diff --git a/.github/workflows/charts-lint.yaml b/.github/workflows/charts-lint.yaml index fc4aeb4e83c..7d745ec8473 100644 --- a/.github/workflows/charts-lint.yaml +++ b/.github/workflows/charts-lint.yaml @@ -102,9 +102,7 @@ jobs: charts_path: "./" run: | CHANGED=$(echo '${{ steps.list-changed.outputs.CHANGED_CHARTS }}' | jq --raw-output '.[]') - for changed in ${CHANGED}; do - .github/scripts/fetch_helm_deps.sh "${changed}" - done + charttool deps ${CHANGED} - name: Run Chart Linting continue-on-error: true diff --git a/.github/workflows/charts-release.yaml b/.github/workflows/charts-release.yaml index db6e3b02592..d8f606da215 100644 --- a/.github/workflows/charts-release.yaml +++ b/.github/workflows/charts-release.yaml @@ -263,10 +263,7 @@ jobs: steps.collect-changes.outputs.changesDetectedAfterTag == 'true' run: | CHARTS=(${{ steps.collect-changes.outputs.modifiedChartsAfterTag }}) - charts_path="charts" - for changed in ${CHARTS[@]}; do - .github/scripts/fetch_helm_deps.sh "${changed}" - done + charttool deps ${CHARTS[@]} pip install yq parthreads=$(($(nproc) * 2)) parallel -j ${parthreads} .github/scripts/build-catalog.sh '2>&1' ::: ${CHARTS[@]} diff --git a/charttool b/charttool index aaa1970436c..7219bb83f86 100755 Binary files a/charttool and b/charttool differ