feat(ci): Download each dep to each own repo dir, and use a cached index instead of fetch each time (#4393)

* download each dep to its repo dir, to avoid confclits

* only fetch index once

* clean index cache on start

* fancy print

* make sure it's always `index.yaml`

* add caches to gitignore

* empty lines

* don't wipe

.github/scripts/fetch_helm_deps.sh

@@ -9,6 +9,8 @@ command -v go-yq >/dev/null 2>&1 || {
 # define defaults
 cache_path=${cache_path:-./tgz_cache}
 charts_path=${charts_path:-./charts}
+# Do NOT persist this directory, in order to always have the latest index for this run.
+index_cache=${index_cache:-./index_cache}
 
 mkdir -p "$cache_path"
 
@@ -27,7 +29,7 @@ deps=$(go-yq '.dependencies' "$charts_path/$train_chart/Chart.yaml")
 # Find how many deps exist, so we can loop through them
 length=$(echo "$deps" | go-yq '. | length')
 
-echo "🔨 Processing <$charts_path/$train_chart>... Dependencies: $length"
+echo "🤖🔨 Processing <$charts_path/$train_chart>... Dependencies: $length"
 echo ""
 
 for idx in $(eval echo "{0..$length}"); do
@@ -39,32 +41,56 @@ for idx in $(eval echo "{0..$length}"); do
         version=$(echo "$curr_dep" | go-yq '.version')
         repo=$(echo "$curr_dep" | go-yq '.repository')
 
+        # Remove http:// or https:// from url to create a dir name
+        repo_dir="${repo#http://}"
+        repo_dir="${repo#https://}"
+
         echo "**********"
         echo "🔗 Dependency: $name"
         echo "🆚 Version: $version"
         echo "🏠 Repository: $repo"
         echo ""
 
-        if [ -f "$cache_path/$name-$version.tgz" ]; then
+        if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then
             echo "✅ Dependency exists in cache..."
         else
             echo "🤷‍♂️ Dependency does not exists in cache..."
 
             repo_url="$repo/index.yaml"
-            echo "🤖 Calculating URL..."
+            if [ -f "$index_cache/$repo_dir/index.yaml" ]; then
+                echo "✅ Index for <$repo> exists!"
+            else
+                echo "⏬ Index for <$repo> is missing. Downloading from <$repo_url>..."
+
+                mkdir -p $index_cache/$repo_dir
+                wget --quiet "$repo_url" -O "$index_cache/$repo_dir/index.yaml"
+                if [ ! $? ]; then
+                    echo "❌ wget encountered an error..."
+                    exit 1
+                fi
+
+                if [ -f "$index_cache/$repo_dir/index.yaml" ]; then
+                    echo "✅ Downloaded index for <$repo>!"
+                else
+                    echo "❌ Failed to download index for <$repo> from <$repo_url>"
+                    exit 1
+                fi
+            fi
+
             # At the time of writing this, only 1 url existed (.urls[0]) pointing to the actual tgz.
             # Extract url from repo_url. It's under .entries.DEP_NAME.urls. We filter the specific version first (.version)
-            dep_url=$(curl -s "$repo_url" | v="$version" n="$name" go-yq '.entries.[env(n)].[] | select (.version == env(v)) | .urls.[0]')
+            dep_url=$(v="$version" n="$name" go-yq '.entries.[env(n)].[] | select (.version == env(v)) | .urls.[0]' "$index_cache/$repo_dir/index.yaml")
 
             echo ""
             echo "⏬ Downloading dependency $name-$version from $dep_url..."
-            wget --quiet "$dep_url" -P "$cache_path/"
+            mkdir -p "$cache_path/$repo_dir"
+            wget --quiet "$dep_url" -P "$cache_path/$repo_dir"
             if [ ! $? ]; then
                 echo "❌ wget encountered an error..."
                 helm dependency build "$charts_path/$train_chart/Chart.yaml" || helm dependency update "$charts_path/$train_chart/Chart.yaml" || exit 1
             fi
 
-            if [ -f "$cache_path/$name-$version.tgz" ]; then
+            if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then
                 echo "✅ Dependency Downloaded!"
             else
                 echo "❌ Failed to download dependency"
@@ -76,7 +102,7 @@ for idx in $(eval echo "{0..$length}"); do
 
         mkdir -p "$charts_path/$train_chart/charts"
         echo "📝 Copying dependency <$name-$version.tgz> to <$charts_path/$train_chart/charts>..."
-        cp "$cache_path/$name-$version.tgz" "$charts_path/$train_chart/charts"
+        cp "$cache_path/$repo_dir/$name-$version.tgz" "$charts_path/$train_chart/charts"
 
         if [ -f "$charts_path/$train_chart/charts/$name-$version.tgz" ]; then
             echo "✅ Dependency copied!"

.gitignore

@@ -54,3 +54,5 @@ chartsrepo/
 .cr-index/
 website/
 pub/
+tgz_cache/
+index_cache/