feat(ci): Download each dep to each own repo dir, and use a cached index instead of fetch each time (#4393)

* download each dep to its repo dir, to avoid confclits

* only fetch index once

* clean index cache on start

* fancy print

* make sure it's always `index.yaml`

* add caches to gitignore

* empty lines

* don't wipe
This commit is contained in:
Stavros Kois 2022-11-12 20:06:22 +02:00 committed by GitHub
parent e5deeaf4fd
commit 881c939dc0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 7 deletions

View File

@ -9,6 +9,8 @@ command -v go-yq >/dev/null 2>&1 || {
# define defaults # define defaults
cache_path=${cache_path:-./tgz_cache} cache_path=${cache_path:-./tgz_cache}
charts_path=${charts_path:-./charts} charts_path=${charts_path:-./charts}
# Do NOT persist this directory, in order to always have the latest index for this run.
index_cache=${index_cache:-./index_cache}
mkdir -p "$cache_path" mkdir -p "$cache_path"
@ -27,7 +29,7 @@ deps=$(go-yq '.dependencies' "$charts_path/$train_chart/Chart.yaml")
# Find how many deps exist, so we can loop through them # Find how many deps exist, so we can loop through them
length=$(echo "$deps" | go-yq '. | length') length=$(echo "$deps" | go-yq '. | length')
echo "🔨 Processing <$charts_path/$train_chart>... Dependencies: $length" echo "🤖🔨 Processing <$charts_path/$train_chart>... Dependencies: $length"
echo "" echo ""
for idx in $(eval echo "{0..$length}"); do for idx in $(eval echo "{0..$length}"); do
@ -39,32 +41,56 @@ for idx in $(eval echo "{0..$length}"); do
version=$(echo "$curr_dep" | go-yq '.version') version=$(echo "$curr_dep" | go-yq '.version')
repo=$(echo "$curr_dep" | go-yq '.repository') repo=$(echo "$curr_dep" | go-yq '.repository')
# Remove http:// or https:// from url to create a dir name
repo_dir="${repo#http://}"
repo_dir="${repo#https://}"
echo "**********" echo "**********"
echo "🔗 Dependency: $name" echo "🔗 Dependency: $name"
echo "🆚 Version: $version" echo "🆚 Version: $version"
echo "🏠 Repository: $repo" echo "🏠 Repository: $repo"
echo "" echo ""
if [ -f "$cache_path/$name-$version.tgz" ]; then if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then
echo "✅ Dependency exists in cache..." echo "✅ Dependency exists in cache..."
else else
echo "🤷‍♂️ Dependency does not exists in cache..." echo "🤷‍♂️ Dependency does not exists in cache..."
repo_url="$repo/index.yaml" repo_url="$repo/index.yaml"
echo "🤖 Calculating URL..." if [ -f "$index_cache/$repo_dir/index.yaml" ]; then
echo "✅ Index for <$repo> exists!"
else
echo "⏬ Index for <$repo> is missing. Downloading from <$repo_url>..."
mkdir -p $index_cache/$repo_dir
wget --quiet "$repo_url" -O "$index_cache/$repo_dir/index.yaml"
if [ ! $? ]; then
echo "❌ wget encountered an error..."
exit 1
fi
if [ -f "$index_cache/$repo_dir/index.yaml" ]; then
echo "✅ Downloaded index for <$repo>!"
else
echo "❌ Failed to download index for <$repo> from <$repo_url>"
exit 1
fi
fi
# At the time of writing this, only 1 url existed (.urls[0]) pointing to the actual tgz. # At the time of writing this, only 1 url existed (.urls[0]) pointing to the actual tgz.
# Extract url from repo_url. It's under .entries.DEP_NAME.urls. We filter the specific version first (.version) # Extract url from repo_url. It's under .entries.DEP_NAME.urls. We filter the specific version first (.version)
dep_url=$(curl -s "$repo_url" | v="$version" n="$name" go-yq '.entries.[env(n)].[] | select (.version == env(v)) | .urls.[0]') dep_url=$(v="$version" n="$name" go-yq '.entries.[env(n)].[] | select (.version == env(v)) | .urls.[0]' "$index_cache/$repo_dir/index.yaml")
echo "" echo ""
echo "⏬ Downloading dependency $name-$version from $dep_url..." echo "⏬ Downloading dependency $name-$version from $dep_url..."
wget --quiet "$dep_url" -P "$cache_path/" mkdir -p "$cache_path/$repo_dir"
wget --quiet "$dep_url" -P "$cache_path/$repo_dir"
if [ ! $? ]; then if [ ! $? ]; then
echo "❌ wget encountered an error..." echo "❌ wget encountered an error..."
helm dependency build "$charts_path/$train_chart/Chart.yaml" || helm dependency update "$charts_path/$train_chart/Chart.yaml" || exit 1 helm dependency build "$charts_path/$train_chart/Chart.yaml" || helm dependency update "$charts_path/$train_chart/Chart.yaml" || exit 1
fi fi
if [ -f "$cache_path/$name-$version.tgz" ]; then if [ -f "$cache_path/$repo_dir/$name-$version.tgz" ]; then
echo "✅ Dependency Downloaded!" echo "✅ Dependency Downloaded!"
else else
echo "❌ Failed to download dependency" echo "❌ Failed to download dependency"
@ -76,7 +102,7 @@ for idx in $(eval echo "{0..$length}"); do
mkdir -p "$charts_path/$train_chart/charts" mkdir -p "$charts_path/$train_chart/charts"
echo "📝 Copying dependency <$name-$version.tgz> to <$charts_path/$train_chart/charts>..." echo "📝 Copying dependency <$name-$version.tgz> to <$charts_path/$train_chart/charts>..."
cp "$cache_path/$name-$version.tgz" "$charts_path/$train_chart/charts" cp "$cache_path/$repo_dir/$name-$version.tgz" "$charts_path/$train_chart/charts"
if [ -f "$charts_path/$train_chart/charts/$name-$version.tgz" ]; then if [ -f "$charts_path/$train_chart/charts/$name-$version.tgz" ]; then
echo "✅ Dependency copied!" echo "✅ Dependency copied!"

2
.gitignore vendored
View File

@ -54,3 +54,5 @@ chartsrepo/
.cr-index/ .cr-index/
website/ website/
pub/ pub/
tgz_cache/
index_cache/