feat(ci): Run release pipeline in parallel (#1463)
* move actual running to function
* add some test charts
* prepare for parallel execution
* enable parallel execution
* use newer devcontainer
* use bash subshell for helm dependency update
* export all the things
* use j2 for parallel
* whoops
* helm template devnull
* use correct tooling
* questions and catalog functions export
* Revert "add some test charts"
This reverts commit d9acc26d6a.
* use correct variables
* nproc*2
* dev null the verbose helm template output
* some trivy scanning cleanup
* fix small mistake
This commit is contained in:
Kjeld Schouten-Lebbing
GitHub
parent
1b6c21ca76
commit
b8c9799765
|
|
@ -159,7 +159,7 @@ jobs:
|
|||
release-test:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: ghcr.io/truecharts/devcontainer:v2.1.0
|
||||
image: ghcr.io/truecharts/devcontainer:v2.1.1
|
||||
steps:
|
||||
- name: Cache helm repo cache
|
||||
id: cache
|
||||
|
|
@ -183,4 +183,4 @@ jobs:
|
|||
|
||||
- name: build-and-run
|
||||
run: |
|
||||
find ./ -type f -name *.yaml -exec sed -i 's/tccr.io/ghcr.io/gI' {} \; && tools/build-release.sh --config .github/cr.yaml
|
||||
tools/build-release.sh --config .github/cr.yaml
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ main() {
|
|||
local production=
|
||||
local charts_repo_url=
|
||||
local token=${CR_TOKEN:-false}
|
||||
local parthreads=$(($(nproc) * 2))
|
||||
|
||||
parse_command_line "$@"
|
||||
if [ "${token}" == "false" ]; then
|
||||
|
|
@ -58,37 +59,8 @@ main() {
|
|||
mkdir -p .cr-index
|
||||
|
||||
prep_helm
|
||||
for chart in "${changed_charts[@]}"; do
|
||||
if [[ -d "$chart" ]]; then
|
||||
echo "Start processing $chart ..."
|
||||
chartversion=$(cat ${chart}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
|
||||
chartname=$(basename ${chart})
|
||||
train=$(basename $(dirname "$chart"))
|
||||
SCALESUPPORT=$(cat ${chart}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r)
|
||||
sync_tag "$chart" "$chartname" "$train" "$chartversion" || echo "Tag sync failed..."
|
||||
helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh
|
||||
helm_sec_scan "$chart" "$chartname" "$train" "$chartversion" || echo "helm-chart security-scan failed..."
|
||||
container_sec_scan "$chart" "$chartname" "$train" "$chartversion" || echo "container security-scan failed..."
|
||||
sec_scan_cleanup "$chart" "$chartname" "$train" "$chartversion" || echo "security-scan cleanup failed..."
|
||||
create_changelog "$chart" "$chartname" "$train" "$chartversion" || echo "changelog generation failed..."
|
||||
generate_docs "$chart" "$chartname" "$train" "$chartversion" || echo "Docs generation failed..."
|
||||
copy_docs "$chart" "$chartname" "$train" "$chartversion" || echo "Docs Copy failed..."
|
||||
helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh || sleep 10 && helm dependency update "${chart}" --skip-refresh
|
||||
package_chart "$chart"
|
||||
if [[ "${SCALESUPPORT}" == "true" ]]; then
|
||||
clean_apps "$chart" "$chartname" "$train" "$chartversion"
|
||||
copy_apps "$chart" "$chartname" "$train" "$chartversion"
|
||||
patch_apps "$chart" "$chartname" "$train" "$chartversion"
|
||||
include_questions "$chart" "$chartname" "$train" "$chartversion"
|
||||
clean_catalog "$chart" "$chartname" "$train" "$chartversion"
|
||||
else
|
||||
echo "Skipping chart ${chart}, no correct SCALE compatibility layer detected"
|
||||
fi
|
||||
else
|
||||
echo "Chart '$chart' no longer exists in repo. Skipping it..."
|
||||
fi
|
||||
echo "Done processing $chart ..."
|
||||
done
|
||||
|
||||
parallel -j ${parthreads} chart_runner ::: ${changed_charts[@]}
|
||||
echo "Starting post-processing"
|
||||
pre_commit
|
||||
validate_catalog
|
||||
|
|
@ -111,6 +83,38 @@ main() {
|
|||
popd > /dev/null
|
||||
}
|
||||
|
||||
chart_runner(){
|
||||
if [[ -d "${1}" ]]; then
|
||||
echo "Start processing ${1} ..."
|
||||
chartversion=$(cat ${1}/Chart.yaml | grep "^version: " | awk -F" " '{ print $2 }')
|
||||
chartname=$(basename ${1})
|
||||
train=$(basename $(dirname "${1}"))
|
||||
SCALESUPPORT=$(cat ${1}/Chart.yaml | yq '.annotations."truecharts.org/SCALE-support"' -r)
|
||||
helm dependency update "${1}" --skip-refresh || (sleep 10 && helm dependency update "${1}" --skip-refresh) || (sleep 10 && helm dependency update "${1}" --skip-refresh)
|
||||
helm_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "helm-chart security-scan failed..."
|
||||
container_sec_scan "${1}" "${chartname}" "$train" "${chartversion}" || echo "container security-scan failed..."
|
||||
sec_scan_cleanup "${1}" "${chartname}" "$train" "${chartversion}" || echo "security-scan cleanup failed..."
|
||||
sync_tag "${1}" "${chartname}" "$train" "${chartversion}" || echo "Tag sync failed..."
|
||||
create_changelog "${1}" "${chartname}" "$train" "${chartversion}" || echo "changelog generation failed..."
|
||||
generate_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs generation failed..."
|
||||
copy_docs "${1}" "${chartname}" "$train" "${chartversion}" || echo "Docs Copy failed..."
|
||||
package_chart "${1}"
|
||||
if [[ "${SCALESUPPORT}" == "true" ]]; then
|
||||
clean_apps "${1}" "${chartname}" "$train" "${chartversion}"
|
||||
copy_apps "${1}" "${chartname}" "$train" "${chartversion}"
|
||||
patch_apps "${1}" "${chartname}" "$train" "${chartversion}"
|
||||
include_questions "${1}" "${chartname}" "$train" "${chartversion}"
|
||||
clean_catalog "${1}" "${chartname}" "$train" "${chartversion}"
|
||||
else
|
||||
echo "Skipping chart ${1}, no correct SCALE compatibility layer detected"
|
||||
fi
|
||||
else
|
||||
echo "Chart '${1}' no longer exists in repo. Skipping it..."
|
||||
fi
|
||||
echo "Done processing ${1} ..."
|
||||
}
|
||||
export -f chart_runner
|
||||
|
||||
include_questions(){
|
||||
local chart="$1"
|
||||
local chartname="$2"
|
||||
|
|
@ -235,6 +239,7 @@ include_questions(){
|
|||
1' templates/questions/resources.yaml ${target}/questions.yaml > tmp && mv tmp ${target}/questions.yaml
|
||||
|
||||
}
|
||||
export -f include_questions
|
||||
|
||||
clean_catalog() {
|
||||
local chart="$1"
|
||||
|
|
@ -251,6 +256,7 @@ clean_catalog() {
|
|||
done
|
||||
cd -
|
||||
}
|
||||
export -f clean_catalog
|
||||
|
||||
# Designed to ensure the appversion in Chart.yaml is in sync with the primary App tag if found
|
||||
sync_tag() {
|
||||
|
|
@ -276,6 +282,7 @@ sync_tag() {
|
|||
tag="${tag%.}"
|
||||
sed -i -e "s|appVersion: .*|appVersion: \"${tag}\"|" "${chart}/Chart.yaml"
|
||||
}
|
||||
export -f sync_tag
|
||||
|
||||
helm_sec_scan() {
|
||||
local chart="$1"
|
||||
|
|
@ -291,10 +298,11 @@ helm_sec_scan() {
|
|||
echo "" >> ${chart}/sec-scan.md
|
||||
echo "##### Scan Results" >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
helm template ${chart} --output-dir ${chart}/render
|
||||
helm template ${chart} --output-dir ${chart}/render > /dev/null
|
||||
trivy config -f template --template "@./templates/trivy.tpl" ${chart}/render >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
}
|
||||
export -f helm_sec_scan
|
||||
|
||||
container_sec_scan() {
|
||||
local chart="$1"
|
||||
|
|
@ -306,20 +314,22 @@ container_sec_scan() {
|
|||
echo "" >> ${chart}/sec-scan.md
|
||||
echo "##### Detected Containers" >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
find ${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp
|
||||
find ./${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp
|
||||
cat ${chart}/render/containers.tmp >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
echo "##### Scan Results" >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
for container in $(cat ${chart}/render/containers.tmp); do
|
||||
ghcrcont="$(echo ${container} | sed 's/tccr.io/ghcr.io/g')"
|
||||
echo "processing container: ${container} using ${ghcrcont}"
|
||||
echo "**Container: ${container}**" >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
ghcrcont=$(echo ${container} | sed "s/tccr.io/ghcr.io/g")
|
||||
trivy image -f template --template "@./templates/trivy.tpl" ${ghcrcont} >> ${chart}/sec-scan.md
|
||||
trivy image -f template --template "@./templates/trivy.tpl" "${ghcrcont}" >> ${chart}/sec-scan.md
|
||||
echo "" >> ${chart}/sec-scan.md
|
||||
done
|
||||
|
||||
}
|
||||
export -f container_sec_scan
|
||||
|
||||
sec_scan_cleanup() {
|
||||
local chart="$1"
|
||||
|
|
@ -329,6 +339,7 @@ sec_scan_cleanup() {
|
|||
rm -rf ${chart}/render
|
||||
sed -i 's/ghcr.io/tccr.io/g' ${chart}/sec-scan.md
|
||||
}
|
||||
export -f sec_scan_cleanup
|
||||
|
||||
pre_commit() {
|
||||
if [[ -z "$standalone" ]]; then
|
||||
|
|
@ -338,6 +349,7 @@ pre_commit() {
|
|||
find . -name '*.sh' | xargs chmod +x
|
||||
fi
|
||||
}
|
||||
export -f pre_commit
|
||||
|
||||
edit_release() {
|
||||
local chart="$1"
|
||||
|
|
@ -347,6 +359,7 @@ edit_release() {
|
|||
# In here we can in the future add code to edit the release notes of the github releases
|
||||
# For example: using the github API: https://docs.github.com/en/rest/reference/repos#update-a-release
|
||||
}
|
||||
export -f edit_release
|
||||
|
||||
create_changelog() {
|
||||
local chart="$1"
|
||||
|
|
@ -371,6 +384,7 @@ create_changelog() {
|
|||
rm ${chart}/app-changelog.md || echo "changelog not found..."
|
||||
fi
|
||||
}
|
||||
export -f create_changelog
|
||||
|
||||
copy_general_docs() {
|
||||
yes | cp -rf index.yaml docs/index.yaml 2>/dev/null || :
|
||||
|
|
@ -385,6 +399,7 @@ copy_general_docs() {
|
|||
yes | cp -rf NOTICE docs/about/legal/NOTICE.md 2>/dev/null || :
|
||||
sed -i '1s/^/# NOTICE<br>\n\n/' docs/about/legal/NOTICE.md
|
||||
}
|
||||
export -f copy_general_docs
|
||||
|
||||
generate_docs() {
|
||||
local chart="$1"
|
||||
|
|
@ -423,6 +438,7 @@ generate_docs() {
|
|||
fi
|
||||
fi
|
||||
}
|
||||
export -f generate_docs
|
||||
|
||||
|
||||
copy_docs() {
|
||||
|
|
@ -447,6 +463,7 @@ copy_docs() {
|
|||
sed -i '1s/^/# License<br>\n\n/' docs/apps/${train}/${chartname}/LICENSE.md 2>/dev/null || :
|
||||
fi
|
||||
}
|
||||
export -f copy_docs
|
||||
|
||||
prep_helm() {
|
||||
if [[ -z "$standalone" ]]; then
|
||||
|
|
@ -458,6 +475,7 @@ prep_helm() {
|
|||
helm repo update
|
||||
fi
|
||||
}
|
||||
export -f prep_helm
|
||||
|
||||
clean_apps() {
|
||||
local chart="$1"
|
||||
|
|
@ -468,6 +486,7 @@ clean_apps() {
|
|||
rm -Rf catalog/${train}/${chartname}/${chartversion} 2>/dev/null || :
|
||||
rm -Rf catalog/${train}/${chartname}/item.yaml 2>/dev/null || :
|
||||
}
|
||||
export -f clean_apps
|
||||
|
||||
patch_apps() {
|
||||
local chart="$1"
|
||||
|
|
@ -495,6 +514,7 @@ patch_apps() {
|
|||
echo "" >> ${target}/app-readme.md
|
||||
echo "This App is supplied by TrueCharts, for more information please visit https://truecharts.org" >> ${target}/app-readme.md
|
||||
}
|
||||
export -f patch_apps
|
||||
|
||||
copy_apps() {
|
||||
local chart="$1"
|
||||
|
|
@ -506,6 +526,7 @@ copy_apps() {
|
|||
cp -Rf ${chart}/* catalog/${train}/${chartname}/${chartversion}/
|
||||
|
||||
}
|
||||
export -f copy_apps
|
||||
|
||||
validate_catalog() {
|
||||
if [[ -z "$standalone" ]]; then
|
||||
|
|
@ -513,6 +534,7 @@ validate_catalog() {
|
|||
/usr/local/bin/catalog_validate validate --path "${PWD}/catalog"
|
||||
fi
|
||||
}
|
||||
export -f validate_catalog
|
||||
|
||||
|
||||
parse_command_line() {
|
||||
|
|
@ -609,6 +631,7 @@ parse_command_line() {
|
|||
charts_repo_url="https://$owner.github.io/$repo"
|
||||
fi
|
||||
}
|
||||
export -f parse_command_line
|
||||
|
||||
lookup_latest_tag() {
|
||||
git fetch --tags > /dev/null 2>&1
|
||||
|
|
@ -617,6 +640,7 @@ lookup_latest_tag() {
|
|||
git rev-list --max-parents=0 --first-parent HEAD
|
||||
fi
|
||||
}
|
||||
export -f lookup_latest_tag
|
||||
|
||||
filter_charts() {
|
||||
while read -r chart; do
|
||||
|
|
@ -628,6 +652,7 @@ filter_charts() {
|
|||
fi
|
||||
done
|
||||
}
|
||||
export -f filter_charts
|
||||
|
||||
lookup_changed_charts() {
|
||||
local commit="$1"
|
||||
|
|
@ -640,6 +665,7 @@ lookup_changed_charts() {
|
|||
|
||||
cut -d '/' -f "$fields" <<< "$changed_files" | uniq | filter_charts
|
||||
}
|
||||
export -f lookup_changed_charts
|
||||
|
||||
package_chart() {
|
||||
local chart="$1"
|
||||
|
|
@ -653,6 +679,7 @@ package_chart() {
|
|||
cr package "${args[@]}"
|
||||
fi
|
||||
}
|
||||
export -f package_chart
|
||||
|
||||
release_charts() {
|
||||
local args=(-o "$owner" -r "$repo" -c "$(git rev-parse HEAD)")
|
||||
|
|
@ -665,6 +692,7 @@ release_charts() {
|
|||
cr upload "${args[@]}"
|
||||
fi
|
||||
}
|
||||
export -f release_charts
|
||||
|
||||
update_index() {
|
||||
local args=(-o "$owner" -r "$repo" -c "$charts_repo_url")
|
||||
|
|
@ -677,5 +705,6 @@ update_index() {
|
|||
cr index "${args[@]}"
|
||||
fi
|
||||
}
|
||||
export -f update_index
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
Loading…
Reference in New Issue