From bcc81487854c98794b8b796cbaac4ce5fb82a4f0 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Date: Tue, 8 Dec 2020 23:23:50 +0500 Subject: [PATCH] Add initial minio helm chart --- test/minio/8.0.5/.helmignore | 23 ++ test/minio/8.0.5/Chart.yaml | 16 + test/minio/8.0.5/README.md | 1 + test/minio/8.0.5/ci/distributed-values.yaml | 1 + test/minio/8.0.5/templates/NOTES.txt | 47 +++ .../8.0.5/templates/_helper_create_bucket.txt | 111 ++++++ test/minio/8.0.5/templates/_helpers.tpl | 182 +++++++++ test/minio/8.0.5/templates/clusterroles.yaml | 20 + test/minio/8.0.5/templates/configmap.yaml | 12 + test/minio/8.0.5/templates/deployment.yaml | 203 +++++++++++ test/minio/8.0.5/templates/ingress.yaml | 45 +++ test/minio/8.0.5/templates/networkpolicy.yaml | 25 ++ .../8.0.5/templates/poddisruptionbudget.yaml | 13 + .../post-install-create-bucket-job.yaml | 87 +++++ .../post-install-prometheus-metrics-job.yaml | 135 +++++++ .../post-install-prometheus-metrics-role.yaml | 38 ++ ...nstall-prometheus-metrics-rolebinding.yaml | 20 + ...all-prometheus-metrics-serviceaccount.yaml | 12 + test/minio/8.0.5/templates/pvc.yaml | 35 ++ test/minio/8.0.5/templates/rolebindings.yaml | 20 + test/minio/8.0.5/templates/secrets.yaml | 32 ++ .../templates/securitycontextconstraints.yaml | 46 +++ test/minio/8.0.5/templates/service.yaml | 47 +++ .../minio/8.0.5/templates/serviceaccount.yaml | 11 + .../minio/8.0.5/templates/servicemonitor.yaml | 41 +++ test/minio/8.0.5/templates/statefulset.yaml | 174 +++++++++ test/minio/8.0.5/values.yaml | 344 ++++++++++++++++++ 27 files changed, 1741 insertions(+) create mode 100644 test/minio/8.0.5/.helmignore create mode 100644 test/minio/8.0.5/Chart.yaml create mode 120000 test/minio/8.0.5/README.md create mode 100644 test/minio/8.0.5/ci/distributed-values.yaml create mode 100644 test/minio/8.0.5/templates/NOTES.txt create mode 100644 test/minio/8.0.5/templates/_helper_create_bucket.txt create mode 100644 test/minio/8.0.5/templates/_helpers.tpl create mode 100644 test/minio/8.0.5/templates/clusterroles.yaml create mode 100644 test/minio/8.0.5/templates/configmap.yaml create mode 100644 test/minio/8.0.5/templates/deployment.yaml create mode 100644 test/minio/8.0.5/templates/ingress.yaml create mode 100644 test/minio/8.0.5/templates/networkpolicy.yaml create mode 100644 test/minio/8.0.5/templates/poddisruptionbudget.yaml create mode 100644 test/minio/8.0.5/templates/post-install-create-bucket-job.yaml create mode 100644 test/minio/8.0.5/templates/post-install-prometheus-metrics-job.yaml create mode 100644 test/minio/8.0.5/templates/post-install-prometheus-metrics-role.yaml create mode 100644 test/minio/8.0.5/templates/post-install-prometheus-metrics-rolebinding.yaml create mode 100644 test/minio/8.0.5/templates/post-install-prometheus-metrics-serviceaccount.yaml create mode 100644 test/minio/8.0.5/templates/pvc.yaml create mode 100644 test/minio/8.0.5/templates/rolebindings.yaml create mode 100644 test/minio/8.0.5/templates/secrets.yaml create mode 100644 test/minio/8.0.5/templates/securitycontextconstraints.yaml create mode 100644 test/minio/8.0.5/templates/service.yaml create mode 100644 test/minio/8.0.5/templates/serviceaccount.yaml create mode 100644 test/minio/8.0.5/templates/servicemonitor.yaml create mode 100644 test/minio/8.0.5/templates/statefulset.yaml create mode 100644 test/minio/8.0.5/values.yaml diff --git a/test/minio/8.0.5/.helmignore b/test/minio/8.0.5/.helmignore new file mode 100644 index 00000000000..a9fe7278811 --- /dev/null +++ b/test/minio/8.0.5/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# OWNERS file for Kubernetes +OWNERS \ No newline at end of file diff --git a/test/minio/8.0.5/Chart.yaml b/test/minio/8.0.5/Chart.yaml new file mode 100644 index 00000000000..dae38ea07d0 --- /dev/null +++ b/test/minio/8.0.5/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +description: High Performance, Kubernetes Native Object Storage +name: minio +version: 8.0.5 +appVersion: master +keywords: +- storage +- object-storage +- S3 +home: https://min.io +icon: https://min.io/resources/img/logo/MINIO_wordmark.png +sources: +- https://github.com/minio/minio +maintainers: +- name: MinIO, Inc + email: dev@minio.io diff --git a/test/minio/8.0.5/README.md b/test/minio/8.0.5/README.md new file mode 120000 index 00000000000..32d46ee883b --- /dev/null +++ b/test/minio/8.0.5/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/test/minio/8.0.5/ci/distributed-values.yaml b/test/minio/8.0.5/ci/distributed-values.yaml new file mode 100644 index 00000000000..e6c46738087 --- /dev/null +++ b/test/minio/8.0.5/ci/distributed-values.yaml @@ -0,0 +1 @@ +mode: distributed diff --git a/test/minio/8.0.5/templates/NOTES.txt b/test/minio/8.0.5/templates/NOTES.txt new file mode 100644 index 00000000000..77efd37ff67 --- /dev/null +++ b/test/minio/8.0.5/templates/NOTES.txt @@ -0,0 +1,47 @@ +{{- if eq .Values.service.type "ClusterIP" "NodePort" }} +Minio can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: +{{ template "minio.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + +To access Minio from localhost, run the below commands: + + 1. export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + + 2. kubectl port-forward $POD_NAME 9000 --namespace {{ .Release.Namespace }} + +Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ + +You can now access Minio server on http://localhost:9000. Follow the below steps to connect to Minio server with mc client: + + 1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide + + 2. Get the ACCESS_KEY=$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.accesskey}" | base64 --decode) and the SECRET_KEY=$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.secretkey}" | base64 --decode) + + 3. mc alias set {{ template "minio.fullname" . }}-local http://localhost:{{ .Values.service.port }} "$ACCESS_KEY" "$SECRET_KEY" --api s3v4 + + 4. mc ls {{ template "minio.fullname" . }}-local + +Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17 +{{- end }} +{{- if eq .Values.service.type "LoadBalancer" }} +Minio can be accessed via port {{ .Values.service.port }} on an external IP address. Get the service external IP address by: +kubectl get svc --namespace {{ .Release.Namespace }} -l app={{ template "minio.fullname" . }} + +Note that the public IP may take a couple of minutes to be available. + +You can now access Minio server on http://:9000. Follow the below steps to connect to Minio server with mc client: + + 1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide + + 2. Get the ACCESS_KEY=$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.accesskey}" | base64 --decode) and the SECRET_KEY=$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.secretkey}" | base64 --decode) + 3. mc alias set {{ template "minio.fullname" . }} http://:{{ .Values.service.port }} "$ACCESS_KEY" "$SECRET_KEY" --api s3v4 + + 4. mc ls {{ template "minio.fullname" . }} + +Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17 +{{- end }} + +{{ if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} +Note: Since NetworkPolicy is enabled, only pods with label +{{ template "minio.fullname" . }}-client=true" +will be able to connect to this minio cluster. +{{- end }} diff --git a/test/minio/8.0.5/templates/_helper_create_bucket.txt b/test/minio/8.0.5/templates/_helper_create_bucket.txt new file mode 100644 index 00000000000..ad2f546b758 --- /dev/null +++ b/test/minio/8.0.5/templates/_helper_create_bucket.txt @@ -0,0 +1,111 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for Minio service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkBucketExists ($bucket) +# Check if the bucket exists, by using the exit code of `mc ls` +checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1) + return $? +} + +# createBucket ($bucket, $policy, $purge) +# Ensure bucket exists, purging if asked to +createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET ; then + echo "Purging bucket '$BUCKET'." + set +e ; # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e ; # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist + if ! checkBucketExists $BUCKET ; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + + + # set versioning for bucket + if [ ! -z $VERSIONING ] ; then + if [ $VERSIONING = true ] ; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ] ; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} policy set $POLICY myminio/$BUCKET +} + +# Try connecting to Minio instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{- if or .Values.defaultBucket.enabled }} +# Create the bucket +createBucket {{ .Values.defaultBucket.name }} {{ .Values.defaultBucket.policy }} {{ .Values.defaultBucket.purge }} {{ .Values.defaultBucket.versioning }} +{{ else if .Values.buckets }} +# Create the buckets +{{- range .Values.buckets }} +createBucket {{ .name }} {{ .policy }} {{ .purge }} {{ .versioning }} +{{- end }} +{{- end }} diff --git a/test/minio/8.0.5/templates/_helpers.tpl b/test/minio/8.0.5/templates/_helpers.tpl new file mode 100644 index 00000000000..227762b13e6 --- /dev/null +++ b/test/minio/8.0.5/templates/_helpers.tpl @@ -0,0 +1,182 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minio.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "minio.networkPolicy.apiVersion" -}} +{{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.Version -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "^1.7-0" .Capabilities.KubeVersion.Version -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "minio.deployment.apiVersion" -}} +{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "minio.statefulset.apiVersion" -}} +{{- if semverCompare "<1.17-0" .Capabilities.KubeVersion.Version -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "minio.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Determine secret name. +*/}} +{{- define "minio.secretName" -}} +{{- if .Values.existingSecret -}} +{{- .Values.existingSecret }} +{{- else -}} +{{- include "minio.fullname" . -}} +{{- end -}} +{{- end -}} + +{{/* +Determine service account name for deployment or statefulset. +*/}} +{{- define "minio.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "minio.fullname" .) .Values.serviceAccount.name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* +Determine name for scc role and rolebinding +*/}} +{{- define "minio.sccRoleName" -}} +{{- printf "%s-%s" "scc" (include "minio.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Properly format optional additional arguments to Minio binary +*/}} +{{- define "minio.extraArgs" -}} +{{- range .Values.extraArgs -}} +{{ " " }}{{ . }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "minio.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- end -}} + +{{/* +Formats volumeMount for Minio tls keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolumeMount" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + mountPath: {{ .Values.certsPath }} +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $casPath := printf "%s/CAs" .Values.certsPath | clean }} +- name: trusted-cert-secret-volume + mountPath: {{ $casPath }} +{{- end }} +{{- end -}} + +{{/* +Formats volume for Minio tls keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolume" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + secret: + secretName: {{ .Values.tls.certSecret }} + items: + - key: {{ .Values.tls.publicCrt }} + path: public.crt + - key: {{ .Values.tls.privateKey }} + path: private.key +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $certSecret := eq .Values.trustedCertsSecret "" | ternary .Values.tls.certSecret .Values.trustedCertsSecret }} +{{- $publicCrt := eq .Values.trustedCertsSecret "" | ternary .Values.tls.publicCrt "" }} +- name: trusted-cert-secret-volume + secret: + secretName: {{ $certSecret }} + {{- if ne $publicCrt "" }} + items: + - key: {{ $publicCrt }} + path: public.crt + {{- end }} +{{- end }} +{{- end -}} diff --git a/test/minio/8.0.5/templates/clusterroles.yaml b/test/minio/8.0.5/templates/clusterroles.yaml new file mode 100644 index 00000000000..c4d9a9371b5 --- /dev/null +++ b/test/minio/8.0.5/templates/clusterroles.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.securityContext.enabled .Values.persistence.enabled (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "minio.serviceAccountName" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - {{ template "minio.fullname" . }} + verbs: + - use +{{- end }} diff --git a/test/minio/8.0.5/templates/configmap.yaml b/test/minio/8.0.5/templates/configmap.yaml new file mode 100644 index 00000000000..cb11fcd7dda --- /dev/null +++ b/test/minio/8.0.5/templates/configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + initialize: |- +{{ include (print $.Template.BasePath "/_helper_create_bucket.txt") . | indent 4 }} diff --git a/test/minio/8.0.5/templates/deployment.yaml b/test/minio/8.0.5/templates/deployment.yaml new file mode 100644 index 00000000000..2fa9edf173e --- /dev/null +++ b/test/minio/8.0.5/templates/deployment.yaml @@ -0,0 +1,203 @@ +{{- if eq .Values.mode "standalone" }} +{{ $scheme := "http" }} +{{- if .Values.tls.enabled }} +{{ $scheme = "https" }} +{{ end }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +apiVersion: {{ template "minio.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + strategy: + type: {{ .Values.DeploymentUpdate.type }} + {{- if eq .Values.DeploymentUpdate.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ .Values.DeploymentUpdate.maxSurge }} + maxUnavailable: {{ .Values.DeploymentUpdate.maxUnavailable }} + {{- end}} + {{- if .Values.nasgateway.enabled }} + replicas: {{ .Values.nasgateway.replicas }} + {{- end }} + {{- if .Values.s3gateway.enabled }} + replicas: {{ .Values.s3gateway.replicas }} + {{- end }} + {{- if .Values.azuregateway.enabled }} + replicas: {{ .Values.azuregateway.replicas }} + {{- end }} + {{- if .Values.gcsgateway.enabled }} + replicas: {{ .Values.gcsgateway.replicas }} + {{- end }} + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} + annotations: + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- if .Values.podAnnotations }} +{{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }} +{{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + serviceAccountName: {{ include "minio.serviceAccountName" . | quote }} +{{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + runAsGroup: {{ .Values.securityContext.runAsGroup }} + fsGroup: {{ .Values.securityContext.fsGroup }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.s3gateway.enabled }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} gateway s3 {{ .Values.s3gateway.serviceEndpoint }} {{- template "minio.extraArgs" . }}" + {{- else }} + {{- if .Values.azuregateway.enabled }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} gateway azure {{- template "minio.extraArgs" . }}" + {{- else }} + {{- if .Values.gcsgateway.enabled }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} gateway gcs {{ .Values.gcsgateway.projectId }} {{- template "minio.extraArgs" . }}" + {{- else }} + {{- if .Values.nasgateway.enabled }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} gateway nas {{ $bucketRoot }} {{- template "minio.extraArgs" . }}" + {{- else }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} server {{ $bucketRoot }} {{- template "minio.extraArgs" . }}" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + volumeMounts: + {{- if and .Values.persistence.enabled (not .Values.gcsgateway.enabled) (not .Values.azuregateway.enabled) (not .Values.s3gateway.enabled) }} + - name: export + mountPath: {{ .Values.mountPath }} + {{- if .Values.persistence.subPath }} + subPath: "{{ .Values.persistence.subPath }}" + {{- end }} + {{- end }} + {{- if or .Values.gcsgateway.enabled .Values.etcd.clientCert .Values.etcd.clientCertKey }} + - name: minio-user + mountPath: "/etc/credentials" + readOnly: true + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + ports: + - name: {{ $scheme }} + containerPort: 9000 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: secretkey + {{- if and .Values.gcsgateway.enabled .Values.gcsgateway.gcsKeyJson }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/credentials/gcs_key.json" + {{- end }} + {{- if .Values.etcd.endpoints }} + - name: MINIO_ETCD_ENDPOINTS + value: {{ join "," .Values.etcd.endpoints | quote }} + {{- if .Values.etcd.clientCert }} + - name: MINIO_ETCD_CLIENT_CERT + value: "/etc/credentials/etcd_client_cert.pem" + {{- end }} + {{- if .Values.etcd.clientCertKey }} + - name: MINIO_ETCD_CLIENT_CERT_KEY + value: "/etc/credentials/etcd_client_cert_key.pem" + {{- end }} + {{- if .Values.etcd.pathPrefix }} + - name: MINIO_ETCD_PATH_PREFIX + value: {{ .Values.etcd.pathPrefix }} + {{- end }} + {{- if .Values.etcd.corednsPathPrefix }} + - name: MINIO_ETCD_COREDNS_PATH + value: {{ .Values.etcd.corednsPathPrefix }} + {{- end }} + {{- end }} + {{- if .Values.s3gateway.enabled -}} + {{- if or .Values.s3gateway.accessKey .Values.existingSecret }} + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: awsAccessKeyId + {{- end }} + {{- if or .Values.s3gateway.secretKey .Values.existingSecret }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: awsSecretAccessKey + {{- end }} + {{- end }} + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end}} + resources: +{{ toYaml .Values.resources | indent 12 }} +{{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} +{{- end }} +{{- include "minio.imagePullSecrets" . | indent 6 }} +{{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + volumes: + {{- if and (not .Values.gcsgateway.enabled) (not .Values.azuregateway.enabled) (not .Values.s3gateway.enabled) }} + - name: export + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "minio.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} +{{- end }} diff --git a/test/minio/8.0.5/templates/ingress.yaml b/test/minio/8.0.5/templates/ingress.yaml new file mode 100644 index 00000000000..2d9bbda0559 --- /dev/null +++ b/test/minio/8.0.5/templates/ingress.yaml @@ -0,0 +1,45 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "minio.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: {{ template "minio.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.ingress.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - http: + paths: + - path: {{ $ingressPath }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- if . }} + host: {{ . | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/test/minio/8.0.5/templates/networkpolicy.yaml b/test/minio/8.0.5/templates/networkpolicy.yaml new file mode 100644 index 00000000000..de57f485fee --- /dev/null +++ b/test/minio/8.0.5/templates/networkpolicy.yaml @@ -0,0 +1,25 @@ +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: {{ template "minio.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + ingress: + - ports: + - port: {{ .Values.service.port }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "minio.name" . }}-client: "true" + {{- end }} +{{- end }} diff --git a/test/minio/8.0.5/templates/poddisruptionbudget.yaml b/test/minio/8.0.5/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000000..1de813b8b80 --- /dev/null +++ b/test/minio/8.0.5/templates/poddisruptionbudget.yaml @@ -0,0 +1,13 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: minio + labels: + app: {{ template "minio.name" . }} +spec: + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + app: {{ template "minio.name" . }} +{{- end }} \ No newline at end of file diff --git a/test/minio/8.0.5/templates/post-install-create-bucket-job.yaml b/test/minio/8.0.5/templates/post-install-create-bucket-job.yaml new file mode 100644 index 00000000000..27e30d4ab73 --- /dev/null +++ b/test/minio/8.0.5/templates/post-install-create-bucket-job.yaml @@ -0,0 +1,87 @@ +{{- if or .Values.defaultBucket.enabled .Values.buckets }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "minio.fullname" . }}-make-bucket-job + labels: + app: {{ template "minio.name" . }}-make-bucket-job + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +{{- with .Values.makeBucketJob.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + template: + metadata: + labels: + app: {{ template "minio.name" . }}-job + release: {{ .Release.Name }} +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} +{{- if .Values.makeBucketJob.podAnnotations }} + annotations: +{{ toYaml .Values.makeBucketJob.podAnnotations | indent 8 }} +{{- end }} + spec: + restartPolicy: OnFailure +{{- include "minio.imagePullSecrets" . | indent 6 }} +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} +{{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.makeBucketJob.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.makeBucketJob.securityContext.runAsUser }} + runAsGroup: {{ .Values.makeBucketJob.securityContext.runAsGroup }} + fsGroup: {{ .Values.makeBucketJob.securityContext.fsGroup }} +{{- end }} + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: {{ template "minio.fullname" . }} + - secret: + name: {{ template "minio.secretName" . }} + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + secret: + secretName: {{ .Values.tls.certSecret }} + items: + - key: {{ .Values.tls.publicCrt }} + path: CAs/public.crt + {{ end }} + serviceAccountName: {{ include "minio.serviceAccountName" . | quote }} + containers: + - name: minio-mc + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{ end }} + resources: +{{ toYaml .Values.makeBucketJob.resources | indent 10 }} +{{- end }} diff --git a/test/minio/8.0.5/templates/post-install-prometheus-metrics-job.yaml b/test/minio/8.0.5/templates/post-install-prometheus-metrics-job.yaml new file mode 100644 index 00000000000..d7590db7c9c --- /dev/null +++ b/test/minio/8.0.5/templates/post-install-prometheus-metrics-job.yaml @@ -0,0 +1,135 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{- $fullName := include "minio.fullname" . -}} +{{ $scheme := "http" }} +{{- if .Values.tls.enabled }} +{{ $scheme = "https" }} +{{ end }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $fullName }}-update-prometheus-secret + labels: + app: {{ template "minio.name" . }}-update-prometheus-secret + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + {{ toYaml .Values.updatePrometheusJob.annotations | indent 4 }} +spec: + template: + metadata: + labels: + app: {{ template "minio.name" . }}-update-prometheus-secret + release: {{ .Release.Name }} +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} +{{- if .Values.updatePrometheusJob.podAnnotations }} + annotations: +{{ toYaml .Values.updatePrometheusJob.podAnnotations | indent 8 }} +{{- end }} + spec: +{{- if .Values.serviceAccount.create }} + serviceAccountName: {{ $fullName }}-update-prometheus-secret +{{- end }} + restartPolicy: OnFailure +{{- include "minio.imagePullSecrets" . | indent 6 }} +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} +{{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.updatePrometheusJob.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.updatePrometheusJob.securityContext.runAsUser }} + runAsGroup: {{ .Values.updatePrometheusJob.securityContext.runAsGroup }} + fsGroup: {{ .Values.updatePrometheusJob.securityContext.fsGroup }} +{{- end }} + volumes: + - name: workdir + emptyDir: {} + initContainers: + - name: minio-mc + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + command: + - /bin/sh + - "-c" + - mc --config-dir {{ .Values.configPathmc }} admin prometheus generate target --json --no-color -q > /workdir/mc.json + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: secretkey + # mc admin prometheus generate don't really connect to remote server, TLS cert isn't required + - name: MC_HOST_target + value: {{ $scheme }}://$(MINIO_ACCESS_KEY):$(MINIO_SECRET_KEY)@{{ $fullName }}:{{ .Values.service.port }} + volumeMounts: + - name: workdir + mountPath: /workdir + resources: +{{ toYaml .Values.resources | indent 12 }} + # extract bearerToken from mc admin output + - name: jq + image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}" + imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }} + command: + - /bin/sh + - "-c" + - jq -e -c -j -r .bearerToken < /workdir/mc.json > /workdir/token + volumeMounts: + - name: workdir + mountPath: /workdir + resources: +{{ toYaml .Values.resources | indent 12 }} + - name: kubectl-create + image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}" + imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }} + command: + - /bin/sh + - "-c" + # The following script does: + # - get the servicemonitor that need this secret and copy some metadata and create the ownerreference for the secret file + # - create the secret + # - merge both json + - > + kubectl -n {{ .Release.Namespace }} get servicemonitor {{ $fullName }} -o json | + jq -c '{metadata: {name: "{{ $fullName }}-prometheus", namespace: .metadata.namespace, labels: {app: .metadata.labels.app, release: .metadata.labels.release}, ownerReferences: [{apiVersion: .apiVersion, kind: .kind, blockOwnerDeletion: true, controller: true, uid: .metadata.uid, name: .metadata.name}]}}' > /workdir/metadata.json && + kubectl create secret generic {{ $fullName }}-prometheus --from-file=token=/workdir/token --dry-run -o json > /workdir/secret.json && + cat /workdir/secret.json /workdir/metadata.json | jq -s add > /workdir/object.json + volumeMounts: + - name: workdir + mountPath: /workdir + resources: +{{ toYaml .Values.resources | indent 12 }} + containers: + - name: kubectl-apply + image: "{{ .Values.helmKubectlJqImage.repository }}:{{ .Values.helmKubectlJqImage.tag }}" + imagePullPolicy: {{ .Values.helmKubectlJqImage.pullPolicy }} + command: + - kubectl + - apply + - "-f" + - /workdir/object.json + volumeMounts: + - name: workdir + mountPath: /workdir + resources: +{{ toYaml .Values.resources | indent 12 }} +{{- end }} diff --git a/test/minio/8.0.5/templates/post-install-prometheus-metrics-role.yaml b/test/minio/8.0.5/templates/post-install-prometheus-metrics-role.yaml new file mode 100644 index 00000000000..26c0ce7edc4 --- /dev/null +++ b/test/minio/8.0.5/templates/post-install-prometheus-metrics-role.yaml @@ -0,0 +1,38 @@ +{{- if .Values.serviceAccount.create -}} +{{- $fullName := include "minio.fullname" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName }}-update-prometheus-secret + labels: + app: {{ template "minio.name" . }}-update-prometheus-secret + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch + resourceNames: + - {{ $fullName }}-prometheus + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + resourceNames: + - {{ $fullName }} +{{- end -}} \ No newline at end of file diff --git a/test/minio/8.0.5/templates/post-install-prometheus-metrics-rolebinding.yaml b/test/minio/8.0.5/templates/post-install-prometheus-metrics-rolebinding.yaml new file mode 100644 index 00000000000..7d0ea75b752 --- /dev/null +++ b/test/minio/8.0.5/templates/post-install-prometheus-metrics-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.serviceAccount.create -}} +{{- $fullName := include "minio.fullname" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName }}-update-prometheus-secret + labels: + app: {{ template "minio.name" . }}-update-prometheus-secret + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $fullName }}-update-prometheus-secret +subjects: + - kind: ServiceAccount + name: {{ $fullName }}-update-prometheus-secret + namespace: {{ .Release.Namespace | quote }} +{{- end -}} \ No newline at end of file diff --git a/test/minio/8.0.5/templates/post-install-prometheus-metrics-serviceaccount.yaml b/test/minio/8.0.5/templates/post-install-prometheus-metrics-serviceaccount.yaml new file mode 100644 index 00000000000..050d368a562 --- /dev/null +++ b/test/minio/8.0.5/templates/post-install-prometheus-metrics-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +{{- $fullName := include "minio.fullname" . -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $fullName }}-update-prometheus-secret + labels: + app: {{ template "minio.name" . }}-update-prometheus-secret + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} \ No newline at end of file diff --git a/test/minio/8.0.5/templates/pvc.yaml b/test/minio/8.0.5/templates/pvc.yaml new file mode 100644 index 00000000000..014f90f3e62 --- /dev/null +++ b/test/minio/8.0.5/templates/pvc.yaml @@ -0,0 +1,35 @@ +{{- if eq .Values.mode "standalone" }} +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: +{{- if and .Values.nasgateway.enabled .Values.nasgateway.pv }} + selector: + matchLabels: + pv: {{ .Values.nasgateway.pv | quote }} +{{- end }} + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + +{{- if .Values.persistence.storageClass }} +{{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" +{{- end }} +{{- end }} +{{- if .Values.persistence.VolumeName }} + volumeName: "{{ .Values.persistence.VolumeName }}" +{{- end }} +{{- end }} +{{- end }} diff --git a/test/minio/8.0.5/templates/rolebindings.yaml b/test/minio/8.0.5/templates/rolebindings.yaml new file mode 100644 index 00000000000..ea8b98c5857 --- /dev/null +++ b/test/minio/8.0.5/templates/rolebindings.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.securityContext.enabled .Values.persistence.enabled (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "minio.serviceAccountName" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "minio.serviceAccountName" . }} +subjects: +- kind: ServiceAccount + name: {{ template "minio.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/test/minio/8.0.5/templates/secrets.yaml b/test/minio/8.0.5/templates/secrets.yaml new file mode 100644 index 00000000000..c254142f92b --- /dev/null +++ b/test/minio/8.0.5/templates/secrets.yaml @@ -0,0 +1,32 @@ +{{- if not .Values.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "minio.secretName" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + accesskey: {{ if .Values.accessKey }}{{ .Values.accessKey | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 20 | b64enc | quote }}{{ end }} + secretkey: {{ if .Values.secretKey }}{{ .Values.secretKey | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 40 | b64enc | quote }}{{ end }} +{{- if and .Values.gcsgateway.enabled .Values.gcsgateway.gcsKeyJson }} + gcs_key.json: {{ .Values.gcsgateway.gcsKeyJson | toString | b64enc }} +{{- end }} +{{- if .Values.s3gateway.enabled -}} +{{- if .Values.s3gateway.accessKey }} + awsAccessKeyId: {{ .Values.s3gateway.accessKey | toString | b64enc | quote }} +{{- end }} +{{- if .Values.s3gateway.secretKey }} + awsSecretAccessKey: {{ .Values.s3gateway.secretKey | toString | b64enc | quote }} +{{- end }} +{{- end }} +{{- if .Values.etcd.clientCert }} + etcd_client_cert.pem: {{ .Values.etcd.clientCert | toString | b64enc | quote }} +{{- end }} +{{- if .Values.etcd.clientCertKey }} + etcd_client_cert_key.pem: {{ .Values.etcd.clientCertKey | toString | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/test/minio/8.0.5/templates/securitycontextconstraints.yaml b/test/minio/8.0.5/templates/securitycontextconstraints.yaml new file mode 100644 index 00000000000..dfde6fb99e9 --- /dev/null +++ b/test/minio/8.0.5/templates/securitycontextconstraints.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.securityContext.enabled .Values.persistence.enabled (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +priority: 10 +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: [] +readOnlyRootFilesystem: false +defaultAddCapabilities: [] +requiredDropCapabilities: +- KILL +- MKNOD +- SETUID +- SETGID +fsGroup: + type: MustRunAs + ranges: + - max: {{ .Values.securityContext.fsGroup }} + min: {{ .Values.securityContext.fsGroup }} +runAsUser: + type: MustRunAs + uid: {{ .Values.securityContext.runAsUser }} +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- end }} diff --git a/test/minio/8.0.5/templates/service.yaml b/test/minio/8.0.5/templates/service.yaml new file mode 100644 index 00000000000..ea681e22079 --- /dev/null +++ b/test/minio/8.0.5/templates/service.yaml @@ -0,0 +1,47 @@ +{{ $scheme := "http" }} +{{- if .Values.tls.enabled }} +{{ $scheme = "https" }} +{{ end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if (or (eq .Values.service.type "ClusterIP" "") (empty .Values.service.type)) }} + type: ClusterIP + {{- if not (empty .Values.service.clusterIP) }} + clusterIP: {{ .Values.service.clusterIP }} + {{end}} +{{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + loadBalancerIP: {{ default "" .Values.service.loadBalancerIP }} +{{- else }} + type: {{ .Values.service.type }} +{{- end }} + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP +{{- if (and (eq .Values.service.type "NodePort") ( .Values.service.nodePort)) }} + nodePort: {{ .Values.service.nodePort }} +{{- else }} + targetPort: 9000 +{{- end}} +{{- if .Values.service.externalIPs }} + externalIPs: +{{- range $i , $ip := .Values.service.externalIPs }} + - {{ $ip }} +{{- end }} +{{- end }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} diff --git a/test/minio/8.0.5/templates/serviceaccount.yaml b/test/minio/8.0.5/templates/serviceaccount.yaml new file mode 100644 index 00000000000..243dfef538f --- /dev/null +++ b/test/minio/8.0.5/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "minio.serviceAccountName" . | quote }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: "{{ .Release.Name }}" +{{- end -}} diff --git a/test/minio/8.0.5/templates/servicemonitor.yaml b/test/minio/8.0.5/templates/servicemonitor.yaml new file mode 100644 index 00000000000..e1259097345 --- /dev/null +++ b/test/minio/8.0.5/templates/servicemonitor.yaml @@ -0,0 +1,41 @@ +{{- if .Values.metrics.serviceMonitor.enabled }} +{{ $scheme := "http" }} +{{- if .Values.tls.enabled }} +{{ $scheme = "https" }} +{{ end }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "minio.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} +{{ toYaml .Values.metrics.serviceMonitor.additionalLabels | indent 4 }} + {{- end }} +spec: + endpoints: + - port: {{ $scheme }} + path: /minio/prometheus/metrics + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + bearerTokenSecret: + name: {{ template "minio.fullname" . }}-prometheus + key: token + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + selector: + matchLabels: + app: {{ include "minio.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/test/minio/8.0.5/templates/statefulset.yaml b/test/minio/8.0.5/templates/statefulset.yaml new file mode 100644 index 00000000000..b1d2d449770 --- /dev/null +++ b/test/minio/8.0.5/templates/statefulset.yaml @@ -0,0 +1,174 @@ +{{- if eq .Values.mode "distributed" }} +{{ $zoneCount := .Values.zones | int }} +{{ $nodeCount := .Values.replicas | int }} +{{ $drivesPerNode := .Values.drivesPerNode | int }} +{{ $scheme := "http" }} +{{- if .Values.tls.enabled }} +{{ $scheme = "https" }} +{{ end }} +{{ $mountPath := .Values.mountPath }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +{{ $subPath := .Values.persistence.subPath }} +{{ $penabled := .Values.persistence.enabled }} +{{ $accessMode := .Values.persistence.accessMode }} +{{ $storageClass := .Values.persistence.storageClass }} +{{ $psize := .Values.persistence.size }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }}-svc + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + publishNotReadyAddresses: true + clusterIP: None + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} +--- +apiVersion: {{ template "minio.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + updateStrategy: + type: {{ .Values.StatefulSetUpdate.updateStrategy }} + podManagementPolicy: "Parallel" + serviceName: {{ template "minio.fullname" . }}-svc + replicas: {{ mul $zoneCount $nodeCount }} + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} +{{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} +{{- end }} + annotations: + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- if .Values.podAnnotations }} +{{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }} +{{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + serviceAccountName: {{ include "minio.serviceAccountName" . | quote }} +{{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + runAsUser: {{ .Values.securityContext.runAsUser }} + runAsGroup: {{ .Values.securityContext.runAsGroup }} + fsGroup: {{ .Values.securityContext.fsGroup }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + + command: [ "/bin/sh", + "-ce", + "/usr/bin/docker-entrypoint.sh minio -S {{ .Values.certsPath }} server {{- range $i := until $zoneCount }}{{ $factor := mul $i $nodeCount }}{{ $endIndex := add $factor $nodeCount }}{{ $beginIndex := mul $i $nodeCount }} {{ $scheme }}://{{ template `minio.fullname` $ }}-{{ `{` }}{{ $beginIndex }}...{{ sub $endIndex 1 }}{{ `}`}}.{{ template `minio.fullname` $ }}-svc.{{ $.Release.Namespace }}.svc.{{ $.Values.clusterDomain }}{{if (gt $drivesPerNode 1)}}{{ $bucketRoot }}-{{ `{` }}0...{{ sub $drivesPerNode 1 }}{{ `}` }}{{else}}{{ $bucketRoot }}{{end}}{{- end}}{{- template `minio.extraArgs` . }}" ] + volumeMounts: + {{- if $penabled }} + {{- if (gt $drivesPerNode 1) }} + {{- range $i := until $drivesPerNode }} + - name: export-{{ $i }} + mountPath: {{ $mountPath }}-{{ $i }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- else }} + - name: export + mountPath: {{ $mountPath }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + ports: + - name: {{ $scheme }} + containerPort: 9000 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: secretkey + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end}} + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} +{{- include "minio.imagePullSecrets" . | indent 6 }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} +{{- if .Values.persistence.enabled }} + volumeClaimTemplates: + {{- if gt $drivesPerNode 1 }} + {{- range $diskId := until $drivesPerNode}} + - metadata: + name: export-{{ $diskId }} + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} + {{- else }} + - metadata: + name: export + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} +{{- end }} +{{- end }} diff --git a/test/minio/8.0.5/values.yaml b/test/minio/8.0.5/values.yaml new file mode 100644 index 00000000000..d18a8cecc7b --- /dev/null +++ b/test/minio/8.0.5/values.yaml @@ -0,0 +1,344 @@ +## Provide a name in place of minio for `app:` labels +## +nameOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## set kubernetes cluster domain where minio is running +## +clusterDomain: cluster.local + +## Set default image, imageTag, and imagePullPolicy. mode is used to indicate the +## +image: + repository: minio/minio + tag: RELEASE.2020-11-19T23-48-16Z + pullPolicy: IfNotPresent + +## Set default image, imageTag, and imagePullPolicy for the `mc` (the minio +## client used to create a default bucket). +## +mcImage: + repository: minio/mc + tag: RELEASE.2020-11-17T00-39-14Z + pullPolicy: IfNotPresent + +## Set default image, imageTag, and imagePullPolicy for the `jq` (the JSON +## process used to create secret for prometheus ServiceMonitor). +## +helmKubectlJqImage: + repository: bskim45/helm-kubectl-jq + tag: 3.1.0 + pullPolicy: IfNotPresent + +## minio server mode, i.e. standalone or distributed. +## Distributed Minio ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide +## +mode: standalone + +## Additional arguments to pass to minio binary +extraArgs: [] + +## Update strategy for Deployments +DeploymentUpdate: + type: RollingUpdate + maxUnavailable: 0 + maxSurge: 100% + +## Update strategy for StatefulSets +StatefulSetUpdate: + updateStrategy: RollingUpdate + +## Pod priority settings +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" + +## Set default accesskey, secretkey, Minio config file path, volume mount path and +## number of nodes (only used for Minio distributed mode) +## AccessKey and secretKey is generated when not set +## Distributed Minio ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide +## +accessKey: "" +secretKey: "" +certsPath: "/etc/minio/certs/" +configPathmc: "/etc/minio/mc/" +mountPath: "/export" + +## Use existing Secret that store following variables: +## +## | Chart var | .data. in Secret | +## |:----------------------|:-------------------------| +## | accessKey | accesskey | +## | secretKey | secretkey | +## | gcsgateway.gcsKeyJson | gcs_key.json | +## | s3gateway.accessKey | awsAccessKeyId | +## | s3gateway.secretKey | awsSecretAccessKey | +## | etcd.clientCert | etcd_client_cert.pem | +## | etcd.clientCertKey | etcd_client_cert_key.pem | +## +## All mentioned variables will be ignored in values file. +## .data.accesskey and .data.secretkey are mandatory, +## others depend on enabled status of corresponding sections. +existingSecret: "" + +## Override the root directory which the minio server should serve from. +## If left empty, it defaults to the value of {{ .Values.mountPath }} +## If defined, it must be a sub-directory of the path specified in {{ .Values.mountPath }} +bucketRoot: "" + +# Number of drives attached to a node +drivesPerNode: 1 +# Number of MinIO containers running +replicas: 4 +# Number of expanded MinIO clusters +zones: 1 + +## TLS Settings for Minio +tls: + enabled: false + ## Create a secret with private.key and public.crt files and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + certSecret: "" + publicCrt: public.crt + privateKey: private.key + +## Trusted Certificates Settings for Minio. Ref: https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls#install-certificates-from-third-party-cas +## Bundle multiple trusted certificates into one secret and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret +## When using self-signed certificates, remember to include Minio's own certificate in the bundle with key public.crt. +## If certSecret is left empty and tls is enabled, this chart installs the public certificate from .Values.tls.certSecret. +trustedCertsSecret: "" + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + enabled: true + + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + existingClaim: "" + + ## minio data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + ## Storage class of PV to bind. By default it looks for standard storage class. + ## If the PV uses a different storage class, specify that here. + storageClass: "" + VolumeName: "" + accessMode: ReadWriteOnce + size: 500Gi + + ## If subPath is set mount a sub folder of a volume instead of the root of the volume. + ## This is especially handy for volume plugins that don't natively support sub mounting (like glusterfs). + ## + subPath: "" + +## Expose the Minio service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## + +service: + type: ClusterIP + clusterIP: ~ + port: 9000 + nodePort: 32000 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + # - externalIp1 + + annotations: {} + # prometheus.io/scrape: 'true' + # prometheus.io/path: '/minio/prometheus/metrics' + # prometheus.io/port: '9000' + +## Configure Ingress based on the documentation here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +## + +imagePullSecrets: [] +# - name: "image-pull-secret" + +ingress: + enabled: false + labels: {} + # node-role.kubernetes.io/ingress: platform + + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.allow-http: "false" + # kubernetes.io/ingress.global-static-ip-name: "" + # nginx.ingress.kubernetes.io/secure-backends: "true" + # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0 + path: / + hosts: + - chart-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +tolerations: [] +affinity: {} + +## Add stateful containers to have security context, if enabled MinIO will run as this +## user and group NOTE: securityContext is only enabled if persistence.enabled=true +securityContext: + enabled: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +# Additational pod annotations +podAnnotations: {} + +# Additional pod labels +podLabels: {} + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 4Gi + +## Create a bucket after minio install +## +defaultBucket: + enabled: false + ## If enabled, must be a string with length > 0 + name: bucket + ## Can be one of none|download|upload|public + policy: none + ## Purge if bucket exists already + purge: false + ## set versioning for bucket true|false + # versioning: false + +## Create multiple buckets after minio install +## Enabling `defaultBucket` will take priority over this list +## +buckets: [] + # - name: bucket1 + # policy: none + # purge: false + # - name: bucket2 + # policy: none + # purge: false + +## Additional Annotations for the Kubernetes Batch (make-bucket-job) +makeBucketJob: + podAnnotations: + annotations: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + resources: + requests: + memory: 128Mi + +## Additional Annotations for the Kubernetes Batch (update-prometheus-secret) +updatePrometheusJob: + podAnnotations: + annotations: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +s3gateway: + enabled: false + replicas: 4 + serviceEndpoint: "" + accessKey: "" + secretKey: "" + +## Use minio as an azure blob gateway, you should disable data persistence so no volume claim are created. +## https://docs.minio.io/docs/minio-gateway-for-azure +azuregateway: + enabled: false + # Number of parallel instances + replicas: 4 + +## Use minio as GCS (Google Cloud Storage) gateway, you should disable data persistence so no volume claim are created. +## https://docs.minio.io/docs/minio-gateway-for-gcs + +gcsgateway: + enabled: false + # Number of parallel instances + replicas: 4 + # credential json file of service account key + gcsKeyJson: "" + # Google cloud project-id + projectId: "" + +## Use minio on NAS backend +## https://docs.minio.io/docs/minio-gateway-for-nas + +nasgateway: + enabled: false + # Number of parallel instances + replicas: 4 + # For NAS Gateway, you may want to bind the PVC to a specific PV. To ensure that happens, PV to bind to should have + # a label like "pv: ", use value here. + pv: ~ + +## Use this field to add environment variables relevant to Minio server. These fields will be passed on to Minio container(s) +## when Chart is deployed +environment: + ## Please refer for comprehensive list https://docs.minio.io/docs/minio-server-configuration-guide.html + +networkPolicy: + enabled: false + allowExternal: true + +## PodDisruptionBudget settings +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ +## +podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +## Specify the service account to use for the Minio pods. If 'create' is set to 'false' +## and 'name' is left unspecified, the account 'default' will be used. +serviceAccount: + create: true + ## The name of the service account to use. If 'create' is 'true', a service account with that name + ## will be created. Otherwise, a name will be auto-generated. + name: + +metrics: + # Metrics can not be disabled yet: https://github.com/minio/minio/issues/7493 + serviceMonitor: + enabled: false + additionalLabels: {} + # namespace: monitoring + # interval: 30s + # scrapeTimeout: 10s + +## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md +## Define endpoints to enable this section. +etcd: + endpoints: [] + pathPrefix: "" + corednsPathPrefix: "" + clientCert: "" + clientCertKey: ""