From 492730454e69b878e2db4a2dec528b238708d607 Mon Sep 17 00:00:00 2001 From: Kjeld Schouten-Lebbing Date: Tue, 9 Feb 2021 19:13:56 +0100 Subject: [PATCH] Small Tests and repo fixes (#80) * Small organisational fixes * Add some new common tests --- .github/PULL_REQUEST_TEMPLATE.md | 8 +- .github/SECURITY.md | 4 +- .gitignore | 3 +- .pre-commit-config.yaml | 9 ++ features_capability.json | 2 +- library/common-test/Chart.yaml | 2 +- library/common-test/ci/basic-values.yaml | 71 ++++++++-- library/common-test/ci/codeserver-values.yaml | 70 ++++++++-- library/common-test/values.yaml | 28 +++- update_dependencies | 128 ++++++++++++++++++ 10 files changed, 284 insertions(+), 41 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 update_dependencies diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ccb1cbd0065..2759fd73fcd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,16 +6,15 @@ Fixes # (issue) **Type of change** +- [ ] Feature/App addition +- [ ] Bugfix - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Refactor of current code -- [ ] This change requires additional documentation update **How Has This Been Tested?** Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration -- Test A -- Test B **Notes:** _Please enter any other relevant information here_ @@ -27,5 +26,4 @@ _Please enter any other relevant information here_ - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings -- [ ] I have added tests to this description that prove my fix is effective or that my feature works -- [ ] Any new files are named using lowercase (to avoid issues on case sensitive file systems) \ No newline at end of file +- [ ] I have added tests to this description that prove my fix is effective or that my feature works \ No newline at end of file diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 550e421c718..6c3247088c5 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -4,8 +4,8 @@ | Branch | TrueNAS version | Supported with updates | | ------- | ------- |------------------ | -| MASTER | BETA | NOT RELEASED YET | -| DEV | 20.21-ALPHA | :white_check_mark: | +| MASTER | 21.02 ALPHA | :white_check_mark: | +| Charts | Standard Kubernetes/Helm | :white_check_mark: | ## Reporting a Vulnerability diff --git a/.gitignore b/.gitignore index 51344b4c4f4..f22d14a95c0 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ npm-debug.log tests/artifacts/ tests/test-report.txt tests/test-report.xml -private_values.yaml \ No newline at end of file +private_values.yaml +temp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..29a73ca4df8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +# See https://pre-commit.com for more information +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + exclude: '\.github(.+)' + - id: trailing-whitespace + exclude: '\.github(.+)' diff --git a/features_capability.json b/features_capability.json index 0a18d90692d..d5fc5074f06 100644 --- a/features_capability.json +++ b/features_capability.json @@ -23,4 +23,4 @@ "stable": {"min": "20.12-ALPHA"}, "nightlies": {"min": "20.12-MASTER"} } -} \ No newline at end of file +} diff --git a/library/common-test/Chart.yaml b/library/common-test/Chart.yaml index 233c343052a..62d4f8e4a05 100644 --- a/library/common-test/Chart.yaml +++ b/library/common-test/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: common-test description: Helper chart to test different use cases of the common library -version: 1.1.0 +version: 1.2.0 keywords: - truecharts - common diff --git a/library/common-test/ci/basic-values.yaml b/library/common-test/ci/basic-values.yaml index 47176dbf2aa..77293628d5b 100644 --- a/library/common-test/ci/basic-values.yaml +++ b/library/common-test/ci/basic-values.yaml @@ -10,21 +10,6 @@ service: ingress: enabled: true -appIngressEnabled: true -appIngress: - main: - enabled: true - -appVolumesEnabled: false -appVolumeMounts: - data: - enabled: true - emptyDir: true - datasetName: "data" - mountPath: "/data" - hostPathEnabled: false - hostPath: "" - probes: liveness: enabled: true @@ -32,3 +17,59 @@ probes: enabled: true startup: enabled: true + + +appIngressEnabled: true +appIngress: + main: + enabled: true + mainhttp: + enabled: true + entrypoint: "web" + certType: "" + existingcert: "" + authForwardURL: "" + annotations: {} + labels: {} + hosts: + - host: app.truecharts.placeholder + paths: + - path: / + # Ignored if not kubeVersion >= 1.14-0 + pathType: Prefix + +appVolumesEnabled: true +appVolumeMounts: + data: + enabled: true + emptyDir: true + datasetName: "data" + mountPath: "/data" + hostPathEnabled: true + hostPath: "/tmp" + test: + enabled: true + emptyDir: true + datasetName: "test" + mountPath: "/test" + hostPathEnabled: true + hostPath: "/tmp" + +appAdditionalServicesEnabled: true +appAdditionalServices: + extra-tcp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: tcp-test + protocol: TCP + targetPort: 51414 + extra-udp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: udp-test + protocol: UDP + targetPort: 51414 \ No newline at end of file diff --git a/library/common-test/ci/codeserver-values.yaml b/library/common-test/ci/codeserver-values.yaml index 42baa9bddee..7149c42ca31 100644 --- a/library/common-test/ci/codeserver-values.yaml +++ b/library/common-test/ci/codeserver-values.yaml @@ -10,27 +10,12 @@ service: ingress: enabled: true -appIngressEnabled: true -appIngress: - main: - enabled: true - persistence: config: enabled: true emptyDir: true mountPath: /www/assets -appVolumesEnabled: false -appVolumeMounts: - data: - enabled: true - emptyDir: true - datasetName: "data" - mountPath: "/data" - hostPathEnabled: false - hostPath: "" - addons: codeserver: enabled: true @@ -40,3 +25,58 @@ addons: volumeMounts: - name: config mountPath: /www/assets + +appIngressEnabled: true +appIngress: + main: + enabled: true + mainhttp: + enabled: true + entrypoint: "web" + certType: "" + existingcert: "" + authForwardURL: "" + annotations: {} + labels: {} + hosts: + - host: app.truecharts.placeholder + paths: + - path: / + # Ignored if not kubeVersion >= 1.14-0 + pathType: Prefix + +appVolumesEnabled: true +appVolumeMounts: + data: + enabled: true + emptyDir: true + datasetName: "data" + mountPath: "/data" + hostPathEnabled: true + hostPath: "/tmp" + test: + enabled: true + emptyDir: true + datasetName: "test" + mountPath: "/test" + hostPathEnabled: true + hostPath: "/tmp" + +appAdditionalServicesEnabled: true +appAdditionalServices: + extra-tcp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: tcp-test + protocol: TCP + targetPort: 51414 + extra-udp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: udp-test + protocol: UDP + targetPort: 51414 \ No newline at end of file diff --git a/library/common-test/values.yaml b/library/common-test/values.yaml index 376829faafe..1eaab4f8eda 100644 --- a/library/common-test/values.yaml +++ b/library/common-test/values.yaml @@ -37,4 +37,30 @@ appVolumeMounts: datasetName: "data" mountPath: "/data" hostPathEnabled: false - hostPath: "" \ No newline at end of file + hostPath: "" + test: + enabled: true + emptyDir: true + datasetName: "test" + mountPath: "/test" + hostPathEnabled: false + hostPath: "" + +appAdditionalServicesEnabled: true +appAdditionalServices: + extra-tcp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: tcp-test + protocol: TCP + targetPort: 51414 + extra-udp: + enabled: true + type: ClusterIP + port: + port: 51414 + name: udp-test + protocol: UDP + targetPort: 51414 \ No newline at end of file diff --git a/update_dependencies b/update_dependencies new file mode 100644 index 00000000000..ac74b60947e --- /dev/null +++ b/update_dependencies @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import errno +import os +import subprocess + +from collections import defaultdict + +ITEMS = defaultdict(lambda: {'success': [], 'error': []}) + + +class ValidationException(Exception): + def __init__(self, error_msg, error_no=errno.EFAULT): + self.errmsg = error_msg + self.errno = error_no + + def get_error_name(self): + return errno.errorcode.get(self.errno) or 'EUNKNOWN' + + def __str__(self): + return f'[{self.get_error_name()}] {self.errmsg}' + + +class NotFoundException(ValidationException): + def __init__(self, error): + super().__init__(error, errno.ENOENT) + + +class TrainNotFoundException(NotFoundException): + def __init__(self): + super(TrainNotFoundException, self).__init__('Failed to find train') + + +class CatalogItemNotFoundException(NotFoundException): + def __init__(self, path): + super(CatalogItemNotFoundException, self).__init__(f'Failed to find {path!r} catalog item') + + +def report_result(): + print('[\033[94mINFO\x1B[0m]\tExecution Complete') + for index, item in enumerate(ITEMS): + index += 1 + data = ITEMS[item] + print(f'\n[\033[94mINFO\x1B[0m]\t{index}) {item}') + if data['success']: + print( + f'[\033[92mOK\x1B[0m]\t - Successfully updated dependencies for {", ".join(data["success"])} versions' + ) + + for i_v, version in enumerate(data['error']): + v_name, error = version + print( + f'[\033[91mFAILED\x1B[0m]\t ({i_v + 1}) Failed to update dependencies for {v_name!r} version: {error}' + ) + + +def update_train_charts(train_path, commit): + # We will gather all charts in the train and then for each chart all it's versions will be updated + if not os.path.exists(train_path): + raise TrainNotFoundException() + + print(f'[\033[94mINFO\x1B[0m]\tProcessing {train_path!r} train') + for item in os.listdir(train_path): + process_catalog_item(os.path.join(train_path, item)) + + report_result() + + if commit and any(ITEMS[item]['success'] for item in ITEMS): + if any(ITEMS[item]['error'] for item in ITEMS): + print(f'[\033[91mFAILED\x1B[0m]\tNot committing changes as failures detected') + else: + commit_msg = f'Updated catalog item dependencies ({train_path.rsplit("/", 1)[-1]} train)\n' \ + 'Following items were updated:\n' + for item in ITEMS: + commit_msg += f'Updated {item} ({", ".join(ITEMS[item]["success"])} versions)\n\n' + + for cmd in ( + ['git', '-C', train_path, 'add', train_path], + ['git', '-C', train_path, 'commit', '-m', commit_msg] + ): + cp = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stderr = cp.communicate()[1] + if cp.returncode: + print(f'[\033[91mFAILED\x1B[0m]\tFailed to execute {" ".join(cmd)}: {stderr.decode()}') + exit(1) + + print('[\033[92mOK\x1B[0m]\tChanges committed successfully') + + +def process_catalog_item(item_path): + if not os.path.exists(item_path): + raise CatalogItemNotFoundException(item_path) + + item_name = item_path.rsplit('/', 1)[-1] + print(f'[\033[94mINFO\x1B[0m]\tProcessing {item_name!r} catalog item') + for item_version in os.listdir(item_path): + if os.path.isdir(os.path.join(item_path, item_version)): + update_item_version(item_name, item_version, os.path.join(item_path, item_version)) + + +def update_item_version(item_name, version, version_path): + cp = subprocess.Popen( + ['helm', 'dependency', 'update', version_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = cp.communicate() + if cp.returncode: + ITEMS[item_name]['error'].append((version, stderr.decode())) + else: + ITEMS[item_name]['success'].append(version) + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help='sub-command help', dest='action') + + parser_setup = subparsers.add_parser('update', help='Update dependencies for specified train') + parser_setup.add_argument('--train', help='Specify train path to update dependencies', required=True) + parser_setup.add_argument('--commit', help='Commit after updating dependencies', default=False) + + args = parser.parse_args() + if args.action == 'update': + update_train_charts(args.train, args.commit) + else: + parser.print_help() + + +if __name__ == '__main__': + main()