Small Tests and repo fixes (#80)

* Small organisational fixes

* Add some new common tests

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,16 +6,15 @@ Fixes # (issue)
 
 **Type of change**
 
+- [ ] Feature/App addition
+- [ ] Bugfix
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] Refactor of current code
-- [ ] This change requires additional documentation update
 
 **How Has This Been Tested?**
 
 Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
 
-- Test A
-- Test B
 
 **Notes:**
 _Please enter any other relevant information here_
@@ -28,4 +27,3 @@ _Please enter any other relevant information here_
 - [ ] I have made corresponding changes to the documentation
 - [ ] My changes generate no new warnings
 - [ ] I have added tests to this description that prove my fix is effective or that my feature works
-- [ ] Any new files are named using lowercase (to avoid issues on case sensitive file systems)

.github/SECURITY.md

@@ -4,8 +4,8 @@
 
 | Branch | TrueNAS version | Supported with updates          |
 | ------- | ------- |------------------ |
-| MASTER   | BETA |  NOT RELEASED YET |
-| DEV   | 20.21-ALPHA |  :white_check_mark: |
+| MASTER   | 21.02 ALPHA | :white_check_mark: |
+| Charts   | Standard Kubernetes/Helm | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

.gitignore

@@ -35,3 +35,4 @@ tests/artifacts/
 tests/test-report.txt
 tests/test-report.xml
 private_values.yaml
+temp/

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+# See https://pre-commit.com for more information
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+  - id: end-of-file-fixer
+    exclude: '\.github(.+)'
+  - id: trailing-whitespace
+    exclude: '\.github(.+)'

library/common-test/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: common-test
 description: Helper chart to test different use cases of the common library
-version: 1.1.0
+version: 1.2.0
 keywords:
   - truecharts
   - common

library/common-test/ci/basic-values.yaml

@@ -10,21 +10,6 @@ service:
 ingress:
   enabled: true
 
-appIngressEnabled: true
-appIngress:
-  main:
-    enabled: true
-
-appVolumesEnabled: false
-appVolumeMounts:
-  data:
-    enabled: true
-    emptyDir: true
-    datasetName: "data"
-    mountPath: "/data"
-    hostPathEnabled: false
-    hostPath: ""
-
 probes:
   liveness:
     enabled: true
@@ -32,3 +17,59 @@ probes:
     enabled: true
   startup:
     enabled: true
+
+
+appIngressEnabled: true
+appIngress:
+  main:
+    enabled: true
+  mainhttp:
+    enabled: true
+    entrypoint: "web"
+    certType: ""
+    existingcert: ""
+    authForwardURL: ""
+    annotations: {}
+    labels: {}
+    hosts:
+      - host: app.truecharts.placeholder
+        paths:
+          - path: /
+            # Ignored if not kubeVersion >= 1.14-0
+            pathType: Prefix
+    
+appVolumesEnabled: true
+appVolumeMounts:
+  data:
+    enabled: true
+    emptyDir: true
+    datasetName: "data"
+    mountPath: "/data"
+    hostPathEnabled: true
+    hostPath: "/tmp"
+  test:
+    enabled: true
+    emptyDir: true
+    datasetName: "test"
+    mountPath: "/test"
+    hostPathEnabled: true
+    hostPath: "/tmp"
+    
+appAdditionalServicesEnabled: true
+appAdditionalServices:
+  extra-tcp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: tcp-test
+      protocol: TCP
+      targetPort: 51414
+  extra-udp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: udp-test
+      protocol: UDP
+      targetPort: 51414

library/common-test/ci/codeserver-values.yaml

@@ -10,27 +10,12 @@ service:
 ingress:
   enabled: true
 
-appIngressEnabled: true
-appIngress:
-  main:
-    enabled: true
-
 persistence:
     config:
     enabled: true
     emptyDir: true
     mountPath: /www/assets
 
-appVolumesEnabled: false
-appVolumeMounts:
-  data:
-    enabled: true
-    emptyDir: true
-    datasetName: "data"
-    mountPath: "/data"
-    hostPathEnabled: false
-    hostPath: ""
-
 addons:
   codeserver:
     enabled: true
@@ -40,3 +25,58 @@ addons:
     volumeMounts:
     - name: config
       mountPath: /www/assets
+
+appIngressEnabled: true
+appIngress:
+  main:
+    enabled: true
+  mainhttp:
+    enabled: true
+    entrypoint: "web"
+    certType: ""
+    existingcert: ""
+    authForwardURL: ""
+    annotations: {}
+    labels: {}
+    hosts:
+      - host: app.truecharts.placeholder
+        paths:
+          - path: /
+            # Ignored if not kubeVersion >= 1.14-0
+            pathType: Prefix
+    
+appVolumesEnabled: true
+appVolumeMounts:
+  data:
+    enabled: true
+    emptyDir: true
+    datasetName: "data"
+    mountPath: "/data"
+    hostPathEnabled: true
+    hostPath: "/tmp"
+  test:
+    enabled: true
+    emptyDir: true
+    datasetName: "test"
+    mountPath: "/test"
+    hostPathEnabled: true
+    hostPath: "/tmp"
+    
+appAdditionalServicesEnabled: true
+appAdditionalServices:
+  extra-tcp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: tcp-test
+      protocol: TCP
+      targetPort: 51414
+  extra-udp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: udp-test
+      protocol: UDP
+      targetPort: 51414

library/common-test/values.yaml

@@ -38,3 +38,29 @@ appVolumeMounts:
     mountPath: "/data"
     hostPathEnabled: false
     hostPath: ""
+  test:
+    enabled: true
+    emptyDir: true
+    datasetName: "test"
+    mountPath: "/test"
+    hostPathEnabled: false
+    hostPath: ""
+    
+appAdditionalServicesEnabled: true
+appAdditionalServices:
+  extra-tcp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: tcp-test
+      protocol: TCP
+      targetPort: 51414
+  extra-udp:
+    enabled: true
+    type: ClusterIP
+    port:
+      port: 51414
+      name: udp-test
+      protocol: UDP
+      targetPort: 51414

update_dependencies

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+import argparse
+import errno
+import os
+import subprocess
+
+from collections import defaultdict
+
+ITEMS = defaultdict(lambda: {'success': [], 'error': []})
+
+
+class ValidationException(Exception):
+    def __init__(self, error_msg, error_no=errno.EFAULT):
+        self.errmsg = error_msg
+        self.errno = error_no
+
+    def get_error_name(self):
+        return errno.errorcode.get(self.errno) or 'EUNKNOWN'
+
+    def __str__(self):
+        return f'[{self.get_error_name()}] {self.errmsg}'
+
+
+class NotFoundException(ValidationException):
+    def __init__(self, error):
+        super().__init__(error, errno.ENOENT)
+
+
+class TrainNotFoundException(NotFoundException):
+    def __init__(self):
+        super(TrainNotFoundException, self).__init__('Failed to find train')
+
+
+class CatalogItemNotFoundException(NotFoundException):
+    def __init__(self, path):
+        super(CatalogItemNotFoundException, self).__init__(f'Failed to find {path!r} catalog item')
+
+
+def report_result():
+    print('[\033[94mINFO\x1B[0m]\tExecution Complete')
+    for index, item in enumerate(ITEMS):
+        index += 1
+        data = ITEMS[item]
+        print(f'\n[\033[94mINFO\x1B[0m]\t{index}) {item}')
+        if data['success']:
+            print(
+                f'[\033[92mOK\x1B[0m]\t - Successfully updated dependencies for {", ".join(data["success"])} versions'
+            )
+
+        for i_v, version in enumerate(data['error']):
+            v_name, error = version
+            print(
+                f'[\033[91mFAILED\x1B[0m]\t ({i_v + 1}) Failed to update dependencies for {v_name!r} version: {error}'
+            )
+
+
+def update_train_charts(train_path, commit):
+    # We will gather all charts in the train and then for each chart all it's versions will be updated
+    if not os.path.exists(train_path):
+        raise TrainNotFoundException()
+
+    print(f'[\033[94mINFO\x1B[0m]\tProcessing {train_path!r} train')
+    for item in os.listdir(train_path):
+        process_catalog_item(os.path.join(train_path, item))
+
+    report_result()
+
+    if commit and any(ITEMS[item]['success'] for item in ITEMS):
+        if any(ITEMS[item]['error'] for item in ITEMS):
+            print(f'[\033[91mFAILED\x1B[0m]\tNot committing changes as failures detected')
+        else:
+            commit_msg = f'Updated catalog item dependencies ({train_path.rsplit("/", 1)[-1]} train)\n' \
+                         'Following items were updated:\n'
+            for item in ITEMS:
+                commit_msg += f'Updated {item} ({", ".join(ITEMS[item]["success"])} versions)\n\n'
+
+            for cmd in (
+                ['git', '-C', train_path, 'add', train_path],
+                ['git', '-C', train_path, 'commit', '-m', commit_msg]
+            ):
+                cp = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+                stderr = cp.communicate()[1]
+                if cp.returncode:
+                    print(f'[\033[91mFAILED\x1B[0m]\tFailed to execute {" ".join(cmd)}: {stderr.decode()}')
+                    exit(1)
+
+            print('[\033[92mOK\x1B[0m]\tChanges committed successfully')
+
+
+def process_catalog_item(item_path):
+    if not os.path.exists(item_path):
+        raise CatalogItemNotFoundException(item_path)
+
+    item_name = item_path.rsplit('/', 1)[-1]
+    print(f'[\033[94mINFO\x1B[0m]\tProcessing {item_name!r} catalog item')
+    for item_version in os.listdir(item_path):
+        if os.path.isdir(os.path.join(item_path, item_version)):
+            update_item_version(item_name, item_version, os.path.join(item_path, item_version))
+
+
+def update_item_version(item_name, version, version_path):
+    cp = subprocess.Popen(
+        ['helm', 'dependency', 'update', version_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    stdout, stderr = cp.communicate()
+    if cp.returncode:
+        ITEMS[item_name]['error'].append((version, stderr.decode()))
+    else:
+        ITEMS[item_name]['success'].append(version)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help='sub-command help', dest='action')
+
+    parser_setup = subparsers.add_parser('update', help='Update dependencies for specified train')
+    parser_setup.add_argument('--train', help='Specify train path to update dependencies', required=True)
+    parser_setup.add_argument('--commit', help='Commit after updating dependencies', default=False)
+
+    args = parser.parse_args()
+    if args.action == 'update':
+        update_train_charts(args.train, args.commit)
+    else:
+        parser.print_help()
+
+
+if __name__ == '__main__':
+    main()