From b8cfb0efacbfc8d9e8393d0f25cba2220388d0fe Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 15:13:34 +0100 Subject: [PATCH 01/23] Start working on validation API --- ESSArch_Core/config/urls.py | 5 +- ESSArch_Core/fixity/serializers.py | 12 ++++ ESSArch_Core/fixity/validation/__init__.py | 20 +++--- .../fixity/validation/backends/checksum.py | 2 + .../fixity/validation/backends/xml.py | 65 +++++++++++++++++++ ESSArch_Core/fixity/views.py | 52 ++++++++++++++- 6 files changed, 146 insertions(+), 10 deletions(-) diff --git a/ESSArch_Core/config/urls.py b/ESSArch_Core/config/urls.py index f5e6505ed..caf76a91d 100644 --- a/ESSArch_Core/config/urls.py +++ b/ESSArch_Core/config/urls.py @@ -38,6 +38,8 @@ ConversionToolViewSet, ValidationFilesViewSet, ValidationViewSet, + ValidatorViewSet, + ValidatorWorkflowViewSet, ) from ESSArch_Core.ip.views import ( ConsignMethodViewSet, @@ -295,7 +297,6 @@ router.register(r'organizations', OrganizationViewSet, basename='organizations') - router.register(r'appraisal-jobs', AppraisalJobViewSet).register( r'information-packages', AppraisalJobInformationPackageViewSet, @@ -321,6 +322,8 @@ router.register(r'conversion-templates', ConversionTemplateViewSet) router.register(r'conversion-tools', ConversionToolViewSet) router.register(r'features', FeatureViewSet, basename='features') +router.register(r'validators', ValidatorViewSet, basename='validators') +router.register(r'validator-workflows', ValidatorWorkflowViewSet, basename='validator-workflows') router.register(r'validations', ValidationViewSet) router.register(r'events', EventIPViewSet) router.register(r'event-types', EventTypeViewSet) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 06d28c3a1..6a6a6a60e 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers from ESSArch_Core.fixity.models import ConversionTool, Validation +from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS class ConversionToolSerializer(serializers.ModelSerializer): @@ -11,6 +12,17 @@ class Meta: fields = ('name', 'form',) +class ValidatorDataSerializer(serializers.Serializer): + name = serializers.ChoiceField(choices=list(AVAILABLE_VALIDATORS.keys())) + data = serializers.JSONField() + + +class ValidatorWorkflowSerializer(serializers.Serializer): + validators = serializers.ListField( + child=ValidatorDataSerializer() + ) + + class ValidationSerializer(serializers.ModelSerializer): specification = serializers.JSONField(read_only=True) diff --git a/ESSArch_Core/fixity/validation/__init__.py b/ESSArch_Core/fixity/validation/__init__.py index 222ef2eb1..b107b8252 100644 --- a/ESSArch_Core/fixity/validation/__init__.py +++ b/ESSArch_Core/fixity/validation/__init__.py @@ -34,6 +34,15 @@ PATH_VARIABLE = "_PATH" +def get_backend(name): + try: + module_name, klass = AVAILABLE_VALIDATORS[name].rsplit('.', 1) + except KeyError: + raise ValueError('Validator "%s" not found' % name) + + return getattr(importlib.import_module(module_name), klass) + + def _validate_file(path, validators, task=None, ip=None, stop_at_failure=True, responsible=None): for validator in validators: included = False @@ -94,13 +103,7 @@ def validate_path(path, validators, profile, data=None, task=None, ip=None, stop validator_instances = [] for name in validators: - try: - module_name, validator_class = AVAILABLE_VALIDATORS[name].rsplit('.', 1) - except KeyError: - raise ValueError('Validator "%s" not found' % name) - - validator = getattr(importlib.import_module(module_name), validator_class) - + validator_klass = get_backend(name) for specification in profile.specification.get(name, []): required = specification.get('required', True) context = specification.get('context') @@ -108,7 +111,8 @@ def validate_path(path, validators, profile, data=None, task=None, ip=None, stop exclude = [os.path.join(path, excluded) for excluded in specification.get('exclude', [])] options = specification.get('options', {}) - validator_instance = validator( + validator_instance = validator_klass( + name, context=context, include=include, exclude=exclude, diff --git a/ESSArch_Core/fixity/validation/backends/checksum.py b/ESSArch_Core/fixity/validation/backends/checksum.py index ac2661654..05224a34e 100644 --- a/ESSArch_Core/fixity/validation/backends/checksum.py +++ b/ESSArch_Core/fixity/validation/backends/checksum.py @@ -25,6 +25,8 @@ class ChecksumValidator(BaseValidator): * ``block_size``: Defaults to 65536 """ + label = 'Checksum Validator' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 5b1ee6357..8bf0aa94e 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -6,6 +6,7 @@ import click from django.utils import timezone from lxml import etree, isoschematron +from rest_framework import serializers from ESSArch_Core.essxml.util import ( find_files, @@ -32,6 +33,70 @@ class DiffCheckValidator(BaseValidator): file_validator = False + @classmethod + def form(cls): + return [ + { + 'key': 'context', + 'type': 'input', + 'templateOptions': { + 'label': 'Metadata file', + 'required': True, + } + }, + { + 'key': 'options.rootdir', + 'type': 'input', + 'templateOptions': { + 'label': 'Directory (leave empty for all files)', + } + }, + { + 'key': 'options.recursive', + 'defaultValue': True, + 'type': 'checkbox', + 'templateOptions': { + 'label': 'Recursive', + 'required': True, + } + }, + { + 'key': 'options.default_algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Default checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + ] + + @classmethod + def get_serializer_class(cls) -> serializers.Serializer: + class OptionsSerializer(serializers.Serializer): + rootdir = serializers.CharField(default='', allow_blank=True) + recursive = serializers.BooleanField(default=True) + default_algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) + + class DiffCheckValidatorSerializer(serializers.Serializer): + context = serializers.CharField(label='Metadata file') + options = OptionsSerializer() + + return DiffCheckValidatorSerializer + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 1c28b26cb..dd255a75b 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -1,7 +1,8 @@ from django.db.models import Exists, Max, Min, OuterRef from django_filters.rest_framework import DjangoFilterBackend -from rest_framework import filters, viewsets +from rest_framework import filters, mixins, status, viewsets from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response from rest_framework_extensions.mixins import NestedViewSetMixin from ESSArch_Core.api.filters import SearchFilter @@ -11,7 +12,13 @@ ConversionToolSerializer, ValidationFilesSerializer, ValidationSerializer, + ValidatorWorkflowSerializer, ) +from ESSArch_Core.fixity.validation import ( + AVAILABLE_VALIDATORS, + get_backend as get_validator, +) +from ESSArch_Core.WorkflowEngine.models import ProcessStep class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): @@ -20,6 +27,49 @@ class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = ConversionToolSerializer +class ValidatorViewSet(viewsets.ViewSet): + permission_classes = () + + def list(self, request, format=None): + validators = {} + for k, _ in AVAILABLE_VALIDATORS.items(): + klass = get_validator(k) + try: + label = klass.label + except AttributeError: + label = klass.__name__ + + try: + form = klass.form() + except AttributeError: + form = {} + + validator = { + 'label': label, + 'form': form, + } + validators[k] = validator + + return Response(validators) + + +class ValidatorWorkflowViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): + queryset = ProcessStep.objects.all() + serializer_class = ValidatorWorkflowSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + for validator in serializer.data['validators']: + klass = get_validator(validator['name']) + klass_serializer = klass.get_serializer_class()(data=validator['data']) + klass_serializer.is_valid(raise_exception=True) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + + class ValidationViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet): queryset = Validation.objects.all().order_by('filename', 'validator') serializer_class = ValidationSerializer From 9cf350482bc61d72612205aeb06175a9f7bc96ad Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 15:39:36 +0100 Subject: [PATCH 02/23] Add information_package field to ValidatorWorkflowSerializer --- ESSArch_Core/fixity/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 6a6a6a60e..14ba58a3d 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -2,6 +2,7 @@ from ESSArch_Core.fixity.models import ConversionTool, Validation from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS +from ESSArch_Core.ip.models import InformationPackage class ConversionToolSerializer(serializers.ModelSerializer): @@ -18,8 +19,9 @@ class ValidatorDataSerializer(serializers.Serializer): class ValidatorWorkflowSerializer(serializers.Serializer): + information_package = serializers.PrimaryKeyRelatedField(queryset=InformationPackage.objects.all()) validators = serializers.ListField( - child=ValidatorDataSerializer() + child=ValidatorDataSerializer(), ) From 428f9b16fb432e54b69bd60a0bb7176059b3c5ef Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 15:56:01 +0100 Subject: [PATCH 03/23] Use empty list as standard form for validators --- ESSArch_Core/fixity/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index dd255a75b..5be43db0b 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -42,7 +42,7 @@ def list(self, request, format=None): try: form = klass.form() except AttributeError: - form = {} + form = [] validator = { 'label': label, From 9832b1cb98069b44ead5d7bbb313a84b34291465 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 22:53:12 +0100 Subject: [PATCH 04/23] Add Validate task and update view and serializers --- ESSArch_Core/config/settings.py | 1 + ESSArch_Core/fixity/serializers.py | 4 +- .../fixity/validation/backends/xml.py | 13 ++++--- ESSArch_Core/fixity/validation/tasks.py | 11 ++++++ ESSArch_Core/fixity/views.py | 39 +++++++++++++++++-- 5 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 ESSArch_Core/fixity/validation/tasks.py diff --git a/ESSArch_Core/config/settings.py b/ESSArch_Core/config/settings.py index 04febb11d..af3318cbd 100644 --- a/ESSArch_Core/config/settings.py +++ b/ESSArch_Core/config/settings.py @@ -412,6 +412,7 @@ RABBITMQ_URL = os.environ.get('RABBITMQ_URL_ESSARCH', 'amqp://guest:guest@localhost:5672') CELERY_BROKER_URL = RABBITMQ_URL CELERY_IMPORTS = ( + "ESSArch_Core.fixity.validation.tasks", "ESSArch_Core.ip.tasks", "ESSArch_Core.maintenance.tasks", "ESSArch_Core.preingest.tasks", diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 14ba58a3d..52b80a518 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -15,7 +15,9 @@ class Meta: class ValidatorDataSerializer(serializers.Serializer): name = serializers.ChoiceField(choices=list(AVAILABLE_VALIDATORS.keys())) - data = serializers.JSONField() + path = serializers.CharField(label='Path to validate', allow_blank=True, default='') + context = serializers.CharField(label='Metadata file') + options = serializers.JSONField(required=False) class ValidatorWorkflowSerializer(serializers.Serializer): diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 8bf0aa94e..a3297afcd 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -31,24 +31,25 @@ class DiffCheckValidator(BaseValidator): the XML was generated. """ + label = "Diff-check validator" file_validator = False @classmethod def form(cls): return [ { - 'key': 'context', + 'key': 'path', 'type': 'input', 'templateOptions': { - 'label': 'Metadata file', - 'required': True, + 'label': 'Path to validate', } }, { - 'key': 'options.rootdir', + 'key': 'context', 'type': 'input', 'templateOptions': { - 'label': 'Directory (leave empty for all files)', + 'label': 'Metadata file', + 'required': True, } }, { @@ -82,7 +83,7 @@ def form(cls): ] @classmethod - def get_serializer_class(cls) -> serializers.Serializer: + def get_options_serializer_class(cls): class OptionsSerializer(serializers.Serializer): rootdir = serializers.CharField(default='', allow_blank=True) recursive = serializers.BooleanField(default=True) diff --git a/ESSArch_Core/fixity/validation/tasks.py b/ESSArch_Core/fixity/validation/tasks.py new file mode 100644 index 000000000..31069b987 --- /dev/null +++ b/ESSArch_Core/fixity/validation/tasks.py @@ -0,0 +1,11 @@ +from ESSArch_Core.fixity.validation import get_backend +from ESSArch_Core.WorkflowEngine.dbtask import DBTask + + +class Validate(DBTask): + def run(self, name, path, context=None, options=None): + options = {} if options is None else options + klass = get_backend(name) + + validator = klass(context=context, ip=self.ip, task=self.get_processtask(), options=options) + validator.validate(path) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 5be43db0b..34b75f560 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -1,3 +1,6 @@ +import os + +from django.db import transaction from django.db.models import Exists, Max, Min, OuterRef from django_filters.rest_framework import DjangoFilterBackend from rest_framework import filters, mixins, status, viewsets @@ -19,6 +22,7 @@ get_backend as get_validator, ) from ESSArch_Core.WorkflowEngine.models import ProcessStep +from ESSArch_Core.WorkflowEngine.util import create_workflow class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): @@ -60,11 +64,38 @@ class ValidatorWorkflowViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet) def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) + workflow_spec = [] + ip = serializer.validated_data['information_package'] + + for validator in serializer.validated_data['validators']: + name = validator['name'] + klass = get_validator(name) + options_serializer = klass.get_options_serializer_class()(data=validator.get('options', {})) + options_serializer.is_valid(raise_exception=True) + options = options_serializer.validated_data + + path = os.path.join(ip.object_path, validator['path']) + context = os.path.join(ip.object_path, validator['context']) + options['rootdir'] = ip.object_path + + task_spec = { + 'name': 'ESSArch_Core.fixity.validation.tasks.Validate', + 'label': 'Validate using {}'.format(klass.label), + 'args': [name, path], + 'params': {'context': context, 'options': options}, + } + + workflow_spec.append(task_spec) + + with transaction.atomic(): + step = { + 'step': True, + 'name': 'Validation', + 'children': workflow_spec + } + workflow = create_workflow([step], ip=ip, name='Validation') - for validator in serializer.data['validators']: - klass = get_validator(validator['name']) - klass_serializer = klass.get_serializer_class()(data=validator['data']) - klass_serializer.is_valid(raise_exception=True) + workflow.run() headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) From dda23efe3707c103c37bc4c51bd49796684028b0 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 23:14:17 +0100 Subject: [PATCH 05/23] form => get_form --- ESSArch_Core/fixity/validation/backends/xml.py | 4 ++-- ESSArch_Core/fixity/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index a3297afcd..c42ca6ef5 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -31,11 +31,11 @@ class DiffCheckValidator(BaseValidator): the XML was generated. """ - label = "Diff-check validator" file_validator = False + label = "Diff-check validator" @classmethod - def form(cls): + def get_form(cls): return [ { 'key': 'path', diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 34b75f560..27f347411 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -44,7 +44,7 @@ def list(self, request, format=None): label = klass.__name__ try: - form = klass.form() + form = klass.get_form() except AttributeError: form = [] From 7c24262826ee3b342cae56b9b084e6079ada081b Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 23:14:30 +0100 Subject: [PATCH 06/23] Add form and serializer to XMLComparisonValidator --- .../fixity/validation/backends/xml.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index c42ca6ef5..5f6226acb 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -308,6 +308,57 @@ def validate(self, path, expected=None): class XMLComparisonValidator(DiffCheckValidator): + label = "XML Comparison validator" + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'First XML', + 'required': True, + } + }, + { + 'key': 'context', + 'type': 'input', + 'templateOptions': { + 'label': 'Second XML', + 'required': True, + } + }, + { + 'key': 'options.recursive', + 'defaultValue': True, + 'type': 'checkbox', + 'templateOptions': { + 'label': 'Recursive', + 'required': True, + } + }, + { + 'key': 'options.default_algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Default checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + ] + def _get_files(self): skip_files = [p.path for p in find_pointers(self.context)] self.logical_files = find_files( From 745f554e5f7d3a3fee0a8bf2feccc56e012f499b Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 23:28:57 +0100 Subject: [PATCH 07/23] Add form and serializer to XMLSyntaxValidator --- ESSArch_Core/fixity/serializers.py | 2 +- .../fixity/validation/backends/xml.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 52b80a518..e779ad6c0 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -16,7 +16,7 @@ class Meta: class ValidatorDataSerializer(serializers.Serializer): name = serializers.ChoiceField(choices=list(AVAILABLE_VALIDATORS.keys())) path = serializers.CharField(label='Path to validate', allow_blank=True, default='') - context = serializers.CharField(label='Metadata file') + context = serializers.CharField(label='Metadata file', allow_blank=True, default='') options = serializers.JSONField(required=False) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 5f6226acb..5bacedd46 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -504,6 +504,27 @@ def cli(path, schema): class XMLSyntaxValidator(BaseValidator): + label = "XML syntax validator" + + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'Path to validate', + } + }, + ] + + @classmethod + def get_options_serializer_class(cls): + class OptionsSerializer(serializers.Serializer): + pass + + return OptionsSerializer + def validate(self, filepath, expected=None): logger.debug('Validating syntax of {xml}'.format(xml=filepath)) From e864aa8833453447f72d01fe88eb66004e2e1883 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 16 Dec 2019 23:59:44 +0100 Subject: [PATCH 08/23] Remove old validation objects when retrying Validate task --- ESSArch_Core/fixity/validation/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ESSArch_Core/fixity/validation/tasks.py b/ESSArch_Core/fixity/validation/tasks.py index 31069b987..54e978c09 100644 --- a/ESSArch_Core/fixity/validation/tasks.py +++ b/ESSArch_Core/fixity/validation/tasks.py @@ -1,9 +1,13 @@ +from ESSArch_Core.fixity.models import Validation from ESSArch_Core.fixity.validation import get_backend from ESSArch_Core.WorkflowEngine.dbtask import DBTask class Validate(DBTask): def run(self, name, path, context=None, options=None): + # delete validations from previous attempts + Validation.objects.filter(task=self.get_processtask()).delete() + options = {} if options is None else options klass = get_backend(name) From 7ee7bf91aabc218e2c2c44475eb9ebd3f6e12c4e Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Tue, 17 Dec 2019 00:00:10 +0100 Subject: [PATCH 09/23] Add purpose to validation workflow serializer --- ESSArch_Core/fixity/serializers.py | 1 + ESSArch_Core/fixity/views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index e779ad6c0..daffc9c8e 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -21,6 +21,7 @@ class ValidatorDataSerializer(serializers.Serializer): class ValidatorWorkflowSerializer(serializers.Serializer): + purpose = serializers.CharField(default='Validation') information_package = serializers.PrimaryKeyRelatedField(queryset=InformationPackage.objects.all()) validators = serializers.ListField( child=ValidatorDataSerializer(), diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 27f347411..d56ef537e 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -90,7 +90,7 @@ def create(self, request, *args, **kwargs): with transaction.atomic(): step = { 'step': True, - 'name': 'Validation', + 'name': serializer.validated_data['purpose'], 'children': workflow_spec } workflow = create_workflow([step], ip=ip, name='Validation') From f262ff5742b00556907df432f360d854183c8708 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Tue, 17 Dec 2019 00:26:36 +0100 Subject: [PATCH 10/23] Remove required field from recursive field in form --- ESSArch_Core/fixity/validation/backends/base.py | 8 ++++++++ ESSArch_Core/fixity/validation/backends/xml.py | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ESSArch_Core/fixity/validation/backends/base.py b/ESSArch_Core/fixity/validation/backends/base.py index b52aafead..35fd591fb 100644 --- a/ESSArch_Core/fixity/validation/backends/base.py +++ b/ESSArch_Core/fixity/validation/backends/base.py @@ -1,4 +1,5 @@ import click +from rest_framework import serializers class BaseValidator: @@ -19,6 +20,13 @@ def __init__(self, context=None, include=None, exclude=None, options=None, self.ip = ip self.responsible = responsible + @classmethod + def get_options_serializer_class(cls): + class OptionsSerializer(serializers.Serializer): + pass + + return OptionsSerializer + def validate(self, filepath, expected=None): raise NotImplementedError('subclasses of BaseValidator must provide a validate() method') diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 5bacedd46..549da4d5e 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -58,7 +58,6 @@ def get_form(cls): 'type': 'checkbox', 'templateOptions': { 'label': 'Recursive', - 'required': True, } }, { @@ -335,7 +334,6 @@ def get_form(cls): 'type': 'checkbox', 'templateOptions': { 'label': 'Recursive', - 'required': True, } }, { @@ -518,13 +516,6 @@ def get_form(cls): }, ] - @classmethod - def get_options_serializer_class(cls): - class OptionsSerializer(serializers.Serializer): - pass - - return OptionsSerializer - def validate(self, filepath, expected=None): logger.debug('Validating syntax of {xml}'.format(xml=filepath)) From ed399ac98f412c0b39376af30ec3d7ced6a5028d Mon Sep 17 00:00:00 2001 From: Petrus Briland Date: Tue, 17 Dec 2019 00:50:10 +0100 Subject: [PATCH 11/23] Add manual validation functionality --- .../frontend/static/frontend/lang/en/index.ts | 2 + .../static/frontend/lang/en/validation.ts | 13 ++ .../frontend/static/frontend/lang/sv/index.ts | 2 + .../static/frontend/lang/sv/validation.ts | 13 ++ .../scripts/components/ValidationComponent.js | 10 ++ .../RemoveValidationModalInstanceCtrl.js | 12 ++ .../scripts/controllers/ValidationCtrl.js | 117 ++++++++++++++++++ .../scripts/directives/sortableTab.js | 84 +++++++++++++ .../modules/essarch.components.module.js | 4 +- .../modules/essarch.controllers.module.js | 2 + .../modules/essarch.directives.module.js | 2 + .../static/frontend/styles/modules/tabs.scss | 13 ++ .../frontend/styles/modules/validation.scss | 3 + .../static/frontend/styles/styles.scss | 1 + .../frontend/views/combined_workarea.html | 5 + .../static/frontend/views/ip_approval.html | 5 + .../views/remove_validation_modal.html | 30 +++++ .../frontend/views/validation_view.html | 89 +++++++++++++ 18 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 ESSArch_Core/frontend/static/frontend/lang/en/validation.ts create mode 100644 ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts create mode 100644 ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js create mode 100644 ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js create mode 100644 ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js create mode 100644 ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js create mode 100644 ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss create mode 100644 ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html create mode 100644 ESSArch_Core/frontend/static/frontend/views/validation_view.html diff --git a/ESSArch_Core/frontend/static/frontend/lang/en/index.ts b/ESSArch_Core/frontend/static/frontend/lang/en/index.ts index 60d8e57ee..86ce74619 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/en/index.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/en/index.ts @@ -17,6 +17,7 @@ import stateTree from './stateTree'; import sysInfo from './sysinfo'; import upload from './upload'; import userSettings from './userSettings'; +import validation from './validation'; export default [ access, @@ -38,4 +39,5 @@ export default [ sysInfo, upload, userSettings, + validation, ]; diff --git a/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts b/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts new file mode 100644 index 000000000..038139057 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/lang/en/validation.ts @@ -0,0 +1,13 @@ +/*@ngInject*/ +export default ($translateProvider: ng.translate.ITranslateProvider) => { + $translateProvider.translations('en', { + VALIDATION_VIEW: { + ADD_VALIDATOR: 'Add validator', + VALIDATOR: 'Validator', + REMOVE_VALIDATION: 'Remove validation', + RUN_VALIDATIONS: 'Run validations', + }, + VALIDATION: 'Validation', + NONE_SELECTED: 'None selected', + }); +}; diff --git a/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts b/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts index 60d8e57ee..86ce74619 100644 --- a/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts +++ b/ESSArch_Core/frontend/static/frontend/lang/sv/index.ts @@ -17,6 +17,7 @@ import stateTree from './stateTree'; import sysInfo from './sysinfo'; import upload from './upload'; import userSettings from './userSettings'; +import validation from './validation'; export default [ access, @@ -38,4 +39,5 @@ export default [ sysInfo, upload, userSettings, + validation, ]; diff --git a/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts b/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts new file mode 100644 index 000000000..0db05109f --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/lang/sv/validation.ts @@ -0,0 +1,13 @@ +/*@ngInject*/ +export default ($translateProvider: ng.translate.ITranslateProvider) => { + $translateProvider.translations('sv', { + VALIDATION_VIEW: { + ADD_VALIDATOR: 'Lägg till validator', + VALIDATOR: 'Validator', + REMOVE_VALIDATION: 'Ta bort validering', + RUN_VALIDATIONS: 'Kör valideringar', + }, + VALIDATION: 'Validering', + NONE_SELECTED: 'Ingen vald', + }); +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js new file mode 100644 index 000000000..50e262245 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js @@ -0,0 +1,10 @@ +import ValidationCtrl from '../controllers/ValidationCtrl'; + +export default { + templateUrl: 'static/frontend/views/validation_view.html', + controller: ['$scope', '$rootScope', 'appConfig', '$translate', '$http', '$timeout', '$uibModal', ValidationCtrl], + controllerAs: 'vm', + bindings: { + ip: '<', + }, +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js new file mode 100644 index 000000000..6b6c53ad4 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/RemoveValidationModalInstanceCtrl.js @@ -0,0 +1,12 @@ +export default class RemoveValidationModalInstanceCtrl { + constructor($uibModalInstance, data) { + const $ctrl = this; + $ctrl.data = data; + $ctrl.remove = () => { + $uibModalInstance.close('remove'); + }; + $ctrl.cancel = function() { + $uibModalInstance.dismiss('cancel'); + }; + } +} diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js new file mode 100644 index 000000000..87030f243 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js @@ -0,0 +1,117 @@ +export default class ValidationCtrl { + constructor($scope, $rootScope, appConfig, $translate, $http, $timeout, $uibModal) { + const vm = this; + vm.flowOptions = {}; + vm.options = {validators: []}; + vm.fields = $scope.mockedValidations; + vm.activeTab = 'validation0'; + vm.purposeField = [ + { + key: 'purpose', + type: 'input', + templateOptions: { + label: $translate.instant('PURPOSE'), + required: true, + }, + }, + ]; + + let tabNumber = 0; + vm.validations = [ + { + id: 0, + label: $translate.instant('VALIDATION') + ' 1', + validator: null, + data: {}, + }, + ]; + vm.currentValidation = vm.validations[0]; + vm.updateValidatorForm = validation => { + vm.currentValidation = validation; + if (validation.validator) { + vm.fields = validation.validator.form; + } else { + vm.fields = []; + } + }; + + vm.getValidators = function(search) { + return $http({ + url: appConfig.djangoUrl + 'validators/', + method: 'GET', + params: {search: search}, + }).then(function(response) { + let validators = []; + Object.keys(response.data).forEach(key => { + validators.push(angular.extend(response.data[key], {name: key})); + }); + vm.options.validators = validators; + return vm.options.validators; + }); + }; + + vm.addValidator = () => { + tabNumber++; + let val = { + id: tabNumber, + label: $translate.instant('VALIDATION') + ' ' + (tabNumber + 1), + validator: null, + data: {}, + }; + vm.validations.push(val); + $timeout(() => { + vm.activeTab = 'validation' + tabNumber; + vm.updateValidatorForm(val); + }); + }; + + vm.removeValidationModal = validation => { + var modalInstance = $uibModal.open({ + animation: true, + ariaLabelledBy: 'modal-title', + ariaDescribedBy: 'modal-body', + templateUrl: 'static/frontend/views/remove_validation_modal.html', + scope: $scope, + controller: 'RemoveValidationModalInstanceCtrl', + controllerAs: '$ctrl', + resolve: { + data: { + validation, + }, + }, + }); + modalInstance.result.then( + () => { + vm.validations.forEach((x, index, array) => { + if (x.id === validation.id) { + array.splice(index, 1); + tabNumber--; + } + }); + }, + function() {} + ); + }; + + vm.startValidation = () => { + if (vm.form.$invalid) { + vm.form.$setSubmitted(); + return; + } + vm.validations = vm.validations.filter(a => { + return a.validator !== null; + }); + let data = angular.extend(vm.flowOptions, { + information_package: vm.ip.id, + validators: vm.validations.map(x => { + let item = x.data; + item.name = x.validator.name; + return item; + }), + }); + $http.post(appConfig.djangoUrl + 'validator-workflows/', data).then(() => { + $rootScope.$broadcast('REFRESH_LIST_VIEW', {}); + }); + }; + } +} diff --git a/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js b/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js new file mode 100644 index 000000000..48f0556c7 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/scripts/directives/sortableTab.js @@ -0,0 +1,84 @@ +export default ($timeout, $document) => { + return { + link: function(scope, element, attrs, controller) { + // Attempt to integrate with ngRepeat + var match = attrs.ngRepeat.match(/^\s*([\s\S]+?)\s+in\s+([\s\S]+?)(?:\s+track\s+by\s+([\s\S]+?))?\s*$/); + var tabs; + scope.$watch(match[2], function(newTabs) { + tabs = newTabs; + }); + + var index = scope.$index; + scope.$watch('$index', function(newIndex) { + index = newIndex; + }); + + attrs.$set('draggable', true); + + // Wrapped in $apply so Angular reacts to changes + var wrappedListeners = { + // On item being dragged + dragstart: function(e) { + e.originalEvent.dataTransfer.effectAllowed = 'move'; + e.originalEvent.dataTransfer.dropEffect = 'move'; + e.originalEvent.dataTransfer.setData('application/json', index); + element.addClass('dragging'); + }, + dragend: function(e) { + //e.stopPropagation(); + element.removeClass('dragging'); + }, + + // On item being dragged over / dropped onto + dragenter: function(e) {}, + dragleave: function(e) { + element.removeClass('hover'); + }, + drop: function(e) { + e.preventDefault(); + e.stopPropagation(); + var sourceIndex = e.originalEvent.dataTransfer.getData('application/json'); + move(sourceIndex, index); + element.removeClass('hover'); + }, + }; + + // For performance purposes, do not + // call $apply for these + var unwrappedListeners = { + dragover: function(e) { + e.preventDefault(); + element.addClass('hover'); + }, + /* Use .hover instead of :hover. :hover doesn't play well with + moving DOM from under mouse when hovered */ + mouseenter: function() { + element.addClass('hover'); + }, + mouseleave: function() { + element.removeClass('hover'); + }, + }; + + angular.forEach(wrappedListeners, function(listener, event) { + element.on(event, wrap(listener)); + }); + + angular.forEach(unwrappedListeners, function(listener, event) { + element.on(event, listener); + }); + + function wrap(fn) { + return function(e) { + scope.$apply(function() { + fn(e); + }); + }; + } + + function move(fromIndex, toIndex) { + tabs.splice(toIndex, 0, tabs.splice(fromIndex, 1)[0]); + } + }, + }; +}; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js index 853fd0939..693761f38 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.components.module.js @@ -24,6 +24,7 @@ import SaEditorComponent from '../components/SaEditorComponent'; import searchFilter from '../components/SearchFilterComponent'; import search from '../components/SearchComponent'; import StateTreeView from '../components/StateTreeViewComponent'; +import Validation from '../components/ValidationComponent'; export default angular .module('essarch.components', ['essarch.controllers']) @@ -52,4 +53,5 @@ export default angular .component('sysInfoComponent', sysInfoComponent) .component('search', search) .component('searchFilter', searchFilter) - .component('userDropdown', UserDropdownComponent).name; + .component('userDropdown', UserDropdownComponent) + .component('validationView', Validation).name; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js index 5b74d3776..7152a2ac6 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.controllers.module.js @@ -86,6 +86,7 @@ import ReceptionCtrl from '../controllers/ReceptionCtrl'; import RemoveNodeModalInstanceCtrl from '../controllers/RemoveNodeModalInstanceCtrl'; import RemoveStructureModalInstanceCtrl from '../controllers/RemoveStructureModalInstanceCtrl'; import RemoveStructureUnitModalInstanceCtrl from '../controllers/RemoveStructureUnitModalInstanceCtrl'; +import RemoveValidationModalInstanceCtrl from '../controllers/RemoveValidationModalInstanceCtrl'; import RequestModalInstanceCtrl from '../controllers/RequestModalInstanceCtrl'; import RestrictedCtrl from '../controllers/RestrictedCtrl'; import RobotInformationCtrl from '../controllers/RobotInformationCtrl'; @@ -964,6 +965,7 @@ export default angular '$translate', RemoveStructureUnitModalInstanceCtrl, ]) + .controller('RemoveValidationModalInstanceCtrl', ['$uibModalInstance', 'data', RemoveValidationModalInstanceCtrl]) .controller('RestrictedCtrl', ['$scope', RestrictedCtrl]) .controller('SavedSearchModalInstanceCtrl', [ '$uibModalInstance', diff --git a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js index 649512392..f077486ee 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/modules/essarch.directives.module.js @@ -2,10 +2,12 @@ import focused from '../directives/focused'; import ngEnter from '../directives/ngEnter'; import treednd from '../directives/dragNdropDirective'; import fileread from '../directives/fileread'; +import sortableTab from '../directives/sortableTab'; export default angular .module('essarch.directives', ['essarch.services']) .directive('fileread', [fileread]) .directive('ngEnter', [ngEnter]) .directive('treednd', ['myService', treednd]) + .directive('sortableTab', ['$timeout', '$document', sortableTab]) .directive('focused', ['$timeout', '$parse', focused]).name; diff --git a/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss b/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss index dd3d9e342..f319c365d 100644 --- a/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss +++ b/ESSArch_Core/frontend/static/frontend/styles/modules/tabs.scss @@ -3,3 +3,16 @@ .no-tabs-available { @include container(10px); } + +.nav-pills > li:hover > a { + background: transparent; + border-color: transparent; +} + +.nav-pills > li.hover > a { + background: #eeeeee; +} + +.nav-pills > li.dragging { + opacity: 0.5; +} diff --git a/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss b/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss new file mode 100644 index 000000000..40f9af184 --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/styles/modules/validation.scss @@ -0,0 +1,3 @@ +.validation-view { + padding: $padding-base-vertical $padding-base-horizontal; +} diff --git a/ESSArch_Core/frontend/static/frontend/styles/styles.scss b/ESSArch_Core/frontend/static/frontend/styles/styles.scss index 1885b2b0b..3c595bfe5 100644 --- a/ESSArch_Core/frontend/static/frontend/styles/styles.scss +++ b/ESSArch_Core/frontend/static/frontend/styles/styles.scss @@ -81,3 +81,4 @@ $icon-font-path: '~bootstrap-sass/assets/fonts/bootstrap/'; @import 'modules/search_filter'; @import 'modules/dashboard'; @import 'modules/form_errors'; +@import 'modules/validation'; diff --git a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html index f87e049f0..1477b5c7d 100644 --- a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html +++ b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html @@ -104,6 +104,11 @@ + +
+ +
+
diff --git a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html index f91d56794..a91ea08f4 100644 --- a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html +++ b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html @@ -105,6 +105,11 @@
+ +
+ +
+
diff --git a/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html b/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html new file mode 100644 index 000000000..d8528a2ce --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/views/remove_validation_modal.html @@ -0,0 +1,30 @@ +
+ +
+
+

{{'VALIDATION' | translate}}

+
+ + + + + + + + + + + + + +
{{'LABEL' | translate}}{{'VALIDATION_VIEW.VALIDATOR' | translate}}
{{$ctrl.data.validation.label}} + {{$ctrl.data.validation.validator ? $ctrl.data.validation.validator.name : ('NONE_SELECTED' | translate)}} +
+
+
+
+ + +
diff --git a/ESSArch_Core/frontend/static/frontend/views/validation_view.html b/ESSArch_Core/frontend/static/frontend/views/validation_view.html new file mode 100644 index 000000000..24739859a --- /dev/null +++ b/ESSArch_Core/frontend/static/frontend/views/validation_view.html @@ -0,0 +1,89 @@ +
+
+ + + +
+ + {{'VALIDATION' | translate}} + +
+ +
+ + {{validation.validator.label}} + +
+ + +
{{'NO_RESULTS_FOUND' | translate}}
+ + +
+
+
+ +
+
+
+ + + + + + + + +
+ +
+ + +
+
+
From 630618a8a472408191441793d1841ea98bdac0e0 Mon Sep 17 00:00:00 2001 From: Petrus Briland Date: Tue, 17 Dec 2019 00:55:26 +0100 Subject: [PATCH 12/23] Do not apply filtering on validations list if there is no validation has selected validator --- .../static/frontend/scripts/controllers/ValidationCtrl.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js index 87030f243..586358bc7 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js @@ -98,9 +98,12 @@ export default class ValidationCtrl { vm.form.$setSubmitted(); return; } - vm.validations = vm.validations.filter(a => { + let validations = vm.validations.filter(a => { return a.validator !== null; }); + if (validations.length > 0) { + vm.validations = validations; + } let data = angular.extend(vm.flowOptions, { information_package: vm.ip.id, validators: vm.validations.map(x => { From b6056c9bd9c6abdb30f9c3ebbadb0331df97f720 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Tue, 17 Dec 2019 01:26:21 +0100 Subject: [PATCH 13/23] Validate path --- ESSArch_Core/api/fields.py | 21 +++++++++++++++++++++ ESSArch_Core/fixity/serializers.py | 14 +++++++++++--- ESSArch_Core/fixity/views.py | 2 +- 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 ESSArch_Core/api/fields.py diff --git a/ESSArch_Core/api/fields.py b/ESSArch_Core/api/fields.py new file mode 100644 index 000000000..6d36cef3a --- /dev/null +++ b/ESSArch_Core/api/fields.py @@ -0,0 +1,21 @@ +import os + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + + +class FilePathField(serializers.CharField): + default_error_messages = { + 'invalid_path': _('{input} is not a valid path.'), + } + + def __init__(self, path, **kwargs): + self.path = path + super().__init__(**kwargs) + + def to_internal_value(self, data): + data = super().to_internal_value(data) + if not os.path.exists(os.path.join(self.path, data)): + self.fail('invalid_path', input=data) + + return data diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index daffc9c8e..23d2c864d 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -1,5 +1,6 @@ from rest_framework import serializers +from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.fixity.models import ConversionTool, Validation from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS from ESSArch_Core.ip.models import InformationPackage @@ -19,13 +20,20 @@ class ValidatorDataSerializer(serializers.Serializer): context = serializers.CharField(label='Metadata file', allow_blank=True, default='') options = serializers.JSONField(required=False) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + request = kwargs['context']['request'] + ip = InformationPackage.objects.get(pk=request.data['information_package']) + self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') + class ValidatorWorkflowSerializer(serializers.Serializer): purpose = serializers.CharField(default='Validation') information_package = serializers.PrimaryKeyRelatedField(queryset=InformationPackage.objects.all()) - validators = serializers.ListField( - child=ValidatorDataSerializer(), - ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['validators'] = serializers.ListField(child=ValidatorDataSerializer(context=kwargs['context'])) class ValidationSerializer(serializers.ModelSerializer): diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index d56ef537e..829ff7b00 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -62,7 +62,7 @@ class ValidatorWorkflowViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet) serializer_class = ValidatorWorkflowSerializer def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) + serializer = self.get_serializer(data=request.data, context={'request': self.request}) serializer.is_valid(raise_exception=True) workflow_spec = [] ip = serializer.validated_data['information_package'] From 36d7001c201d5972c12744e7946bb5bd4b0a7fad Mon Sep 17 00:00:00 2001 From: Petrus Briland Date: Tue, 17 Dec 2019 01:28:44 +0100 Subject: [PATCH 14/23] Make purpose not required --- .../static/frontend/scripts/controllers/ValidationCtrl.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js index 586358bc7..8f1de75ea 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js @@ -11,7 +11,6 @@ export default class ValidationCtrl { type: 'input', templateOptions: { label: $translate.instant('PURPOSE'), - required: true, }, }, ]; @@ -104,6 +103,9 @@ export default class ValidationCtrl { if (validations.length > 0) { vm.validations = validations; } + if (!angular.isUndefined(vm.flowOptions.purpose) && vm.flowOptions.purpose === '') { + delete vm.flowOptions.purpose; + } let data = angular.extend(vm.flowOptions, { information_package: vm.ip.id, validators: vm.validations.map(x => { From 526b6dc4e96c1ba1c6e165be1604147314e46e03 Mon Sep 17 00:00:00 2001 From: Petrus Briland Date: Tue, 7 Jan 2020 14:28:07 +0100 Subject: [PATCH 15/23] Add base url attribute to validation component --- .../static/frontend/scripts/components/ValidationComponent.js | 1 + .../static/frontend/scripts/controllers/ValidationCtrl.js | 4 ++-- .../frontend/static/frontend/views/combined_workarea.html | 2 +- ESSArch_Core/frontend/static/frontend/views/ip_approval.html | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js index 50e262245..20fe2b7c0 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/components/ValidationComponent.js @@ -6,5 +6,6 @@ export default { controllerAs: 'vm', bindings: { ip: '<', + baseUrl: '@', }, }; diff --git a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js index 8f1de75ea..90f1dc525 100644 --- a/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js +++ b/ESSArch_Core/frontend/static/frontend/scripts/controllers/ValidationCtrl.js @@ -107,14 +107,14 @@ export default class ValidationCtrl { delete vm.flowOptions.purpose; } let data = angular.extend(vm.flowOptions, { - information_package: vm.ip.id, validators: vm.validations.map(x => { let item = x.data; item.name = x.validator.name; return item; }), }); - $http.post(appConfig.djangoUrl + 'validator-workflows/', data).then(() => { + let id = vm.baseUrl === 'workareas' ? vm.ip.workarea[0].id : vm.ip.id; + $http.post(appConfig.djangoUrl + vm.baseUrl + '/' + id + '/validate/', data).then(() => { $rootScope.$broadcast('REFRESH_LIST_VIEW', {}); }); }; diff --git a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html index 1477b5c7d..6c75cc51f 100644 --- a/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html +++ b/ESSArch_Core/frontend/static/frontend/views/combined_workarea.html @@ -106,7 +106,7 @@
- +
diff --git a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html index a91ea08f4..c042cdf92 100644 --- a/ESSArch_Core/frontend/static/frontend/views/ip_approval.html +++ b/ESSArch_Core/frontend/static/frontend/views/ip_approval.html @@ -107,7 +107,7 @@
- +
From 5f4b9ea637afc72bf32064dd20dc0b37dcb455ae Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 10 Jan 2020 19:19:17 +0100 Subject: [PATCH 16/23] Add tests for FilePathField --- ESSArch_Core/api/tests/test_fields.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ESSArch_Core/api/tests/test_fields.py diff --git a/ESSArch_Core/api/tests/test_fields.py b/ESSArch_Core/api/tests/test_fields.py new file mode 100644 index 000000000..45a88760b --- /dev/null +++ b/ESSArch_Core/api/tests/test_fields.py @@ -0,0 +1,20 @@ +import os + +from rest_framework import serializers +from rest_framework.test import APITestCase + +from ESSArch_Core.api.fields import FilePathField + + +class FilePathFieldTests(APITestCase): + @classmethod + def setUpTestData(cls): + cls.field = FilePathField(os.path.abspath(os.path.dirname(__file__))) + + def test_valid_path(self): + self.assertEqual(self.field.run_validation(__file__), __file__) + self.assertEqual(self.field.run_validation(os.path.basename(__file__)), os.path.basename(__file__)) + + def test_invalid_path(self): + with self.assertRaises(serializers.ValidationError): + self.field.run_validation('invalid_file') From 78a7db7606c9821599831ba71fc68853c3f4dad9 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 10 Jan 2020 19:43:29 +0100 Subject: [PATCH 17/23] Add tests for validation viewsets --- ESSArch_Core/fixity/serializers.py | 5 ++- ESSArch_Core/fixity/tests/test_views.py | 45 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 ESSArch_Core/fixity/tests/test_views.py diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 23d2c864d..eb06199d8 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -33,7 +33,10 @@ class ValidatorWorkflowSerializer(serializers.Serializer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['validators'] = serializers.ListField(child=ValidatorDataSerializer(context=kwargs['context'])) + self.fields['validators'] = serializers.ListField( + min_length=1, + child=ValidatorDataSerializer(context=kwargs['context']), + ) class ValidationSerializer(serializers.ModelSerializer): diff --git a/ESSArch_Core/fixity/tests/test_views.py b/ESSArch_Core/fixity/tests/test_views.py new file mode 100644 index 000000000..dfa8ec5b2 --- /dev/null +++ b/ESSArch_Core/fixity/tests/test_views.py @@ -0,0 +1,45 @@ +from django.contrib.auth import get_user_model +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from ESSArch_Core.ip.models import InformationPackage +from ESSArch_Core.testing.runner import TaskRunner + +User = get_user_model() + + +class ValidatorViewSetTests(APITestCase): + def test_list(self): + url = reverse('validators-list') + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +class ValidatorWorkflowViewSetTests(APITestCase): + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create() + cls.superuser = User.objects.create(username='superuser', is_superuser=True) + cls.url = reverse('validator-workflows-list') + + def test_without_permission(self): + self.client.force_authenticate(self.user) + response = self.client.post(self.url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + @TaskRunner() + def test_create_and_run_workflow(self): + self.client.force_authenticate(self.superuser) + + ip = InformationPackage.objects.create() + + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + } + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) From 5f109efa51d1c1b4d3f6f8832cd9e41ac8dc7e1c Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Thu, 19 Mar 2020 16:16:51 +0100 Subject: [PATCH 18/23] Remove DiffCheckValidatorSerializer --- ESSArch_Core/fixity/validation/backends/xml.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 549da4d5e..e588a1882 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -91,11 +91,7 @@ class OptionsSerializer(serializers.Serializer): default='SHA-256', ) - class DiffCheckValidatorSerializer(serializers.Serializer): - context = serializers.CharField(label='Metadata file') - options = OptionsSerializer() - - return DiffCheckValidatorSerializer + return OptionsSerializer def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From 337acac48ba989d094ca67907be8dc1581c7d4d8 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 20 Mar 2020 10:09:04 +0100 Subject: [PATCH 19/23] Fix serializers, validators and tests --- ESSArch_Core/fixity/serializers.py | 48 +++++----- ESSArch_Core/fixity/tests/test_views.py | 92 ++++++++++++++++--- .../fixity/validation/backends/base.py | 7 ++ .../fixity/validation/backends/checksum.py | 31 +++++++ ESSArch_Core/fixity/validation/tasks.py | 7 +- ESSArch_Core/fixity/views.py | 3 +- 6 files changed, 152 insertions(+), 36 deletions(-) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index eb06199d8..05d0b75b7 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -1,8 +1,7 @@ from rest_framework import serializers -from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.fixity.models import ConversionTool, Validation -from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS +from ESSArch_Core.fixity.validation import get_backend as get_validator from ESSArch_Core.ip.models import InformationPackage @@ -14,29 +13,36 @@ class Meta: fields = ('name', 'form',) -class ValidatorDataSerializer(serializers.Serializer): - name = serializers.ChoiceField(choices=list(AVAILABLE_VALIDATORS.keys())) - path = serializers.CharField(label='Path to validate', allow_blank=True, default='') - context = serializers.CharField(label='Metadata file', allow_blank=True, default='') - options = serializers.JSONField(required=False) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - request = kwargs['context']['request'] - ip = InformationPackage.objects.get(pk=request.data['information_package']) - self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') - - class ValidatorWorkflowSerializer(serializers.Serializer): purpose = serializers.CharField(default='Validation') information_package = serializers.PrimaryKeyRelatedField(queryset=InformationPackage.objects.all()) + validators = serializers.ListField(min_length=1, child=serializers.JSONField()) + + def validate_validators(self, validators): + new_data = [] + sub_context = {'information_package': self.context['request'].data.get('information_package', None)} + + for validator in validators: + name = validator.pop('name') + klass = get_validator(name) + options_serializer = klass.get_options_serializer_class()( + data=validator.pop('options', {}), + context=sub_context, + ) + serializer = klass.get_serializer_class()( + data=validator, context=sub_context, + ) + + serializer.is_valid(True) + options_serializer.is_valid(True) + + data = serializer.validated_data + data['name'] = name + data['options'] = options_serializer.validated_data + + new_data.append(data) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.fields['validators'] = serializers.ListField( - min_length=1, - child=ValidatorDataSerializer(context=kwargs['context']), - ) + return new_data class ValidationSerializer(serializers.ModelSerializer): diff --git a/ESSArch_Core/fixity/tests/test_views.py b/ESSArch_Core/fixity/tests/test_views.py index dfa8ec5b2..e12da9185 100644 --- a/ESSArch_Core/fixity/tests/test_views.py +++ b/ESSArch_Core/fixity/tests/test_views.py @@ -1,10 +1,18 @@ +import os +import shutil +import tempfile + +from celery import states as celery_states from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase +from ESSArch_Core.configuration.models import Path +from ESSArch_Core.fixity.checksum import calculate_checksum from ESSArch_Core.ip.models import InformationPackage from ESSArch_Core.testing.runner import TaskRunner +from ESSArch_Core.WorkflowEngine.models import ProcessTask User = get_user_model() @@ -23,23 +31,83 @@ def setUpTestData(cls): cls.superuser = User.objects.create(username='superuser', is_superuser=True) cls.url = reverse('validator-workflows-list') + def setUp(self): + self.datadir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.datadir) + + Path.objects.create(entity='temp', value=tempfile.mkdtemp(dir=self.datadir)) + def test_without_permission(self): self.client.force_authenticate(self.user) response = self.client.post(self.url) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - @TaskRunner() + @TaskRunner(False) def test_create_and_run_workflow(self): self.client.force_authenticate(self.superuser) - ip = InformationPackage.objects.create() - - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - } - ], - }) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) + ip = InformationPackage.objects.create( + object_path=tempfile.mkdtemp(dir=self.datadir), + ) + os.makedirs(os.path.join(ip.object_path, 'content')) + + with open(os.path.join(ip.object_path, 'content', 'foo.txt'), 'w') as f: + f.write('hello') + + expected = calculate_checksum(os.path.join(ip.object_path, 'content', 'foo.txt'), 'SHA-224') + + with self.subTest('invalid file path'): + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': expected, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(ProcessTask.objects.exists()) + + with self.subTest('valid path and expected value'): + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': expected, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 0) + + with self.subTest('valid path and unexpected value'): + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': 'incorrect', + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) diff --git a/ESSArch_Core/fixity/validation/backends/base.py b/ESSArch_Core/fixity/validation/backends/base.py index 35fd591fb..8d90451d8 100644 --- a/ESSArch_Core/fixity/validation/backends/base.py +++ b/ESSArch_Core/fixity/validation/backends/base.py @@ -20,6 +20,13 @@ def __init__(self, context=None, include=None, exclude=None, options=None, self.ip = ip self.responsible = responsible + @classmethod + def get_serializer_class(cls): + class Serializer(serializers.Serializer): + pass + + return Serializer + @classmethod def get_options_serializer_class(cls): class OptionsSerializer(serializers.Serializer): diff --git a/ESSArch_Core/fixity/validation/backends/checksum.py b/ESSArch_Core/fixity/validation/backends/checksum.py index 05224a34e..ec181c69d 100644 --- a/ESSArch_Core/fixity/validation/backends/checksum.py +++ b/ESSArch_Core/fixity/validation/backends/checksum.py @@ -2,7 +2,9 @@ import traceback from django.utils import timezone +from rest_framework import serializers +from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.essxml.util import find_file from ESSArch_Core.exceptions import ValidationError from ESSArch_Core.fixity.checksum import calculate_checksum @@ -27,6 +29,35 @@ class ChecksumValidator(BaseValidator): label = 'Checksum Validator' + @classmethod + def get_serializer_class(cls): + class Serializer(serializers.Serializer): + context = serializers.ChoiceField(choices=['checksum_str', 'checksum_file', 'xml_file']) + block_size = serializers.IntegerField(default=65536) + + def __init__(self, *args, **kwargs): + from ESSArch_Core.ip.models import InformationPackage + + super().__init__(*args, **kwargs) + ip_pk = kwargs['context']['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') + + return Serializer + + @classmethod + def get_options_serializer_class(cls): + class OptionsSerializer(serializers.Serializer): + expected = serializers.CharField() + rootdir = serializers.CharField(default='', allow_blank=True) + recursive = serializers.BooleanField(default=True) + algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) + + return OptionsSerializer + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/ESSArch_Core/fixity/validation/tasks.py b/ESSArch_Core/fixity/validation/tasks.py index 54e978c09..d592cf989 100644 --- a/ESSArch_Core/fixity/validation/tasks.py +++ b/ESSArch_Core/fixity/validation/tasks.py @@ -11,5 +11,10 @@ def run(self, name, path, context=None, options=None): options = {} if options is None else options klass = get_backend(name) - validator = klass(context=context, ip=self.ip, task=self.get_processtask(), options=options) + validator = klass( + context=context, + ip=self.get_information_package(), + task=self.get_processtask(), + options=options, + ) validator.validate(path) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 829ff7b00..25f783ee1 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -75,14 +75,13 @@ def create(self, request, *args, **kwargs): options = options_serializer.validated_data path = os.path.join(ip.object_path, validator['path']) - context = os.path.join(ip.object_path, validator['context']) options['rootdir'] = ip.object_path task_spec = { 'name': 'ESSArch_Core.fixity.validation.tasks.Validate', 'label': 'Validate using {}'.format(klass.label), 'args': [name, path], - 'params': {'context': context, 'options': options}, + 'params': {'context': validator['context'], 'options': options}, } workflow_spec.append(task_spec) From 3fec58c09ac7589a744c45bfb86bdaf4cba463d2 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 20 Mar 2020 10:59:55 +0100 Subject: [PATCH 20/23] Generalize serializers, more testing --- ESSArch_Core/api/fields.py | 2 +- ESSArch_Core/api/tests/test_fields.py | 2 +- ESSArch_Core/fixity/tests/test_views.py | 71 +++++++++++++++++-- .../fixity/validation/backends/base.py | 26 ++++--- .../fixity/validation/backends/checksum.py | 41 ++++------- .../fixity/validation/backends/xml.py | 30 +++++--- 6 files changed, 118 insertions(+), 54 deletions(-) diff --git a/ESSArch_Core/api/fields.py b/ESSArch_Core/api/fields.py index 6d36cef3a..0d4ea4bec 100644 --- a/ESSArch_Core/api/fields.py +++ b/ESSArch_Core/api/fields.py @@ -18,4 +18,4 @@ def to_internal_value(self, data): if not os.path.exists(os.path.join(self.path, data)): self.fail('invalid_path', input=data) - return data + return os.path.join(self.path, data) diff --git a/ESSArch_Core/api/tests/test_fields.py b/ESSArch_Core/api/tests/test_fields.py index 45a88760b..f02110d4e 100644 --- a/ESSArch_Core/api/tests/test_fields.py +++ b/ESSArch_Core/api/tests/test_fields.py @@ -13,7 +13,7 @@ def setUpTestData(cls): def test_valid_path(self): self.assertEqual(self.field.run_validation(__file__), __file__) - self.assertEqual(self.field.run_validation(os.path.basename(__file__)), os.path.basename(__file__)) + self.assertEqual(self.field.run_validation(os.path.basename(__file__)), __file__) def test_invalid_path(self): with self.assertRaises(serializers.ValidationError): diff --git a/ESSArch_Core/fixity/tests/test_views.py b/ESSArch_Core/fixity/tests/test_views.py index e12da9185..14f98de99 100644 --- a/ESSArch_Core/fixity/tests/test_views.py +++ b/ESSArch_Core/fixity/tests/test_views.py @@ -9,6 +9,7 @@ from rest_framework.test import APITestCase from ESSArch_Core.configuration.models import Path +from ESSArch_Core.exceptions import ValidationError from ESSArch_Core.fixity.checksum import calculate_checksum from ESSArch_Core.ip.models import InformationPackage from ESSArch_Core.testing.runner import TaskRunner @@ -42,7 +43,7 @@ def test_without_permission(self): response = self.client.post(self.url) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - @TaskRunner(False) + @TaskRunner() def test_create_and_run_workflow(self): self.client.force_authenticate(self.superuser) @@ -55,6 +56,8 @@ def test_create_and_run_workflow(self): f.write('hello') expected = calculate_checksum(os.path.join(ip.object_path, 'content', 'foo.txt'), 'SHA-224') + with open(os.path.join(ip.object_path, 'metadata.xml'), 'w') as f: + f.write('') with self.subTest('invalid file path'): response = self.client.post(self.url, { @@ -94,6 +97,57 @@ def test_create_and_run_workflow(self): self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 0) with self.subTest('valid path and unexpected value'): + with self.assertRaises(ValidationError): + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': 'incorrect', + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) + + with self.subTest('multiple validators'): + with self.assertRaises(ValidationError): + response = self.client.post(self.url, { + 'information_package': str(ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': expected, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': 'content/foo.txt', + 'context': 'metadata.xml', + 'options': { + 'expected': expected, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 2) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) + + with open(os.path.join(ip.object_path, 'metadata.xml'), 'w') as f: + f.write('') + response = self.client.post(self.url, { 'information_package': str(ip.pk), 'validators': [ @@ -102,12 +156,21 @@ def test_create_and_run_workflow(self): 'path': 'content/foo.txt', 'context': 'checksum_str', 'options': { - 'expected': 'incorrect', + 'expected': expected, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': 'content/foo.txt', + 'context': 'metadata.xml', + 'options': { + 'expected': expected, 'algorithm': 'SHA-224', } }, ], }) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 4) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) diff --git a/ESSArch_Core/fixity/validation/backends/base.py b/ESSArch_Core/fixity/validation/backends/base.py index 8d90451d8..76f77734b 100644 --- a/ESSArch_Core/fixity/validation/backends/base.py +++ b/ESSArch_Core/fixity/validation/backends/base.py @@ -1,6 +1,8 @@ import click from rest_framework import serializers +from ESSArch_Core.api.fields import FilePathField + class BaseValidator: file_validator = True # Does the validator operate on single files or entire directories? @@ -20,19 +22,27 @@ def __init__(self, context=None, include=None, exclude=None, options=None, self.ip = ip self.responsible = responsible + class Serializer(serializers.Serializer): + context = serializers.CharField() + + def __init__(self, *args, **kwargs): + from ESSArch_Core.ip.models import InformationPackage + + super().__init__(*args, **kwargs) + ip_pk = kwargs['context']['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') + + class OptionsSerializer(serializers.Serializer): + pass + @classmethod def get_serializer_class(cls): - class Serializer(serializers.Serializer): - pass - - return Serializer + return cls.Serializer @classmethod def get_options_serializer_class(cls): - class OptionsSerializer(serializers.Serializer): - pass - - return OptionsSerializer + return cls.OptionsSerializer def validate(self, filepath, expected=None): raise NotImplementedError('subclasses of BaseValidator must provide a validate() method') diff --git a/ESSArch_Core/fixity/validation/backends/checksum.py b/ESSArch_Core/fixity/validation/backends/checksum.py index ec181c69d..258d02cff 100644 --- a/ESSArch_Core/fixity/validation/backends/checksum.py +++ b/ESSArch_Core/fixity/validation/backends/checksum.py @@ -4,7 +4,6 @@ from django.utils import timezone from rest_framework import serializers -from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.essxml.util import find_file from ESSArch_Core.exceptions import ValidationError from ESSArch_Core.fixity.checksum import calculate_checksum @@ -29,34 +28,18 @@ class ChecksumValidator(BaseValidator): label = 'Checksum Validator' - @classmethod - def get_serializer_class(cls): - class Serializer(serializers.Serializer): - context = serializers.ChoiceField(choices=['checksum_str', 'checksum_file', 'xml_file']) - block_size = serializers.IntegerField(default=65536) - - def __init__(self, *args, **kwargs): - from ESSArch_Core.ip.models import InformationPackage - - super().__init__(*args, **kwargs) - ip_pk = kwargs['context']['information_package'] - ip = InformationPackage.objects.get(pk=ip_pk) - self.fields['path'] = FilePathField(ip.object_path, allow_blank=True, default='') - - return Serializer - - @classmethod - def get_options_serializer_class(cls): - class OptionsSerializer(serializers.Serializer): - expected = serializers.CharField() - rootdir = serializers.CharField(default='', allow_blank=True) - recursive = serializers.BooleanField(default=True) - algorithm = serializers.ChoiceField( - choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], - default='SHA-256', - ) - - return OptionsSerializer + class Serializer(BaseValidator.Serializer): + context = serializers.ChoiceField(choices=['checksum_str', 'checksum_file', 'xml_file']) + block_size = serializers.IntegerField(default=65536) + + class OptionsSerializer(BaseValidator.OptionsSerializer): + expected = serializers.CharField() + rootdir = serializers.CharField(default='', allow_blank=True) + recursive = serializers.BooleanField(default=True) + algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index e588a1882..3304886b2 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -8,6 +8,7 @@ from lxml import etree, isoschematron from rest_framework import serializers +from ESSArch_Core.api.fields import FilePathField from ESSArch_Core.essxml.util import ( find_files, find_pointers, @@ -81,17 +82,24 @@ def get_form(cls): }, ] - @classmethod - def get_options_serializer_class(cls): - class OptionsSerializer(serializers.Serializer): - rootdir = serializers.CharField(default='', allow_blank=True) - recursive = serializers.BooleanField(default=True) - default_algorithm = serializers.ChoiceField( - choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], - default='SHA-256', - ) + class Serializer(BaseValidator.Serializer): + context = serializers.CharField() + + def __init__(self, *args, **kwargs): + from ESSArch_Core.ip.models import InformationPackage - return OptionsSerializer + super().__init__(*args, **kwargs) + ip_pk = kwargs['context']['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + self.fields['context'] = FilePathField(ip.object_path, allow_blank=True, default='') + + class OptionsSerializer(BaseValidator.OptionsSerializer): + rootdir = serializers.CharField(default='', allow_blank=True) + recursive = serializers.BooleanField(default=True) + default_algorithm = serializers.ChoiceField( + choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], + default='SHA-256', + ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -151,7 +159,7 @@ def _create_obj(self, filename, passed, msg): validator=self.__class__.__name__, required=self.required, task=self.task, - information_package_id=self.ip, + information_package=self.ip, responsible=self.responsible, message=msg, passed=passed, From f30bb334914bc6dd64d12c564acf21c67dd6f0ff Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 20 Mar 2020 12:10:52 +0100 Subject: [PATCH 21/23] Fix ValidateLogicalPhysicalRepresentation and CompareXMLFiles --- ESSArch_Core/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ESSArch_Core/tasks.py b/ESSArch_Core/tasks.py index e877c9531..6c4dcc414 100644 --- a/ESSArch_Core/tasks.py +++ b/ESSArch_Core/tasks.py @@ -433,9 +433,9 @@ def run(self, path, xmlfile, skip_files=None, relpath=None): else: rootdir = os.path.dirname(path) - ip = InformationPackage.objects.get(pk=self.ip) + ip = self.get_information_package() validator = DiffCheckValidator(context=xmlfile, exclude=skip_files, options={'rootdir': rootdir}, - task=self.get_processtask(), ip=self.ip, responsible=ip.responsible) + task=self.get_processtask(), ip=ip, responsible=ip.responsible) validator.validate(path) def event_outcome_success(self, result, path, xmlfile, skip_files=None, relpath=None): @@ -452,7 +452,7 @@ class CompareXMLFiles(DBTask): def run(self, first, second, rootdir=None, recursive=True): Validation.objects.filter(task=self.get_processtask()).delete() first, second = self.parse_params(first, second) - ip = InformationPackage.objects.get(pk=self.ip) + ip = self.get_information_package() if rootdir is None: rootdir = ip.object_path else: @@ -462,7 +462,7 @@ def run(self, first, second, rootdir=None, recursive=True): context=first, options={'rootdir': rootdir, 'recursive': recursive}, task=self.get_processtask(), - ip=self.ip, + ip=ip, responsible=ip.responsible, ) validator.validate(second) From 0510e0d23dff4aebcc952a8ff3457c4af9d432b5 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 20 Mar 2020 17:40:49 +0100 Subject: [PATCH 22/23] Working GUI, improved error reporting and parallel validation --- ESSArch_Core/WorkflowEngine/util.py | 1 + ESSArch_Core/config/urls.py | 2 - ESSArch_Core/fixity/serializers.py | 24 ++++--- .../fixity/validation/backends/checksum.py | 62 ++++++++++++++--- .../fixity/validation/backends/xml.py | 8 ++- ESSArch_Core/fixity/views.py | 51 +------------- ESSArch_Core/ip/views.py | 69 ++++++++++--------- ESSArch_Core/tasks.py | 62 ----------------- 8 files changed, 114 insertions(+), 165 deletions(-) diff --git a/ESSArch_Core/WorkflowEngine/util.py b/ESSArch_Core/WorkflowEngine/util.py index 7e5e504ea..a74645ff8 100644 --- a/ESSArch_Core/WorkflowEngine/util.py +++ b/ESSArch_Core/WorkflowEngine/util.py @@ -55,6 +55,7 @@ def _create_step(parent_step, flow, ip, responsible, context=None): child_s = ProcessStep.objects.create( name=flow_entry['name'], + parallel=flow_entry['parallel'], parent_step=parent_step, parent_step_pos=e_idx, eager=parent_step.eager, diff --git a/ESSArch_Core/config/urls.py b/ESSArch_Core/config/urls.py index caf76a91d..1c04e0220 100644 --- a/ESSArch_Core/config/urls.py +++ b/ESSArch_Core/config/urls.py @@ -39,7 +39,6 @@ ValidationFilesViewSet, ValidationViewSet, ValidatorViewSet, - ValidatorWorkflowViewSet, ) from ESSArch_Core.ip.views import ( ConsignMethodViewSet, @@ -323,7 +322,6 @@ router.register(r'conversion-tools', ConversionToolViewSet) router.register(r'features', FeatureViewSet, basename='features') router.register(r'validators', ValidatorViewSet, basename='validators') -router.register(r'validator-workflows', ValidatorWorkflowViewSet, basename='validator-workflows') router.register(r'validations', ValidationViewSet) router.register(r'events', EventIPViewSet) router.register(r'event-types', EventTypeViewSet) diff --git a/ESSArch_Core/fixity/serializers.py b/ESSArch_Core/fixity/serializers.py index 05d0b75b7..7f08fd877 100644 --- a/ESSArch_Core/fixity/serializers.py +++ b/ESSArch_Core/fixity/serializers.py @@ -20,24 +20,32 @@ class ValidatorWorkflowSerializer(serializers.Serializer): def validate_validators(self, validators): new_data = [] - sub_context = {'information_package': self.context['request'].data.get('information_package', None)} + ip = self.context['request'].data.get('information_package', None) + sub_context = {'information_package': ip} + sub_context.update(self.context) for validator in validators: name = validator.pop('name') klass = get_validator(name) - options_serializer = klass.get_options_serializer_class()( - data=validator.pop('options', {}), - context=sub_context, - ) + serializer = klass.get_serializer_class()( data=validator, context=sub_context, ) - serializer.is_valid(True) - options_serializer.is_valid(True) - data = serializer.validated_data data['name'] = name + + options_data = validator.pop('options', {}) + options_context = { + 'information_package': ip, + 'base_data': data, + } + options_serializer = klass.get_options_serializer_class()( + data=options_data, + context=options_context, + ) + + options_serializer.is_valid(True) data['options'] = options_serializer.validated_data new_data.append(data) diff --git a/ESSArch_Core/fixity/validation/backends/checksum.py b/ESSArch_Core/fixity/validation/backends/checksum.py index 258d02cff..298ce49cb 100644 --- a/ESSArch_Core/fixity/validation/backends/checksum.py +++ b/ESSArch_Core/fixity/validation/backends/checksum.py @@ -1,5 +1,5 @@ import logging -import traceback +import os from django.utils import timezone from rest_framework import serializers @@ -28,14 +28,52 @@ class ChecksumValidator(BaseValidator): label = 'Checksum Validator' + @classmethod + def get_form(cls): + return [ + { + 'key': 'path', + 'type': 'input', + 'templateOptions': { + 'label': 'Path to validate', + 'required': True, + } + }, + { + 'key': 'options.algorithm', + 'type': 'select', + 'defaultValue': 'SHA-256', + 'templateOptions': { + 'label': 'Checksum algorithm', + 'required': True, + 'labelProp': 'name', + 'valueProp': 'value', + 'options': [ + {'name': 'MD5', 'value': 'MD5'}, + {'name': 'SHA-1', 'value': 'SHA-1'}, + {'name': 'SHA-224', 'value': 'SHA-224'}, + {'name': 'SHA-256', 'value': 'SHA-256'}, + {'name': 'SHA-384', 'value': 'SHA-384'}, + {'name': 'SHA-512', 'value': 'SHA-512'}, + ] + } + }, + { + 'key': 'options.expected', + 'type': 'input', + 'templateOptions': { + 'label': 'Checksum', + 'required': True, + } + }, + ] + class Serializer(BaseValidator.Serializer): - context = serializers.ChoiceField(choices=['checksum_str', 'checksum_file', 'xml_file']) + context = serializers.CharField(default='checksum_str') block_size = serializers.IntegerField(default=65536) class OptionsSerializer(BaseValidator.OptionsSerializer): expected = serializers.CharField() - rootdir = serializers.CharField(default='', allow_blank=True) - recursive = serializers.BooleanField(default=True) algorithm = serializers.ChoiceField( choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], default='SHA-256', @@ -52,8 +90,14 @@ def __init__(self, *args, **kwargs): def validate(self, filepath, expected=None): logger.debug('Validating checksum of %s' % filepath) + + if self.ip is not None: + relpath = os.path.relpath(filepath, self.ip.object_path) + else: + relpath = filepath + val_obj = Validation.objects.create( - filename=filepath, + filename=relpath, time_started=timezone.now(), validator=self.__class__.__name__, required=self.required, @@ -82,14 +126,14 @@ def validate(self, filepath, expected=None): actual_checksum = calculate_checksum(filepath, algorithm=self.algorithm, block_size=self.block_size) if actual_checksum != checksum: raise ValidationError("checksum for %s is not valid (%s != %s)" % ( - filepath, checksum, actual_checksum + relpath, checksum, actual_checksum )) passed = True - except Exception: - val_obj.message = traceback.format_exc() + except Exception as e: + val_obj.message = str(e) raise else: - message = 'Successfully validated checksum of %s' % filepath + message = 'Successfully validated checksum of %s' % relpath val_obj.message = message logger.info(message) finally: diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 3304886b2..2f3fd2494 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -94,13 +94,19 @@ def __init__(self, *args, **kwargs): self.fields['context'] = FilePathField(ip.object_path, allow_blank=True, default='') class OptionsSerializer(BaseValidator.OptionsSerializer): - rootdir = serializers.CharField(default='', allow_blank=True) + rootdir = serializers.CharField(required=False) recursive = serializers.BooleanField(default=True) default_algorithm = serializers.ChoiceField( choices=['MD5', 'SHA-1', 'SHA-224', 'SHA-256', 'SHA-384', 'SHA-512'], default='SHA-256', ) + def validate(self2, data): + if 'rootdir' not in data: + data['rootdir'] = self2.context['base_data']['path'] + + return data + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/ESSArch_Core/fixity/views.py b/ESSArch_Core/fixity/views.py index 25f783ee1..edef59cff 100644 --- a/ESSArch_Core/fixity/views.py +++ b/ESSArch_Core/fixity/views.py @@ -1,9 +1,6 @@ -import os - -from django.db import transaction from django.db.models import Exists, Max, Min, OuterRef from django_filters.rest_framework import DjangoFilterBackend -from rest_framework import filters, mixins, status, viewsets +from rest_framework import filters, viewsets from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework_extensions.mixins import NestedViewSetMixin @@ -15,14 +12,11 @@ ConversionToolSerializer, ValidationFilesSerializer, ValidationSerializer, - ValidatorWorkflowSerializer, ) from ESSArch_Core.fixity.validation import ( AVAILABLE_VALIDATORS, get_backend as get_validator, ) -from ESSArch_Core.WorkflowEngine.models import ProcessStep -from ESSArch_Core.WorkflowEngine.util import create_workflow class ConversionToolViewSet(viewsets.ReadOnlyModelViewSet): @@ -57,49 +51,6 @@ def list(self, request, format=None): return Response(validators) -class ValidatorWorkflowViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): - queryset = ProcessStep.objects.all() - serializer_class = ValidatorWorkflowSerializer - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data, context={'request': self.request}) - serializer.is_valid(raise_exception=True) - workflow_spec = [] - ip = serializer.validated_data['information_package'] - - for validator in serializer.validated_data['validators']: - name = validator['name'] - klass = get_validator(name) - options_serializer = klass.get_options_serializer_class()(data=validator.get('options', {})) - options_serializer.is_valid(raise_exception=True) - options = options_serializer.validated_data - - path = os.path.join(ip.object_path, validator['path']) - options['rootdir'] = ip.object_path - - task_spec = { - 'name': 'ESSArch_Core.fixity.validation.tasks.Validate', - 'label': 'Validate using {}'.format(klass.label), - 'args': [name, path], - 'params': {'context': validator['context'], 'options': options}, - } - - workflow_spec.append(task_spec) - - with transaction.atomic(): - step = { - 'step': True, - 'name': serializer.validated_data['purpose'], - 'children': workflow_spec - } - workflow = create_workflow([step], ip=ip, name='Validation') - - workflow.run() - - headers = self.get_success_headers(serializer.data) - return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) - - class ValidationViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet): queryset = Validation.objects.all().order_by('filename', 'validator') serializer_class = ValidationSerializer diff --git a/ESSArch_Core/ip/views.py b/ESSArch_Core/ip/views.py index 10ef39621..b0d8b344b 100644 --- a/ESSArch_Core/ip/views.py +++ b/ESSArch_Core/ip/views.py @@ -64,8 +64,12 @@ from ESSArch_Core.essxml.util import get_objectpath, parse_submit_description from ESSArch_Core.exceptions import Conflict, NoFileChunksFound from ESSArch_Core.fixity.format import FormatIdentifier +from ESSArch_Core.fixity.serializers import ValidatorWorkflowSerializer from ESSArch_Core.fixity.transformation import AVAILABLE_TRANSFORMERS -from ESSArch_Core.fixity.validation import AVAILABLE_VALIDATORS +from ESSArch_Core.fixity.validation import ( + AVAILABLE_VALIDATORS, + get_backend as get_validator, +) from ESSArch_Core.fixity.validation.backends.checksum import ChecksumValidator from ESSArch_Core.ip.filters import ( AgentFilter, @@ -1923,46 +1927,45 @@ def unlock_profile(self, request, pk=None): ) }) - @transaction.atomic @action(detail=True, methods=['post'], url_path='validate') def validate(self, request, pk=None): ip = self.get_object() - prepare = Path.objects.get(entity="ingest_workarea").value - xmlfile = os.path.join(prepare, "%s.xml" % pk) + request.data['information_package'] = str(ip.pk) + serializer = ValidatorWorkflowSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + workflow_spec = [] + ip = serializer.validated_data['information_package'] - step = ProcessStep.objects.create( - name="Validation", - information_package=ip - ) + for validator in serializer.validated_data['validators']: + name = validator['name'] + klass = get_validator(name) - step.add_tasks( - ProcessTask.objects.create( - name="ESSArch_Core.tasks.ValidateXMLFile", - params={ - "xml_filename": xmlfile - }, - log=EventIP, - information_package=ip, - responsible=self.request.user, - ), - ProcessTask.objects.create( - name="ESSArch_Core.tasks.ValidateFiles", - params={ - "mets_path": xmlfile, - "validate_fileformat": True, - "validate_integrity": True, - }, - log=EventIP, - processstep_pos=0, - information_package=ip, - responsible=self.request.user, - ) - ) + options = validator['options'] + path = os.path.join(ip.object_path, validator['path']) - step.run() + task_spec = { + 'name': 'ESSArch_Core.fixity.validation.tasks.Validate', + 'label': 'Validate using {}'.format(klass.label), + 'args': [name, path], + 'params': {'context': validator['context'], 'options': options}, + } - return Response("Validating IP") + workflow_spec.append(task_spec) + + with transaction.atomic(): + step = { + 'step': True, + 'name': serializer.validated_data['purpose'], + 'parallel': True, + 'children': workflow_spec + } + workflow = create_workflow([step], ip=ip, name='Validation') + + workflow.run() + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) def update(self, request, *args, **kwargs): ip = self.get_object() diff --git a/ESSArch_Core/tasks.py b/ESSArch_Core/tasks.py index 6c4dcc414..889b49662 100644 --- a/ESSArch_Core/tasks.py +++ b/ESSArch_Core/tasks.py @@ -33,7 +33,6 @@ import requests from django.conf import settings from django.contrib.auth import get_user_model -from django.core.exceptions import ValidationError from django.core.mail import EmailMessage from django.db import transaction from django.utils import timezone @@ -47,7 +46,6 @@ findElementWithoutNamespace, ) from ESSArch_Core.essxml.util import find_pointers, get_premis_ref -from ESSArch_Core.fixity import validation from ESSArch_Core.fixity.models import Validation from ESSArch_Core.fixity.transformation import get_backend as get_transformer from ESSArch_Core.fixity.validation.backends.xml import ( @@ -309,66 +307,6 @@ def event_outcome_success(self, result, filename=None, format_name=None, ) -class ValidateWorkarea(DBTask): - queue = 'validation' - - def create_notification(self, ip): - errcount = Validation.objects.filter(information_package=ip, passed=False, required=True).count() - - if errcount: - Notification.objects.create( - message='Validation of "{ip}" failed with {errcount} error(s)'.format( - ip=ip.object_identifier_value, errcount=errcount - ), - level=logging.ERROR, - user_id=self.responsible, - refresh=True - ) - else: - Notification.objects.create( - message='"{ip}" was successfully validated'.format( - ip=ip.object_identifier_value - ), - level=logging.INFO, - user_id=self.responsible, - refresh=True - ) - - def run(self, workarea, validators, stop_at_failure=True): - workarea = Workarea.objects.get(pk=workarea) - workarea.successfully_validated = {} - - for validator in validators: - workarea.successfully_validated[validator] = None - - workarea.save(update_fields=['successfully_validated']) - ip = workarea.ip - sa = ip.submission_agreement - validation_profile = ip.get_profile('validation') - profile_data = fill_specification_data(data=ip.get_profile_data('validation'), sa=sa, ip=ip) - responsible = User.objects.get(pk=self.responsible) - - try: - validation.validate_path(workarea.path, validators, validation_profile, data=profile_data, ip=ip, - task=self.get_processtask(), stop_at_failure=stop_at_failure, - responsible=responsible) - except ValidationError: - self.create_notification(ip) - else: - self.create_notification(ip) - finally: - validations = ip.validation_set.all() - failed_validators = validations.values('validator').filter( - passed=False, required=True - ).values_list('validator', flat=True) - - for k, _v in workarea.successfully_validated.items(): - class_name = validation.AVAILABLE_VALIDATORS[k].split('.')[-1] - workarea.successfully_validated[k] = class_name not in failed_validators - - workarea.save(update_fields=['successfully_validated']) - - class TransformWorkarea(DBTask): def run(self, backend, workarea): workarea = Workarea.objects.select_related('ip__submission_agreement').get(pk=workarea) From 5d35bf3ec095abbb72dd51aec5318abc0f2785cf Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 23 Mar 2020 10:36:37 +0100 Subject: [PATCH 23/23] Fix tests --- ESSArch_Core/fixity/tests/test_views.py | 163 ----------------- .../fixity/validation/backends/xml.py | 11 +- ESSArch_Core/ip/tests/test_views.py | 169 +++++++++++++++++- ESSArch_Core/ip/views.py | 5 +- 4 files changed, 181 insertions(+), 167 deletions(-) diff --git a/ESSArch_Core/fixity/tests/test_views.py b/ESSArch_Core/fixity/tests/test_views.py index 14f98de99..2b59ca694 100644 --- a/ESSArch_Core/fixity/tests/test_views.py +++ b/ESSArch_Core/fixity/tests/test_views.py @@ -1,20 +1,8 @@ -import os -import shutil -import tempfile - -from celery import states as celery_states from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase -from ESSArch_Core.configuration.models import Path -from ESSArch_Core.exceptions import ValidationError -from ESSArch_Core.fixity.checksum import calculate_checksum -from ESSArch_Core.ip.models import InformationPackage -from ESSArch_Core.testing.runner import TaskRunner -from ESSArch_Core.WorkflowEngine.models import ProcessTask - User = get_user_model() @@ -23,154 +11,3 @@ def test_list(self): url = reverse('validators-list') response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) - - -class ValidatorWorkflowViewSetTests(APITestCase): - @classmethod - def setUpTestData(cls): - cls.user = User.objects.create() - cls.superuser = User.objects.create(username='superuser', is_superuser=True) - cls.url = reverse('validator-workflows-list') - - def setUp(self): - self.datadir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.datadir) - - Path.objects.create(entity='temp', value=tempfile.mkdtemp(dir=self.datadir)) - - def test_without_permission(self): - self.client.force_authenticate(self.user) - response = self.client.post(self.url) - self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - - @TaskRunner() - def test_create_and_run_workflow(self): - self.client.force_authenticate(self.superuser) - - ip = InformationPackage.objects.create( - object_path=tempfile.mkdtemp(dir=self.datadir), - ) - os.makedirs(os.path.join(ip.object_path, 'content')) - - with open(os.path.join(ip.object_path, 'content', 'foo.txt'), 'w') as f: - f.write('hello') - - expected = calculate_checksum(os.path.join(ip.object_path, 'content', 'foo.txt'), 'SHA-224') - with open(os.path.join(ip.object_path, 'metadata.xml'), 'w') as f: - f.write('') - - with self.subTest('invalid file path'): - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - 'path': 'foo.txt', - 'context': 'checksum_str', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - ], - }) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertFalse(ProcessTask.objects.exists()) - - with self.subTest('valid path and expected value'): - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - 'path': 'content/foo.txt', - 'context': 'checksum_str', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - ], - }) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 0) - - with self.subTest('valid path and unexpected value'): - with self.assertRaises(ValidationError): - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - 'path': 'content/foo.txt', - 'context': 'checksum_str', - 'options': { - 'expected': 'incorrect', - 'algorithm': 'SHA-224', - } - }, - ], - }) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) - - with self.subTest('multiple validators'): - with self.assertRaises(ValidationError): - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - 'path': 'content/foo.txt', - 'context': 'checksum_str', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - { - 'name': 'diff_check', - 'path': 'content/foo.txt', - 'context': 'metadata.xml', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - ], - }) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 2) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) - - with open(os.path.join(ip.object_path, 'metadata.xml'), 'w') as f: - f.write('') - - response = self.client.post(self.url, { - 'information_package': str(ip.pk), - 'validators': [ - { - 'name': 'checksum', - 'path': 'content/foo.txt', - 'context': 'checksum_str', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - { - 'name': 'diff_check', - 'path': 'content/foo.txt', - 'context': 'metadata.xml', - 'options': { - 'expected': expected, - 'algorithm': 'SHA-224', - } - }, - ], - }) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 4) - self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) diff --git a/ESSArch_Core/fixity/validation/backends/xml.py b/ESSArch_Core/fixity/validation/backends/xml.py index 2f3fd2494..2becc173a 100644 --- a/ESSArch_Core/fixity/validation/backends/xml.py +++ b/ESSArch_Core/fixity/validation/backends/xml.py @@ -101,9 +101,16 @@ class OptionsSerializer(BaseValidator.OptionsSerializer): default='SHA-256', ) - def validate(self2, data): + def validate(self, data): if 'rootdir' not in data: - data['rootdir'] = self2.context['base_data']['path'] + if self.context['base_data']['path']: + data['rootdir'] = self.context['base_data']['path'] + else: + from ESSArch_Core.ip.models import InformationPackage + + ip_pk = self.context['information_package'] + ip = InformationPackage.objects.get(pk=ip_pk) + data['rootdir'] = ip.object_path return data diff --git a/ESSArch_Core/ip/tests/test_views.py b/ESSArch_Core/ip/tests/test_views.py index 6e50be859..980144c84 100644 --- a/ESSArch_Core/ip/tests/test_views.py +++ b/ESSArch_Core/ip/tests/test_views.py @@ -2,12 +2,13 @@ import shutil import tempfile +from celery import states as celery_states from django.contrib.auth import get_user_model from django.contrib.auth.models import Permission from django.test import TestCase from django.urls import reverse from rest_framework import status -from rest_framework.test import APIClient +from rest_framework.test import APIClient, APITestCase from ESSArch_Core.auth.models import ( Group, @@ -16,6 +17,8 @@ GroupType, ) from ESSArch_Core.configuration.models import Path, StoragePolicy +from ESSArch_Core.exceptions import ValidationError +from ESSArch_Core.fixity.checksum import calculate_checksum from ESSArch_Core.ip.models import InformationPackage, Workarea from ESSArch_Core.profiles.models import SubmissionAgreement from ESSArch_Core.storage.models import ( @@ -28,6 +31,8 @@ StorageObject, StorageTarget, ) +from ESSArch_Core.testing.runner import TaskRunner +from ESSArch_Core.WorkflowEngine.models import ProcessTask User = get_user_model() @@ -314,3 +319,165 @@ def test_get_migratable_new_storage_target(self): ip_exists = InformationPackage.objects.migratable().exists() self.assertFalse(ip_exists) + + +class InformationPackageValidationTests(APITestCase): + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create(username='superuser', is_superuser=True) + + def setUp(self): + self.datadir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.datadir) + + Path.objects.create(entity='temp', value=tempfile.mkdtemp(dir=self.datadir)) + + self.ip = InformationPackage.objects.create( + object_path=tempfile.mkdtemp(dir=self.datadir), + ) + + @TaskRunner() + def test_validate(self): + self.client.force_authenticate(self.user) + url = reverse('informationpackage-validate', args=(str(self.ip.pk),)) + + os.makedirs(os.path.join(self.ip.object_path, 'content')) + + with open(os.path.join(self.ip.object_path, 'content', 'foo.txt'), 'w') as f: + f.write('hello') + + chksum = calculate_checksum(os.path.join(self.ip.object_path, 'content', 'foo.txt'), 'SHA-224') + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write('') + + with self.subTest('invalid file path'): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(ProcessTask.objects.exists()) + + with self.subTest('valid path and expected value'): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 0) + + with self.subTest('valid path and unexpected value'): + with self.assertRaises(ValidationError): + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': 'incorrect', + 'algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 1) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 1) + + with self.subTest('multiple validators'): + with self.assertRaises(ValidationError): + response = self.client.post(url, { + 'information_package': str(self.ip.pk), + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': '', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 2) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) + + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write(f'') + + response = self.client.post(url, { + 'validators': [ + { + 'name': 'checksum', + 'path': 'content/foo.txt', + 'context': 'checksum_str', + 'options': { + 'expected': chksum, + 'algorithm': 'SHA-224', + } + }, + { + 'name': 'diff_check', + 'path': '', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 4) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) + + with open(os.path.join(self.ip.object_path, 'metadata.xml'), 'w') as f: + f.write(f'') + + response = self.client.post(url, { + 'validators': [ + { + 'name': 'diff_check', + 'path': 'content', + 'context': 'metadata.xml', + 'options': { + 'recursive': True, + 'default_algorithm': 'SHA-224', + } + }, + ], + }) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.SUCCESS).count(), 5) + self.assertEqual(ProcessTask.objects.filter(status=celery_states.FAILURE).count(), 2) diff --git a/ESSArch_Core/ip/views.py b/ESSArch_Core/ip/views.py index b0d8b344b..f8fb6bbb7 100644 --- a/ESSArch_Core/ip/views.py +++ b/ESSArch_Core/ip/views.py @@ -1942,7 +1942,10 @@ def validate(self, request, pk=None): klass = get_validator(name) options = validator['options'] - path = os.path.join(ip.object_path, validator['path']) + if validator['path']: + path = os.path.join(ip.object_path, validator['path']) + else: + path = ip.object_path task_spec = { 'name': 'ESSArch_Core.fixity.validation.tasks.Validate',