From a0331e0236ceb5639f6d3dc9b59f59eeb9151ee8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 5 Jul 2019 21:26:45 -0400 Subject: [PATCH 001/402] Add support for icon shadows Signed-off-by: Roberto Rosario --- HISTORY.rst | 4 + docs/releases/3.3.rst | 101 ++++++++++++++++++ docs/releases/index.rst | 1 + mayan/apps/appearance/classes.py | 74 +++++++------ .../appearance/icons/font_awesome_layers.html | 3 + .../appearance/icons/font_awesome_symbol.html | 9 +- .../templatetags/appearance_tags.py | 5 + mayan/apps/documents/icons.py | 2 +- .../navigation/large_button_link.html | 6 +- 9 files changed, 167 insertions(+), 38 deletions(-) create mode 100644 docs/releases/3.3.rst diff --git a/HISTORY.rst b/HISTORY.rst index 33c632cde12..a4aaac26044 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,7 @@ +3.3 (2019-XX-XX) +================ +* Add support for icon shadows. + 3.2.5 (2019-07-05) ================== * Don't error out if the EXTRA_APPS or the DISABLED_APPS settings diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst new file mode 100644 index 00000000000..229f7a466cf --- /dev/null +++ b/docs/releases/3.3.rst @@ -0,0 +1,101 @@ +Version 3.3 +=========== + +Released: XX XX, 2019 + + +Changes +------- + +- Add support for icon shadows. + +Removals +-------- + +- None + + +Upgrading from a previous version +--------------------------------- + +If installed via Python's PIP +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Remove deprecated requirements:: + + $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + +Type in the console:: + + $ pip install mayan-edms==3.3 + +the requirements will also be updated automatically. + + +Using Git +^^^^^^^^^ + +If you installed Mayan EDMS by cloning the Git repository issue the commands:: + + $ git reset --hard HEAD + $ git pull + +otherwise download the compressed archived and uncompress it overriding the +existing installation. + +Remove deprecated requirements:: + + $ pip uninstall -y -r removals.txt + +Next upgrade/add the new requirements:: + + $ pip install --upgrade -r requirements.txt + + +Common steps +^^^^^^^^^^^^ + +Perform these steps after updating the code from either step above. + +Make a backup of your supervisord file:: + + sudo cp /etc/supervisor/conf.d/mayan.conf /etc/supervisor/conf.d/mayan.conf.bck + +Update the supervisord configuration file. Replace the environment +variables values show here with your respective settings. This step will refresh +the supervisord configuration file with the new queues and the latest +recommended layout:: + + sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ + MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ + MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf + +Edit the supervisord configuration file and update any setting the template +generator missed:: + + sudo vi /etc/supervisor/conf.d/mayan.conf + +Migrate existing database schema with:: + + $ mayan-edms.py performupgrade + +Add new static media:: + + $ mayan-edms.py preparestatic --noinput + +The upgrade procedure is now complete. + + +Backward incompatible changes +----------------------------- + +- None + + +Bugs fixed or issues closed +--------------------------- + +- :gitlab-issue:`XX` + +.. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/docs/releases/index.rst b/docs/releases/index.rst index 80bdb19c1ed..33d838bd3da 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -20,6 +20,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.3 3.2.5 3.2.4 3.2.3 diff --git a/mayan/apps/appearance/classes.py b/mayan/apps/appearance/classes.py index 9b05a39fc34..7f1ad604ecd 100644 --- a/mayan/apps/appearance/classes.py +++ b/mayan/apps/appearance/classes.py @@ -4,6 +4,7 @@ class IconDriver(object): + context = {} _registry = {} @classmethod @@ -14,6 +15,17 @@ def get(cls, name): def register(cls, driver_class): cls._registry[driver_class.name] = driver_class + def get_context(self): + return self.context + + def render(self, extra_context=None): + context = self.get_context() + if extra_context: + context.update(extra_context) + return get_template(template_name=self.template_name).render( + context=context + ) + class FontAwesomeDriver(IconDriver): name = 'fontawesome' @@ -22,10 +34,8 @@ class FontAwesomeDriver(IconDriver): def __init__(self, symbol): self.symbol = symbol - def render(self): - return get_template(template_name=self.template_name).render( - context={'symbol': self.symbol} - ) + def get_context(self): + return {'symbol': self.symbol} class FontAwesomeDualDriver(IconDriver): @@ -36,23 +46,21 @@ def __init__(self, primary_symbol, secondary_symbol): self.primary_symbol = primary_symbol self.secondary_symbol = secondary_symbol - def render(self): - return get_template(template_name=self.template_name).render( - context={ - 'data': ( - { - 'class': 'fas fa-circle', - 'transform': 'down-3 right-10', - 'mask': 'fas fa-{}'.format(self.primary_symbol) - }, - {'class': 'far fa-circle', 'transform': 'down-3 right-10'}, - { - 'class': 'fas fa-{}'.format(self.secondary_symbol), - 'transform': 'shrink-4 down-3 right-10' - }, - ) - } - ) + def get_context(self): + return { + 'data': ( + { + 'class': 'fas fa-circle', + 'transform': 'down-3 right-10', + 'mask': 'fas fa-{}'.format(self.primary_symbol) + }, + {'class': 'far fa-circle', 'transform': 'down-3 right-10'}, + { + 'class': 'fas fa-{}'.format(self.secondary_symbol), + 'transform': 'shrink-4 down-3 right-10' + }, + ) + } class FontAwesomeCSSDriver(IconDriver): @@ -62,10 +70,8 @@ class FontAwesomeCSSDriver(IconDriver): def __init__(self, css_classes): self.css_classes = css_classes - def render(self): - return get_template(template_name=self.template_name).render( - context={'css_classes': self.css_classes} - ) + def get_context(self): + return {'css_classes': self.css_classes} class FontAwesomeMasksDriver(IconDriver): @@ -75,23 +81,23 @@ class FontAwesomeMasksDriver(IconDriver): def __init__(self, data): self.data = data - def render(self): - return get_template(template_name=self.template_name).render( - context={'data': self.data} - ) + def get_context(self): + return {'data': self.data} class FontAwesomeLayersDriver(IconDriver): name = 'fontawesome-layers' template_name = 'appearance/icons/font_awesome_layers.html' - def __init__(self, data): + def __init__(self, data, shadow_class=None): self.data = data + self.shadow_class = shadow_class - def render(self): - return get_template(template_name=self.template_name).render( - context={'data': self.data} - ) + def get_context(self): + return { + 'data': self.data, + 'shadow_class': self.shadow_class, + } class Icon(object): diff --git a/mayan/apps/appearance/templates/appearance/icons/font_awesome_layers.html b/mayan/apps/appearance/templates/appearance/icons/font_awesome_layers.html index 4356ad89e14..8a8b4549666 100644 --- a/mayan/apps/appearance/templates/appearance/icons/font_awesome_layers.html +++ b/mayan/apps/appearance/templates/appearance/icons/font_awesome_layers.html @@ -1,4 +1,7 @@ + {% if enable_shadow %} + + {% endif %} {% for entry in data %} {% endfor %} diff --git a/mayan/apps/appearance/templates/appearance/icons/font_awesome_symbol.html b/mayan/apps/appearance/templates/appearance/icons/font_awesome_symbol.html index fac6aeca4db..84e7b87eb84 100644 --- a/mayan/apps/appearance/templates/appearance/icons/font_awesome_symbol.html +++ b/mayan/apps/appearance/templates/appearance/icons/font_awesome_symbol.html @@ -1 +1,8 @@ - +{% if enable_shadow %} + + + + +{% else %} + +{% endif %} diff --git a/mayan/apps/appearance/templatetags/appearance_tags.py b/mayan/apps/appearance/templatetags/appearance_tags.py index 05c3ba043d7..eb9cf1bc02a 100644 --- a/mayan/apps/appearance/templatetags/appearance_tags.py +++ b/mayan/apps/appearance/templatetags/appearance_tags.py @@ -7,6 +7,11 @@ register = Library() +@register.simple_tag +def appearance_icon_render(icon_class, enable_shadow=False): + return icon_class.render(extra_context={'enable_shadow': enable_shadow}) + + @register.filter def get_choice_value(field): try: diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index 7d411a30f05..8b9b4067d9c 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -7,7 +7,7 @@ driver_name='fontawesome-layers', data=[ {'class': 'fas fa-circle', 'transform': 'shrink-12 up-2'}, {'class': 'fas fa-cog', 'transform': 'shrink-6 up-2', 'mask': 'fas fa-torah'} - ] + ], shadow_class='fas fa-torah' ) icon_menu_documents = Icon(driver_name='fontawesome', symbol='book') diff --git a/mayan/apps/navigation/templates/navigation/large_button_link.html b/mayan/apps/navigation/templates/navigation/large_button_link.html index dacbe785d90..e9c2d84f975 100644 --- a/mayan/apps/navigation/templates/navigation/large_button_link.html +++ b/mayan/apps/navigation/templates/navigation/large_button_link.html @@ -1,6 +1,8 @@ +{% load appearance_tags %} +
- - {% if link.icon_class %}{{ link.icon_class.render }}{% endif %} + + {% if link.icon_class %}{% appearance_icon_render link.icon_class enable_shadow=True %}{% endif %}
{{ link.text }} From 300bdbfc8a399c28be99d04e66d5e24aed9d3345 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 5 Jul 2019 21:34:20 -0400 Subject: [PATCH 002/402] Tweak setup buttom border and tag shadows Signed-off-by: Roberto Rosario --- .../appearance/static/appearance/css/base.css | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 688f8e6e51d..2e57afddb92 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -70,7 +70,8 @@ img.lazy-load-carousel { } .label-tag { - text-shadow: 0px 0px 2px #000 + text-shadow: 0px 0px 2px #000; + box-shadow: 3px 3px 5px rgba(0, 0, 0, 0.5); } .fancybox-nav span { @@ -88,18 +89,20 @@ hr { } .btn-block { + border-top: 2px solid rgba(255, 255, 255, 0.7); + border-left: 2px solid rgba(255, 255, 255, 0.7); + border-right: 2px solid rgba(0, 0, 0, 0.7); + border-bottom: 2px solid rgba(0, 0, 0, 0.7); + box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.5); margin-bottom: 15px; - white-space: normal; min-height: 120px; - padding-top: 20px; padding-bottom: 1px; -} - -.btn-block .fa { + padding-top: 20px; text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); + white-space: normal; } -.btn-block { +.btn-block .fa { text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); } @@ -112,12 +115,12 @@ a i { } .dashboard-widget { - box-shadow: 1px 1px 1px rgba(0,0,0,0.3); + box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); border: 1px solid black; } .dashboard-widget .panel-heading i { - text-shadow: 1px 1px 1px rgba(0,0,0,0.3); + text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); } .dashboard-widget-icon { @@ -170,7 +173,7 @@ a i { } .navbar-collapse { border-top: 1px solid transparent; - box-shadow: inset 0 1px 0 rgba(255,255,255,0.1); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1); } .navbar-fixed-top { top: 0; From 5146c6d2021576b53bcb1024d3d8dd8fddbaa2f3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 5 Jul 2019 21:34:20 -0400 Subject: [PATCH 003/402] Tweak setup buttom border and tag shadows Signed-off-by: Roberto Rosario --- HISTORY.rst | 743 +++++++++--------- docs/releases/3.3.rst | 2 + .../appearance/static/appearance/css/base.css | 23 +- mayan/apps/common/icons.py | 10 +- mayan/apps/common/links.py | 12 +- mayan/apps/common/views.py | 10 +- 6 files changed, 407 insertions(+), 393 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a4aaac26044..f59ef7f5d51 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,612 +1,611 @@ 3.3 (2019-XX-XX) ================ -* Add support for icon shadows. +- Add support for icon shadows. +- Add icons and no-result template to the object error log view and + links. 3.2.5 (2019-07-05) ================== -* Don't error out if the EXTRA_APPS or the DISABLED_APPS settings +- Don't error out if the EXTRA_APPS or the DISABLED_APPS settings are set to blank. -* Update troubleshooting documentation topic. -* Add data migration to the file metadata app. Synchronizes the +- Update troubleshooting documentation topic. +- Add data migration to the file metadata app. Synchronizes the document type settings model of existing document types. -* Fix cabinet and tags upload wizard steps missing some entries. +- Fix cabinet and tags upload wizard steps missing some entries. GitLab issue #632. Thanks to Matthias Urhahn (@d4rken) for the report. -* Add alert when settings are changed and util the installation +- Add alert when settings are changed and util the installation is restarted. GitLab issue #605. Thanks to Vikas Kedia (@vikaskedia) to the report. -* Update Django to version 1.11.22, PyYAML to version 5.1.1, +- Update Django to version 1.11.22, PyYAML to version 5.1.1, django-widget-tweaks to version 1.4.5, pathlib2 to version 2.3.4, Werkzeug to version 0.15.4, django-extensions to version 2.1.9, django-rosetta to version 0.9.3, psutil to version 5.6.3. 3.2.4 (2019-06-29) ================== -* Support configurable GUnicorn timeouts. Defaults to +- Support configurable GUnicorn timeouts. Defaults to current value of 120 seconds. -* Fix help text of the platformtemplate command. -* Fix IMAP4 mailbox.store flags argument. Python's documentation +- Fix help text of the platformtemplate command. +- Fix IMAP4 mailbox.store flags argument. Python's documentation incorrectly state it is named flag_list. Closes GitLab issue #606. -* Improve the workflow preview generation. Use polylines +- Improve the workflow preview generation. Use polylines instead of splines. Add state actions to the preview. Highlight the initial state. -* Add help text to the workflow transition form comment field. -* Fix direct deployment instructions. -* Add user, group, and role dashboard widgets. -* Add test mixin detect database connection leaks. -* Remove tag create event registration from the tag +- Add help text to the workflow transition form comment field. +- Fix direct deployment instructions. +- Add user, group, and role dashboard widgets. +- Add test mixin detect database connection leaks. +- Remove tag create event registration from the tag instances. The tag create event is not applicable to existing tags. -* Add proper redirection after moving a document to the +- Add proper redirection after moving a document to the trash. -* Remove the INSTALLED_APPS setting. Replace it with +- Remove the INSTALLED_APPS setting. Replace it with the new COMMON_EXTRA_APPS and COMMON_DISABLED_APPS. -* Improve email metadata support. Can now work on +- Improve email metadata support. Can now work on email with nested parts. Also the metadata.yaml attachment no longer needs to be the first attachment. 3.2.3 (2019-06-21) ================== -* Add support for disabling the random primary key +- Add support for disabling the random primary key test mixin. -* Fix mailing profile log columns mappings. +- Fix mailing profile log columns mappings. GitLab issue #626. Thanks to Jesaja Everling (@jeverling) for the report. -* Fix the Django SMTP backend username field name. +- Fix the Django SMTP backend username field name. GitLab issue #625. Thanks to Jesaja Everling (@jeverling) for the report and the research. -* Increase the Django STMP username. +- Increase the Django STMP username. GitLab issue #625. Thanks to Jesaja Everling (@jeverling) for the report and the research. 3.2.2 (2019-06-19) ================== -* Fix document type change view. Closes GitLab issue #614 +- Fix document type change view. Closes GitLab issue #614 Thanks to Christoph Roeder (@brightdroid) for the report. -* Fix document parsing tool view typo. Closes GitLab issue #615. +- Fix document parsing tool view typo. Closes GitLab issue #615. Thanks to Tyler Page (@iamtpage) for the report. -* Update the task_check_interval_source reference +- Update the task_check_interval_source reference GitLab issue #617. Thanks to Lukas Gill (@lukkigi) for the report and debug information. 3.2.1 (2019-06-14) ================== -* Fix sub cabinet creation view. Thanks to Frédéric Sheedy +- Fix sub cabinet creation view. Thanks to Frédéric Sheedy (@fsheedy) for the report. -* Add PostgreSQL troubleshooting entry. Closes GitLab +- Add PostgreSQL troubleshooting entry. Closes GitLab issues #523 and #602 -* Use YAML SafeDumper to avoid adding YAML datatype tags. +- Use YAML SafeDumper to avoid adding YAML datatype tags. Closes GitLab issue #599. Thanks to Frédéric Sheedy (@fsheedy) for the report and debug information. -* Add check for app references and point users to release notes for details. +- Add check for app references and point users to release notes for details. GitLab issue #603. Thanks to Vikas Kedia (@vikaskedia) for the report. -* Remove sidebar floar right. +- Remove sidebar floar right. Fixed GitLab issue #600. Thanks to Frédéric Sheedy (@fsheedy) for the report and debug information. -* Collapse sidebar on small screen +- Collapse sidebar on small screen Display sidebar at the bottom of the screen on small displays. 3.2 (2019-06-13) ================ -* Split sources models into separate modules. -* Add support for subfolder scanning to watchfolders. Closes +- Split sources models into separate modules. +- Add support for subfolder scanning to watchfolders. Closes GitLab issue #498 and #563. -* Updated the source check behavior to allow checking a source +- Updated the source check behavior to allow checking a source even when the source is disabled and to not deleted processed files during a check. -* Switch to full app paths. -* Split document app models into separate modules. -* Split workflow views into separate modules. -* Add custom DatabaseWarning to tag the SQLite usage warning. -* Add keyword arguments to add_to_class instances. -* Move add_to_class function to their own module called methods.py -* Remove catch all exception handling for the check in and +- Switch to full app paths. +- Split document app models into separate modules. +- Split workflow views into separate modules. +- Add custom DatabaseWarning to tag the SQLite usage warning. +- Add keyword arguments to add_to_class instances. +- Move add_to_class function to their own module called methods.py +- Remove catch all exception handling for the check in and check out views. -* Improve checkouts tests code reducing redundant code. -* Change how the HOME_VIEW setting is defined. -* Remove the role permission grant and revoke permission. -* Split trashed document views into their own module. -* Show entire sys trace when an App import exception is raised. -* Remove Django suit from requirements. -* Remove development URLs from main URL file. -* Move API documentation generation from the root URLs module +- Improve checkouts tests code reducing redundant code. +- Change how the HOME_VIEW setting is defined. +- Remove the role permission grant and revoke permission. +- Split trashed document views into their own module. +- Show entire sys trace when an App import exception is raised. +- Remove Django suit from requirements. +- Remove development URLs from main URL file. +- Move API documentation generation from the root URLs module to the REST API app's URLs module. -* Update Pillow to version 6.0.0 -* Update PyYAML to version 5.1. Update use of safe_load and +- Update Pillow to version 6.0.0 +- Update PyYAML to version 5.1. Update use of safe_load and safe_dump to load and dump using the SafeLoader. -* Add SilenceLoggerTestCaseMixin to lower level of loggers +- Add SilenceLoggerTestCaseMixin to lower level of loggers during tests. -* New default value for setting DOCUMENTS_HASH_BLOCK_SIZE is +- New default value for setting DOCUMENTS_HASH_BLOCK_SIZE is 65535. -* New default value for setting MIMETYPE_FILE_READ_SIZE is +- New default value for setting MIMETYPE_FILE_READ_SIZE is 1024. -* Add workaround for Tesseract bug 1670 +- Add workaround for Tesseract bug 1670 https://github.com/tesseract-ocr/tesseract/issues/1670 https://github.com/tesseract-ocr/tesseract/commit/3292484f67af8bdda23aa5e510918d0115785291 https://gitlab.gnome.org/World/OpenPaperwork/pyocr/issues/104 -* Move setting COMMON_TEMPORARY_DIRECTORY to the storage app. +- Move setting COMMON_TEMPORARY_DIRECTORY to the storage app. The setting is now STORAGE_TEMPORARY_DIRECTORY. -* Move file related utilities to the storage app. -* Backport and remove unused code from the permission app. -* Move the navigation and authentication templates to their +- Move file related utilities to the storage app. +- Backport and remove unused code from the permission app. +- Move the navigation and authentication templates to their respective apps. -* Add dashboard app. -* Remove queryset slicing hack from the Document list view. +- Add dashboard app. +- Remove queryset slicing hack from the Document list view. And slice the Recently Added Document queryset itself. -* Move stub filtering to the Document model manager. -* Increase the default number of recently added documents and +- Move stub filtering to the Document model manager. +- Increase the default number of recently added documents and recently accessed documents from 40 to 400. -* Integrate django-autoadmin into the core apps. -* Update middleware to new style classes. -* Add server side invalid document template. -* Move tag specific JavaScript to the tags app. -* Reduce form boilerplate code with new FormOptions class. -* Use FormOptions for the DetailForm class. -* DetailForm now support help text on extra fields. -* Add FilteredSelectionForm class. -* Use FilteredSelectionForm for TagMultipleSelectionForm. -* Use FilteredSelectionForm for the class CabinetListForm. -* Add keyword arguments to URL definitions. -* Use FilteredSelectionForm to add a new ACLCreateForm. -* Rename IndexListForm to IndexTemplateFilteredForm. -* Use FilteredSelectionForm for IndexTemplateFilteredForm. -* Use FilteredSelectionForm for DocumentVersionSignatureCreateForm. -* Improve document signatures tests. -* Add docstrings to most models. -* Add support to the mailing profiles for specifying a from +- Integrate django-autoadmin into the core apps. +- Update middleware to new style classes. +- Add server side invalid document template. +- Move tag specific JavaScript to the tags app. +- Reduce form boilerplate code with new FormOptions class. +- Use FormOptions for the DetailForm class. +- DetailForm now support help text on extra fields. +- Add FilteredSelectionForm class. +- Use FilteredSelectionForm for TagMultipleSelectionForm. +- Use FilteredSelectionForm for the class CabinetListForm. +- Add keyword arguments to URL definitions. +- Use FilteredSelectionForm to add a new ACLCreateForm. +- Rename IndexListForm to IndexTemplateFilteredForm. +- Use FilteredSelectionForm for IndexTemplateFilteredForm. +- Use FilteredSelectionForm for DocumentVersionSignatureCreateForm. +- Improve document signatures tests. +- Add docstrings to most models. +- Add support to the mailing profiles for specifying a from address. Closes GitLab issue #522. -* Expose new Django settings: AUTH_PASSWORD_VALIDATORS, DEFAULT_FROM_EMAIL, +- Expose new Django settings: AUTH_PASSWORD_VALIDATORS, DEFAULT_FROM_EMAIL, EMAIL_TIMEOUT, INTERNAL_IPS, LANGUAGES, LANGUAGE_CODE, STATIC_URL, STATICFILES_STORAGE, TIME_ZONE, WSGI_APPLICATION. -* Convert language choices into a function. -* Move language choices generation to documents.utils. -* Remove support for generating documents images in base 64 +- Convert language choices into a function. +- Move language choices generation to documents.utils. +- Remove support for generating documents images in base 64 format. -* Move Pillow initialization from the module to the backend +- Move Pillow initialization from the module to the backend class initialization. -* Remove star import from the ACL and Common apps. -* Add dependencies app -* Convert the document tags widget to use HTML templates. -* Move Tag app HTML widgets to their own module. -* Move the document index app widgets to the html_widget.py +- Remove star import from the ACL and Common apps. +- Add dependencies app +- Convert the document tags widget to use HTML templates. +- Move Tag app HTML widgets to their own module. +- Move the document index app widgets to the html_widget.py module. -* Update group members view permission. The group edit and +- Update group members view permission. The group edit and user edit permission are now required. -* Add keyword arguments to messages uses. -* Add keyword arguments to the reverse use in views. -* Add MERCs 5 and 6. -* Update authentication function views to use Django's new class +- Add keyword arguments to messages uses. +- Add keyword arguments to the reverse use in views. +- Add MERCs 5 and 6. +- Update authentication function views to use Django's new class based authentication views. -* Expose Django's LOGOUT_REDIRECT_URL setting. -* Move current user views from the common app to the user +- Expose Django's LOGOUT_REDIRECT_URL setting. +- Move current user views from the common app to the user management app. -* Move the purge permission logic to the StorePermission +- Move the purge permission logic to the StorePermission manager. -* Remove the MIMETYPE_FILE_READ_SIZE setting. -* Use copyfileobj in the document parsers. -* Backport list facet menu code. -* Backport sidebar code. -* CSS updates to maximize usable width. -* Improve partial navigation error messages and display. -* Add user created and user edited events. -* Add group created and group edited events. -* Add support for SourceColumn widgets. -* Improve styling of the template debug view. -* Add support for showing the current user's events. -* Add support kwargs to the SourceColumn class. -* Improve the event widgets, views and tests. -* Add mailer use event. -* Remove the include fontawesome and download it from +- Remove the MIMETYPE_FILE_READ_SIZE setting. +- Use copyfileobj in the document parsers. +- Backport list facet menu code. +- Backport sidebar code. +- CSS updates to maximize usable width. +- Improve partial navigation error messages and display. +- Add user created and user edited events. +- Add group created and group edited events. +- Add support for SourceColumn widgets. +- Improve styling of the template debug view. +- Add support for showing the current user's events. +- Add support kwargs to the SourceColumn class. +- Improve the event widgets, views and tests. +- Add mailer use event. +- Remove the include fontawesome and download it from the NPMregistry. -* Fix issue installing scoped NPM packages. -* Add new icons classes and templates. -* Add support for icon composition. -* Add support for link icon path imports. -* Remove support for link icon strings. -* Split document app form into separate modules. -* Move the favorite document views to their own module. -* Replace DocumentTypeSelectioForm with an improved +- Fix issue installing scoped NPM packages. +- Add new icons classes and templates. +- Add support for icon composition. +- Add support for link icon path imports. +- Remove support for link icon strings. +- Split document app form into separate modules. +- Move the favorite document views to their own module. +- Replace DocumentTypeSelectioForm with an improved version that does filtering. -* Update OCR links activation. -* Update document parsing link activation. -* Add favorite document views tests. -* Add document state action view test. -* Remove sidebar menu instance. The secondary menu and the +- Update OCR links activation. +- Update document parsing link activation. +- Add favorite document views tests. +- Add document state action view test. +- Remove sidebar menu instance. The secondary menu and the previour sidebar menu now perform the same function. -* Backport source column identifiable and sortable +- Backport source column identifiable and sortable improvements. -* Update the way the no-result template is shown. -* Improve TwoStateWidget to use a template. Make +- Update the way the no-result template is shown. +- Improve TwoStateWidget to use a template. Make it compatible with the SourceColumn. -* Update SourceColumn to support related attributes. -* Add support for display for empty values for +- Update SourceColumn to support related attributes. +- Add support for display for empty values for source columns. -* Add support for source column object or attribute +- Add support for source column object or attribute absolute URLs. -* Add sortable columns to all apps. -* Remove permission list display from the ACL list view. +- Add sortable columns to all apps. +- Remove permission list display from the ACL list view. Reduces clutter and unpredictable column size. -* Remove the full name from the user list. -* Add the first name and last name to the user list. -* Add file metadata app. -* Add support for submitting forms by pressing the +- Remove the full name from the user list. +- Add the first name and last name to the user list. +- Add file metadata app. +- Add support for submitting forms by pressing the Enter key or by double clicking. -* Rename form template 'form_class' to 'form_css_classes'. -* Add support for adding form button aside from the +- Rename form template 'form_class' to 'form_css_classes'. +- Add support for adding form button aside from the default submit and cancel. -* Update ChoiceForm to be full height. -* Add AddRemoveView to replace AssignRemoveView -* Update the group roles view to use the new AddRemoveView. -* Add role create and edit events. -* Sort users by lastname, firstname. -* Switch user groups and group users views to AddRemoveView. -* Commit user edit event when an user is added or removed +- Update ChoiceForm to be full height. +- Add AddRemoveView to replace AssignRemoveView +- Update the group roles view to use the new AddRemoveView. +- Add role create and edit events. +- Sort users by lastname, firstname. +- Switch user groups and group users views to AddRemoveView. +- Commit user edit event when an user is added or removed from a group. -* Commit the group edit event when a group is added or remove +- Commit the group edit event when a group is added or remove from an user. -* Require dual permissions when add or removing users to and +- Require dual permissions when add or removing users to and from group. Same with group to users. -* Backport search improvements. -* Remove search elapsed time calculation. -* Remove SEARCH_LIMIT setting. -* Use the 'handler' prefix for all the signal handler functions. -* Remove custom email widget and use Django's. -* Increase default maximum number of favorite documents to 400. -* Update the role group list view to use the new AddRemoveView. -* Commit the group event in conjunction with the role event +- Backport search improvements. +- Remove search elapsed time calculation. +- Remove SEARCH_LIMIT setting. +- Use the 'handler' prefix for all the signal handler functions. +- Remove custom email widget and use Django's. +- Increase default maximum number of favorite documents to 400. +- Update the role group list view to use the new AddRemoveView. +- Commit the group event in conjunction with the role event when a group is added or remove from role. -* Update the role permission view to use the new AddRemoveView. -* Rename transformation manager method add_for_model to +- Update the role permission view to use the new AddRemoveView. +- Rename transformation manager method add_for_model to add_to_object. -* Rename transformation manager method get_for_model to +- Rename transformation manager method get_for_model to get_for_object. -* Load the converter class on demand. -* Remove app top level star imports. -* Monkeypatch group and user models to make their fields +- Load the converter class on demand. +- Remove app top level star imports. +- Monkeypatch group and user models to make their fields translatable. -* Add new and default Tesseract OCR backend to avoid +- Add new and default Tesseract OCR backend to avoid Tesseract bug 1670 (https://github.com/tesseract-ocr/tesseract/issues/1670) -* Load only one language in the document properties form. -* Convert title calculation form to a template tag. -* Show the full title as a hover title even when truncated. -* Increase default title truncation length to 120 characters. -* Improve inherited permission computation. -* Add test case mixin that produces ephimeral models. -* Update ACL permissions view to use the new AddRemoveView class. -* Add ACL created and edited events. -* Update index document types view to use the new AddRemoveView +- Load only one language in the document properties form. +- Convert title calculation form to a template tag. +- Show the full title as a hover title even when truncated. +- Increase default title truncation length to 120 characters. +- Improve inherited permission computation. +- Add test case mixin that produces ephimeral models. +- Update ACL permissions view to use the new AddRemoveView class. +- Add ACL created and edited events. +- Update index document types view to use the new AddRemoveView class. -* Add index create and edit events. -* Allow overloading the action_add and action_remove methods +- Add index create and edit events. +- Allow overloading the action_add and action_remove methods from the AddRemoveView. -* Add view to link document type and indexes from the document +- Add view to link document type and indexes from the document type side. -* Update smart link document type selection view to use +- Update smart link document type selection view to use AddRemoveView class. -* Add smart link created and edited events. -* Fix smart link ACL support. -* Update JavaScript downloader to work with Python 3. -* Improve speed of the NPM package hash verification. -* Add view to enable smart links for documents types +- Add smart link created and edited events. +- Fix smart link ACL support. +- Update JavaScript downloader to work with Python 3. +- Improve speed of the NPM package hash verification. +- Add view to enable smart links for documents types from the document type side. -* Enable list link icons. -* Add outline links CSS for facets. -* Add a bottom margin to list links. -* Use copyfileobj to save documents to files -* Add user logged in and logged out events. -* Add transaction handling in more places. -* Update ACLs tests to use ephimeral models. -* Add new app to handle all dependencies. -* Remove the licenses.py module and replace +- Enable list link icons. +- Add outline links CSS for facets. +- Add a bottom margin to list links. +- Use copyfileobj to save documents to files +- Add user logged in and logged out events. +- Add transaction handling in more places. +- Update ACLs tests to use ephimeral models. +- Add new app to handle all dependencies. +- Remove the licenses.py module and replace it with a dependencies.py module. -* Backport ACL computation improvements. -* Remove model permission proxy models. -* Remove related access control argument. This is +- Backport ACL computation improvements. +- Remove model permission proxy models. +- Remove related access control argument. This is now handled by the related field registration. -* Allow nested access control checking. -* check_access's permissions argument must now be +- Allow nested access control checking. +- check_access's permissions argument must now be an interable. -* Remove permissions_related from links. -* Remove mayan_permission_attribute_check from +- Remove permissions_related from links. +- Remove mayan_permission_attribute_check from API permission. -* Update Bootstrap and Bootswatch to version 3.4.1. -* Convert the workflow document types view to use +- Update Bootstrap and Bootswatch to version 3.4.1. +- Convert the workflow document types view to use the new AddRemove view. -* Add the workflow created and edited events. -* Remove AssignRemove View. -* Add view to setup workflows per document type +- Add the workflow created and edited events. +- Remove AssignRemove View. +- Add view to setup workflows per document type from the document type side. -* Make workflows, workflows states, workflow +- Make workflows, workflows states, workflow transitions column sortable. -* Show completion and intial state in the +- Show completion and intial state in the workflow proxy instance menu list. -* Fix translation of the source upload forms +- Fix translation of the source upload forms using dropzone.js -* Rename get_object_list to get_source_queryset. -* Add uniqueness validation to SingleObjectCreateView. -* Remove MultipleInstanceActionMixin. -* Backport MultipleObjectMixin improvements. -* Remove ObjectListPermissionFilterMixin. -* Add deprecation warning to convertdb -* Add the preparestatic command. -* Remove the related attribute of check_access. -* Remove filter_by_access. Replaced by restrict_queryset. -* Move the user set password views to the authentication app. -* All views redirect to common's home view instead of the +- Rename get_object_list to get_source_queryset. +- Add uniqueness validation to SingleObjectCreateView. +- Remove MultipleInstanceActionMixin. +- Backport MultipleObjectMixin improvements. +- Remove ObjectListPermissionFilterMixin. +- Add deprecation warning to convertdb +- Add the preparestatic command. +- Remove the related attribute of check_access. +- Remove filter_by_access. Replaced by restrict_queryset. +- Move the user set password views to the authentication app. +- All views redirect to common's home view instead of the REDIRECT_URL setting. -* Update tag document list and the document tag list +- Update tag document list and the document tag list views to require the view permissions for both objects. -* Install and server static content to and from the image. -* Add support for editing document comments. -* Remove Internet Explorer specific markup. -* Fix optional metadata remove when mixed with required +- Install and server static content to and from the image. +- Add support for editing document comments. +- Remove Internet Explorer specific markup. +- Fix optional metadata remove when mixed with required metadata. -* Create intermedia file cache folder. Fixes preview errors +- Create intermedia file cache folder. Fixes preview errors when the first document uploaded is an office file. -* Move queue and task registration to the CeleryQueue class. +- Move queue and task registration to the CeleryQueue class. The .queues.py module is now loaded automatically. -* Allow setting the Docker user UID and GUID. -* Add task path validation. -* Increase dropzone upload file size limit to 2GB. -* Add cabinet created and edited events. -* Show a null mailer backend if there is backend with an +- Allow setting the Docker user UID and GUID. +- Add task path validation. +- Increase dropzone upload file size limit to 2GB. +- Add cabinet created and edited events. +- Show a null mailer backend if there is backend with an invalid path. Due to the app full path change, existing mailer setups need to be recreated. -* The document link URL when mailed is now composed of the +- The document link URL when mailed is now composed of the COMMON_PROJECT_URL + document URL instead of the Site domain. -* Add the checkdependencies command. -* Add comment and make file target to generate all requirement +- Add the checkdependencies command. +- Add comment and make file target to generate all requirement files. -* Place deletion policies units before periods for clarity. -* Remove repeated EMAIL_TIMEOUT setting. -* Invert order to the Action Object and Target columns for +- Place deletion policies units before periods for clarity. +- Remove repeated EMAIL_TIMEOUT setting. +- Invert order to the Action Object and Target columns for clarity. -* Add note about the new preparestatic command. -* Add no-result template for workflow instance detail view. -* Update HTTP workflow action to new requests API. -* Remove the included Lato font. The font is now downloaded +- Add note about the new preparestatic command. +- Add no-result template for workflow instance detail view. +- Update HTTP workflow action to new requests API. +- Remove the included Lato font. The font is now downloaded at install time. -* Add support for Google Fonts dependencies. -* Add support for patchin dependency files using rewriting rules. -* Allow searching documents by UUID. -* Improve search negation logic. -* Add support for search field transformations. -* Disable hiding page navigation on idle. -* Display namespace in the transition trigger view. -* Sort events list in the transition trigger view. -* Add support for form media to DynamicFormMixin. -* Fix tag attach and remove action form media. -* Sort content type list of the access grant and remove action. -* Use select2 for the content type filed of the access +- Add support for Google Fonts dependencies. +- Add support for patchin dependency files using rewriting rules. +- Allow searching documents by UUID. +- Improve search negation logic. +- Add support for search field transformations. +- Disable hiding page navigation on idle. +- Display namespace in the transition trigger view. +- Sort events list in the transition trigger view. +- Add support for form media to DynamicFormMixin. +- Fix tag attach and remove action form media. +- Sort content type list of the access grant and remove action. +- Use select2 for the content type filed of the access grant and remove action. -* Add Latvian translation. -* Support search model selection. -* Support passing a queryset factory to the search model. -* Add workflow actions to grant or remove permissions to +- Add Latvian translation. +- Support search model selection. +- Support passing a queryset factory to the search model. +- Add workflow actions to grant or remove permissions to a document. -* Add support for locked files for watchfolder. - -3.1.11 (2019-04-XX) -=================== -* Fix multiple tag selection wizard step. -* Change the required permission for the checkout info link from +- Add support for locked files for watchfolder. +- Fix multiple tag selection wizard step. +- Change the required permission for the checkout info link from document check in to document checkout details view. -* Lower the log severity when links don't resolve. -* Add DOCUMENTS_HASH_BLOCK_SIZE to control the size of the file +- Lower the log severity when links don't resolve. +- Add DOCUMENTS_HASH_BLOCK_SIZE to control the size of the file block when calculating a document's checksum. 3.1.10 (2019-04-04) =================== -* Backport test case improvements from the development branch. Add random +- Backport test case improvements from the development branch. Add random primary key mixin. Split test case code into mixins. Make the view test case and the API test cases part of the same class hierarchy. Update tests that failed due to the new import locations. -* Add support for disabling the content type checking test case mixin. -* Update document indexing tests to be order agnostic. GitLab issue #559. -* Add test for the advanced search API. -* Apply merge !36 by Simeon Walker (@simeon-walker) to fix the advanced search +- Add support for disabling the content type checking test case mixin. +- Update document indexing tests to be order agnostic. GitLab issue #559. +- Add test for the advanced search API. +- Apply merge !36 by Simeon Walker (@simeon-walker) to fix the advanced search API. -* Apply merge !35 by Manoel Brunnen (@mbru) to fix building the Docker image +- Apply merge !35 by Manoel Brunnen (@mbru) to fix building the Docker image on the armv7l platform (RasperryPi, Odroid XU4, Odroid HC2). Also fixes assertion errors from pip (https://github.com/pypa/pip/issues/6197). -* Apply merge !37 by Roger Hunwicks (@roger.hunwicks) to allow +- Apply merge !37 by Roger Hunwicks (@roger.hunwicks) to allow TestViewTestCaseMixin to work with a custom ROOT_URLCONF. GitLab issue #566. -* Apply merge !40 by Roger Hunwicks (@/roger.hunwicks) to pin the Tornado +- Apply merge !40 by Roger Hunwicks (@/roger.hunwicks) to pin the Tornado version used to 6.0 and continue supporting Python 2.7. GitLab issue #568. -* Apply merge !41 by Jorge E. Gomez (@jorgeegomez) to fix the compressed class +- Apply merge !41 by Jorge E. Gomez (@jorgeegomez) to fix the compressed class method name. GitLab issue #572. -* Remove notification badge AJAX setup. Individual link AJAX workers are +- Remove notification badge AJAX setup. Individual link AJAX workers are obsolete now that the menu is being rendered by its own AJAX renderer. GitLab issue #562. -* Add support for server side link badges. -* Add API to list all templates. -* Remove newlines from the rendered templates. -* Reject emails attachments of size 0. Thanks to Robert Schoeftner +- Add support for server side link badges. +- Add API to list all templates. +- Remove newlines from the rendered templates. +- Reject emails attachments of size 0. Thanks to Robert Schoeftner (@robert.schoeftner)for the report and solution. GitLab issue #574. -* Add missing document index API view create permission. -* Fix index list API view. Add index create, delete, detail API tests. +- Add missing document index API view create permission. +- Fix index list API view. Add index create, delete, detail API tests. GitLab issue #564. Thanks to the Stéphane (@shoyu) for the report and debug information. -* Validate the state completion value before saving. Thanks to Manoel Brunnen +- Validate the state completion value before saving. Thanks to Manoel Brunnen (@mbru) for the report and debug information. GitLab issue #557. -* Add the MIMETYPE_FILE_READ_SIZE setting to limit the number of bytes read +- Add the MIMETYPE_FILE_READ_SIZE setting to limit the number of bytes read to determine the MIME type of a new document. -* Force object to text when raising PermissionDenied to avoid +- Force object to text when raising PermissionDenied to avoid UnicodeDecodeError. Thanks to Mathias Behrle (@mbehrle) for the report and the debug information. GitLab issue #576. -* Add support for skipping a default set of tests. +- Add support for skipping a default set of tests. 3.1.9 (2018-11-01) ================== -* Convert the furl instance to text to allow serializing it into +- Convert the furl instance to text to allow serializing it into JSON to be passed as arguments to the background task. 3.1.8 (2018-10-31) ================== -* Reorganize documentation into topics and chapters. -* Add Workflows and API chapters. -* Add new material from the Wiki to the documentation. -* Add data migrations to the sources app migraton 0019 to ensure all labels +- Reorganize documentation into topics and chapters. +- Add Workflows and API chapters. +- Add new material from the Wiki to the documentation. +- Add data migrations to the sources app migraton 0019 to ensure all labels are unique before performing the schema migations. -* Add improvements to the metadata URL encoding and decoding to support +- Add improvements to the metadata URL encoding and decoding to support ampersand characters as part of the metadata value. GitLab issue #529. Thanks to Mark Maglana @relaxdiego for the report. -* Add custom validator for multiple emails in a single text field. +- Add custom validator for multiple emails in a single text field. Change the widget of the email fields in the mailer app to avoid browser side email validation. Closes GitLab issue #530. Thanks to Mark Maglana @relaxdiego for the report. -* Add configuration option to change the project/installation URL. +- Add configuration option to change the project/installation URL. This is used in the password reset emails and in the default document mailing templates. -* Increase the size of the workflow preview image. -* Center the workflow preview image. -* Move the noop OCR backend to the right place. -* Add new management command to display the current configuration +- Increase the size of the workflow preview image. +- Center the workflow preview image. +- Move the noop OCR backend to the right place. +- Add new management command to display the current configuration settings. -* Default the YAML flow format to False which never uses inline. -* Add support for reindexing documents when their base properties like +- Default the YAML flow format to False which never uses inline. +- Add support for reindexing documents when their base properties like the label and description are edited. 3.1.7 (2018-10-14) ================== -* Fix an issue with some browsers not firing the .load event on cached +- Fix an issue with some browsers not firing the .load event on cached images. Ref: http://api.jquery.com/load-event/ -* Remove duplicate YAML loading of environment variables. -* Don't load development apps if they are already loaded. -* Make sure all key used as input for the cache key hash are +- Remove duplicate YAML loading of environment variables. + Don't load development apps if they are already loaded. +- Make sure all key used as input for the cache key hash are bytes and not unicode. GitLab issue #520. Thanks to TheOneValen @TheOneValen for the report. -* Ignore document stub from the index mirror. GitLab issue +- Ignore document stub from the index mirror. GitLab issue #520. Thanks to TheOneValen @TheOneValen for the report. -* Fix for the Docker image INSTALL_FLAG path. Thanks to +- Fix for the Docker image INSTALL_FLAG path. Thanks to Mark Maglana @relaxdiego for the report and to Hamish Farroq @farroq_HAM for the patch. GitLab issue #525. -* Fix the typo in the Docker variable for worker concurrency. Thanks to +- Fix the typo in the Docker variable for worker concurrency. Thanks to Mark Maglana @relaxdiego for the report and to Hamish Farroq @farroq_HAM for the patch. GitLab issue #527. -* Add a noop OCR backend that disables OCR and the check for the +- Add a noop OCR backend that disables OCR and the check for the Tesseract OCR binaries. Set the OCR_BACKEND setting or MAYAN_OCR_BACKEND environment variable to ocr.backends.pyocr.PyOCR to use this. -* All tests pass on Python 3. -* documentation: Add Docker installation method using a dedicated +- All tests pass on Python 3. +- documentation: Add Docker installation method using a dedicated Docker network. -* documentation: Add scaling up chapter. -* documentation: Add S3 storage configuration section. +- documentation: Add scaling up chapter. +- documentation: Add S3 storage configuration section. 3.1.6 (2018-10-09) ================== -* Improve index mirroring value clean up code to remove the spaces at the +- Improve index mirroring value clean up code to remove the spaces at the starts and at the end of directories. Closes again GitLab issue #520 Thanks to TheOneValen @ for the report. -* Improve index mirroring cache class to use the hash of the keys +- Improve index mirroring cache class to use the hash of the keys instead of the literal keys. Avoid warning about invalid key characters. Closes GitLab issue #518. Thanks to TheOneValen @ for the report. -* Only render the Template API view for authenticated users. +- Only render the Template API view for authenticated users. Thanks rgarcia for the report. -* Add icon to the cabinet "Add new level" link. -* Display the cabinet "Add new level" link in the top level view too. +- Add icon to the cabinet "Add new level" link. +- Display the cabinet "Add new level" link in the top level view too. 3.1.5 (2018-10-08) ================== -* Consolidate some document indexing test code into a new mixin. -* Split the code of the mountindex command to be able to add tests. -* Fix the way the children of IndexInstanceNode are accessed. Fixes GitLab +- Consolidate some document indexing test code into a new mixin. +- Split the code of the mountindex command to be able to add tests. +- Fix the way the children of IndexInstanceNode are accessed. Fixes GitLab issue #518. Thanks to TheOneValen @TheOneValen for the report. -* Remove newlines from the index name levels before using them as FUSE +- Remove newlines from the index name levels before using them as FUSE directories. -* Fixed duplicated FUSE directory removal. -* Add link and view to show the parsed content of each document page. -* Add a modelform for adding and editing transformation and perform YAML +- Fixed duplicated FUSE directory removal. +- Add link and view to show the parsed content of each document page. +- Add a modelform for adding and editing transformation and perform YAML validation of arguments. -* Add stricted error checking to the crop transformation. -* Update compressed files class module to work with Python 3. -* Update document parsing app tests to work with Python 3. -* Handle office files in explicit binary mode for Python 3. -* Return a proper list of SearchModel instances (Python 3). -* Specify FUSE literals in explicit octal notation (Python 3). -* URL quote the encoded names of the staging files using Django's compat +- Add stricted error checking to the crop transformation. +- Update compressed files class module to work with Python 3. +- Update document parsing app tests to work with Python 3. +- Handle office files in explicit binary mode for Python 3. +- Return a proper list of SearchModel instances (Python 3). +- Specify FUSE literals in explicit octal notation (Python 3). +- URL quote the encoded names of the staging files using Django's compat module. (Python 3) -* Open staging file in explicit binary mode. (Python 3) -* Add separate Python 2 and Python 3 versions of the MetadataType model +- Open staging file in explicit binary mode. (Python 3) +- Add separate Python 2 and Python 3 versions of the MetadataType model .comma_splitter() static method. -* Update the metadata app tests to work on Python 3. -* Make sure metadata lookup choices are a list to be able to add the +- Update the metadata app tests to work on Python 3. +- Make sure metadata lookup choices are a list to be able to add the optional marker (Python 3). -* Make sure the image in the document preview view is centered when it is +- Make sure the image in the document preview view is centered when it is smaller than the viewport. -* Restore use of the .store_body variable accidentally remove in +- Restore use of the .store_body variable accidentally remove in 63a77d0235ffef3cd49924ba280879313c622682. Closes GitLab issue #519. Thanks to TheOneValen @TheOneValen for the report. -* Add shared cache class and add mounted index cache invalidation when +- Add shared cache class and add mounted index cache invalidation when document and index instance nodes are updated or deleted. -* Fix document metadata app view error when adding multiple optional +- Fix document metadata app view error when adding multiple optional metadata types. Closes GitLab issue #521. Thanks to the TheOneValen @TheOneValen for the report. 3.1.4 (2018-10-04) ================== -* Fix the link to the documenation. Closes GitLab issue #516. +- Fix the link to the documenation. Closes GitLab issue #516. Thanks to Matthias Urlichs @smurfix for the report. -* Update related links. Add links to the new Wiki and Forum. -* Add Redis config entries in the Docker images to disable +- Update related links. Add links to the new Wiki and Forum. +- Add Redis config entries in the Docker images to disable saving the database and to only provision 1 database. -* Remove use of hard coded font icon for document page +- Remove use of hard coded font icon for document page rendering busy indicator. -* Disable the fancybox caption link if the document is +- Disable the fancybox caption link if the document is in the trash. -* Load the DropZone CSS from package and remove the +- Load the DropZone CSS from package and remove the hard code CSS from appearance/base.css. -* Add support for indexing on OCR content changes. -* Add support for reindexing document on content parsing +- Add support for indexing on OCR content changes. +- Add support for reindexing document on content parsing changes. -* Strip HTML entities from the browser's window title. +- Strip HTML entities from the browser's window title. Closes GitLab issue #517. Thanks to Daniel Carrico @daniel1113 for the report. -* Improve search app. Refactored to resolve search queries +- Improve search app. Refactored to resolve search queries by terms first then by field. -* Add explanation to the launch workflows tool. +- Add explanation to the launch workflows tool. 3.1.3 (2018-09-27) ================== -* Make sure template API renders in non US languages. -* Fix user groups view. -* Add no results help text to the document type -> metadata type +- Make sure template API renders in non US languages. +- Fix user groups view. +- Add no results help text to the document type -> metadata type association view. -* Expose the Django INSTALLED_APPS setting. -* Add support for changing the concurrency of the Celery workers in the +- Expose the Django INSTALLED_APPS setting. +- Add support for changing the concurrency of the Celery workers in the Docker image. Add environment variables MAYAN_WORKER_FAST_CONCURRENCY, MAYAN_WORKER_MEDIUM_CONCURRENCY and MAYAN_WORKER_SLOW_CONCURRENCY. -* Add latest translation updates. -* Fixes a few text typos. -* Documentation updates in the deployment and docker chapters. +- Add latest translation updates. +- Fixes a few text typos. +- Documentation updates in the deployment and docker chapters. 3.1.2 (2018-09-21) ================== -* Database access in data migrations defaults to the 'default' database. +- Database access in data migrations defaults to the 'default' database. Force it to the user selected database instead. -* Don't use a hardcoded database alias for the destination of the database +- Don't use a hardcoded database alias for the destination of the database conversion. -* Improve natural key support in the UserOptions model. -* Update from Django 1.11.11 to 1.11.15. -* Add support to the convertdb command to operate on specified apps too. -* Add test mixin to test the db conversion (dumping and loading) of a specific app. -* Add an user test mixin to group user testing. -* Add test the user managament app for database conversion. -* Add support for natural keys to the DocumentPageImageCache model. -* Add database conversion test to the common app. -* Fix label display for resolved smart links when not using a dynamic label. -* Only show smart link resolution errors to the user with the smart link edit +- Improve natural key support in the UserOptions model. +- Update from Django 1.11.11 to 1.11.15. +- Add support to the convertdb command to operate on specified apps too. +- Add test mixin to test the db conversion (dumping and loading) of a specific app. +- Add an user test mixin to group user testing. +- Add test the user managament app for database conversion. +- Add support for natural keys to the DocumentPageImageCache model. +- Add database conversion test to the common app. +- Fix label display for resolved smart links when not using a dynamic label. +- Only show smart link resolution errors to the user with the smart link edit permission. -* Intercept document list view exception and display them as an error message. +- Intercept document list view exception and display them as an error message. 3.1.1 (2018-09-18) ================== -* CSS tweak to make sure the AJAX spinner stays in place. -* Fix 90, 180 and 270 degrees rotation transformations. +- CSS tweak to make sure the AJAX spinner stays in place. +- Fix 90, 180 and 270 degrees rotation transformations. 3.1 (2018-09-17) ================ diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 229f7a466cf..88e4d1e09b8 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -8,6 +8,8 @@ Changes ------- - Add support for icon shadows. +- Add icons and no-result template to the object error log view and + links. Removals -------- diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 688f8e6e51d..2e57afddb92 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -70,7 +70,8 @@ img.lazy-load-carousel { } .label-tag { - text-shadow: 0px 0px 2px #000 + text-shadow: 0px 0px 2px #000; + box-shadow: 3px 3px 5px rgba(0, 0, 0, 0.5); } .fancybox-nav span { @@ -88,18 +89,20 @@ hr { } .btn-block { + border-top: 2px solid rgba(255, 255, 255, 0.7); + border-left: 2px solid rgba(255, 255, 255, 0.7); + border-right: 2px solid rgba(0, 0, 0, 0.7); + border-bottom: 2px solid rgba(0, 0, 0, 0.7); + box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.5); margin-bottom: 15px; - white-space: normal; min-height: 120px; - padding-top: 20px; padding-bottom: 1px; -} - -.btn-block .fa { + padding-top: 20px; text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); + white-space: normal; } -.btn-block { +.btn-block .fa { text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); } @@ -112,12 +115,12 @@ a i { } .dashboard-widget { - box-shadow: 1px 1px 1px rgba(0,0,0,0.3); + box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); border: 1px solid black; } .dashboard-widget .panel-heading i { - text-shadow: 1px 1px 1px rgba(0,0,0,0.3); + text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); } .dashboard-widget-icon { @@ -170,7 +173,7 @@ a i { } .navbar-collapse { border-top: 1px solid transparent; - box-shadow: inset 0 1px 0 rgba(255,255,255,0.1); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1); } .navbar-fixed-top { top: 0; diff --git a/mayan/apps/common/icons.py b/mayan/apps/common/icons.py index fc72d63d70f..0c8919ec0c2 100644 --- a/mayan/apps/common/icons.py +++ b/mayan/apps/common/icons.py @@ -35,8 +35,14 @@ icon_menu_user = Icon( driver_name='fontawesome', symbol='user-circle' ) -icon_object_error_list_with_icon = Icon( - driver_name='fontawesome', symbol='lock' +icon_object_errors = Icon( + driver_name='fontawesome', symbol='exclamation-triangle' +) +icon_object_error_list = Icon( + driver_name='fontawesome', symbol='exclamation-triangle' +) +icon_object_error_list_clear = Icon( + driver_name='fontawesome', symbol='times' ) icon_ok = Icon( driver_name='fontawesome', symbol='check' diff --git a/mayan/apps/common/links.py b/mayan/apps/common/links.py index 25a74bf903a..5fbea67244b 100644 --- a/mayan/apps/common/links.py +++ b/mayan/apps/common/links.py @@ -8,8 +8,8 @@ from .icons import ( icon_about, icon_current_user_locale_profile_details, icon_current_user_locale_profile_edit, icon_documentation, - icon_forum, icon_license, icon_object_error_list_with_icon, - icon_setup, icon_source_code, icon_support, icon_tools + icon_forum, icon_license, icon_setup, icon_source_code, icon_support, + icon_tools ) from .permissions_runtime import permission_error_log_view @@ -50,21 +50,17 @@ def get_kwargs(context): text=_('Documentation'), url='https://docs.mayan-edms.com' ) link_object_error_list = Link( + icon_class_path='mayan.apps.common.icons.icon_object_error_list', kwargs=get_kwargs_factory('resolved_object'), permissions=(permission_error_log_view,), text=_('Errors'), view='common:object_error_list', ) link_object_error_list_clear = Link( + icon_class_path='mayan.apps.common.icons.icon_object_error_list_clear', kwargs=get_kwargs_factory('resolved_object'), permissions=(permission_error_log_view,), text=_('Clear all'), view='common:object_error_list_clear', ) -link_object_error_list_with_icon = Link( - kwargs=get_kwargs_factory('resolved_object'), - icon_class=icon_object_error_list_with_icon, - permissions=(permission_error_log_view,), text=_('Errors'), - view='common:error_list', -) link_forum = Link( icon_class=icon_forum, tags='new_window', text=_('Forum'), url='https://forum.mayan-edms.com' diff --git a/mayan/apps/common/views.py b/mayan/apps/common/views.py index 426109fe1b8..30f16cc0f47 100644 --- a/mayan/apps/common/views.py +++ b/mayan/apps/common/views.py @@ -21,7 +21,7 @@ from .generics import ( ConfirmView, SingleObjectEditView, SingleObjectListView, SimpleView ) -from .icons import icon_setup +from .icons import icon_object_errors, icon_setup from .menus import menu_tools, menu_setup from .permissions_runtime import permission_error_log_view from .settings import setting_home_view @@ -155,6 +155,14 @@ def get_extra_context(self): {'name': _('Result'), 'attribute': 'result'}, ), 'hide_object': True, + 'no_results_icon': icon_object_errors, + 'no_results_text': _( + 'This view displays the error log of different object. ' + 'An empty list is a good thing.' + ), + 'no_results_title': _( + 'There are no error log entries' + ), 'object': self.get_object(), 'title': _('Error log entries for: %s' % self.get_object()), } From 109fcba7955fe5ac87a939e7cb37fe1f9c7cc88d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 5 Jul 2019 23:26:11 -0400 Subject: [PATCH 004/402] Use Select2 for the document type selection form Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index f59ef7f5d51..cedc1152ba5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,7 @@ - Add support for icon shadows. - Add icons and no-result template to the object error log view and links. +- Use Select2 widget for the document type selection form. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 88e4d1e09b8..47bcc0a2d0d 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -10,6 +10,7 @@ Changes - Add support for icon shadows. - Add icons and no-result template to the object error log view and links. +- Use Select2 widget for the document type selection form. Removals -------- From 72a380735480846059a834344b8f88ed0e504a92 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 01:50:57 -0400 Subject: [PATCH 005/402] Add vertical main menu Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 6 + .../appearance/static/appearance/css/base.css | 121 +++++++++++++++++- .../static/appearance/js/mayan_app.js | 12 ++ .../templates/appearance/menu_main.html | 70 ++++++++++ .../{main_menu.html => menu_topbar.html} | 26 +--- .../appearance/templates/appearance/root.html | 12 +- mayan/apps/common/apps.py | 9 +- mayan/apps/common/menus.py | 1 + mayan/apps/events/apps.py | 4 +- 10 files changed, 234 insertions(+), 28 deletions(-) create mode 100644 mayan/apps/appearance/templates/appearance/menu_main.html rename mayan/apps/appearance/templates/appearance/{main_menu.html => menu_topbar.html} (59%) diff --git a/HISTORY.rst b/HISTORY.rst index cedc1152ba5..64cd15ae0e6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,7 @@ - Add icons and no-result template to the object error log view and links. - Use Select2 widget for the document type selection form. +- Backport the vertical main menu update. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 47bcc0a2d0d..b75446368b7 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -11,6 +11,12 @@ Changes - Add icons and no-result template to the object error log view and links. - Use Select2 widget for the document type selection form. +- Backport the vertical main menu update. This update splits the previous + main menu into a new menu in the same location as the previous one + now called the top bar, and a new vertical main menu on the left side. + The vertical menu remain open even when clicking on items and upon + a browser refresh will also restore its state to match the selected + view. Removals -------- diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 2e57afddb92..0959dddf573 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -12,7 +12,7 @@ } body { - padding-top: 70px; + padding-top: 60px; } .navbar-brand { @@ -331,7 +331,7 @@ a i { .main { padding-right: 0px; padding-left: 0px; - /*margin-left: 210px;*/ + margin-left: 210px; } } @@ -413,3 +413,120 @@ a i { .btn-list { margin-bottom: 2px; } + + +/* + * Top navigation + * Hide default border to remove 1px line. + */ +.navbar-fixed-top { + border: 0; +} + + +/* menu_main */ +/* Hide for mobile, show later */ + +#menu-main { + display: none; + background-color: #2c3e50; + border-right: 1px solid #18bc9c; + bottom: 0; + left: 0; + overflow-x: hidden; + overflow-y: auto; + padding-top: 10px; + position: fixed; + top: 51px; + width: 210px; + z-index: 1000; +} + +@media (min-width: 768px) { + #menu-main { + display: block; + } + + .navbar-brand { + text-align: center; + width: 210px; + } +} + +.main .page-header { + margin-top: 0; +} + +.navbar-brand { +} + +.navbar-brand { + outline: none; +} + +.container-fluid { + margin-right: 0px; + margin-left: 0px; + width: 100%; +} + +#accordion-sidebar a { + padding: 10px 15px; +} + +#accordion-sidebar a[aria-expanded="true"] { + background: #1a242f; +} + +#accordion-sidebar .panel { + border: 0px; +} + +#accordion-sidebar a { + text-decoration: none; + outline: none; + position: relative; + display: block; +} + +#accordion-sidebar .panel-heading { + background-color: #2c3e50; + color: white; + padding: 0px; +} + +#accordion-sidebar .panel-heading:hover { + background-color: #517394; +} + +#accordion-sidebar > .panel > div > .panel-body > ul > li > a:hover { + background-color: #517394; +} + +#accordion-sidebar > .panel > div > .panel-body > ul > li.active { + background: #1a242f; +} + +#accordion-sidebar .panel-title { + font-size: 15px; +} + +#accordion-sidebar .panel-body { + font-size: 13px; + border: 0px; + background-color: #2c3e50; + padding-top: 5px; + padding-left: 20px; + padding-right: 0px; + padding-bottom: 0px; +} + +#accordion-sidebar .panel-body li { + padding: 0px; +} + +#accordion-sidebar .panel-body a { + color: white; + text-decoration: none; + padding: 9px; +} diff --git a/mayan/apps/appearance/static/appearance/js/mayan_app.js b/mayan/apps/appearance/static/appearance/js/mayan_app.js index 4011ce59ca0..d96c7f4c9b5 100644 --- a/mayan/apps/appearance/static/appearance/js/mayan_app.js +++ b/mayan/apps/appearance/static/appearance/js/mayan_app.js @@ -41,6 +41,17 @@ class MayanApp { } } + static setupNavBarState () { + $('body').on('click', '.a-main-menu-accordion-link', function (event) { + console.log('ad'); + $('.a-main-menu-accordion-link').each(function (index, value) { + $(this).parent().removeClass('active'); + }); + + $(this).parent().addClass('active'); + }); + } + static updateNavbarState () { var uri = new URI(window.location.hash); var uriFragment = uri.fragment(); @@ -160,6 +171,7 @@ class MayanApp { this.setupFullHeightResizing(); this.setupItemsSelector(); this.setupNavbarCollapse(); + MayanApp.setupNavBarState(); this.setupNewWindowAnchor(); $.each(this.ajaxMenusOptions, function(index, value) { value.app = self; diff --git a/mayan/apps/appearance/templates/appearance/menu_main.html b/mayan/apps/appearance/templates/appearance/menu_main.html new file mode 100644 index 00000000000..d9e356607ad --- /dev/null +++ b/mayan/apps/appearance/templates/appearance/menu_main.html @@ -0,0 +1,70 @@ +{% load i18n %} + +{% load navigation_tags %} +{% load smart_settings_tags %} + +{% load common_tags %} +{% load navigation_tags %} + +{% spaceless %} +
+ {% navigation_resolve_menu name='main' as main_menus_results %} + {% for main_menu_results in main_menus_results %} + {% for link_group in main_menu_results.link_groups %} + {% for link in link_group.links %} + {% with 'active' as li_class_active %} + {% with ' ' as link_classes %} + {% if link|get_type == "" %} +
+ +
+
+
    + {% navigation_resolve_menu name=link.name as sub_menus_results %} + {% for sub_menu_results in sub_menus_results %} + {% for link_group in sub_menu_results.link_groups %} + {% with '' as link_class_active %} + {% with 'a-main-menu-accordion-link' as link_classes %} + {% with 'true' as as_li %} + {% with link_group.links as object_navigation_links %} + {% include 'navigation/generic_navigation.html' %} + {% endwith %} + {% endwith %} + {% endwith %} + {% endwith %} + {% endfor %} + {% endfor %} +
+
+
+
+ {% else %} +
+ +
+ {% endif %} + {% endwith %} + {% endwith %} + {% endfor %} + {% endfor %} + {% endfor %} +
+{% endspaceless %} diff --git a/mayan/apps/appearance/templates/appearance/main_menu.html b/mayan/apps/appearance/templates/appearance/menu_topbar.html similarity index 59% rename from mayan/apps/appearance/templates/appearance/main_menu.html rename to mayan/apps/appearance/templates/appearance/menu_topbar.html index 5d41c7ee1b7..b9401818a12 100644 --- a/mayan/apps/appearance/templates/appearance/main_menu.html +++ b/mayan/apps/appearance/templates/appearance/menu_topbar.html @@ -3,10 +3,11 @@ {% load navigation_tags %} {% load smart_settings_tags %} +{% spaceless %} +{% endspaceless %} diff --git a/mayan/apps/appearance/templates/appearance/root.html b/mayan/apps/appearance/templates/appearance/root.html index be3f89e235a..548108a1e8f 100644 --- a/mayan/apps/appearance/templates/appearance/root.html +++ b/mayan/apps/appearance/templates/appearance/root.html @@ -32,8 +32,11 @@ {% if appearance_type == 'plain' %} {% block content_plain %}{% endblock %} {% else %} +
@@ -101,11 +104,18 @@

{% blocktrans %}Server communication error{% endblocktrans %}< var app = new MayanApp({ ajaxMenusOptions: [ { + callback: MayanApp.updateNavbarState, interval: 5000, menuSelector: '#menu-main', name: 'menu_main', url: '{% url "rest_api:template-detail" "menu_main" %}' }, + { + interval: 5000, + menuSelector: '#menu-topbar', + name: 'menu_topbar', + url: '{% url "rest_api:template-detail" "menu_topbar" %}' + }, ] }); diff --git a/mayan/apps/common/apps.py b/mayan/apps/common/apps.py index 3d7f633675d..f0971536614 100644 --- a/mayan/apps/common/apps.py +++ b/mayan/apps/common/apps.py @@ -28,7 +28,7 @@ from .literals import MESSAGE_SQLITE_WARNING from .menus import ( - menu_about, menu_main, menu_secondary, menu_user + menu_about, menu_main, menu_secondary, menu_topbar, menu_user ) from .settings import ( setting_auto_logging, setting_production_error_log_path, @@ -97,7 +97,10 @@ def ready(self): ) Template( - name='menu_main', template_name='appearance/main_menu.html' + name='menu_main', template_name='appearance/menu_main.html' + ) + Template( + name='menu_topbar', template_name='appearance/menu_topbar.html' ) menu_user.bind_links( @@ -112,7 +115,7 @@ def ready(self): ) ) - menu_main.bind_links(links=(menu_about, menu_user,), position=99) + menu_topbar.bind_links(links=(menu_about, menu_user,), position=99) menu_secondary.bind_links( links=(link_object_error_list_clear,), sources=( 'common:object_error_list', diff --git a/mayan/apps/common/menus.py b/mayan/apps/common/menus.py index c25064cb404..d1c39545a54 100644 --- a/mayan/apps/common/menus.py +++ b/mayan/apps/common/menus.py @@ -17,6 +17,7 @@ menu_secondary = Menu(label=_('Secondary'), name='secondary') menu_setup = Menu(name='setup') menu_tools = Menu(name='tools') +menu_topbar = Menu(name='topbar') menu_user = Menu( icon_class=icon_menu_user, name='user', label=_('User') ) diff --git a/mayan/apps/events/apps.py b/mayan/apps/events/apps.py index 164f3011313..6ec59d3b90a 100644 --- a/mayan/apps/events/apps.py +++ b/mayan/apps/events/apps.py @@ -6,7 +6,7 @@ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.html_widgets import TwoStateWidget from mayan.apps.common.menus import ( - menu_main, menu_object, menu_secondary, menu_tools, menu_user + menu_main, menu_object, menu_secondary, menu_tools, menu_topbar, menu_user ) from mayan.apps.navigation.classes import SourceColumn @@ -85,7 +85,7 @@ def ready(self): source=Notification, widget=TwoStateWidget ) - menu_main.bind_links( + menu_topbar.bind_links( links=(link_user_notifications_list,), position=99 ) menu_object.bind_links( From 9e068c3e8399d9bb61f97d6750ee7dbff1854372 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 02:01:48 -0400 Subject: [PATCH 006/402] Add topbar shadow Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 0959dddf573..818e81c133e 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -530,3 +530,7 @@ a i { text-decoration: none; padding: 9px; } + +.navbar-fixed-top { + box-shadow: 0px 3px 5px rgba(0, 0, 0, 0.5); +} From fbb0f0b9bdb8dfa6eaf23fa00046fa02fdc33d92 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 02:41:16 -0400 Subject: [PATCH 007/402] Backport workflow preview refactor GitLab issue #532. Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 3 +- mayan/apps/document_states/api_views.py | 42 +++++++++++++++++ mayan/apps/document_states/fields.py | 9 ++++ mayan/apps/document_states/forms.py | 6 +-- mayan/apps/document_states/literals.py | 1 + mayan/apps/document_states/models.py | 45 +++++++++++++++++++ mayan/apps/document_states/queues.py | 17 +++++-- mayan/apps/document_states/settings.py | 32 +++++++++++++ mayan/apps/document_states/storages.py | 12 +++++ mayan/apps/document_states/tasks.py | 11 +++++ .../forms/widgets/workflow_image.html | 2 + mayan/apps/document_states/urls.py | 18 ++++---- .../document_states/views/workflow_views.py | 22 +++------ mayan/apps/document_states/widgets.py | 28 +++--------- 15 files changed, 194 insertions(+), 55 deletions(-) create mode 100644 mayan/apps/document_states/fields.py create mode 100644 mayan/apps/document_states/settings.py create mode 100644 mayan/apps/document_states/storages.py create mode 100644 mayan/apps/document_states/templates/document_states/forms/widgets/workflow_image.html diff --git a/HISTORY.rst b/HISTORY.rst index 64cd15ae0e6..8ea2bef8053 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,7 @@ links. - Use Select2 widget for the document type selection form. - Backport the vertical main menu update. +- Backport workflow preview refactor. GitLab issue #532. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index b75446368b7..9edd482f8ba 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -17,6 +17,7 @@ Changes The vertical menu remain open even when clicking on items and upon a browser refresh will also restore its state to match the selected view. +- Backport workflow preview refactor. GitLab issue #532. Removals -------- @@ -105,6 +106,6 @@ Backward incompatible changes Bugs fixed or issues closed --------------------------- -- :gitlab-issue:`XX` +- :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified .. _PyPI: https://pypi.python.org/pypi/mayan-edms/ diff --git a/mayan/apps/document_states/api_views.py b/mayan/apps/document_states/api_views.py index 78e4e318935..5f998db1156 100644 --- a/mayan/apps/document_states/api_views.py +++ b/mayan/apps/document_states/api_views.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, unicode_literals +from django.http import HttpResponse from django.shortcuts import get_object_or_404 +from django.views.decorators.cache import cache_control, patch_cache_control from rest_framework import generics @@ -10,6 +12,7 @@ from mayan.apps.rest_api.filters import MayanObjectPermissionsFilter from mayan.apps.rest_api.permissions import MayanPermission +from .literals import WORKFLOW_IMAGE_TASK_TIMEOUT from .models import Workflow from .permissions import ( permission_workflow_create, permission_workflow_delete, @@ -23,6 +26,10 @@ WritableWorkflowTransitionSerializer ) +from .settings import settings_workflow_image_cache_time +from .storages import storage_workflowimagecache +from .tasks import task_generate_workflow_image + class APIDocumentTypeWorkflowListView(generics.ListAPIView): """ @@ -172,6 +179,41 @@ def perform_destroy(self, instance): self.get_workflow().document_types.remove(instance) +class APIWorkflowImageView(generics.RetrieveAPIView): + """ + get: Returns an image representation of the selected workflow. + """ + filter_backends = (MayanObjectPermissionsFilter,) + mayan_object_permissions = { + 'GET': (permission_workflow_view,), + } + queryset = Workflow.objects.all() + + def get_serializer(self, *args, **kwargs): + return None + + def get_serializer_class(self): + return None + + @cache_control(private=True) + def retrieve(self, request, *args, **kwargs): + task = task_generate_workflow_image.apply_async( + kwargs=dict( + document_state_id=self.get_object().pk, + ) + ) + + cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT) + with storage_workflowimagecache.open(cache_filename) as file_object: + response = HttpResponse(file_object.read(), content_type='image') + if '_hash' in request.GET: + patch_cache_control( + response, + max_age=settings_workflow_image_cache_time.value + ) + return response + + class APIWorkflowListView(generics.ListCreateAPIView): """ get: Returns a list of all the workflows. diff --git a/mayan/apps/document_states/fields.py b/mayan/apps/document_states/fields.py new file mode 100644 index 00000000000..75a3e7b958c --- /dev/null +++ b/mayan/apps/document_states/fields.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import, unicode_literals + +from django import forms + +from .widgets import WorkflowImageWidget + + +class WorfklowImageField(forms.fields.Field): + widget = WorkflowImageWidget diff --git a/mayan/apps/document_states/forms.py b/mayan/apps/document_states/forms.py index 971b17bd2d2..930dbf49e9e 100644 --- a/mayan/apps/document_states/forms.py +++ b/mayan/apps/document_states/forms.py @@ -12,10 +12,10 @@ from mayan.apps.common.forms import DynamicModelForm from .classes import WorkflowAction +from .fields import WorfklowImageField from .models import ( Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition ) -from .widgets import WorkflowImageWidget class WorkflowActionSelectionForm(forms.Form): @@ -188,9 +188,9 @@ def __init__(self, *args, **kwargs): class WorkflowPreviewForm(forms.Form): - preview = forms.CharField(widget=WorkflowImageWidget()) + workflow = WorfklowImageField() def __init__(self, *args, **kwargs): instance = kwargs.pop('instance', None) super(WorkflowPreviewForm, self).__init__(*args, **kwargs) - self.fields['preview'].initial = instance + self.fields['workflow'].initial = instance diff --git a/mayan/apps/document_states/literals.py b/mayan/apps/document_states/literals.py index 79fc51f828a..674bbeebefc 100644 --- a/mayan/apps/document_states/literals.py +++ b/mayan/apps/document_states/literals.py @@ -9,3 +9,4 @@ (WORKFLOW_ACTION_ON_ENTRY, _('On entry')), (WORKFLOW_ACTION_ON_EXIT, _('On exit')), ) +WORKFLOW_IMAGE_TASK_TIMEOUT = 60 diff --git a/mayan/apps/document_states/models.py b/mayan/apps/document_states/models.py index a5da8ab106d..edfa4316916 100644 --- a/mayan/apps/document_states/models.py +++ b/mayan/apps/document_states/models.py @@ -1,12 +1,16 @@ from __future__ import absolute_import, unicode_literals +import hashlib import json import logging +from furl import furl from graphviz import Digraph from django.conf import settings +from django.core import serializers from django.core.exceptions import PermissionDenied, ValidationError +from django.core.files.base import ContentFile from django.db import IntegrityError, models, transaction from django.db.models import F, Max, Q from django.urls import reverse @@ -27,6 +31,7 @@ ) from .managers import WorkflowManager from .permissions import permission_workflow_transition +from .storages import storage_workflowimagecache logger = logging.getLogger(__name__) @@ -63,9 +68,49 @@ class Meta: def __str__(self): return self.label + def generate_image(self): + cache_filename = '{}-{}'.format(self.id, self.get_hash()) + image = self.render() + + # Since open "wb+" doesn't create files, check if the file + # exists, if not then create it + if not storage_workflowimagecache.exists(cache_filename): + storage_workflowimagecache.save( + name=cache_filename, content=ContentFile(content='') + ) + + with storage_workflowimagecache.open(cache_filename, mode='wb+') as file_object: + file_object.write(image) + + return cache_filename + + def get_api_image_url(self, *args, **kwargs): + final_url = furl() + final_url.args = kwargs + final_url.path = reverse( + viewname='rest_api:workflow-image', + kwargs={'pk': self.pk} + ) + final_url.args['_hash'] = self.get_hash() + + return final_url.tostr() + def get_document_types_not_in_workflow(self): return DocumentType.objects.exclude(pk__in=self.document_types.all()) + def get_hash(self): + objects_lists = list( + Workflow.objects.filter(pk=self.pk) + ) + list( + WorkflowState.objects.filter(workflow__pk=self.pk) + ) + list( + WorkflowTransition.objects.filter(workflow__pk=self.pk) + ) + + return hashlib.sha256( + serializers.serialize('json', objects_lists) + ).hexdigest() + def get_initial_state(self): try: return self.states.get(initial=True) diff --git a/mayan/apps/document_states/queues.py b/mayan/apps/document_states/queues.py index 6b269d5b999..b68354e8b21 100644 --- a/mayan/apps/document_states/queues.py +++ b/mayan/apps/document_states/queues.py @@ -3,12 +3,21 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.task_manager.classes import CeleryQueue -from mayan.apps.task_manager.workers import worker_slow +from mayan.apps.task_manager.workers import worker_fast, worker_slow queue_document_states = CeleryQueue( - name='document_states', label=_('Document states'), worker=worker_slow + label=_('Document states'), name='document_states', worker=worker_slow ) +queue_document_states_fast = CeleryQueue( + label=_('Document states fast'), name='document_states_fast', + worker=worker_fast +) + queue_document_states.add_task_type( - dotted_path='mayan.apps.document_states.tasks.task_launch_all_workflows', - label=_('Launch all workflows') + label=_('Launch all workflows'), + dotted_path='mayan.apps.document_states.tasks.task_launch_all_workflows' +) +queue_document_states_fast.add_task_type( + label=_('Generate workflow previews'), + dotted_path='mayan.apps.document_states.tasks.task_generate_workflow_image' ) diff --git a/mayan/apps/document_states/settings.py b/mayan/apps/document_states/settings.py new file mode 100644 index 00000000000..b90ae5e3821 --- /dev/null +++ b/mayan/apps/document_states/settings.py @@ -0,0 +1,32 @@ +from __future__ import unicode_literals + +import os + +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings import Namespace + +namespace = Namespace(label=_('Workflows'), name='document_states') + +settings_workflow_image_cache_time = namespace.add_setting( + global_name='WORKFLOWS_IMAGE_CACHE_TIME', default='31556926', + help_text=_( + 'Time in seconds that the browser should cache the supplied workflow ' + 'images. The default of 31559626 seconds corresponde to 1 year.' + ) +) +setting_workflowimagecache_storage = namespace.add_setting( + global_name='WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND', + default='django.core.files.storage.FileSystemStorage', help_text=_( + 'Path to the Storage subclass to use when storing the cached ' + 'workflow image files.' + ) +) +setting_workflowimagecache_storage_arguments = namespace.add_setting( + global_name='WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND_ARGUMENTS', + default={'location': os.path.join(settings.MEDIA_ROOT, 'workflows')}, + help_text=_( + 'Arguments to pass to the WORKFLOWS_IMAGE_CACHE_STORAGE_BACKEND.' + ) +) diff --git a/mayan/apps/document_states/storages.py b/mayan/apps/document_states/storages.py new file mode 100644 index 00000000000..8a689634c70 --- /dev/null +++ b/mayan/apps/document_states/storages.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals + +from django.utils.module_loading import import_string + +from .settings import ( + setting_workflowimagecache_storage, + setting_workflowimagecache_storage_arguments +) + +storage_workflowimagecache = import_string( + dotted_path=setting_workflowimagecache_storage.value +)(**setting_workflowimagecache_storage_arguments.value) diff --git a/mayan/apps/document_states/tasks.py b/mayan/apps/document_states/tasks.py index 2156a45ead1..3ec8d90acbe 100644 --- a/mayan/apps/document_states/tasks.py +++ b/mayan/apps/document_states/tasks.py @@ -9,6 +9,17 @@ logger = logging.getLogger(__name__) +@app.task() +def task_generate_workflow_image(document_state_id): + Workflow = apps.get_model( + app_label='document_states', model_name='Workflow' + ) + + workflow = Workflow.objects.get(pk=document_state_id) + + return workflow.generate_image() + + @app.task(ignore_result=True) def task_launch_all_workflows(): Document = apps.get_model(app_label='documents', model_name='Document') diff --git a/mayan/apps/document_states/templates/document_states/forms/widgets/workflow_image.html b/mayan/apps/document_states/templates/document_states/forms/widgets/workflow_image.html new file mode 100644 index 00000000000..6aa520bc92d --- /dev/null +++ b/mayan/apps/document_states/templates/document_states/forms/widgets/workflow_image.html @@ -0,0 +1,2 @@ + + diff --git a/mayan/apps/document_states/urls.py b/mayan/apps/document_states/urls.py index e139e3ff8a0..f37af454523 100644 --- a/mayan/apps/document_states/urls.py +++ b/mayan/apps/document_states/urls.py @@ -4,9 +4,10 @@ from .api_views import ( APIDocumentTypeWorkflowListView, APIWorkflowDocumentTypeList, - APIWorkflowDocumentTypeView, APIWorkflowInstanceListView, - APIWorkflowInstanceView, APIWorkflowInstanceLogEntryListView, - APIWorkflowListView, APIWorkflowStateListView, APIWorkflowStateView, + APIWorkflowDocumentTypeView, APIWorkflowImageView, + APIWorkflowInstanceListView, APIWorkflowInstanceView, + APIWorkflowInstanceLogEntryListView, APIWorkflowListView, + APIWorkflowStateListView, APIWorkflowStateView, APIWorkflowTransitionListView, APIWorkflowTransitionView, APIWorkflowView ) from .views import ( @@ -22,7 +23,7 @@ SetupWorkflowTransitionEditView, SetupWorkflowTransitionTriggerEventListView, ToolLaunchAllWorkflows, WorkflowDocumentListView, WorkflowInstanceDetailView, - WorkflowImageView, WorkflowInstanceTransitionView, WorkflowListView, + WorkflowInstanceTransitionView, WorkflowListView, WorkflowPreviewView, WorkflowStateDocumentListView, WorkflowStateListView, ) from .views.workflow_views import SetupDocumentTypeWorkflowsView @@ -167,11 +168,6 @@ view=WorkflowStateListView.as_view(), name='workflow_state_list' ), - url( - regex=r'^(?P\d+)/image/$', - view=WorkflowImageView.as_view(), - name='workflow_image' - ), url( regex=r'^(?P\d+)/preview/$', view=WorkflowPreviewView.as_view(), @@ -204,6 +200,10 @@ view=APIWorkflowDocumentTypeView.as_view(), name='workflow-document-type-detail' ), + url( + regex=r'^workflows/(?P\d+)/image/$', + name='workflow-image', view=APIWorkflowImageView.as_view() + ), url( regex=r'^workflows/(?P[0-9]+)/states/$', view=APIWorkflowStateListView.as_view(), name='workflowstate-list' diff --git a/mayan/apps/document_states/views/workflow_views.py b/mayan/apps/document_states/views/workflow_views.py index 423134e0b1d..4abab873864 100644 --- a/mayan/apps/document_states/views/workflow_views.py +++ b/mayan/apps/document_states/views/workflow_views.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, unicode_literals from django.contrib import messages -from django.core.files.base import ContentFile from django.db import transaction from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 @@ -13,7 +12,7 @@ AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, SingleObjectDeleteView, SingleObjectDetailView, SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, - SingleObjectDownloadView, SingleObjectEditView, SingleObjectListView + SingleObjectEditView, SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.events import event_document_type_edited @@ -49,7 +48,6 @@ from ..tasks import task_launch_all_workflows __all__ = ( - 'WorkflowImageView', 'WorkflowPreviewView', 'SetupWorkflowListView', 'SetupWorkflowCreateView', 'SetupWorkflowEditView', 'SetupWorkflowDeleteView', 'SetupWorkflowDocumentTypesView', 'SetupWorkflowStateActionCreateView', 'SetupWorkflowStateActionDeleteView', @@ -59,7 +57,8 @@ 'SetupWorkflowStateListView', 'SetupWorkflowTransitionCreateView', 'SetupWorkflowTransitionDeleteView', 'SetupWorkflowTransitionEditView', 'SetupWorkflowTransitionListView', - 'SetupWorkflowTransitionTriggerEventListView', 'ToolLaunchAllWorkflows' + 'SetupWorkflowTransitionTriggerEventListView', 'ToolLaunchAllWorkflows', + 'WorkflowPreviewView' ) @@ -750,26 +749,15 @@ def view_action(self): ) -class WorkflowImageView(SingleObjectDownloadView): - attachment = False - model = Workflow - object_permission = permission_workflow_view - - def get_file(self): - workflow = self.get_object() - return ContentFile(workflow.render(), name=workflow.label) - - def get_mimetype(self): - return 'image' - - class WorkflowPreviewView(SingleObjectDetailView): form_class = WorkflowPreviewForm model = Workflow object_permission = permission_workflow_view + pk_url_kwarg = 'pk' def get_extra_context(self): return { 'hide_labels': True, + 'object': self.get_object(), 'title': _('Preview of: %s') % self.get_object() } diff --git a/mayan/apps/document_states/widgets.py b/mayan/apps/document_states/widgets.py index 75e563baa1a..92e761b7b9f 100644 --- a/mayan/apps/document_states/widgets.py +++ b/mayan/apps/document_states/widgets.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals from django import forms -from django.urls import reverse -from django.utils.html import format_html_join, mark_safe +from django.utils.html import format_html_join def widget_transition_events(transition): @@ -15,23 +14,10 @@ def widget_transition_events(transition): ) -def widget_workflow_diagram(workflow): - return mark_safe( - ''.format( - reverse( - viewname='document_states:workflow_image', kwargs={ - 'pk': workflow.pk - } - ) - ) - ) - - class WorkflowImageWidget(forms.widgets.Widget): - def render(self, name, value, attrs=None): - if value: - output = [] - output.append(widget_workflow_diagram(value)) - return mark_safe(''.join(output)) - else: - return '' + template_name = 'document_states/forms/widgets/workflow_image.html' + + def format_value(self, value): + if value == '' or value is None: + return None + return value From 6cd857e2bfe34eb0e9bb8ed375e4bc65a2146c08 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 02:44:00 -0400 Subject: [PATCH 008/402] Use Select2 widget for the document type selection form This was committed in 109fcba7955fe5ac87a939e7cb37fe1f9c7cc88d without adding the actual change. Signed-off-by: Roberto Rosario --- mayan/apps/documents/forms/document_type_forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/documents/forms/document_type_forms.py b/mayan/apps/documents/forms/document_type_forms.py index cc8a029a036..7484cf23c3e 100644 --- a/mayan/apps/documents/forms/document_type_forms.py +++ b/mayan/apps/documents/forms/document_type_forms.py @@ -41,7 +41,8 @@ def __init__(self, *args, **kwargs): self.fields['document_type'] = field_class( help_text=help_text, label=_('Document type'), queryset=queryset, required=True, - widget=widget_class(attrs={'size': 10}), **extra_kwargs + widget=widget_class(attrs={'class': 'select2', 'size': 10}), + **extra_kwargs ) From 06c3ef658368d5cf6d5a776adaaa95a71035fd74 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 04:09:44 -0400 Subject: [PATCH 009/402] Add source column inheritance and exclusions Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + docs/releases/3.3.rst | 2 + mayan/apps/navigation/classes.py | 91 ++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 34 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8ea2bef8053..0fb21226a70 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,8 @@ - Use Select2 widget for the document type selection form. - Backport the vertical main menu update. - Backport workflow preview refactor. GitLab issue #532. +- Add support for source column inheritance. +- Add support for source column exclusion. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 9edd482f8ba..980763c0a03 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -18,6 +18,8 @@ Changes a browser refresh will also restore its state to match the selected view. - Backport workflow preview refactor. GitLab issue #532. +- Add support for source column inheritance. +- Add support for source column exclusion. Removals -------- diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index 1cc9b1c3aff..a070bafa27f 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -577,44 +577,63 @@ def sort(columns): @classmethod def get_for_source(cls, context, source, exclude_identifier=False, only_identifier=False): + columns = [] + + source_classes = set() + + if hasattr(source, '_meta'): + source_classes.add(source._meta.model) + else: + source_classes.add(source) + try: - result = cls._registry[source] + columns.extend(cls._registry[source]) + except KeyError: + pass + + try: + # Might be an instance, try its class + columns.extend(cls._registry[source.__class__]) except KeyError: try: - # Might be an instance, try its class - result = cls._registry[source.__class__] + # Might be a subclass, try its root class + columns.extend(cls._registry[source.__class__.__mro__[-2]]) except KeyError: - try: - # Might be a subclass, try its root class - result = cls._registry[source.__class__.__mro__[-2]] - except KeyError: - try: - # Might be an inherited class insance, try its source class - result = cls._registry[source.source_ptr.__class__] - except (KeyError, AttributeError): - try: - # Try it as a queryset - result = cls._registry[source.model] - except AttributeError: - try: - # Special case for queryset items produced from - # .defer() or .only() optimizations - result = cls._registry[list(source._meta.parents.items())[0][0]] - except (AttributeError, KeyError, IndexError): - result = () - except TypeError: - # unhashable type: list - result = () + pass + + try: + # Might be an inherited class instance, try its source class + columns.extend(cls._registry[source.source_ptr.__class__]) + except (KeyError, AttributeError): + pass + + try: + # Try it as a queryset + columns.extend(cls._registry[source.model]) + except AttributeError: + try: + # Special case for queryset items produced from + # .defer() or .only() optimizations + result = cls._registry[list(source._meta.parents.items())[0][0]] + except (AttributeError, KeyError, IndexError): + pass + else: + # Second level special case for model subclasses from + # .defer and .only querysets + # Examples: Workflow runtime proxy and index instances in 3.2.x + for column in result: + if not source_classes.intersection(set(column.exclude)): + columns.append(column) - result = SourceColumn.sort(columns=result) + columns = SourceColumn.sort(columns=columns) if exclude_identifier: - result = [item for item in result if not item.is_identifier] + columns = [column for column in columns if not column.is_identifier] else: if only_identifier: - for item in result: - if item.is_identifier: - return item + for column in columns: + if column.is_identifier: + return column return None final_result = [] @@ -635,12 +654,12 @@ def get_for_source(cls, context, source, exclude_identifier=False, only_identifi return result current_view_name = get_current_view_name(request=request) - for item in result: - if item.views: - if current_view_name in item.views: - final_result.append(item) + for column in columns: + if column.views: + if current_view_name in column.views: + final_result.append(column) else: - final_result.append(item) + final_result.append(column) return final_result @@ -655,6 +674,7 @@ def __init__( self._label = label self.attribute = attribute self.empty_value = empty_value + self.exclude = () self.func = func self.is_attribute_absolute_url = is_attribute_absolute_url self.is_object_absolute_url = is_object_absolute_url @@ -697,6 +717,9 @@ def _calculate_label(self): self.label = self._label + def add_exclude(self, source): + self.exclude = self.exclude + (source,) + def get_absolute_url(self, obj): if self.is_object_absolute_url: return obj.get_absolute_url() From 97804b255bcf121671bd68805d0e86900b85cff8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 04:10:41 -0400 Subject: [PATCH 010/402] Add and exclude Index instance columns Exclude inherited columns from the Index models. Add the label columns to Index instances. Signed-off-by: Roberto Rosario --- mayan/apps/document_indexing/apps.py | 11 +++++++++-- mayan/apps/document_indexing/views.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mayan/apps/document_indexing/apps.py b/mayan/apps/document_indexing/apps.py index c9b9dc5cbd6..38a20c153a0 100644 --- a/mayan/apps/document_indexing/apps.py +++ b/mayan/apps/document_indexing/apps.py @@ -100,17 +100,24 @@ def ready(self): model=IndexInstanceNode, related='index_template_node__index' ) - SourceColumn( + column_index_label = SourceColumn( attribute='label', is_identifier=True, is_sortable=True, source=Index ) + column_index_label.add_exclude(source=IndexInstance) SourceColumn( + attribute='label', is_object_absolute_url=True, is_identifier=True, + is_sortable=True, source=IndexInstance + ) + column_index_slug = SourceColumn( attribute='slug', is_sortable=True, source=Index ) - SourceColumn( + column_index_slug.add_exclude(IndexInstance) + column_index_enabled = SourceColumn( attribute='enabled', is_sortable=True, source=Index, widget=TwoStateWidget ) + column_index_enabled.add_exclude(source=IndexInstance) SourceColumn( func=lambda context: context[ diff --git a/mayan/apps/document_indexing/views.py b/mayan/apps/document_indexing/views.py index 5972da68706..55e79cc3c36 100644 --- a/mayan/apps/document_indexing/views.py +++ b/mayan/apps/document_indexing/views.py @@ -278,6 +278,7 @@ class IndexListView(SingleObjectListView): def get_extra_context(self): return { + 'hide_object': True, 'hide_links': True, 'no_results_icon': icon_index, 'no_results_main_link': link_index_template_create.resolve( From 941356ed698613ffbc39512473879ee8b0747e70 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 04:11:43 -0400 Subject: [PATCH 011/402] Add a general use YAML validator Signed-off-by: Roberto Rosario --- mayan/apps/common/validators.py | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/mayan/apps/common/validators.py b/mayan/apps/common/validators.py index 84ff982615a..97a4db33266 100644 --- a/mayan/apps/common/validators.py +++ b/mayan/apps/common/validators.py @@ -1,9 +1,18 @@ from __future__ import unicode_literals +import json import re +import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +from django.core.exceptions import ValidationError from django.core.validators import RegexValidator from django.utils import six +from django.utils.deconstruct import deconstructible from django.utils.functional import SimpleLazyObject from django.utils.translation import ugettext_lazy as _ @@ -23,6 +32,54 @@ def _compile(): return SimpleLazyObject(_compile) +@deconstructible +class JSONValidator(object): + """ + Validates that the input is JSON compliant. + """ + def __call__(self, value): + value = value.strip() + try: + json.loads(stream=value) + except ValueError: + raise ValidationError( + _('Enter a valid JSON value.'), + code='invalid' + ) + + def __eq__(self, other): + return ( + isinstance(other, JSONValidator) + ) + + def __ne__(self, other): + return not (self == other) + + +@deconstructible +class YAMLValidator(object): + """ + Validates that the input is YAML compliant. + """ + def __call__(self, value): + value = value.strip() + try: + yaml.load(stream=value, Loader=SafeLoader) + except yaml.error.YAMLError: + raise ValidationError( + _('Enter a valid YAML value.'), + code='invalid' + ) + + def __eq__(self, other): + return ( + isinstance(other, YAMLValidator) + ) + + def __ne__(self, other): + return not (self == other) + + internal_name_re = _lazy_re_compile(r'^[a-zA-Z0-9_]+\Z') validate_internal_name = RegexValidator( internal_name_re, _( From 4c212f6ea4f667087cc6b6d886ec2d75868a15e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 04:13:26 -0400 Subject: [PATCH 012/402] Backport workflow context and field support Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + docs/releases/3.3.rst | 2 + mayan/apps/document_states/apps.py | 86 +++++++- mayan/apps/document_states/forms.py | 13 +- mayan/apps/document_states/html_widgets.py | 27 +++ mayan/apps/document_states/icons.py | 38 +++- mayan/apps/document_states/links.py | 31 ++- mayan/apps/document_states/literals.py | 21 ++ .../migrations/0014_auto_20190701_0454.py | 46 +++++ .../migrations/0015_auto_20190701_1311.py | 31 +++ mayan/apps/document_states/models.py | 187 +++++++++++++++--- .../templates/document_states/extra_data.html | 8 + mayan/apps/document_states/tests/literals.py | 7 + mayan/apps/document_states/tests/mixins.py | 7 +- .../tests/test_workflow_transition_views.py | 129 +++++++++++- mayan/apps/document_states/urls.py | 47 ++++- .../views/workflow_instance_views.py | 112 +++++++++-- .../document_states/views/workflow_views.py | 126 +++++++++++- mayan/apps/document_states/widgets.py | 11 -- 19 files changed, 837 insertions(+), 94 deletions(-) create mode 100644 mayan/apps/document_states/html_widgets.py create mode 100644 mayan/apps/document_states/migrations/0014_auto_20190701_0454.py create mode 100644 mayan/apps/document_states/migrations/0015_auto_20190701_1311.py create mode 100644 mayan/apps/document_states/templates/document_states/extra_data.html diff --git a/HISTORY.rst b/HISTORY.rst index 0fb21226a70..d14468108c6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,8 @@ - Backport workflow preview refactor. GitLab issue #532. - Add support for source column inheritance. - Add support for source column exclusion. +- Backport workflow context support. +- Backport workflow transitions field support. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 980763c0a03..6a66ffeb194 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -20,6 +20,8 @@ Changes - Backport workflow preview refactor. GitLab issue #532. - Add support for source column inheritance. - Add support for source column exclusion. +- Backport workflow context support. +- Backport workflow transitions field support. Removals -------- diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index f49310b8364..0e35c0d59c2 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -27,6 +27,7 @@ from .handlers import ( handler_index_document, handler_launch_workflow, handler_trigger_transition ) +from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events from .links import ( link_document_workflow_instance_list, link_setup_document_type_workflows, link_setup_workflow_document_types, link_setup_workflow_create, @@ -40,6 +41,10 @@ link_setup_workflow_state_edit, link_setup_workflow_transitions, link_setup_workflow_transition_create, link_setup_workflow_transition_delete, link_setup_workflow_transition_edit, + link_setup_workflow_transition_field_create, + link_setup_workflow_transition_field_delete, + link_setup_workflow_transition_field_edit, + link_setup_workflow_transition_field_list, link_tool_launch_all_workflows, link_workflow_instance_detail, link_workflow_instance_transition, link_workflow_runtime_proxy_document_list, link_workflow_runtime_proxy_list, link_workflow_preview, @@ -50,7 +55,6 @@ permission_workflow_delete, permission_workflow_edit, permission_workflow_transition, permission_workflow_view ) -from .widgets import widget_transition_events class DocumentStatesApp(MayanAppConfig): @@ -86,6 +90,7 @@ def ready(self): WorkflowStateAction = self.get_model('WorkflowStateAction') WorkflowStateRuntimeProxy = self.get_model('WorkflowStateRuntimeProxy') WorkflowTransition = self.get_model('WorkflowTransition') + WorkflowTransitionField = self.get_model('WorkflowTransitionField') WorkflowTransitionTriggerEvent = self.get_model( 'WorkflowTransitionTriggerEvent' ) @@ -152,6 +157,9 @@ def ready(self): ModelPermission.register_inheritance( model=WorkflowTransition, related='workflow', ) + ModelPermission.register_inheritance( + model=WorkflowTransitionField, related='transition', + ) ModelPermission.register_inheritance( model=WorkflowTransitionTriggerEvent, related='transition__workflow', @@ -160,9 +168,10 @@ def ready(self): SourceColumn( attribute='label', is_sortable=True, source=Workflow ) - SourceColumn( + column_workflow_internal_name = SourceColumn( attribute='internal_name', is_sortable=True, source=Workflow ) + column_workflow_internal_name.add_exclude(source=WorkflowRuntimeProxy) SourceColumn( attribute='get_initial_state', empty_value=_('None'), source=Workflow @@ -203,12 +212,25 @@ def ready(self): source=WorkflowInstanceLogEntry, label=_('User'), attribute='user' ) SourceColumn( - source=WorkflowInstanceLogEntry, label=_('Transition'), - attribute='transition' + source=WorkflowInstanceLogEntry, + attribute='transition__origin_state', is_sortable=True + ) + SourceColumn( + source=WorkflowInstanceLogEntry, + attribute='transition', is_sortable=True + ) + SourceColumn( + source=WorkflowInstanceLogEntry, + attribute='transition__destination_state', is_sortable=True + ) + SourceColumn( + source=WorkflowInstanceLogEntry, + attribute='comment', is_sortable=True ) SourceColumn( - source=WorkflowInstanceLogEntry, label=_('Comment'), - attribute='comment' + source=WorkflowInstanceLogEntry, + attribute='get_extra_data', label=_('Additional details'), + widget=WorkflowLogExtraDataWidget ) SourceColumn( @@ -256,6 +278,43 @@ def ready(self): ) ) + SourceColumn( + attribute='name', is_identifier=True, is_sortable=True, + source=WorkflowTransitionField + ) + SourceColumn( + attribute='label', is_sortable=True, source=WorkflowTransitionField + ) + SourceColumn( + attribute='get_field_type_display', label=_('Type'), + source=WorkflowTransitionField + ) + SourceColumn( + attribute='required', is_sortable=True, + source=WorkflowTransitionField, widget=TwoStateWidget + ) + SourceColumn( + attribute='get_widget_display', label=_('Widget'), + is_sortable=False, source=WorkflowTransitionField + ) + SourceColumn( + attribute='widget_kwargs', is_sortable=True, + source=WorkflowTransitionField + ) + + SourceColumn( + source=WorkflowRuntimeProxy, label=_('Documents'), + func=lambda context: context['object'].get_document_count( + user=context['request'].user + ), order=99 + ) + SourceColumn( + source=WorkflowStateRuntimeProxy, label=_('Documents'), + func=lambda context: context['object'].get_document_count( + user=context['request'].user + ), order=99 + ) + menu_facet.bind_links( links=(link_document_workflow_instance_list,), sources=(Document,) ) @@ -291,10 +350,17 @@ def ready(self): menu_object.bind_links( links=( link_setup_workflow_transition_edit, - link_workflow_transition_events, link_acl_list, + link_workflow_transition_events, + link_setup_workflow_transition_field_list, link_acl_list, link_setup_workflow_transition_delete ), sources=(WorkflowTransition,) ) + menu_object.bind_links( + links=( + link_setup_workflow_transition_field_delete, + link_setup_workflow_transition_field_edit + ), sources=(WorkflowTransitionField,) + ) menu_object.bind_links( links=( link_workflow_instance_detail, @@ -328,6 +394,12 @@ def ready(self): 'document_states:setup_workflow_list' ) ) + menu_secondary.bind_links( + links=(link_setup_workflow_transition_field_create,), + sources=( + WorkflowTransition, + ) + ) menu_secondary.bind_links( links=(link_workflow_runtime_proxy_list,), sources=( diff --git a/mayan/apps/document_states/forms.py b/mayan/apps/document_states/forms.py index 930dbf49e9e..e2320e2b46f 100644 --- a/mayan/apps/document_states/forms.py +++ b/mayan/apps/document_states/forms.py @@ -165,26 +165,19 @@ def save(self): ) -class WorkflowInstanceTransitionForm(forms.Form): +class WorkflowInstanceTransitionSelectForm(forms.Form): def __init__(self, *args, **kwargs): user = kwargs.pop('user') workflow_instance = kwargs.pop('workflow_instance') - super(WorkflowInstanceTransitionForm, self).__init__(*args, **kwargs) + super(WorkflowInstanceTransitionSelectForm, self).__init__(*args, **kwargs) self.fields[ 'transition' ].queryset = workflow_instance.get_transition_choices(_user=user) transition = forms.ModelChoiceField( + help_text=_('Select a transition to execute in the next step.'), label=_('Transition'), queryset=WorkflowTransition.objects.none() ) - comment = forms.CharField( - help_text=_('Optional comment to attach to the transition.'), - label=_('Comment'), required=False, widget=forms.widgets.Textarea( - attrs={ - 'rows': 3 - } - ) - ) class WorkflowPreviewForm(forms.Form): diff --git a/mayan/apps/document_states/html_widgets.py b/mayan/apps/document_states/html_widgets.py new file mode 100644 index 00000000000..98ad2d817f2 --- /dev/null +++ b/mayan/apps/document_states/html_widgets.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +from django import forms +from django.template.loader import render_to_string +from django.urls import reverse +from django.utils.html import format_html_join, mark_safe + + +def widget_transition_events(transition): + return format_html_join( + sep='\n', format_string='
{}
', args_generator=( + ( + transition_trigger.event_type.label, + ) for transition_trigger in transition.trigger_events.all() + ) + ) + + +class WorkflowLogExtraDataWidget(object): + template_name = 'document_states/extra_data.html' + + def render(self, name=None, value=None): + return render_to_string( + template_name=self.template_name, context={ + 'value': value + } + ) diff --git a/mayan/apps/document_states/icons.py b/mayan/apps/document_states/icons.py index 8ae3b8e990c..9fa519fac1a 100644 --- a/mayan/apps/document_states/icons.py +++ b/mayan/apps/document_states/icons.py @@ -3,7 +3,6 @@ from mayan.apps.appearance.classes import Icon from mayan.apps.documents.icons import icon_document_type - icon_workflow = Icon(driver_name='fontawesome', symbol='sitemap') icon_document_type_workflow_list = icon_workflow @@ -25,8 +24,9 @@ icon_workflow_list = Icon(driver_name='fontawesome', symbol='sitemap') icon_workflow_preview = Icon(driver_name='fontawesome', symbol='eye') - -icon_workflow_instance_detail = Icon(driver_name='fontawesome', symbol='sitemap') +icon_workflow_instance_detail = Icon( + driver_name='fontawesome', symbol='sitemap' +) icon_workflow_instance_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) @@ -58,13 +58,19 @@ icon_workflow_state_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') icon_workflow_state_action = Icon(driver_name='fontawesome', symbol='code') -icon_workflow_state_action_delete = Icon(driver_name='fontawesome', symbol='times') -icon_workflow_state_action_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') +icon_workflow_state_action_delete = Icon( + driver_name='fontawesome', symbol='times' +) +icon_workflow_state_action_edit = Icon( + driver_name='fontawesome', symbol='pencil-alt' +) icon_workflow_state_action_selection = Icon( driver_name='fontawesome-dual', primary_symbol='code', secondary_symbol='plus' ) -icon_workflow_state_action_list = Icon(driver_name='fontawesome', symbol='code') +icon_workflow_state_action_list = Icon( + driver_name='fontawesome', symbol='code' +) icon_workflow_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) @@ -72,10 +78,28 @@ driver_name='fontawesome-dual', primary_symbol='arrows-alt-h', secondary_symbol='plus' ) -icon_workflow_transition_delete = Icon(driver_name='fontawesome', symbol='times') +icon_workflow_transition_delete = Icon( + driver_name='fontawesome', symbol='times' +) icon_workflow_transition_edit = Icon( driver_name='fontawesome', symbol='pencil-alt' ) + +icon_workflow_transition_field = Icon(driver_name='fontawesome', symbol='table') +icon_workflow_transition_field_delete = Icon( + driver_name='fontawesome', symbol='times' +) +icon_workflow_transition_field_edit = Icon( + driver_name='fontawesome', symbol='pencil-alt' +) +icon_workflow_transition_field_create = Icon( + driver_name='fontawesome-dual', primary_symbol='table', + secondary_symbol='plus' +) +icon_workflow_transition_field_list = Icon( + driver_name='fontawesome', symbol='table' +) + icon_workflow_transition_triggers = Icon( driver_name='fontawesome', symbol='bolt' ) diff --git a/mayan/apps/document_states/links.py b/mayan/apps/document_states/links.py index 3d23efec95c..1404ff2644b 100644 --- a/mayan/apps/document_states/links.py +++ b/mayan/apps/document_states/links.py @@ -129,6 +129,35 @@ text=_('Transition triggers'), view='document_states:setup_workflow_transition_events' ) + +# Workflow transition fields +link_setup_workflow_transition_field_create = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field', + permissions=(permission_workflow_edit,), text=_('Create field'), + view='document_states:setup_workflow_transition_field_create', +) +link_setup_workflow_transition_field_delete = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_delete', + permissions=(permission_workflow_edit,), + tags='dangerous', text=_('Delete'), + view='document_states:setup_workflow_transition_field_delete', +) +link_setup_workflow_transition_field_edit = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_edit', + permissions=(permission_workflow_edit,), + text=_('Edit'), view='document_states:setup_workflow_transition_field_edit', +) +link_setup_workflow_transition_field_list = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_list', + permissions=(permission_workflow_edit,), + text=_('Fields'), + view='document_states:setup_workflow_transition_field_list', +) + link_workflow_preview = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_preview', @@ -159,7 +188,7 @@ args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_transition', text=_('Transition'), - view='document_states:workflow_instance_transition', + view='document_states:workflow_instance_transition_selection', ) # Runtime proxies diff --git a/mayan/apps/document_states/literals.py b/mayan/apps/document_states/literals.py index 674bbeebefc..ad6906fd3bb 100644 --- a/mayan/apps/document_states/literals.py +++ b/mayan/apps/document_states/literals.py @@ -2,6 +2,27 @@ from django.utils.translation import ugettext_lazy as _ +FIELD_TYPE_CHOICE_CHAR = 1 +FIELD_TYPE_CHOICE_INTEGER = 2 +FIELD_TYPE_CHOICES = ( + (FIELD_TYPE_CHOICE_CHAR, _('Character')), + (FIELD_TYPE_CHOICE_INTEGER, _('Number (Integer)')), +) + +FIELD_TYPE_MAPPING = { + FIELD_TYPE_CHOICE_CHAR: 'django.forms.CharField', + FIELD_TYPE_CHOICE_INTEGER: 'django.forms.IntegerField', +} + +WIDGET_CLASS_TEXTAREA = 1 +WIDGET_CLASS_CHOICES = ( + (WIDGET_CLASS_TEXTAREA, _('Text area')), +) + +WIDGET_CLASS_MAPPING = { + WIDGET_CLASS_TEXTAREA: 'django.forms.widgets.Textarea', +} + WORKFLOW_ACTION_ON_ENTRY = 1 WORKFLOW_ACTION_ON_EXIT = 2 diff --git a/mayan/apps/document_states/migrations/0014_auto_20190701_0454.py b/mayan/apps/document_states/migrations/0014_auto_20190701_0454.py new file mode 100644 index 00000000000..6ede77fcd0c --- /dev/null +++ b/mayan/apps/document_states/migrations/0014_auto_20190701_0454.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-01 04:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('document_states', '0013_auto_20190423_0810'), + ] + + operations = [ + migrations.CreateModel( + name='WorkflowTransitionField', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field_type', models.PositiveIntegerField(choices=[(1, 'Character'), (2, 'Number (Integer)')], verbose_name='Type')), + ('name', models.CharField(help_text='The name that will be used to identify this field in other parts of the workflow system.', max_length=128, verbose_name='Internal name')), + ('label', models.CharField(help_text='The field name that will be shown on the user interface.', max_length=128, verbose_name='Label')), + ('help_text', models.TextField(blank=True, help_text='An optional message that will help users better understand the purpose of the field and data to provide.', verbose_name='Help text')), + ('required', models.BooleanField(default=False, help_text='Whether this fields needs to be filled out or not to proceed.', verbose_name='Required')), + ('transition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='fields', to='document_states.WorkflowTransition', verbose_name='Transition')), + ], + options={ + 'verbose_name': 'Workflow transition trigger event', + 'verbose_name_plural': 'Workflow transitions trigger events', + }, + ), + migrations.AddField( + model_name='workflowinstance', + name='context', + field=models.TextField(blank=True, verbose_name='Backend data'), + ), + migrations.AddField( + model_name='workflowinstancelogentry', + name='extra_data', + field=models.TextField(blank=True, verbose_name='Extra data'), + ), + migrations.AlterUniqueTogether( + name='workflowtransitionfield', + unique_together=set([('transition', 'name')]), + ), + ] diff --git a/mayan/apps/document_states/migrations/0015_auto_20190701_1311.py b/mayan/apps/document_states/migrations/0015_auto_20190701_1311.py new file mode 100644 index 00000000000..baef5fc8867 --- /dev/null +++ b/mayan/apps/document_states/migrations/0015_auto_20190701_1311.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-01 13:11 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.common.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('document_states', '0014_auto_20190701_0454'), + ] + + operations = [ + migrations.AddField( + model_name='workflowtransitionfield', + name='widget', + field=models.PositiveIntegerField(blank=True, choices=[(1, 'Text area')], help_text='An optional class to change the default presentation of the field.', null=True, verbose_name='Widget class'), + ), + migrations.AddField( + model_name='workflowtransitionfield', + name='widget_kwargs', + field=models.TextField(blank=True, help_text='A group of keyword arguments to customize the widget. Use YAML format.', validators=[mayan.apps.common.validators.YAMLValidator()], verbose_name='Widget keyword arguments'), + ), + migrations.AlterField( + model_name='workflowinstance', + name='context', + field=models.TextField(blank=True, verbose_name='Context'), + ), + ] diff --git a/mayan/apps/document_states/models.py b/mayan/apps/document_states/models.py index edfa4316916..d3d41038146 100644 --- a/mayan/apps/document_states/models.py +++ b/mayan/apps/document_states/models.py @@ -6,6 +6,11 @@ from furl import furl from graphviz import Digraph +import yaml +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader from django.conf import settings from django.core import serializers @@ -19,15 +24,16 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.models import AccessControlList -from mayan.apps.common.validators import validate_internal_name +from mayan.apps.common.validators import YAMLValidator, validate_internal_name from mayan.apps.documents.models import Document, DocumentType +from mayan.apps.documents.permissions import permission_document_view from mayan.apps.events.models import StoredEventType from .error_logs import error_log_state_actions from .events import event_workflow_created, event_workflow_edited from .literals import ( - WORKFLOW_ACTION_WHEN_CHOICES, WORKFLOW_ACTION_ON_ENTRY, - WORKFLOW_ACTION_ON_EXIT + FIELD_TYPE_CHOICES, WIDGET_CLASS_CHOICES, WORKFLOW_ACTION_WHEN_CHOICES, + WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT ) from .managers import WorkflowManager from .permissions import permission_workflow_transition @@ -407,6 +413,61 @@ def __str__(self): return self.label +@python_2_unicode_compatible +class WorkflowTransitionField(models.Model): + transition = models.ForeignKey( + on_delete=models.CASCADE, related_name='fields', + to=WorkflowTransition, verbose_name=_('Transition') + ) + field_type = models.PositiveIntegerField( + choices=FIELD_TYPE_CHOICES, verbose_name=_('Type') + ) + name = models.CharField( + help_text=_( + 'The name that will be used to identify this field in other parts ' + 'of the workflow system.' + ), max_length=128, verbose_name=_('Internal name') + ) + label = models.CharField( + help_text=_( + 'The field name that will be shown on the user interface.' + ), max_length=128, verbose_name=_('Label')) + help_text = models.TextField( + blank=True, help_text=_( + 'An optional message that will help users better understand the ' + 'purpose of the field and data to provide.' + ), verbose_name=_('Help text') + ) + required = models.BooleanField( + default=False, help_text=_( + 'Whether this fields needs to be filled out or not to proceed.' + ), verbose_name=_('Required') + ) + widget = models.PositiveIntegerField( + blank=True, choices=WIDGET_CLASS_CHOICES, help_text=_( + 'An optional class to change the default presentation of the field.' + ), null=True, verbose_name=_('Widget class') + ) + widget_kwargs = models.TextField( + blank=True, help_text=_( + 'A group of keyword arguments to customize the widget. ' + 'Use YAML format.' + ), validators=[YAMLValidator()], + verbose_name=_('Widget keyword arguments') + ) + + class Meta: + unique_together = ('transition', 'name') + verbose_name = _('Workflow transition trigger event') + verbose_name_plural = _('Workflow transitions trigger events') + + def __str__(self): + return self.label + + def get_widget_kwargs(self): + return yaml.load(stream=self.widget_kwargs, Loader=SafeLoader) + + @python_2_unicode_compatible class WorkflowTransitionTriggerEvent(models.Model): transition = models.ForeignKey( @@ -436,6 +497,9 @@ class WorkflowInstance(models.Model): on_delete=models.CASCADE, related_name='workflows', to=Document, verbose_name=_('Document') ) + context = models.TextField( + blank=True, verbose_name=_('Context') + ) class Meta: ordering = ('workflow',) @@ -446,15 +510,30 @@ class Meta: def __str__(self): return force_text(self.workflow) - def do_transition(self, transition, user=None, comment=None): - try: - if transition in self.get_current_state().origin_transitions.all(): - self.log_entries.create( - comment=comment or '', transition=transition, user=user - ) - except AttributeError: - # No initial state has been set for this workflow - pass + def do_transition(self, transition, extra_data=None, user=None, comment=None): + with transaction.atomic(): + try: + if transition in self.get_current_state().origin_transitions.all(): + if extra_data: + context = self.loads() + context.update(extra_data) + self.dumps(context=context) + + self.log_entries.create( + comment=comment or '', + extra_data=json.dumps(extra_data or {}), + transition=transition, user=user + ) + except AttributeError: + # No initial state has been set for this workflow + pass + + def dumps(self, context): + """ + Serialize the context data. + """ + self.context = json.dumps(context) + self.save() def get_absolute_url(self): return reverse( @@ -464,10 +543,12 @@ def get_absolute_url(self): ) def get_context(self): - return { + context = { 'document': self.document, 'workflow': self.workflow, 'workflow_instance': self, } + context['workflow_instance_context'] = self.loads() + return context def get_current_state(self): """ @@ -533,6 +614,12 @@ def get_transition_choices(self, _user=None): """ return WorkflowTransition.objects.none() + def loads(self): + """ + Deserialize the context data. + """ + return json.loads(self.context or '{}') + @python_2_unicode_compatible class WorkflowInstanceLogEntry(models.Model): @@ -559,6 +646,7 @@ class WorkflowInstanceLogEntry(models.Model): to=settings.AUTH_USER_MODEL, verbose_name=_('User') ) comment = models.TextField(blank=True, verbose_name=_('Comment')) + extra_data = models.TextField(blank=True, verbose_name=_('Extra data')) class Meta: ordering = ('datetime',) @@ -572,32 +660,46 @@ def clean(self): if self.transition not in self.workflow_instance.get_transition_choices(_user=self.user): raise ValidationError(_('Not a valid transition choice.')) - def save(self, *args, **kwargs): - result = super(WorkflowInstanceLogEntry, self).save(*args, **kwargs) - context = self.workflow_instance.get_context() - context.update( - { - 'entry_log': self - } - ) + def get_extra_data(self): + result = {} + for key, value in self.loads().items(): + result[self.transition.fields.get(name=key).label] = value - for action in self.transition.origin_state.exit_actions.filter(enabled=True): - context.update( - { - 'action': action, - } - ) - action.execute(context=context) + return result - for action in self.transition.destination_state.entry_actions.filter(enabled=True): + def loads(self): + """ + Deserialize the context data. + """ + return json.loads(self.extra_data or '{}') + + def save(self, *args, **kwargs): + with transaction.atomic(): + result = super(WorkflowInstanceLogEntry, self).save(*args, **kwargs) + context = self.workflow_instance.get_context() context.update( { - 'action': action, + 'entry_log': self } ) - action.execute(context=context) - return result + for action in self.transition.origin_state.exit_actions.filter(enabled=True): + context.update( + { + 'action': action, + } + ) + action.execute(context=context) + + for action in self.transition.destination_state.entry_actions.filter(enabled=True): + context.update( + { + 'action': action, + } + ) + action.execute(context=context) + + return result class WorkflowRuntimeProxy(Workflow): @@ -606,9 +708,30 @@ class Meta: verbose_name = _('Workflow runtime proxy') verbose_name_plural = _('Workflow runtime proxies') + def get_document_count(self, user): + """ + Return the numeric count of documents executing this workflow. + The count is filtered by access. + """ + return AccessControlList.objects.restrict_queryset( + permission=permission_document_view, + queryset=Document.objects.filter(workflows__workflow=self), + user=user + ).count() + class WorkflowStateRuntimeProxy(WorkflowState): class Meta: proxy = True verbose_name = _('Workflow state runtime proxy') verbose_name_plural = _('Workflow state runtime proxies') + + def get_document_count(self, user): + """ + Return the numeric count of documents at this workflow state. + The count is filtered by access. + """ + return AccessControlList.objects.restrict_queryset( + permission=permission_document_view, queryset=self.get_documents(), + user=user + ).count() diff --git a/mayan/apps/document_states/templates/document_states/extra_data.html b/mayan/apps/document_states/templates/document_states/extra_data.html new file mode 100644 index 00000000000..b996601769c --- /dev/null +++ b/mayan/apps/document_states/templates/document_states/extra_data.html @@ -0,0 +1,8 @@ +{% if value %} +
    + {% for key, value in value.items %} +
  • {{ key }}: {{ value }}
    • + {% endfor %} +
+{% endif %} + diff --git a/mayan/apps/document_states/tests/literals.py b/mayan/apps/document_states/tests/literals.py index 212427d2d30..c17a870e9cf 100644 --- a/mayan/apps/document_states/tests/literals.py +++ b/mayan/apps/document_states/tests/literals.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +from ..literals import FIELD_TYPE_CHOICE_CHAR + TEST_INDEX_LABEL = 'test workflow index' TEST_WORKFLOW_LABEL = 'test workflow label' @@ -11,6 +13,10 @@ TEST_WORKFLOW_STATE_LABEL = 'test state label' TEST_WORKFLOW_STATE_LABEL_EDITED = 'test state label edited' TEST_WORKFLOW_STATE_COMPLETION = 66 +TEST_WORKFLOW_TRANSITION_FIELD_HELP_TEXT = 'test workflow transition field help test' +TEST_WORKFLOW_TRANSITION_FIELD_LABEL = 'test workflow transition field' +TEST_WORKFLOW_TRANSITION_FIELD_NAME = 'test_workflow_transition_field' +TEST_WORKFLOW_TRANSITION_FIELD_TYPE = FIELD_TYPE_CHOICE_CHAR TEST_WORKFLOW_TRANSITION_LABEL = 'test transition label' TEST_WORKFLOW_TRANSITION_LABEL_2 = 'test transition label 2' TEST_WORKFLOW_TRANSITION_LABEL_EDITED = 'test transition label edited' @@ -18,3 +24,4 @@ TEST_INDEX_TEMPLATE_METADATA_EXPRESSION = '{{{{ document.workflow.{}.get_current_state }}}}'.format( TEST_WORKFLOW_INTERNAL_NAME ) + diff --git a/mayan/apps/document_states/tests/mixins.py b/mayan/apps/document_states/tests/mixins.py index 76a1865ed49..f8ea985a1c2 100644 --- a/mayan/apps/document_states/tests/mixins.py +++ b/mayan/apps/document_states/tests/mixins.py @@ -152,9 +152,10 @@ def _request_test_workflow_transition_list_view(self): def _request_test_workflow_transition(self): return self.post( - viewname='document_states:workflow_instance_transition', - kwargs={'pk': self.test_workflow_instance.pk}, data={ - 'transition': self.test_workflow_transition.pk, + viewname='document_states:workflow_instance_transition_execute', + kwargs={ + 'workflow_instance_pk': self.test_workflow_instance.pk, + 'workflow_transition_pk': self.test_workflow_transition.pk, } ) diff --git a/mayan/apps/document_states/tests/test_workflow_transition_views.py b/mayan/apps/document_states/tests/test_workflow_transition_views.py index 1eb8dfb1333..c298e5dde4f 100644 --- a/mayan/apps/document_states/tests/test_workflow_transition_views.py +++ b/mayan/apps/document_states/tests/test_workflow_transition_views.py @@ -10,7 +10,10 @@ ) from .literals import ( - TEST_WORKFLOW_TRANSITION_LABEL, TEST_WORKFLOW_TRANSITION_LABEL_EDITED + TEST_WORKFLOW_TRANSITION_FIELD_HELP_TEXT, + TEST_WORKFLOW_TRANSITION_FIELD_LABEL, TEST_WORKFLOW_TRANSITION_FIELD_NAME, + TEST_WORKFLOW_TRANSITION_FIELD_TYPE, TEST_WORKFLOW_TRANSITION_LABEL, + TEST_WORKFLOW_TRANSITION_LABEL_EDITED ) from .mixins import ( WorkflowTestMixin, WorkflowViewTestMixin, WorkflowTransitionViewTestMixin @@ -160,7 +163,7 @@ def test_transition_workflow_no_access(self): permission. """ response = self._request_test_workflow_transition() - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 404) # Workflow should remain in the same initial state self.assertEqual( @@ -232,3 +235,125 @@ def test_workflow_transition_event_list_with_access(self): response = self._request_test_workflow_transition_event_list_view() self.assertEqual(response.status_code, 200) + + +class WorkflowTransitionFieldViewTestCase( + WorkflowTestMixin, WorkflowTransitionViewTestMixin, GenericViewTestCase +): + def setUp(self): + super(WorkflowTransitionFieldViewTestCase, self).setUp() + self._create_test_workflow() + self._create_test_workflow_states() + self._create_test_workflow_transition() + + def _create_test_workflow_transition_field(self): + self.test_workflow_transition_field = self.test_workflow_transition.fields.create( + field_type=TEST_WORKFLOW_TRANSITION_FIELD_TYPE, + name=TEST_WORKFLOW_TRANSITION_FIELD_NAME, + label=TEST_WORKFLOW_TRANSITION_FIELD_LABEL, + help_text=TEST_WORKFLOW_TRANSITION_FIELD_HELP_TEXT + ) + + def _request_test_workflow_transition_field_list_view(self): + return self.get( + viewname='document_states:setup_workflow_transition_field_list', + kwargs={'pk': self.test_workflow_transition.pk} + ) + + def test_workflow_transition_field_list_view_no_permission(self): + self._create_test_workflow_transition_field() + + response = self._request_test_workflow_transition_field_list_view() + self.assertNotContains( + response=response, + text=self.test_workflow_transition_field.label, + status_code=404 + ) + + def test_workflow_transition_field_list_view_with_access(self): + self._create_test_workflow_transition_field() + + self.grant_access( + obj=self.test_workflow, permission=permission_workflow_edit + ) + + response = self._request_test_workflow_transition_field_list_view() + self.assertContains( + response=response, + text=self.test_workflow_transition_field.label, + status_code=200 + ) + + def _request_workflow_transition_field_create_view(self): + return self.post( + viewname='document_states:setup_workflow_transition_field_create', + kwargs={'pk': self.test_workflow_transition.pk}, + data={ + 'field_type': TEST_WORKFLOW_TRANSITION_FIELD_TYPE, + 'name': TEST_WORKFLOW_TRANSITION_FIELD_NAME, + 'label': TEST_WORKFLOW_TRANSITION_FIELD_LABEL, + 'help_text': TEST_WORKFLOW_TRANSITION_FIELD_HELP_TEXT + } + ) + + def test_workflow_transition_field_create_view_no_permission(self): + workflow_transition_field_count = self.test_workflow_transition.fields.count() + + response = self._request_workflow_transition_field_create_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual( + self.test_workflow_transition.fields.count(), + workflow_transition_field_count + ) + + def test_workflow_transition_field_create_view_with_access(self): + workflow_transition_field_count = self.test_workflow_transition.fields.count() + + self.grant_access( + obj=self.test_workflow, permission=permission_workflow_edit + ) + + response = self._request_workflow_transition_field_create_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual( + self.test_workflow_transition.fields.count(), + workflow_transition_field_count + 1 + ) + + def _request_workflow_transition_field_delete_view(self): + return self.post( + viewname='document_states:setup_workflow_transition_field_delete', + kwargs={'pk': self.test_workflow_transition_field.pk}, + ) + + def test_workflow_transition_field_delete_view_no_permission(self): + self._create_test_workflow_transition_field() + + workflow_transition_field_count = self.test_workflow_transition.fields.count() + + response = self._request_workflow_transition_field_delete_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual( + self.test_workflow_transition.fields.count(), + workflow_transition_field_count + ) + + def test_workflow_transition_field_delete_view_with_access(self): + self._create_test_workflow_transition_field() + + workflow_transition_field_count = self.test_workflow_transition.fields.count() + + self.grant_access( + obj=self.test_workflow, permission=permission_workflow_edit + ) + + response = self._request_workflow_transition_field_delete_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual( + self.test_workflow_transition.fields.count(), + workflow_transition_field_count - 1 + ) diff --git a/mayan/apps/document_states/urls.py b/mayan/apps/document_states/urls.py index f37af454523..ebc3e6acd92 100644 --- a/mayan/apps/document_states/urls.py +++ b/mayan/apps/document_states/urls.py @@ -23,10 +23,15 @@ SetupWorkflowTransitionEditView, SetupWorkflowTransitionTriggerEventListView, ToolLaunchAllWorkflows, WorkflowDocumentListView, WorkflowInstanceDetailView, - WorkflowInstanceTransitionView, WorkflowListView, - WorkflowPreviewView, WorkflowStateDocumentListView, WorkflowStateListView, + WorkflowInstanceTransitionExecuteView, WorkflowInstanceTransitionSelectView, + WorkflowListView, WorkflowPreviewView, WorkflowStateDocumentListView, + WorkflowStateListView, +) +from .views.workflow_views import ( + SetupDocumentTypeWorkflowsView, SetupWorkflowTransitionFieldCreateView, + SetupWorkflowTransitionFieldDeleteView, + SetupWorkflowTransitionFieldEditView, SetupWorkflowTransitionFieldListView ) -from .views.workflow_views import SetupDocumentTypeWorkflowsView urlpatterns_workflows = [ url( @@ -36,6 +41,29 @@ ), ] +urlpatterns_workflow_transition_fields = [ + url( + regex=r'^setup/workflows/transitions/(?P\d+)/fields/create/$', + view=SetupWorkflowTransitionFieldCreateView.as_view(), + name='setup_workflow_transition_field_create' + ), + url( + regex=r'^setup/workflows/transitions/(?P\d+)/fields/$', + view=SetupWorkflowTransitionFieldListView.as_view(), + name='setup_workflow_transition_field_list' + ), + url( + regex=r'^setup/workflows/transitions/fields/(?P\d+)/delete/$', + view=SetupWorkflowTransitionFieldDeleteView.as_view(), + name='setup_workflow_transition_field_delete' + ), + url( + regex=r'^setup/workflows/transitions/fields/(?P\d+)/edit/$', + view=SetupWorkflowTransitionFieldEditView.as_view(), + name='setup_workflow_transition_field_edit' + ), +] + urlpatterns = [ url( regex=r'^document/(?P\d+)/workflows/$', @@ -48,9 +76,14 @@ name='workflow_instance_detail' ), url( - regex=r'^document/workflows/(?P\d+)/transition/$', - view=WorkflowInstanceTransitionView.as_view(), - name='workflow_instance_transition' + regex=r'^document/workflows/(?P\d+)/transitions/select/$', + view=WorkflowInstanceTransitionSelectView.as_view(), + name='workflow_instance_transition_selection' + ), + url( + regex=r'^document/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', + view=WorkflowInstanceTransitionExecuteView.as_view(), + name='workflow_instance_transition_execute' ), url( regex=r'^setup/all/$', view=SetupWorkflowListView.as_view(), @@ -179,7 +212,9 @@ name='workflow_state_document_list' ), ] + urlpatterns.extend(urlpatterns_workflows) +urlpatterns.extend(urlpatterns_workflow_transition_fields) api_urls = [ url( diff --git a/mayan/apps/document_states/views/workflow_instance_views.py b/mayan/apps/document_states/views/workflow_instance_views.py index 66750fc4689..bed57fdcbc4 100644 --- a/mayan/apps/document_states/views/workflow_instance_views.py +++ b/mayan/apps/document_states/views/workflow_instance_views.py @@ -4,21 +4,26 @@ from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.template import RequestContext +from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.models import AccessControlList +from mayan.apps.common.forms import DynamicForm from mayan.apps.common.generics import FormView, SingleObjectListView +from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.models import Document -from ..forms import WorkflowInstanceTransitionForm +from ..forms import WorkflowInstanceTransitionSelectForm from ..icons import icon_workflow_instance_detail, icon_workflow_list from ..links import link_workflow_instance_transition +from ..literals import FIELD_TYPE_MAPPING, WIDGET_CLASS_MAPPING from ..models import WorkflowInstance from ..permissions import permission_workflow_view __all__ = ( 'DocumentWorkflowInstanceListView', 'WorkflowInstanceDetailView', - 'WorkflowInstanceTransitionView' + 'WorkflowInstanceTransitionSelectView', + 'WorkflowInstanceTransitionExecuteView' ) @@ -100,14 +105,17 @@ def get_workflow_instance(self): return get_object_or_404(klass=WorkflowInstance, pk=self.kwargs['pk']) -class WorkflowInstanceTransitionView(FormView): - form_class = WorkflowInstanceTransitionForm +class WorkflowInstanceTransitionExecuteView(FormView): + form_class = DynamicForm template_name = 'appearance/generic_form.html' def form_valid(self, form): + form_data = form.cleaned_data + comment = form_data.pop('comment') + self.get_workflow_instance().do_transition( - comment=form.cleaned_data['comment'], - transition=form.cleaned_data['transition'], user=self.request.user + comment=comment, extra_data=form_data, + transition=self.get_workflow_transition(), user=self.request.user, ) messages.success( self.request, _( @@ -122,19 +130,99 @@ def get_extra_context(self): 'object': self.get_workflow_instance().document, 'submit_label': _('Submit'), 'title': _( - 'Do transition for workflow: %s' - ) % self.get_workflow_instance(), + 'Execute transition "%(transition)s" for workflow: %(workflow)s' + ) % { + 'transition': self.get_workflow_transition(), + 'workflow': self.get_workflow_instance(), + }, 'workflow_instance': self.get_workflow_instance(), } def get_form_extra_kwargs(self): - return { - 'user': self.request.user, - 'workflow_instance': self.get_workflow_instance() + schema = { + 'fields': { + 'comment': { + 'label': _('Comment'), + 'class': 'django.forms.CharField', 'kwargs': { + 'help_text': _( + 'Optional comment to attach to the transition.' + ), + 'required': False, + } + } + }, + 'widgets': { + 'comment': { + 'class': 'django.forms.widgets.Textarea', + 'kwargs': { + 'attrs': { + 'rows': 3 + } + } + } + } } + for field in self.get_workflow_transition().fields.all(): + schema['fields'][field.name] = { + 'class': FIELD_TYPE_MAPPING[field.field_type], + 'help_text': field.help_text, + 'label': field.label, + 'required': field.required, + } + if field.widget: + schema['widgets'][field.name] = { + 'class': WIDGET_CLASS_MAPPING[field.widget], + 'kwargs': field.get_widget_kwargs() + } + + return {'schema': schema} + def get_success_url(self): return self.get_workflow_instance().get_absolute_url() def get_workflow_instance(self): - return get_object_or_404(klass=WorkflowInstance, pk=self.kwargs['pk']) + return get_object_or_404( + klass=WorkflowInstance, pk=self.kwargs['workflow_instance_pk'] + ) + + def get_workflow_transition(self): + return get_object_or_404( + klass=self.get_workflow_instance().get_transition_choices( + _user=self.request.user + ), pk=self.kwargs['workflow_transition_pk'] + ) + + +class WorkflowInstanceTransitionSelectView(ExternalObjectMixin, FormView): + external_object_class = WorkflowInstance + form_class = WorkflowInstanceTransitionSelectForm + template_name = 'appearance/generic_form.html' + + def form_valid(self, form): + return HttpResponseRedirect( + redirect_to=reverse( + viewname='document_states:workflow_instance_transition_execute', + kwargs={ + 'workflow_instance_pk': self.external_object.pk, + 'workflow_transition_pk': form.cleaned_data['transition'].pk + } + ) + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.external_object.document, + 'submit_label': _('Select'), + 'title': _( + 'Select transition for workflow: %s' + ) % self.external_object, + 'workflow_instance': self.external_object, + } + + def get_form_extra_kwargs(self): + return { + 'user': self.request.user, + 'workflow_instance': self.external_object + } diff --git a/mayan/apps/document_states/views/workflow_views.py b/mayan/apps/document_states/views/workflow_views.py index 4abab873864..008c83a108b 100644 --- a/mayan/apps/document_states/views/workflow_views.py +++ b/mayan/apps/document_states/views/workflow_views.py @@ -30,15 +30,17 @@ ) from ..icons import ( icon_workflow_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition + icon_workflow_transition, icon_workflow_transition_field ) from ..links import ( link_setup_workflow_create, link_setup_workflow_state_create, link_setup_workflow_state_action_selection, - link_setup_workflow_transition_create + link_setup_workflow_transition_create, + link_setup_workflow_transition_field_create, ) from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField ) from ..permissions import ( permission_workflow_create, permission_workflow_delete, @@ -731,6 +733,124 @@ def get_post_action_redirect(self): ) +# Transition fields + +class SetupWorkflowTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + def get_extra_context(self): + return { + 'navigation_object_list': ('transition', 'workflow'), + 'transition': self.external_object, + 'title': _( + 'Create a field for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow + } + + def get_instance_extra_data(self): + return { + 'transition': self.external_object, + } + + def get_queryset(self): + return self.external_object.fields.all() + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:setup_workflow_transition_field_list', + kwargs={'pk': self.external_object.pk} + ) + + +class SetupWorkflowTransitionFieldDeleteView(SingleObjectDeleteView): + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Delete workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:setup_workflow_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class SetupWorkflowTransitionFieldEditView(SingleObjectEditView): + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Edit workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:setup_workflow_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class SetupWorkflowTransitionFieldListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'hide_object': True, + 'navigation_object_list': ('object', 'workflow'), + 'no_results_icon': icon_workflow_transition_field, + 'no_results_main_link': link_setup_workflow_transition_field_create.resolve( + context=RequestContext( + request=self.request, dict_={ + 'object': self.external_object + } + ) + ), + 'no_results_text': _( + 'Workflow transition fields allow adding data to the ' + 'workflow\'s context. This additional context data can then ' + 'be used by other elements of the workflow system like the ' + 'workflow state actions.' + ), + 'no_results_title': _( + 'There are no fields for this workflow transition' + ), + 'object': self.external_object, + 'title': _( + 'Fields for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow, + } + + def get_source_queryset(self): + return self.external_object.fields.all() + + class ToolLaunchAllWorkflows(ConfirmView): extra_context = { 'title': _('Launch all workflows?'), diff --git a/mayan/apps/document_states/widgets.py b/mayan/apps/document_states/widgets.py index 92e761b7b9f..a1ccdb57d7f 100644 --- a/mayan/apps/document_states/widgets.py +++ b/mayan/apps/document_states/widgets.py @@ -1,17 +1,6 @@ from __future__ import unicode_literals from django import forms -from django.utils.html import format_html_join - - -def widget_transition_events(transition): - return format_html_join( - sep='\n', format_string='
{}
', args_generator=( - ( - transition_trigger.event_type.label, - ) for transition_trigger in transition.trigger_events.all() - ) - ) class WorkflowImageWidget(forms.widgets.Widget): From bb5324ef50d8fff14d314bbf2068b898f1a528b0 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 6 Jul 2019 17:14:44 -0400 Subject: [PATCH 013/402] Encode settings YAML before hashing Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/classes.py | 6 ++++-- .../apps/smart_settings/tests/test_classes.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index ccedbd5083c..9f35323608e 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -17,7 +17,9 @@ from django.apps import apps from django.conf import settings from django.utils.functional import Promise -from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.encoding import ( + force_bytes, force_text, python_2_unicode_compatible +) logger = logging.getLogger(__name__) @@ -141,7 +143,7 @@ def get_all(cls): @classmethod def get_hash(cls): return force_text( - hashlib.sha256(cls.dump_data()).hexdigest() + hashlib.sha256(force_bytes(cls.dump_data())).hexdigest() ) @classmethod diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index e4c919accea..6e88607f7f3 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -11,12 +11,13 @@ from mayan.apps.common.tests import BaseTestCase from mayan.apps.storage.utils import fs_cleanup -from ..classes import Setting +from ..classes import Namespace, Setting from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE +from .mixins import SmartSettingTestMixin -class ClassesTestCase(BaseTestCase): +class ClassesTestCase(SmartSettingTestMixin, BaseTestCase): def test_environment_variable(self): os.environ[ 'MAYAN_{}'.format(ENVIRONMENT_TEST_NAME) @@ -39,3 +40,16 @@ def test_config_backup_creation_no_tags(self): with path_config_backup.open(mode='r') as file_object: self.assertFalse('!!python/' in file_object.read()) + + def test_setting_check_changed(self): + self._create_test_settings_namespace() + test_setting = self.test_settings_namespace.add_setting( + global_name='SMART_SETTINGS_TEST_SETTING', + default='test value' + ) + # Initialize hash cache + Setting.check_changed() + self.assertFalse(Setting.check_changed()) + test_setting.value = 'test value edited' + self.assertTrue(Setting.check_changed()) + From d7d77fcb55a7332036037a1df56b91d911853b6c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 00:27:29 -0400 Subject: [PATCH 014/402] Backport workflow email action Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 1 + mayan/apps/mailer/tests/literals.py | 4 +- mayan/apps/mailer/tests/test_actions.py | 180 ++++++++++++++++++++++++ mayan/apps/mailer/tests/test_models.py | 13 +- mayan/apps/mailer/workflow_actions.py | 124 ++++++++++++++++ 6 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 mayan/apps/mailer/tests/test_actions.py create mode 100644 mayan/apps/mailer/workflow_actions.py diff --git a/HISTORY.rst b/HISTORY.rst index d14468108c6..d5c54b33bb6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - Add support for source column exclusion. - Backport workflow context support. - Backport workflow transitions field support. +- Backport workflow email action. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 6a66ffeb194..d5425852008 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -22,6 +22,7 @@ Changes - Add support for source column exclusion. - Backport workflow context support. - Backport workflow transitions field support. +- Backport workflow email action. Removals -------- diff --git a/mayan/apps/mailer/tests/literals.py b/mayan/apps/mailer/tests/literals.py index ae3b6033554..e3ea1038a00 100644 --- a/mayan/apps/mailer/tests/literals.py +++ b/mayan/apps/mailer/tests/literals.py @@ -1,8 +1,10 @@ from __future__ import unicode_literals -TEST_BODY_HTML = 'test body' TEST_EMAIL_ADDRESS = 'test@example.com' +TEST_EMAIL_BODY = 'test body' +TEST_EMAIL_BODY_HTML = 'test body' TEST_EMAIL_FROM_ADDRESS = 'from.test@example.com' +TEST_EMAIL_SUBJECT = 'test subject' TEST_RECIPIENTS_MULTIPLE_COMMA = 'test@example.com,test2@example.com' TEST_RECIPIENTS_MULTIPLE_COMMA_RESULT = [ 'test@example.com', 'test2@example.com' diff --git a/mayan/apps/mailer/tests/test_actions.py b/mayan/apps/mailer/tests/test_actions.py new file mode 100644 index 00000000000..eed6d9f49ad --- /dev/null +++ b/mayan/apps/mailer/tests/test_actions.py @@ -0,0 +1,180 @@ +from __future__ import unicode_literals + +import json + +from django.core import mail + +from mayan.apps.common.tests import GenericViewTestCase +from mayan.apps.documents.tests.mixins import DocumentTestMixin +from mayan.apps.document_states.literals import WORKFLOW_ACTION_ON_ENTRY +from mayan.apps.document_states.tests.mixins import WorkflowTestMixin +from mayan.apps.document_states.tests.test_actions import ActionTestCase +from mayan.apps.metadata.tests.mixins import MetadataTypeTestMixin + +from ..permissions import permission_user_mailer_use +from ..workflow_actions import EmailAction + +from .literals import ( + TEST_EMAIL_ADDRESS, TEST_EMAIL_BODY, TEST_EMAIL_FROM_ADDRESS, + TEST_EMAIL_SUBJECT +) +from .mixins import MailerTestMixin + + +class EmailActionTestCase(MailerTestMixin, WorkflowTestMixin, ActionTestCase): + def test_email_action_literal_text(self): + self._create_test_user_mailer() + + action = EmailAction( + form_data={ + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': TEST_EMAIL_ADDRESS, + 'subject': TEST_EMAIL_SUBJECT, + 'body': TEST_EMAIL_BODY, + } + ) + action.execute(context={'document': self.test_document}) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) + self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + + def test_email_action_workflow_execute(self): + self._create_test_workflow() + self._create_test_workflow_state() + self._create_test_user_mailer() + + self.test_workflow_state.actions.create( + action_data=json.dumps( + { + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': TEST_EMAIL_ADDRESS, + 'subject': TEST_EMAIL_SUBJECT, + 'body': TEST_EMAIL_BODY, + } + ), + action_path='mayan.apps.mailer.workflow_actions.EmailAction', + label='test email action', when=WORKFLOW_ACTION_ON_ENTRY, + ) + + self.test_workflow_state.initial = True + self.test_workflow_state.save() + self.test_workflow.document_types.add(self.test_document_type) + + self.upload_document() + + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) + self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + + +class EmailActionTemplateTestCase(MetadataTypeTestMixin, MailerTestMixin, WorkflowTestMixin, ActionTestCase): + def test_email_action_recipient_template(self): + self._create_test_metadata_type() + self.test_document_type.metadata.create(metadata_type=self.test_metadata_type) + self.test_document.metadata.create(metadata_type=self.test_metadata_type, value=TEST_EMAIL_ADDRESS) + + self._create_test_user_mailer() + + action = EmailAction( + form_data={ + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': '{{{{ document.metadata_value_of.{} }}}}'.format(self.test_metadata_type.name), + 'subject': TEST_EMAIL_SUBJECT, + 'body': '', + } + ) + action.execute(context={'document': self.test_document}) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) + self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + + def test_email_action_subject_template(self): + self._create_test_metadata_type() + self.test_document_type.metadata.create(metadata_type=self.test_metadata_type) + self.test_document.metadata.create(metadata_type=self.test_metadata_type, value=TEST_EMAIL_SUBJECT) + + self._create_test_user_mailer() + + action = EmailAction( + form_data={ + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': TEST_EMAIL_ADDRESS, + 'subject': '{{{{ document.metadata_value_of.{} }}}}'.format(self.test_metadata_type.name), + 'body': '', + } + ) + action.execute(context={'document': self.test_document}) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) + self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + + def test_email_action_body_template(self): + self._create_test_metadata_type() + self.test_document_type.metadata.create(metadata_type=self.test_metadata_type) + self.test_document.metadata.create(metadata_type=self.test_metadata_type, value=TEST_EMAIL_BODY) + + self._create_test_user_mailer() + + action = EmailAction( + form_data={ + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': TEST_EMAIL_ADDRESS, + 'subject': TEST_EMAIL_SUBJECT, + 'body': '{{{{ document.metadata_value_of.{} }}}}'.format(self.test_metadata_type.name), + } + ) + action.execute(context={'document': self.test_document}) + self.assertEqual(len(mail.outbox), 1) + self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) + self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) + self.assertEqual(mail.outbox[0].body, TEST_EMAIL_BODY) + + +class EmailActionViewTestCase(DocumentTestMixin, MailerTestMixin, WorkflowTestMixin, GenericViewTestCase): + auto_upload_document = False + + def test_email_action_create_get_view(self): + self._create_test_workflow() + self._create_test_workflow_state() + self._create_test_user_mailer() + + response = self.get( + viewname='document_states:setup_workflow_state_action_create', + kwargs={ + 'pk': self.test_workflow_state.pk, + 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', + } + ) + self.assertEqual(response.status_code, 200) + + self.assertEqual(self.test_workflow_state.actions.count(), 0) + + def _request_email_action_create_post_view(self): + return self.post( + viewname='document_states:setup_workflow_state_action_create', + kwargs={ + 'pk': self.test_workflow_state.pk, + 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', + }, data={ + 'when': WORKFLOW_ACTION_ON_ENTRY, + 'label': 'test email action', + 'mailing_profile': self.test_user_mailer.pk, + 'recipient': TEST_EMAIL_ADDRESS, + 'subject': TEST_EMAIL_SUBJECT, + 'body': TEST_EMAIL_BODY, + } + ) + + def test_email_action_create_post_view(self): + self._create_test_workflow() + self._create_test_workflow_state() + self._create_test_user_mailer() + + self.grant_access( + obj=self.test_user_mailer, permission=permission_user_mailer_use + ) + + response = self._request_email_action_create_post_view() + self.assertEqual(response.status_code, 302) + + self.assertEqual(self.test_workflow_state.actions.count(), 1) diff --git a/mayan/apps/mailer/tests/test_models.py b/mayan/apps/mailer/tests/test_models.py index 376e4ba669c..02f3cab3b11 100644 --- a/mayan/apps/mailer/tests/test_models.py +++ b/mayan/apps/mailer/tests/test_models.py @@ -5,7 +5,7 @@ from mayan.apps.documents.tests.test_models import GenericDocumentTestCase from .literals import ( - TEST_BODY_HTML, TEST_EMAIL_ADDRESS, TEST_EMAIL_FROM_ADDRESS, + TEST_EMAIL_BODY_HTML, TEST_EMAIL_ADDRESS, TEST_EMAIL_FROM_ADDRESS, TEST_RECIPIENTS_MULTIPLE_COMMA, TEST_RECIPIENTS_MULTIPLE_COMMA_RESULT, TEST_RECIPIENTS_MULTIPLE_SEMICOLON, TEST_RECIPIENTS_MULTIPLE_SEMICOLON_RESULT, TEST_RECIPIENTS_MULTIPLE_MIXED, @@ -25,17 +25,22 @@ def test_send_simple(self): def test_send_simple_with_html(self): self._create_test_user_mailer() - self.test_user_mailer.send(to=TEST_EMAIL_ADDRESS, body=TEST_BODY_HTML) + self.test_user_mailer.send( + to=TEST_EMAIL_ADDRESS, body=TEST_EMAIL_BODY_HTML + ) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].from_email, TEST_EMAIL_FROM_ADDRESS) self.assertEqual(mail.outbox[0].to, [TEST_EMAIL_ADDRESS]) - self.assertEqual(mail.outbox[0].alternatives[0][0], TEST_BODY_HTML) + self.assertEqual( + mail.outbox[0].alternatives[0][0], TEST_EMAIL_BODY_HTML + ) def test_send_attachment(self): self._create_test_user_mailer() self.test_user_mailer.send_document( - to=TEST_EMAIL_ADDRESS, document=self.test_document, as_attachment=True + to=TEST_EMAIL_ADDRESS, document=self.test_document, + as_attachment=True ) self.assertEqual(len(mail.outbox), 1) diff --git a/mayan/apps/mailer/workflow_actions.py b/mayan/apps/mailer/workflow_actions.py new file mode 100644 index 00000000000..c344082ac53 --- /dev/null +++ b/mayan/apps/mailer/workflow_actions.py @@ -0,0 +1,124 @@ +from __future__ import absolute_import, unicode_literals + +import logging + +from django.template import Template, Context +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.acls.models import AccessControlList +from mayan.apps.document_states.classes import WorkflowAction +from mayan.apps.document_states.exceptions import WorkflowStateActionError + +from .models import UserMailer +from .permissions import permission_user_mailer_use + +__all__ = ('EmailAction',) +logger = logging.getLogger(__name__) + + +class EmailAction(WorkflowAction): + fields = { + 'mailing_profile': { + 'label': _('Mailing profile'), + 'class': 'django.forms.ModelChoiceField', 'kwargs': { + 'help_text': _('Mailing profile to use when sending the email.'), + 'queryset': UserMailer.objects.none(), 'required': True + } + }, + 'recipient': { + 'label': _('Recipient'), + 'class': 'django.forms.CharField', 'kwargs': { + 'help_text': _( + 'Email address of the recipient. Can be multiple addresses ' + 'separated by comma or semicolon. A template can be used ' + 'to reference properties of the document.' + ), + 'required': True + } + }, + 'subject': { + 'label': _('Subject'), + 'class': 'django.forms.CharField', 'kwargs': { + 'help_text': _( + 'Subject of the email. Can be a string or a template.' + ), + 'required': True + } + }, + 'body': { + 'label': _('Body'), + 'class': 'django.forms.CharField', 'kwargs': { + 'help_text': _( + 'Body of the email to send. Can be a string or a template.' + ), + 'required': True + } + }, + } + field_order = ('mailing_profile', 'recipient', 'subject', 'body') + label = _('Send email') + widgets = { + 'body': { + 'class': 'django.forms.widgets.Textarea', 'kwargs': {} + } + } + permission = permission_user_mailer_use + + def execute(self, context): + try: + recipient = Template(self.form_data['recipient']).render( + context=Context(context) + ) + except Exception as exception: + raise WorkflowStateActionError( + _('Recipient template error: %s') % exception + ) + else: + logger.debug('Recipient result: %s', recipient) + + try: + subject = Template(self.form_data['subject']).render( + context=Context(context) + ) + except Exception as exception: + raise WorkflowStateActionError( + _('Subject template error: %s') % exception + ) + else: + logger.debug('Subject result: %s', subject) + + try: + body = Template(self.form_data['body']).render( + context=Context(context) + ) + except Exception as exception: + raise WorkflowStateActionError( + _('Body template error: %s') % exception + ) + else: + logger.debug('Body result: %s', body) + + user_mailer = self.get_user_mailer() + user_mailer.send( + to=recipient, subject=subject, body=body, + ) + + def get_form_schema(self, request): + user = request.user + logger.debug('user: %s', user) + + queryset = AccessControlList.objects.restrict_queryset( + permission=self.permission, queryset=UserMailer.objects.all(), + user=user + ) + + self.fields['mailing_profile']['kwargs']['queryset'] = queryset + + return { + 'field_order': self.field_order, + 'fields': self.fields, + 'widgets': self.widgets + } + + def get_user_mailer(self): + return UserMailer.objects.get(pk=self.form_data['mailing_profile']) From 08ee07e65209a3f6c0b7756f4ec85f71a22afcda Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 00:37:47 -0400 Subject: [PATCH 015/402] Remove duplicated trashed document previews Side effect of source column inheritance added in 06c3ef658368d5cf6d5a776adaaa95a71035fd74. Signed-off-by: Roberto Rosario --- mayan/apps/documents/apps.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 5e8639c4668..0b83d72ce43 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -310,14 +310,6 @@ def ready(self): attribute='label', is_identifier=True, is_sortable=True, source=DeletedDocument ) - SourceColumn( - func=lambda context: document_page_thumbnail_widget.render( - instance=context['object'] - ), label=_('Thumbnail'), source=DeletedDocument - ) - SourceColumn( - attribute='document_type', is_sortable=True, source=DeletedDocument - ) SourceColumn( attribute='deleted_date_time', source=DeletedDocument ) From 080553c797ce4948b3c24e985b24001d31772940 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 00:38:47 -0400 Subject: [PATCH 016/402] Add trashed date time label and position Signed-off-by: Roberto Rosario --- mayan/apps/documents/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 0b83d72ce43..0032127a92a 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -311,7 +311,8 @@ def ready(self): source=DeletedDocument ) SourceColumn( - attribute='deleted_date_time', source=DeletedDocument + attribute='deleted_date_time', include_label=True, order=99, + source=DeletedDocument ) # DocumentVersion From ec6a3bd9606143dd54a3e7421fc1e4ffddd1161a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 00:43:14 -0400 Subject: [PATCH 017/402] Move AJAX spinner to the left of the top bar Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 818e81c133e..b63d146d709 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -264,8 +264,8 @@ a i { #ajax-spinner { position: fixed; - top: 12px; - right: 10px; + top: 16px; + left: 10px; z-index: 9999; width: 25px; height: 25px; From 1c86ea5b5ba2c364ebe22e77a6dd3a690e5dde6c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 01:03:39 -0400 Subject: [PATCH 018/402] Backport individual index rebuild support Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 1 + mayan/apps/document_indexing/apps.py | 8 ++- mayan/apps/document_indexing/links.py | 6 ++ mayan/apps/document_indexing/tests/mixins.py | 7 +++ .../document_indexing/tests/test_views.py | 58 +++++++++++++++++-- mayan/apps/document_indexing/urls.py | 8 ++- mayan/apps/document_indexing/views.py | 36 +++++++++++- 8 files changed, 112 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d5c54b33bb6..389a9ccc5c6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ - Backport workflow context support. - Backport workflow transitions field support. - Backport workflow email action. +- Backport individual index rebuild support. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index d5425852008..ba872b7fdb4 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -23,6 +23,7 @@ Changes - Backport workflow context support. - Backport workflow transitions field support. - Backport workflow email action. +- Backport individual index rebuild support. Removals -------- diff --git a/mayan/apps/document_indexing/apps.py b/mayan/apps/document_indexing/apps.py index 38a20c153a0..52df061a63c 100644 --- a/mayan/apps/document_indexing/apps.py +++ b/mayan/apps/document_indexing/apps.py @@ -31,9 +31,10 @@ ) from .links import ( link_document_index_instance_list, link_document_type_index_templates, - link_index_instance_menu, link_index_template_setup, - link_index_template_create, link_index_template_document_types, - link_index_template_delete, link_index_template_edit, link_index_template_list, + link_index_instance_menu, link_index_instance_rebuild, + link_index_template_setup, link_index_template_create, + link_index_template_document_types, link_index_template_delete, + link_index_template_edit, link_index_template_list, link_index_template_node_tree_view, link_index_instances_rebuild, link_index_template_node_create, link_index_template_node_delete, link_index_template_node_edit @@ -199,6 +200,7 @@ def ready(self): menu_object.bind_links( links=( link_index_template_delete, link_index_template_edit, + link_index_instance_rebuild ), sources=(Index,) ) menu_object.bind_links( diff --git a/mayan/apps/document_indexing/links.py b/mayan/apps/document_indexing/links.py index fc1f8320230..a9441e3e9c9 100644 --- a/mayan/apps/document_indexing/links.py +++ b/mayan/apps/document_indexing/links.py @@ -49,6 +49,12 @@ def is_not_root_node(context): ), text=_('Rebuild indexes'), view='indexing:rebuild_index_instances' ) +link_index_instance_rebuild = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_indexing.icons.icon_index_instances_rebuild', + permissions=(permission_document_indexing_rebuild,), + text=_('Rebuild index'), view='indexing:index_setup_rebuild' +) link_index_template_setup = Link( condition=get_cascade_condition( diff --git a/mayan/apps/document_indexing/tests/mixins.py b/mayan/apps/document_indexing/tests/mixins.py index a13b2fa2f42..339982fea7e 100644 --- a/mayan/apps/document_indexing/tests/mixins.py +++ b/mayan/apps/document_indexing/tests/mixins.py @@ -50,3 +50,10 @@ def _request_test_index_edit_view(self): 'label': TEST_INDEX_LABEL_EDITED, 'slug': TEST_INDEX_SLUG } ) + + def _request_test_index_rebuild_view(self): + return self.post( + viewname='indexing:index_setup_rebuild', kwargs={ + 'pk': self.test_index.pk + } + ) diff --git a/mayan/apps/document_indexing/tests/test_views.py b/mayan/apps/document_indexing/tests/test_views.py index 633939b5766..06854039b4b 100644 --- a/mayan/apps/document_indexing/tests/test_views.py +++ b/mayan/apps/document_indexing/tests/test_views.py @@ -2,7 +2,7 @@ from mayan.apps.documents.tests import GenericDocumentViewTestCase -from ..models import Index +from ..models import Index, IndexInstanceNode from ..permissions import ( permission_document_indexing_create, permission_document_indexing_delete, permission_document_indexing_edit, @@ -10,11 +10,16 @@ permission_document_indexing_rebuild ) -from .literals import TEST_INDEX_LABEL, TEST_INDEX_LABEL_EDITED +from .literals import ( + TEST_INDEX_LABEL, TEST_INDEX_LABEL_EDITED, + TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION +) from .mixins import IndexTestMixin, IndexViewTestMixin class IndexViewTestCase(IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase): + auto_upload_document = False + def test_index_create_view_no_permission(self): response = self._request_test_index_create_view() self.assertEqual(response.status_code, 403) @@ -72,6 +77,45 @@ def test_index_edit_view_with_access(self): self.test_index.refresh_from_db() self.assertEqual(self.test_index.label, TEST_INDEX_LABEL_EDITED) + def test_index_rebuild_view_no_permission(self): + self.upload_document() + self._create_test_index() + + self.test_index.node_templates.create( + parent=self.test_index.template_root, + expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, + link_documents=True + ) + + response = self._request_test_index_rebuild_view() + self.assertEqual(response.status_code, 404) + + self.assertEqual(IndexInstanceNode.objects.count(), 0) + + def test_index_rebuild_view_with_access(self): + self.upload_document() + self._create_test_index() + + self.test_index.node_templates.create( + parent=self.test_index.template_root, + expression=TEST_INDEX_TEMPLATE_DOCUMENT_LABEL_EXPRESSION, + link_documents=True + ) + + self.grant_access( + obj=self.test_index, + permission=permission_document_indexing_rebuild + ) + + response = self._request_test_index_rebuild_view() + self.assertEqual(response.status_code, 302) + + self.assertNotEqual(IndexInstanceNode.objects.count(), 0) + + +class IndexInstaceViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): def _request_index_instance_node_view(self, index_instance_node): return self.get( viewname='indexing:index_instance_node_view', kwargs={ @@ -100,9 +144,13 @@ def test_index_instance_node_view_with_access(self): ) self.assertContains(response, text=TEST_INDEX_LABEL, status_code=200) + +class IndexToolsViewTestCase( + IndexTestMixin, IndexViewTestMixin, GenericDocumentViewTestCase +): def _request_index_rebuild_get_view(self): return self.get( - viewname='indexing:rebuild_index_instances', + viewname='indexing:rebuild_index_instances' ) def _request_index_rebuild_post_view(self): @@ -112,7 +160,7 @@ def _request_index_rebuild_post_view(self): } ) - def test_index_rebuild_no_permission(self): + def test_indexes_rebuild_no_permission(self): self._create_test_index(rebuild=False) response = self._request_index_rebuild_get_view() @@ -128,7 +176,7 @@ def test_index_rebuild_no_permission(self): self.test_index.instance_root.get_children_count(), 0 ) - def test_index_rebuild_with_access(self): + def test_indexes_rebuild_with_access(self): self._create_test_index(rebuild=False) self.grant_access( diff --git a/mayan/apps/document_indexing/urls.py b/mayan/apps/document_indexing/urls.py index 380e1d7fdc2..d34650c1b5c 100644 --- a/mayan/apps/document_indexing/urls.py +++ b/mayan/apps/document_indexing/urls.py @@ -11,8 +11,8 @@ DocumentIndexNodeListView, DocumentTypeIndexesView, IndexInstanceNodeView, IndexListView, IndexesRebuildView, SetupIndexDocumentTypesView, SetupIndexCreateView, SetupIndexDeleteView, SetupIndexEditView, - SetupIndexListView, SetupIndexTreeTemplateListView, TemplateNodeCreateView, - TemplateNodeDeleteView, TemplateNodeEditView + SetupIndexListView, SetupIndexRebuildView, SetupIndexTreeTemplateListView, + TemplateNodeCreateView, TemplateNodeDeleteView, TemplateNodeEditView ) urlpatterns = [ @@ -46,6 +46,10 @@ view=SetupIndexDocumentTypesView.as_view(), name='index_setup_document_types' ), + url( + regex=r'^setup/index/(?P\d+)/rebuild/$', + view=SetupIndexRebuildView.as_view(), name='index_setup_rebuild' + ), url( regex=r'^setup/template/node/(?P\d+)/create/child/$', view=TemplateNodeCreateView.as_view(), name='template_node_create' diff --git a/mayan/apps/document_indexing/views.py b/mayan/apps/document_indexing/views.py index 55e79cc3c36..33d6047d2a2 100644 --- a/mayan/apps/document_indexing/views.py +++ b/mayan/apps/document_indexing/views.py @@ -9,8 +9,8 @@ from mayan.apps.acls.models import AccessControlList from mayan.apps.common.generics import ( - AddRemoveView, FormView, SingleObjectCreateView, SingleObjectDeleteView, - SingleObjectEditView, SingleObjectListView + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectEditView, SingleObjectListView ) from mayan.apps.documents.events import event_document_type_edited from mayan.apps.documents.models import Document, DocumentType @@ -32,7 +32,7 @@ permission_document_indexing_create, permission_document_indexing_delete, permission_document_indexing_edit, permission_document_indexing_instance_view, - permission_document_indexing_view + permission_document_indexing_rebuild, permission_document_indexing_view ) from .tasks import task_rebuild_index @@ -150,6 +150,36 @@ def get_extra_context(self): } +class SetupIndexRebuildView(ConfirmView): + post_action_redirect = reverse_lazy( + viewname='indexing:index_setup_list' + ) + + def get_extra_context(self): + return { + 'object': self.get_object(), + 'title': _('Rebuild index: %s') % self.get_object() + } + + def get_object(self): + return get_object_or_404(klass=self.get_queryset(), pk=self.kwargs['pk']) + + def get_queryset(self): + return AccessControlList.objects.restrict_queryset( + permission=permission_document_indexing_rebuild, + queryset=Index.objects.all(), user=self.request.user + ) + + def view_action(self): + task_rebuild_index.apply_async( + kwargs=dict(index_id=self.get_object().pk) + ) + + messages.success( + message='Index queued for rebuild.', request=self.request + ) + + class SetupIndexDocumentTypesView(AddRemoveView): main_object_method_add = 'document_types_add' main_object_method_remove = 'document_types_remove' From 7913b5ddcc29e90de785674adbadbbd7d39e9fc8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 01:06:58 -0400 Subject: [PATCH 019/402] Sort dictionary entry Signed-off-by: Roberto Rosario --- mayan/apps/document_indexing/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/document_indexing/views.py b/mayan/apps/document_indexing/views.py index 33d6047d2a2..384e45064f5 100644 --- a/mayan/apps/document_indexing/views.py +++ b/mayan/apps/document_indexing/views.py @@ -308,8 +308,8 @@ class IndexListView(SingleObjectListView): def get_extra_context(self): return { - 'hide_object': True, 'hide_links': True, + 'hide_object': True, 'no_results_icon': icon_index, 'no_results_main_link': link_index_template_create.resolve( context=RequestContext(request=self.request) From 0e972eff06a7d2dde3b1b1b2ebc29b5a357b2755 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 01:12:25 -0400 Subject: [PATCH 020/402] Fix typos and PEP8 warnings Signed-off-by: Roberto Rosario --- mayan/apps/common/apps.py | 4 +--- mayan/apps/document_states/html_widgets.py | 4 +--- mayan/apps/document_states/tests/literals.py | 1 - mayan/apps/document_states/views/workflow_views.py | 1 + mayan/apps/events/apps.py | 2 +- mayan/apps/smart_settings/tests/test_classes.py | 3 +-- mayan/apps/tags/models.py | 2 +- 7 files changed, 6 insertions(+), 11 deletions(-) diff --git a/mayan/apps/common/apps.py b/mayan/apps/common/apps.py index f0971536614..324f33dc4ef 100644 --- a/mayan/apps/common/apps.py +++ b/mayan/apps/common/apps.py @@ -27,9 +27,7 @@ ) from .literals import MESSAGE_SQLITE_WARNING -from .menus import ( - menu_about, menu_main, menu_secondary, menu_topbar, menu_user -) +from .menus import menu_about, menu_secondary, menu_topbar, menu_user from .settings import ( setting_auto_logging, setting_production_error_log_path, setting_production_error_logging diff --git a/mayan/apps/document_states/html_widgets.py b/mayan/apps/document_states/html_widgets.py index 98ad2d817f2..58d3da17739 100644 --- a/mayan/apps/document_states/html_widgets.py +++ b/mayan/apps/document_states/html_widgets.py @@ -1,9 +1,7 @@ from __future__ import unicode_literals -from django import forms from django.template.loader import render_to_string -from django.urls import reverse -from django.utils.html import format_html_join, mark_safe +from django.utils.html import format_html_join def widget_transition_events(transition): diff --git a/mayan/apps/document_states/tests/literals.py b/mayan/apps/document_states/tests/literals.py index c17a870e9cf..63d758efc33 100644 --- a/mayan/apps/document_states/tests/literals.py +++ b/mayan/apps/document_states/tests/literals.py @@ -24,4 +24,3 @@ TEST_INDEX_TEMPLATE_METADATA_EXPRESSION = '{{{{ document.workflow.{}.get_current_state }}}}'.format( TEST_WORKFLOW_INTERNAL_NAME ) - diff --git a/mayan/apps/document_states/views/workflow_views.py b/mayan/apps/document_states/views/workflow_views.py index 008c83a108b..104560a805e 100644 --- a/mayan/apps/document_states/views/workflow_views.py +++ b/mayan/apps/document_states/views/workflow_views.py @@ -742,6 +742,7 @@ class SetupWorkflowTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCr 'name', 'label', 'field_type', 'help_text', 'required', 'widget', 'widget_kwargs' ) + def get_extra_context(self): return { 'navigation_object_list': ('transition', 'workflow'), diff --git a/mayan/apps/events/apps.py b/mayan/apps/events/apps.py index 6ec59d3b90a..046f98968b2 100644 --- a/mayan/apps/events/apps.py +++ b/mayan/apps/events/apps.py @@ -6,7 +6,7 @@ from mayan.apps.common.apps import MayanAppConfig from mayan.apps.common.html_widgets import TwoStateWidget from mayan.apps.common.menus import ( - menu_main, menu_object, menu_secondary, menu_tools, menu_topbar, menu_user + menu_object, menu_secondary, menu_tools, menu_topbar, menu_user ) from mayan.apps.navigation.classes import SourceColumn diff --git a/mayan/apps/smart_settings/tests/test_classes.py b/mayan/apps/smart_settings/tests/test_classes.py index 6e88607f7f3..dc472dd245d 100644 --- a/mayan/apps/smart_settings/tests/test_classes.py +++ b/mayan/apps/smart_settings/tests/test_classes.py @@ -11,7 +11,7 @@ from mayan.apps.common.tests import BaseTestCase from mayan.apps.storage.utils import fs_cleanup -from ..classes import Namespace, Setting +from ..classes import Setting from .literals import ENVIRONMENT_TEST_NAME, ENVIRONMENT_TEST_VALUE from .mixins import SmartSettingTestMixin @@ -52,4 +52,3 @@ def test_setting_check_changed(self): self.assertFalse(Setting.check_changed()) test_setting.value = 'test value edited' self.assertTrue(Setting.check_changed()) - diff --git a/mayan/apps/tags/models.py b/mayan/apps/tags/models.py index 470a5f838d1..abc00dd33ca 100644 --- a/mayan/apps/tags/models.py +++ b/mayan/apps/tags/models.py @@ -61,7 +61,7 @@ def get_absolute_url(self): def get_document_count(self, user): """ Return the numeric count of documents that have this tag attached. - The count if filtered by access. + The count is filtered by access. """ queryset = AccessControlList.objects.restrict_queryset( permission=permission_document_view, queryset=self.documents, From f36f99c5fbd125d4b1a7afeceb77532c9c52b234 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 01:23:49 -0400 Subject: [PATCH 021/402] Split workflow URL patterns Signed-off-by: Roberto Rosario --- mayan/apps/document_states/urls.py | 190 +++++++++++++++-------------- 1 file changed, 101 insertions(+), 89 deletions(-) diff --git a/mayan/apps/document_states/urls.py b/mayan/apps/document_states/urls.py index ebc3e6acd92..9dab2abd22f 100644 --- a/mayan/apps/document_states/urls.py +++ b/mayan/apps/document_states/urls.py @@ -33,153 +33,162 @@ SetupWorkflowTransitionFieldEditView, SetupWorkflowTransitionFieldListView ) -urlpatterns_workflows = [ +urlpatterns_workflow_templates = [ url( - regex=r'^document_type/(?P\d+)/workflows/$', - view=SetupDocumentTypeWorkflowsView.as_view(), - name='document_type_workflows' + regex=r'^setup/workflows/$', view=SetupWorkflowListView.as_view(), + name='setup_workflow_list' ), -] - -urlpatterns_workflow_transition_fields = [ url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/create/$', - view=SetupWorkflowTransitionFieldCreateView.as_view(), - name='setup_workflow_transition_field_create' + regex=r'^setup/workflows/create/$', view=SetupWorkflowCreateView.as_view(), + name='setup_workflow_create' ), url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/$', - view=SetupWorkflowTransitionFieldListView.as_view(), - name='setup_workflow_transition_field_list' + regex=r'^setup/workflows/(?P\d+)/delete/$', + view=SetupWorkflowDeleteView.as_view(), name='setup_workflow_delete' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/delete/$', - view=SetupWorkflowTransitionFieldDeleteView.as_view(), - name='setup_workflow_transition_field_delete' + regex=r'^setup/workflows/(?P\d+)/edit/$', + view=SetupWorkflowEditView.as_view(), name='setup_workflow_edit' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/edit/$', - view=SetupWorkflowTransitionFieldEditView.as_view(), - name='setup_workflow_transition_field_edit' + regex=r'^setup/document_types/(?P\d+)/workflows/$', + view=SetupDocumentTypeWorkflowsView.as_view(), + name='document_type_workflows' ), ] -urlpatterns = [ +urlpatterns_workflow_states = [ url( - regex=r'^document/(?P\d+)/workflows/$', - view=DocumentWorkflowInstanceListView.as_view(), - name='document_workflow_instance_list' - ), - url( - regex=r'^document/workflows/(?P\d+)/$', - view=WorkflowInstanceDetailView.as_view(), - name='workflow_instance_detail' - ), - url( - regex=r'^document/workflows/(?P\d+)/transitions/select/$', - view=WorkflowInstanceTransitionSelectView.as_view(), - name='workflow_instance_transition_selection' - ), - url( - regex=r'^document/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', - view=WorkflowInstanceTransitionExecuteView.as_view(), - name='workflow_instance_transition_execute' + regex=r'^setup/workflows/(?P\d+)/states/$', + view=SetupWorkflowStateListView.as_view(), + name='setup_workflow_state_list' ), url( - regex=r'^setup/all/$', view=SetupWorkflowListView.as_view(), - name='setup_workflow_list' + regex=r'^setup/workflows/(?P\d+)/states/create/$', + view=SetupWorkflowStateCreateView.as_view(), + name='setup_workflow_state_create' ), url( - regex=r'^setup/create/$', view=SetupWorkflowCreateView.as_view(), - name='setup_workflow_create' + regex=r'^setup/workflows/states/(?P\d+)/delete/$', + view=SetupWorkflowStateDeleteView.as_view(), + name='setup_workflow_state_delete' ), url( - regex=r'^setup/workflow/(?P\d+)/edit/$', - view=SetupWorkflowEditView.as_view(), name='setup_workflow_edit' + regex=r'^setup/workflows/states/(?P\d+)/edit/$', + view=SetupWorkflowStateEditView.as_view(), + name='setup_workflow_state_edit' ), +] + +urlpatterns_workflow_state_actions = [ url( - regex=r'^setup/workflow/(?P\d+)/delete/$', - view=SetupWorkflowDeleteView.as_view(), name='setup_workflow_delete' + regex=r'^setup/workflows/states/(?P\d+)/actions/$', + view=SetupWorkflowStateActionListView.as_view(), + name='setup_workflow_state_action_list' ), url( - regex=r'^setup/workflow/(?P\d+)/documents/$', - view=WorkflowDocumentListView.as_view(), - name='setup_workflow_document_list' + regex=r'^setup/workflows/states/(?P\d+)/actions/selection/$', + view=SetupWorkflowStateActionSelectionView.as_view(), + name='setup_workflow_state_action_selection' ), url( - regex=r'^setup/workflow/(?P\d+)/document_types/$', - view=SetupWorkflowDocumentTypesView.as_view(), - name='setup_workflow_document_types' + regex=r'^setup/workflows/states/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', + view=SetupWorkflowStateActionCreateView.as_view(), + name='setup_workflow_state_action_create' ), url( - regex=r'^setup/workflow/(?P\d+)/states/$', - view=SetupWorkflowStateListView.as_view(), - name='setup_workflow_state_list' + regex=r'^setup/workflows/states/actions/(?P\d+)/delete/$', + view=SetupWorkflowStateActionDeleteView.as_view(), + name='setup_workflow_state_action_delete' ), url( - regex=r'^setup/workflow/(?P\d+)/states/create/$', - view=SetupWorkflowStateCreateView.as_view(), - name='setup_workflow_state_create' + regex=r'^setup/workflows/states/actions/(?P\d+)/edit/$', + view=SetupWorkflowStateActionEditView.as_view(), + name='setup_workflow_state_action_edit' ), +] + +urlpatterns_workflow_transitions = [ url( - regex=r'^setup/workflow/(?P\d+)/transitions/$', + regex=r'^setup/workflows/(?P\d+)/transitions/$', view=SetupWorkflowTransitionListView.as_view(), name='setup_workflow_transition_list' ), url( - regex=r'^setup/workflow/(?P\d+)/transitions/create/$', + regex=r'^setup/workflows/(?P\d+)/transitions/create/$', view=SetupWorkflowTransitionCreateView.as_view(), name='setup_workflow_transition_create' ), url( - regex=r'^setup/workflow/(?P\d+)/transitions/events/$', + regex=r'^setup/workflows/(?P\d+)/transitions/events/$', view=SetupWorkflowTransitionTriggerEventListView.as_view(), name='setup_workflow_transition_events' ), url( - regex=r'^setup/workflow/state/(?P\d+)/delete/$', - view=SetupWorkflowStateDeleteView.as_view(), - name='setup_workflow_state_delete' + regex=r'^setup/workflows/transitions/(?P\d+)/delete/$', + view=SetupWorkflowTransitionDeleteView.as_view(), + name='setup_workflow_transition_delete' ), url( - regex=r'^setup/workflow/state/(?P\d+)/edit/$', - view=SetupWorkflowStateEditView.as_view(), - name='setup_workflow_state_edit' + regex=r'^setup/workflows/transitions/(?P\d+)/edit/$', + view=SetupWorkflowTransitionEditView.as_view(), + name='setup_workflow_transition_edit' ), url( - regex=r'^setup/workflow/state/(?P\d+)/actions/$', - view=SetupWorkflowStateActionListView.as_view(), - name='setup_workflow_state_action_list' + regex=r'^documents/workflows/(?P\d+)/transitions/select/$', + view=WorkflowInstanceTransitionSelectView.as_view(), + name='workflow_instance_transition_selection' ), url( - regex=r'^setup/workflow/state/(?P\d+)/actions/selection/$', - view=SetupWorkflowStateActionSelectionView.as_view(), - name='setup_workflow_state_action_selection' + regex=r'^documents/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', + view=WorkflowInstanceTransitionExecuteView.as_view(), + name='workflow_instance_transition_execute' ), +] + +urlpatterns_workflow_transition_fields = [ url( - regex=r'^setup/workflow/state/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', - view=SetupWorkflowStateActionCreateView.as_view(), - name='setup_workflow_state_action_create' + regex=r'^setup/workflows/transitions/(?P\d+)/fields/create/$', + view=SetupWorkflowTransitionFieldCreateView.as_view(), + name='setup_workflow_transition_field_create' ), url( - regex=r'^setup/workflow/state/actions/(?P\d+)/delete/$', - view=SetupWorkflowStateActionDeleteView.as_view(), - name='setup_workflow_state_action_delete' + regex=r'^setup/workflows/transitions/(?P\d+)/fields/$', + view=SetupWorkflowTransitionFieldListView.as_view(), + name='setup_workflow_transition_field_list' ), url( - regex=r'^setup/workflow/state/actions/(?P\d+)/edit/$', - view=SetupWorkflowStateActionEditView.as_view(), - name='setup_workflow_state_action_edit' + regex=r'^setup/workflows/transitions/fields/(?P\d+)/delete/$', + view=SetupWorkflowTransitionFieldDeleteView.as_view(), + name='setup_workflow_transition_field_delete' ), url( - regex=r'^setup/workflow/transitions/(?P\d+)/delete/$', - view=SetupWorkflowTransitionDeleteView.as_view(), - name='setup_workflow_transition_delete' + regex=r'^setup/workflows/transitions/fields/(?P\d+)/edit/$', + view=SetupWorkflowTransitionFieldEditView.as_view(), + name='setup_workflow_transition_field_edit' ), +] + +urlpatterns = [ url( - regex=r'^setup/workflow/transitions/(?P\d+)/edit/$', - view=SetupWorkflowTransitionEditView.as_view(), - name='setup_workflow_transition_edit' + regex=r'^document/(?P\d+)/workflows/$', + view=DocumentWorkflowInstanceListView.as_view(), + name='document_workflow_instance_list' + ), + url( + regex=r'^document/workflows/(?P\d+)/$', + view=WorkflowInstanceDetailView.as_view(), + name='workflow_instance_detail' + ), + url( + regex=r'^setup/workflow/(?P\d+)/documents/$', + view=WorkflowDocumentListView.as_view(), + name='setup_workflow_document_list' + ), + url( + regex=r'^setup/workflow/(?P\d+)/document_types/$', + view=SetupWorkflowDocumentTypesView.as_view(), + name='setup_workflow_document_types' ), url( regex=r'^tools/workflow/all/launch/$', @@ -213,7 +222,10 @@ ), ] -urlpatterns.extend(urlpatterns_workflows) +urlpatterns.extend(urlpatterns_workflow_states) +urlpatterns.extend(urlpatterns_workflow_state_actions) +urlpatterns.extend(urlpatterns_workflow_templates) +urlpatterns.extend(urlpatterns_workflow_transitions) urlpatterns.extend(urlpatterns_workflow_transition_fields) api_urls = [ From 8811c8269f81cb82979239dcb89fd99bf492ed30 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 02:05:13 -0400 Subject: [PATCH 022/402] Rename document states apps view and URLs. Object layout: WorkflowTemplate, WorkflowInstance, WorkflowRuntimeProxy, WorkflowTemplateState, WorkflowTemplateTransition. Signed-off-by: Roberto Rosario --- mayan/apps/document_states/api_views.py | 6 +- mayan/apps/document_states/apps.py | 94 +- mayan/apps/document_states/icons.py | 57 +- mayan/apps/document_states/links.py | 172 ++-- mayan/apps/document_states/tests/mixins.py | 28 +- .../tests/test_workflow_state_action_views.py | 2 +- .../tests/test_workflow_transition_views.py | 8 +- .../tests/test_workflow_views.py | 14 +- mayan/apps/document_states/urls.py | 289 +++--- mayan/apps/document_states/views/__init__.py | 4 +- .../views/workflow_instance_views.py | 14 +- .../views/workflow_proxy_views.py | 31 +- .../views/workflow_template_state_views.py | 326 +++++++ .../workflow_template_transition_views.py | 372 ++++++++ .../views/workflow_template_views.py | 261 ++++++ .../document_states/views/workflow_views.py | 884 ------------------ mayan/apps/mailer/tests/test_actions.py | 4 +- mayan/apps/tags/tests/test_actions.py | 4 +- 18 files changed, 1334 insertions(+), 1236 deletions(-) create mode 100644 mayan/apps/document_states/views/workflow_template_state_views.py create mode 100644 mayan/apps/document_states/views/workflow_template_transition_views.py create mode 100644 mayan/apps/document_states/views/workflow_template_views.py delete mode 100644 mayan/apps/document_states/views/workflow_views.py diff --git a/mayan/apps/document_states/api_views.py b/mayan/apps/document_states/api_views.py index 5f998db1156..90df8863cb3 100644 --- a/mayan/apps/document_states/api_views.py +++ b/mayan/apps/document_states/api_views.py @@ -31,7 +31,7 @@ from .tasks import task_generate_workflow_image -class APIDocumentTypeWorkflowListView(generics.ListAPIView): +class APIDocumentTypeWorkflowRuntimeProxyListView(generics.ListAPIView): """ get: Returns a list of all the document type workflows. """ @@ -214,7 +214,7 @@ def retrieve(self, request, *args, **kwargs): return response -class APIWorkflowListView(generics.ListCreateAPIView): +class APIWorkflowRuntimeProxyListView(generics.ListCreateAPIView): """ get: Returns a list of all the workflows. post: Create a new workflow. @@ -229,7 +229,7 @@ def get_serializer(self, *args, **kwargs): if not self.request: return None - return super(APIWorkflowListView, self).get_serializer(*args, **kwargs) + return super(APIWorkflowRuntimeProxyListView, self).get_serializer(*args, **kwargs) def get_serializer_class(self): if self.request.method == 'GET': diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index 0e35c0d59c2..065105db199 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -29,27 +29,27 @@ ) from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events from .links import ( - link_document_workflow_instance_list, link_setup_document_type_workflows, - link_setup_workflow_document_types, link_setup_workflow_create, - link_setup_workflow_delete, link_setup_workflow_edit, - link_setup_workflow_list, link_setup_workflow_states, - link_setup_workflow_state_action_delete, - link_setup_workflow_state_action_edit, - link_setup_workflow_state_action_list, - link_setup_workflow_state_action_selection, - link_setup_workflow_state_create, link_setup_workflow_state_delete, - link_setup_workflow_state_edit, link_setup_workflow_transitions, - link_setup_workflow_transition_create, - link_setup_workflow_transition_delete, link_setup_workflow_transition_edit, - link_setup_workflow_transition_field_create, - link_setup_workflow_transition_field_delete, - link_setup_workflow_transition_field_edit, - link_setup_workflow_transition_field_list, - link_tool_launch_all_workflows, link_workflow_instance_detail, + link_workflow_instance_list, link_document_type_workflow_templates, + link_workflow_template_document_types, link_workflow_template_create, + link_workflow_template_delete, link_workflow_template_edit, + link_workflow_template_list, link_workflow_template_state_list, + link_workflow_template_state_action_delete, + link_workflow_template_state_action_edit, + link_workflow_template_state_action_list, + link_workflow_template_state_action_selection, + link_workflow_template_state_create, link_workflow_template_state_delete, + link_workflow_template_state_edit, link_workflow_template_transition_list, + link_workflow_template_transition_create, + link_workflow_template_transition_delete, link_workflow_template_transition_edit, + link_workflow_template_transition_field_create, + link_workflow_template_transition_field_delete, + link_workflow_template_transition_field_edit, + link_workflow_template_transition_field_list, + link_tool_launch_workflows, link_workflow_instance_detail, link_workflow_instance_transition, link_workflow_runtime_proxy_document_list, - link_workflow_runtime_proxy_list, link_workflow_preview, + link_workflow_runtime_proxy_list, link_workflow_template_preview, link_workflow_runtime_proxy_state_document_list, link_workflow_runtime_proxy_state_list, - link_workflow_transition_events + link_workflow_template_transition_events ) from .permissions import ( permission_workflow_delete, permission_workflow_edit, @@ -316,49 +316,49 @@ def ready(self): ) menu_facet.bind_links( - links=(link_document_workflow_instance_list,), sources=(Document,) + links=(link_workflow_instance_list,), sources=(Document,) ) menu_list_facet.bind_links( links=( link_acl_list, link_events_for_object, link_object_event_types_user_subcriptions_list, - link_setup_workflow_document_types, - link_setup_workflow_states, link_setup_workflow_transitions, - link_workflow_preview + link_workflow_template_document_types, + link_workflow_template_state_list, link_workflow_template_transition_list, + link_workflow_template_preview ), sources=(Workflow,) ) menu_list_facet.bind_links( links=( - link_setup_document_type_workflows, + link_document_type_workflow_templates, ), sources=(DocumentType,) ) menu_main.bind_links(links=(link_workflow_runtime_proxy_list,), position=10) menu_object.bind_links( links=( - link_setup_workflow_delete, link_setup_workflow_edit + link_workflow_template_delete, link_workflow_template_edit ), sources=(Workflow,) ) menu_object.bind_links( links=( - link_setup_workflow_state_edit, - link_setup_workflow_state_action_list, - link_setup_workflow_state_delete + link_workflow_template_state_edit, + link_workflow_template_state_action_list, + link_workflow_template_state_delete ), sources=(WorkflowState,) ) menu_object.bind_links( links=( - link_setup_workflow_transition_edit, - link_workflow_transition_events, - link_setup_workflow_transition_field_list, link_acl_list, - link_setup_workflow_transition_delete + link_workflow_template_transition_edit, + link_workflow_template_transition_events, + link_workflow_template_transition_field_list, link_acl_list, + link_workflow_template_transition_delete ), sources=(WorkflowTransition,) ) menu_object.bind_links( links=( - link_setup_workflow_transition_field_delete, - link_setup_workflow_transition_field_edit + link_workflow_template_transition_field_delete, + link_workflow_template_transition_field_edit ), sources=(WorkflowTransitionField,) ) menu_object.bind_links( @@ -381,21 +381,21 @@ def ready(self): ) menu_object.bind_links( links=( - link_setup_workflow_state_action_edit, + link_workflow_template_state_action_edit, link_object_error_list, - link_setup_workflow_state_action_delete, + link_workflow_template_state_action_delete, ), sources=(WorkflowStateAction,) ) menu_secondary.bind_links( - links=(link_setup_workflow_list, link_setup_workflow_create), + links=(link_workflow_template_list, link_workflow_template_create), sources=( - Workflow, 'document_states:setup_workflow_create', - 'document_states:setup_workflow_list' + Workflow, 'document_states:workflow_template_create', + 'document_states:workflow_template_list' ) ) menu_secondary.bind_links( - links=(link_setup_workflow_transition_field_create,), + links=(link_workflow_template_transition_field_create,), sources=( WorkflowTransition, ) @@ -407,31 +407,31 @@ def ready(self): ) ) menu_secondary.bind_links( - links=(link_setup_workflow_state_action_selection,), + links=(link_workflow_template_state_action_selection,), sources=( WorkflowState, ) ) menu_secondary.bind_links( links=( - link_setup_workflow_transition_create, + link_workflow_template_transition_create, ), sources=( WorkflowTransition, - 'document_states:setup_workflow_transition_list', + 'document_states:workflow_template_transition_list', ) ) menu_secondary.bind_links( links=( - link_setup_workflow_state_create, + link_workflow_template_state_create, ), sources=( WorkflowState, - 'document_states:setup_workflow_state_list', + 'document_states:workflow_template_state_list', ) ) - menu_setup.bind_links(links=(link_setup_workflow_list,)) + menu_setup.bind_links(links=(link_workflow_template_list,)) - menu_tools.bind_links(links=(link_tool_launch_all_workflows,)) + menu_tools.bind_links(links=(link_tool_launch_workflows,)) post_save.connect( dispatch_uid='workflows_handler_launch_workflow', diff --git a/mayan/apps/document_states/icons.py b/mayan/apps/document_states/icons.py index 9fa519fac1a..3115901935a 100644 --- a/mayan/apps/document_states/icons.py +++ b/mayan/apps/document_states/icons.py @@ -1,45 +1,44 @@ from __future__ import absolute_import, unicode_literals from mayan.apps.appearance.classes import Icon -from mayan.apps.documents.icons import icon_document_type +from mayan.apps.documents.icons import icon_document, icon_document_type icon_workflow = Icon(driver_name='fontawesome', symbol='sitemap') -icon_document_type_workflow_list = icon_workflow +icon_tool_launch_workflows = icon_workflow -icon_document_workflow_instance_list = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_setup_workflow_list = Icon(driver_name='fontawesome', symbol='sitemap') -icon_tool_launch_all_workflows = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_workflow_create = Icon( +icon_document_type_workflow_list = icon_workflow +icon_workflow_template_create = Icon( driver_name='fontawesome-dual', primary_symbol='sitemap', secondary_symbol='plus' ) -icon_workflow_delete = Icon(driver_name='fontawesome', symbol='times') -icon_workflow_document_type_list = icon_document_type -icon_workflow_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') -icon_workflow_list = Icon(driver_name='fontawesome', symbol='sitemap') -icon_workflow_preview = Icon(driver_name='fontawesome', symbol='eye') - -icon_workflow_instance_detail = Icon( - driver_name='fontawesome', symbol='sitemap' +icon_workflow_template_delete = Icon(driver_name='fontawesome', symbol='times') +icon_workflow_template_document_type_list = icon_document_type +icon_workflow_template_edit = Icon( + driver_name='fontawesome', symbol='pencil-alt' ) +icon_workflow_template_list = icon_workflow +icon_workflow_template_preview = Icon(driver_name='fontawesome', symbol='eye') + +# Workflow instances + +icon_workflow_instance_detail = icon_workflow +icon_workflow_instance_list = icon_workflow icon_workflow_instance_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) -icon_workflow_runtime_proxy_document_list = icon_document_type -icon_workflow_runtime_proxy_list = Icon( - driver_name='fontawesome', symbol='sitemap' -) -icon_workflow_runtime_proxy_state_document_list = icon_document_type +# Workflow runtime proxies + +icon_workflow_runtime_proxy_document_list = icon_document +icon_workflow_runtime_proxy_list = icon_workflow +icon_workflow_runtime_proxy_state_document_list = icon_document icon_workflow_runtime_proxy_state_list = Icon( driver_name='fontawesome', symbol='circle' ) +# Workflow transition states + icon_workflow_state_action_delete = Icon( driver_name='fontawesome', symbol='times' ) @@ -57,6 +56,8 @@ icon_workflow_state_delete = Icon(driver_name='fontawesome', symbol='times') icon_workflow_state_edit = Icon(driver_name='fontawesome', symbol='pencil-alt') +# Workflow transition state actions + icon_workflow_state_action = Icon(driver_name='fontawesome', symbol='code') icon_workflow_state_action_delete = Icon( driver_name='fontawesome', symbol='times' @@ -71,6 +72,9 @@ icon_workflow_state_action_list = Icon( driver_name='fontawesome', symbol='code' ) + +# Workflow transitions + icon_workflow_transition = Icon( driver_name='fontawesome', symbol='arrows-alt-h' ) @@ -85,7 +89,11 @@ driver_name='fontawesome', symbol='pencil-alt' ) -icon_workflow_transition_field = Icon(driver_name='fontawesome', symbol='table') +# Workflow transition fields + +icon_workflow_transition_field = Icon( + driver_name='fontawesome', symbol='table' +) icon_workflow_transition_field_delete = Icon( driver_name='fontawesome', symbol='times' ) @@ -99,7 +107,6 @@ icon_workflow_transition_field_list = Icon( driver_name='fontawesome', symbol='table' ) - icon_workflow_transition_triggers = Icon( driver_name='fontawesome', symbol='bolt' ) diff --git a/mayan/apps/document_states/links.py b/mayan/apps/document_states/links.py index 1404ff2644b..08e8d001cda 100644 --- a/mayan/apps/document_states/links.py +++ b/mayan/apps/document_states/links.py @@ -11,179 +11,185 @@ permission_workflow_view, ) -link_setup_document_type_workflows = Link( +# Workflow templates + +link_document_type_workflow_templates = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_document_type_workflow_list', permissions=(permission_document_type_edit,), text=_('Workflows'), - view='document_states:document_type_workflows', + view='document_states:document_type_workflow_templates', ) -link_setup_workflow_create = Link( - icon_class_path='mayan.apps.document_states.icons.icon_workflow_create', +link_workflow_template_create = Link( + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_create', permissions=(permission_workflow_create,), - text=_('Create workflow'), view='document_states:setup_workflow_create' + text=_('Create workflow'), view='document_states:workflow_template_create' ) -link_setup_workflow_delete = Link( +link_workflow_template_delete = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_delete', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_delete', permissions=(permission_workflow_delete,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_delete', + view='document_states:workflow_template_delete', ) -link_setup_workflow_document_types = Link( +link_workflow_template_document_types = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_document_type_list', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_document_type_list', permissions=(permission_workflow_edit,), text=_('Document types'), - view='document_states:setup_workflow_document_types', + view='document_states:workflow_template_document_types', ) -link_setup_workflow_edit = Link( +link_workflow_template_edit = Link( args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_edit', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_edit', + text=_('Edit'), view='document_states:workflow_template_edit', ) -link_setup_workflow_list = Link( - icon_class_path='mayan.apps.document_states.icons.icon_setup_workflow_list', +link_workflow_template_list = Link( + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_list', permissions=(permission_workflow_view,), text=_('Workflows'), - view='document_states:setup_workflow_list' + view='document_states:workflow_template_list' ) -link_setup_workflow_state_action_delete = Link( +link_workflow_template_preview = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_template_preview', + permissions=(permission_workflow_view,), + text=_('Preview'), view='document_states:workflow_template_preview' +) + +# Workflow template state actions + +link_workflow_template_state_action_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_state_action_delete', + view='document_states:workflow_template_state_action_delete', ) -link_setup_workflow_state_action_edit = Link( +link_workflow_template_state_action_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_state_action_edit', + text=_('Edit'), view='document_states:workflow_template_state_action_edit', ) -link_setup_workflow_state_action_list = Link( +link_workflow_template_state_action_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action_list', permissions=(permission_workflow_edit,), text=_('Actions'), - view='document_states:setup_workflow_state_action_list', + view='document_states:workflow_template_state_action_list', ) -link_setup_workflow_state_action_selection = Link( +link_workflow_template_state_action_selection = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_action', permissions=(permission_workflow_edit,), text=_('Create action'), - view='document_states:setup_workflow_state_action_selection', + view='document_states:workflow_template_state_action_selection', ) -link_setup_workflow_state_create = Link( + +# Workflow template states + +link_workflow_template_state_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_create', permissions=(permission_workflow_edit,), text=_('Create state'), - view='document_states:setup_workflow_state_create', + view='document_states:workflow_template_state_create', ) -link_setup_workflow_state_delete = Link( +link_workflow_template_state_delete = Link( args='object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_state_delete', + view='document_states:workflow_template_state_delete', ) -link_setup_workflow_state_edit = Link( +link_workflow_template_state_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_state_edit', + text=_('Edit'), view='document_states:workflow_template_state_edit', ) -link_setup_workflow_states = Link( +link_workflow_template_state_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_state', permissions=(permission_workflow_view,), text=_('States'), - view='document_states:setup_workflow_state_list', + view='document_states:workflow_template_state_list', ) -link_setup_workflow_transition_create = Link( + +# Workflow template transitions + +link_workflow_template_transition_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_create', permissions=(permission_workflow_edit,), text=_('Create transition'), - view='document_states:setup_workflow_transition_create', + view='document_states:workflow_template_transition_create', ) -link_setup_workflow_transition_delete = Link( +link_workflow_template_transition_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_transition_delete', + view='document_states:workflow_template_transition_delete', ) -link_setup_workflow_transition_edit = Link( +link_workflow_template_transition_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_transition_edit', + text=_('Edit'), view='document_states:workflow_template_transition_edit', ) -link_setup_workflow_transitions = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition', - permissions=(permission_workflow_view,), text=_('Transitions'), - view='document_states:setup_workflow_transition_list', -) -link_workflow_transition_events = Link( +link_workflow_template_transition_events = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_triggers', permissions=(permission_workflow_edit,), text=_('Transition triggers'), - view='document_states:setup_workflow_transition_events' + view='document_states:workflow_template_transition_events' +) +link_workflow_template_transition_list = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition', + permissions=(permission_workflow_view,), text=_('Transitions'), + view='document_states:workflow_template_transition_list', ) # Workflow transition fields -link_setup_workflow_transition_field_create = Link( + +link_workflow_template_transition_field_create = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field', permissions=(permission_workflow_edit,), text=_('Create field'), - view='document_states:setup_workflow_transition_field_create', + view='document_states:workflow_template_transition_field_create', ) -link_setup_workflow_transition_field_delete = Link( +link_workflow_template_transition_field_delete = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_delete', permissions=(permission_workflow_edit,), tags='dangerous', text=_('Delete'), - view='document_states:setup_workflow_transition_field_delete', + view='document_states:workflow_template_transition_field_delete', ) -link_setup_workflow_transition_field_edit = Link( +link_workflow_template_transition_field_edit = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_edit', permissions=(permission_workflow_edit,), - text=_('Edit'), view='document_states:setup_workflow_transition_field_edit', + text=_('Edit'), view='document_states:workflow_template_transition_field_edit', ) -link_setup_workflow_transition_field_list = Link( +link_workflow_template_transition_field_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_transition_field_list', permissions=(permission_workflow_edit,), text=_('Fields'), - view='document_states:setup_workflow_transition_field_list', -) - -link_workflow_preview = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_workflow_preview', - permissions=(permission_workflow_view,), - text=_('Preview'), view='document_states:workflow_preview' -) -link_tool_launch_all_workflows = Link( - icon_class_path='mayan.apps.document_states.icons.icon_tool_launch_all_workflows', - permissions=(permission_workflow_tools,), - text=_('Launch all workflows'), - view='document_states:tool_launch_all_workflows' + view='document_states:workflow_template_transition_field_list', ) # Document workflow instances -link_document_workflow_instance_list = Link( - args='resolved_object.pk', - icon_class_path='mayan.apps.document_states.icons.icon_document_workflow_instance_list', - permissions=(permission_workflow_view,), text=_('Workflows'), - view='document_states:document_workflow_instance_list', -) + link_workflow_instance_detail = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_detail', permissions=(permission_workflow_view,), text=_('Detail'), view='document_states:workflow_instance_detail', ) +link_workflow_instance_list = Link( + args='resolved_object.pk', + icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_list', + permissions=(permission_workflow_view,), text=_('Workflows'), + view='document_states:workflow_instance_list', +) link_workflow_instance_transition = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_instance_transition', @@ -192,28 +198,38 @@ ) # Runtime proxies + link_workflow_runtime_proxy_document_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_document_list', permissions=(permission_workflow_view,), text=_('Workflow documents'), - view='document_states:workflow_document_list', + view='document_states:workflow_runtime_proxy_document_list', ) link_workflow_runtime_proxy_list = Link( icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_list', permissions=(permission_workflow_view,), - text=_('Workflows'), view='document_states:workflow_list' + text=_('Workflows'), view='document_states:workflow_runtime_proxy_list' ) link_workflow_runtime_proxy_state_document_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_state_document_list', permissions=(permission_workflow_view,), text=_('State documents'), - view='document_states:workflow_state_document_list', + view='document_states:workflow_runtime_proxy_state_document_list', ) link_workflow_runtime_proxy_state_list = Link( args='resolved_object.pk', icon_class_path='mayan.apps.document_states.icons.icon_workflow_runtime_proxy_state_list', permissions=(permission_workflow_view,), - text=_('States'), view='document_states:workflow_state_list', + text=_('States'), view='document_states:workflow_runtime_proxy_state_list', +) + +# Tools + +link_tool_launch_workflows = Link( + icon_class_path='mayan.apps.document_states.icons.icon_tool_launch_workflows', + permissions=(permission_workflow_tools,), + text=_('Launch all workflows'), + view='document_states:tool_launch_workflows' ) diff --git a/mayan/apps/document_states/tests/mixins.py b/mayan/apps/document_states/tests/mixins.py index f8ea985a1c2..ff098444252 100644 --- a/mayan/apps/document_states/tests/mixins.py +++ b/mayan/apps/document_states/tests/mixins.py @@ -38,19 +38,19 @@ def _request_test_workflow_state_create_view(self, extra_data=None): data.update(extra_data) return self.post( - viewname='document_states:setup_workflow_state_create', + viewname='document_states:workflow_template_state_create', kwargs={'pk': self.test_workflow.pk}, data=data ) def _request_test_workflow_state_delete_view(self): return self.post( - viewname='document_states:setup_workflow_state_delete', + viewname='document_states:workflow_template_state_delete', kwargs={'pk': self.test_workflow_state_1.pk} ) def _request_test_workflow_state_edit_view(self): return self.post( - viewname='document_states:setup_workflow_state_edit', + viewname='document_states:workflow_template_state_edit', kwargs={'pk': self.test_workflow_state_1.pk}, data={ 'label': TEST_WORKFLOW_STATE_LABEL_EDITED } @@ -58,7 +58,7 @@ def _request_test_workflow_state_edit_view(self): def _request_test_workflow_state_list_view(self): return self.get( - viewname='document_states:setup_workflow_state_list', + viewname='document_states:workflow_template_state_list', kwargs={'pk': self.test_workflow.pk} ) @@ -120,7 +120,7 @@ def _create_test_workflow_instance_log_entry(self): class WorkflowTransitionViewTestMixin(object): def _request_test_workflow_transition_create_view(self): return self.post( - viewname='document_states:setup_workflow_transition_create', + viewname='document_states:workflow_template_transition_create', kwargs={'pk': self.test_workflow.pk}, data={ 'label': TEST_WORKFLOW_TRANSITION_LABEL, 'origin_state': self.test_workflow_state_1.pk, @@ -130,13 +130,13 @@ def _request_test_workflow_transition_create_view(self): def _request_test_workflow_transition_delete_view(self): return self.post( - viewname='document_states:setup_workflow_transition_delete', + viewname='document_states:workflow_template_transition_delete', kwargs={'pk': self.test_workflow_transition.pk} ) def _request_test_workflow_transition_edit_view(self): return self.post( - viewname='document_states:setup_workflow_transition_edit', + viewname='document_states:workflow_template_transition_edit', kwargs={'pk': self.test_workflow_transition.pk}, data={ 'label': TEST_WORKFLOW_TRANSITION_LABEL_EDITED, 'origin_state': self.test_workflow_state_1.pk, @@ -146,7 +146,7 @@ def _request_test_workflow_transition_edit_view(self): def _request_test_workflow_transition_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_list', + viewname='document_states:workflow_template_transition_list', kwargs={'pk': self.test_workflow.pk} ) @@ -163,7 +163,7 @@ def _request_test_workflow_transition(self): class WorkflowViewTestMixin(object): def _request_test_workflow_create_view(self): return self.post( - viewname='document_states:setup_workflow_create', data={ + viewname='document_states:workflow_template_create', data={ 'label': TEST_WORKFLOW_LABEL, 'internal_name': TEST_WORKFLOW_INTERNAL_NAME, } @@ -171,14 +171,14 @@ def _request_test_workflow_create_view(self): def _request_test_workflow_delete_view(self): return self.post( - viewname='document_states:setup_workflow_delete', kwargs={ + viewname='document_states:workflow_template_delete', kwargs={ 'pk': self.test_workflow.pk } ) def _request_test_workflow_edit_view(self): return self.post( - viewname='document_states:setup_workflow_edit', kwargs={ + viewname='document_states:workflow_template_edit', kwargs={ 'pk': self.test_workflow.pk, }, data={ 'label': TEST_WORKFLOW_LABEL_EDITED, @@ -188,12 +188,12 @@ def _request_test_workflow_edit_view(self): def _request_test_workflow_list_view(self): return self.get( - viewname='document_states:setup_workflow_list', + viewname='document_states:workflow_template_list', ) - def _request_test_workflow_preview_view(self): + def _request_test_workflow_template_preview_view(self): return self.get( - viewname='document_states:workflow_preview', kwargs={ + viewname='document_states:workflow_template_preview', kwargs={ 'pk': self.test_workflow.pk, } ) diff --git a/mayan/apps/document_states/tests/test_workflow_state_action_views.py b/mayan/apps/document_states/tests/test_workflow_state_action_views.py index e79853fd900..c1b54093a83 100644 --- a/mayan/apps/document_states/tests/test_workflow_state_action_views.py +++ b/mayan/apps/document_states/tests/test_workflow_state_action_views.py @@ -15,7 +15,7 @@ def setUp(self): def _request_test_document_state_action_view(self): return self.get( - viewname='document_states:setup_workflow_state_action_list', + viewname='document_states:workflow_template_state_action_list', kwargs={'pk': self.test_workflow_state.pk} ) diff --git a/mayan/apps/document_states/tests/test_workflow_transition_views.py b/mayan/apps/document_states/tests/test_workflow_transition_views.py index c298e5dde4f..c4a004871c6 100644 --- a/mayan/apps/document_states/tests/test_workflow_transition_views.py +++ b/mayan/apps/document_states/tests/test_workflow_transition_views.py @@ -212,7 +212,7 @@ class WorkflowTransitionEventViewTestCase( ): def _request_test_workflow_transition_event_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_events', + viewname='document_states:workflow_template_transition_events', kwargs={'pk': self.test_workflow_transition.pk} ) @@ -256,7 +256,7 @@ def _create_test_workflow_transition_field(self): def _request_test_workflow_transition_field_list_view(self): return self.get( - viewname='document_states:setup_workflow_transition_field_list', + viewname='document_states:workflow_template_transition_field_list', kwargs={'pk': self.test_workflow_transition.pk} ) @@ -286,7 +286,7 @@ def test_workflow_transition_field_list_view_with_access(self): def _request_workflow_transition_field_create_view(self): return self.post( - viewname='document_states:setup_workflow_transition_field_create', + viewname='document_states:workflow_template_transition_field_create', kwargs={'pk': self.test_workflow_transition.pk}, data={ 'field_type': TEST_WORKFLOW_TRANSITION_FIELD_TYPE, @@ -324,7 +324,7 @@ def test_workflow_transition_field_create_view_with_access(self): def _request_workflow_transition_field_delete_view(self): return self.post( - viewname='document_states:setup_workflow_transition_field_delete', + viewname='document_states:workflow_template_transition_field_delete', kwargs={'pk': self.test_workflow_transition_field.pk}, ) diff --git a/mayan/apps/document_states/tests/test_workflow_views.py b/mayan/apps/document_states/tests/test_workflow_views.py index 24df83a0951..1e738d7d538 100644 --- a/mayan/apps/document_states/tests/test_workflow_views.py +++ b/mayan/apps/document_states/tests/test_workflow_views.py @@ -93,31 +93,31 @@ def test_workflow_list_view_with_access(self): self.assertEqual(response.status_code, 200) self.assertContains(response, text=self.test_workflow.label) - def test_workflow_preview_view_no_access(self): + def test_workflow_template_preview_view_no_access(self): self._create_test_workflow() - response = self._request_test_workflow_preview_view() + response = self._request_test_workflow_template_preview_view() self.assertEqual(response.status_code, 404) self.assertTrue(self.test_workflow in Workflow.objects.all()) - def test_workflow_preview_view_with_access(self): + def test_workflow_template_preview_view_with_access(self): self._create_test_workflow() self.grant_access( obj=self.test_workflow, permission=permission_workflow_view ) - response = self._request_test_workflow_preview_view() + response = self._request_test_workflow_template_preview_view() self.assertEqual(response.status_code, 200) class WorkflowToolViewTestCase(WorkflowTestMixin, GenericDocumentViewTestCase): def _request_workflow_launch_view(self): return self.post( - viewname='document_states:tool_launch_all_workflows', + viewname='document_states:tool_launch_workflows', ) - def test_tool_launch_all_workflows_view_no_permission(self): + def test_tool_launch_workflows_view_no_permission(self): self._create_test_workflow(add_document_type=True) self._create_test_workflow_states() self._create_test_workflow_transition() @@ -129,7 +129,7 @@ def test_tool_launch_all_workflows_view_no_permission(self): self.assertEqual(self.test_document.workflows.count(), 0) - def test_tool_launch_all_workflows_view_with_permission(self): + def test_tool_launch_workflows_view_with_permission(self): self._create_test_workflow(add_document_type=True) self._create_test_workflow_states() self._create_test_workflow_transition() diff --git a/mayan/apps/document_states/urls.py b/mayan/apps/document_states/urls.py index 9dab2abd22f..6b36dd490cb 100644 --- a/mayan/apps/document_states/urls.py +++ b/mayan/apps/document_states/urls.py @@ -3,225 +3,238 @@ from django.conf.urls import url from .api_views import ( - APIDocumentTypeWorkflowListView, APIWorkflowDocumentTypeList, + APIDocumentTypeWorkflowRuntimeProxyListView, APIWorkflowDocumentTypeList, APIWorkflowDocumentTypeView, APIWorkflowImageView, APIWorkflowInstanceListView, APIWorkflowInstanceView, - APIWorkflowInstanceLogEntryListView, APIWorkflowListView, + APIWorkflowInstanceLogEntryListView, APIWorkflowRuntimeProxyListView, APIWorkflowStateListView, APIWorkflowStateView, APIWorkflowTransitionListView, APIWorkflowTransitionView, APIWorkflowView ) -from .views import ( - DocumentWorkflowInstanceListView, SetupWorkflowCreateView, - SetupWorkflowDeleteView, SetupWorkflowDocumentTypesView, - SetupWorkflowEditView, SetupWorkflowListView, - SetupWorkflowStateActionCreateView, SetupWorkflowStateActionDeleteView, - SetupWorkflowStateActionEditView, SetupWorkflowStateActionListView, - SetupWorkflowStateActionSelectionView, SetupWorkflowStateCreateView, - SetupWorkflowStateDeleteView, SetupWorkflowStateEditView, - SetupWorkflowStateListView, SetupWorkflowTransitionListView, - SetupWorkflowTransitionCreateView, SetupWorkflowTransitionDeleteView, - SetupWorkflowTransitionEditView, - SetupWorkflowTransitionTriggerEventListView, ToolLaunchAllWorkflows, - WorkflowDocumentListView, WorkflowInstanceDetailView, - WorkflowInstanceTransitionExecuteView, WorkflowInstanceTransitionSelectView, - WorkflowListView, WorkflowPreviewView, WorkflowStateDocumentListView, - WorkflowStateListView, +from .views.workflow_instance_views import ( + WorkflowInstanceDetailView, WorkflowInstanceListView, + WorkflowInstanceTransitionSelectView, + WorkflowInstanceTransitionExecuteView ) -from .views.workflow_views import ( - SetupDocumentTypeWorkflowsView, SetupWorkflowTransitionFieldCreateView, - SetupWorkflowTransitionFieldDeleteView, - SetupWorkflowTransitionFieldEditView, SetupWorkflowTransitionFieldListView +from .views.workflow_proxy_views import ( + WorkflowRuntimeProxyDocumentListView, + WorkflowRuntimeProxyListView, WorkflowRuntimeProxyStateDocumentListView, + WorkflowRuntimeProxyStateListView +) +from .views.workflow_template_views import ( + DocumentTypeWorkflowTemplatesView, ToolLaunchWorkflows, + WorkflowTemplateCreateView, WorkflowTemplateDeleteView, + WorkflowTemplateEditView, WorkflowTemplateListView, + WorkflowTemplatePreviewView, WorkflowTemplateDocumentTypesView +) +from .views.workflow_template_state_views import ( + WorkflowTemplateStateActionCreateView, + WorkflowTemplateStateActionDeleteView, WorkflowTemplateStateActionEditView, + WorkflowTemplateStateActionListView, + WorkflowTemplateStateActionSelectionView, WorkflowTemplateStateCreateView, + WorkflowTemplateStateDeleteView, WorkflowTemplateStateEditView, + WorkflowTemplateStateListView +) +from .views.workflow_template_transition_views import ( + WorkflowTemplateTransitionCreateView, WorkflowTemplateTransitionDeleteView, + WorkflowTemplateTransitionEditView, WorkflowTemplateTransitionListView, + WorkflowTemplateTransitionTriggerEventListView, + WorkflowTemplateTransitionFieldCreateView, + WorkflowTemplateTransitionFieldDeleteView, + WorkflowTemplateTransitionFieldEditView, + WorkflowTemplateTransitionFieldListView ) -urlpatterns_workflow_templates = [ - url( - regex=r'^setup/workflows/$', view=SetupWorkflowListView.as_view(), - name='setup_workflow_list' - ), +urlpatterns_workflow_instances = [ url( - regex=r'^setup/workflows/create/$', view=SetupWorkflowCreateView.as_view(), - name='setup_workflow_create' + regex=r'^documents/(?P\d+)/workflows/$', + view=WorkflowInstanceListView.as_view(), + name='workflow_instance_list' ), url( - regex=r'^setup/workflows/(?P\d+)/delete/$', - view=SetupWorkflowDeleteView.as_view(), name='setup_workflow_delete' + regex=r'^documents/workflows/(?P\d+)/$', + view=WorkflowInstanceDetailView.as_view(), + name='workflow_instance_detail' ), url( - regex=r'^setup/workflows/(?P\d+)/edit/$', - view=SetupWorkflowEditView.as_view(), name='setup_workflow_edit' + regex=r'^documents/workflows/(?P\d+)/transitions/select/$', + view=WorkflowInstanceTransitionSelectView.as_view(), + name='workflow_instance_transition_selection' ), url( - regex=r'^setup/document_types/(?P\d+)/workflows/$', - view=SetupDocumentTypeWorkflowsView.as_view(), - name='document_type_workflows' + regex=r'^documents/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', + view=WorkflowInstanceTransitionExecuteView.as_view(), + name='workflow_instance_transition_execute' ), ] -urlpatterns_workflow_states = [ +urlpatterns_workflow_runtime_proxies = [ url( - regex=r'^setup/workflows/(?P\d+)/states/$', - view=SetupWorkflowStateListView.as_view(), - name='setup_workflow_state_list' + regex=r'workflow_runtime_proxies/$', + view=WorkflowRuntimeProxyListView.as_view(), + name='workflow_runtime_proxy_list' ), url( - regex=r'^setup/workflows/(?P\d+)/states/create/$', - view=SetupWorkflowStateCreateView.as_view(), - name='setup_workflow_state_create' + regex=r'^workflow_runtime_proxies/(?P\d+)/documents/$', + view=WorkflowRuntimeProxyDocumentListView.as_view(), + name='workflow_runtime_proxy_document_list' ), url( - regex=r'^setup/workflows/states/(?P\d+)/delete/$', - view=SetupWorkflowStateDeleteView.as_view(), - name='setup_workflow_state_delete' + regex=r'^workflow_runtime_proxies/(?P\d+)/states/$', + view=WorkflowRuntimeProxyStateListView.as_view(), + name='workflow_runtime_proxy_state_list' ), url( - regex=r'^setup/workflows/states/(?P\d+)/edit/$', - view=SetupWorkflowStateEditView.as_view(), - name='setup_workflow_state_edit' + regex=r'^workflow_runtime_proxies/states/(?P\d+)/documents/$', + view=WorkflowRuntimeProxyStateDocumentListView.as_view(), + name='workflow_runtime_proxy_state_document_list' ), ] -urlpatterns_workflow_state_actions = [ - url( - regex=r'^setup/workflows/states/(?P\d+)/actions/$', - view=SetupWorkflowStateActionListView.as_view(), - name='setup_workflow_state_action_list' - ), +urlpatterns_workflow_states = [ url( - regex=r'^setup/workflows/states/(?P\d+)/actions/selection/$', - view=SetupWorkflowStateActionSelectionView.as_view(), - name='setup_workflow_state_action_selection' + regex=r'^workflow_templates/(?P\d+)/states/$', + view=WorkflowTemplateStateListView.as_view(), + name='workflow_template_state_list' ), url( - regex=r'^setup/workflows/states/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', - view=SetupWorkflowStateActionCreateView.as_view(), - name='setup_workflow_state_action_create' + regex=r'^workflow_templates/(?P\d+)/states/create/$', + view=WorkflowTemplateStateCreateView.as_view(), + name='workflow_template_state_create' ), url( - regex=r'^setup/workflows/states/actions/(?P\d+)/delete/$', - view=SetupWorkflowStateActionDeleteView.as_view(), - name='setup_workflow_state_action_delete' + regex=r'^workflow_templates/states/(?P\d+)/delete/$', + view=WorkflowTemplateStateDeleteView.as_view(), + name='workflow_template_state_delete' ), url( - regex=r'^setup/workflows/states/actions/(?P\d+)/edit/$', - view=SetupWorkflowStateActionEditView.as_view(), - name='setup_workflow_state_action_edit' + regex=r'^workflow_templates/states/(?P\d+)/edit/$', + view=WorkflowTemplateStateEditView.as_view(), + name='workflow_template_state_edit' ), ] -urlpatterns_workflow_transitions = [ +urlpatterns_workflow_state_actions = [ + url( + regex=r'^workflow_templates/states/(?P\d+)/actions/$', + view=WorkflowTemplateStateActionListView.as_view(), + name='workflow_template_state_action_list' + ), url( - regex=r'^setup/workflows/(?P\d+)/transitions/$', - view=SetupWorkflowTransitionListView.as_view(), - name='setup_workflow_transition_list' + regex=r'^workflow_templates/states/(?P\d+)/actions/selection/$', + view=WorkflowTemplateStateActionSelectionView.as_view(), + name='workflow_template_state_action_selection' ), url( - regex=r'^setup/workflows/(?P\d+)/transitions/create/$', - view=SetupWorkflowTransitionCreateView.as_view(), - name='setup_workflow_transition_create' + regex=r'^workflow_templates/states/(?P\d+)/actions/(?P[a-zA-Z0-9_.]+)/create/$', + view=WorkflowTemplateStateActionCreateView.as_view(), + name='workflow_template_state_action_create' ), url( - regex=r'^setup/workflows/(?P\d+)/transitions/events/$', - view=SetupWorkflowTransitionTriggerEventListView.as_view(), - name='setup_workflow_transition_events' + regex=r'^workflow_templates/states/actions/(?P\d+)/delete/$', + view=WorkflowTemplateStateActionDeleteView.as_view(), + name='workflow_template_state_action_delete' ), url( - regex=r'^setup/workflows/transitions/(?P\d+)/delete/$', - view=SetupWorkflowTransitionDeleteView.as_view(), - name='setup_workflow_transition_delete' + regex=r'^workflow_templates/states/actions/(?P\d+)/edit/$', + view=WorkflowTemplateStateActionEditView.as_view(), + name='workflow_template_state_action_edit' ), +] + +urlpatterns_workflow_templates = [ url( - regex=r'^setup/workflows/transitions/(?P\d+)/edit/$', - view=SetupWorkflowTransitionEditView.as_view(), - name='setup_workflow_transition_edit' + regex=r'^workflow_templates/$', view=WorkflowTemplateListView.as_view(), + name='workflow_template_list' ), url( - regex=r'^documents/workflows/(?P\d+)/transitions/select/$', - view=WorkflowInstanceTransitionSelectView.as_view(), - name='workflow_instance_transition_selection' + regex=r'^workflow_templates/create/$', view=WorkflowTemplateCreateView.as_view(), + name='workflow_template_create' ), url( - regex=r'^documents/workflows/(?P\d+)/transitions/(?P\d+)/execute/$', - view=WorkflowInstanceTransitionExecuteView.as_view(), - name='workflow_instance_transition_execute' + regex=r'^workflow_templates/(?P\d+)/delete/$', + view=WorkflowTemplateDeleteView.as_view(), name='workflow_template_delete' ), -] - -urlpatterns_workflow_transition_fields = [ url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/create/$', - view=SetupWorkflowTransitionFieldCreateView.as_view(), - name='setup_workflow_transition_field_create' + regex=r'^workflow_templates/(?P\d+)/document_types/$', + view=WorkflowTemplateDocumentTypesView.as_view(), + name='workflow_template_document_types' ), url( - regex=r'^setup/workflows/transitions/(?P\d+)/fields/$', - view=SetupWorkflowTransitionFieldListView.as_view(), - name='setup_workflow_transition_field_list' + regex=r'^workflow_templates/(?P\d+)/edit/$', + view=WorkflowTemplateEditView.as_view(), name='workflow_template_edit' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/delete/$', - view=SetupWorkflowTransitionFieldDeleteView.as_view(), - name='setup_workflow_transition_field_delete' + regex=r'^workflow_templates/(?P\d+)/preview/$', + view=WorkflowTemplatePreviewView.as_view(), + name='workflow_template_preview' ), url( - regex=r'^setup/workflows/transitions/fields/(?P\d+)/edit/$', - view=SetupWorkflowTransitionFieldEditView.as_view(), - name='setup_workflow_transition_field_edit' + regex=r'^document_types/(?P\d+)/workflow_templates/$', + view=DocumentTypeWorkflowTemplatesView.as_view(), + name='document_type_workflow_templates' ), ] - -urlpatterns = [ +urlpatterns_workflow_transitions = [ url( - regex=r'^document/(?P\d+)/workflows/$', - view=DocumentWorkflowInstanceListView.as_view(), - name='document_workflow_instance_list' + regex=r'^workflow_templates/(?P\d+)/transitions/$', + view=WorkflowTemplateTransitionListView.as_view(), + name='workflow_template_transition_list' ), url( - regex=r'^document/workflows/(?P\d+)/$', - view=WorkflowInstanceDetailView.as_view(), - name='workflow_instance_detail' + regex=r'^workflow_templates/(?P\d+)/transitions/create/$', + view=WorkflowTemplateTransitionCreateView.as_view(), + name='workflow_template_transition_create' ), url( - regex=r'^setup/workflow/(?P\d+)/documents/$', - view=WorkflowDocumentListView.as_view(), - name='setup_workflow_document_list' + regex=r'^workflow_templates/(?P\d+)/transitions/events/$', + view=WorkflowTemplateTransitionTriggerEventListView.as_view(), + name='workflow_template_transition_events' ), url( - regex=r'^setup/workflow/(?P\d+)/document_types/$', - view=SetupWorkflowDocumentTypesView.as_view(), - name='setup_workflow_document_types' + regex=r'^workflow_templates/transitions/(?P\d+)/delete/$', + view=WorkflowTemplateTransitionDeleteView.as_view(), + name='workflow_template_transition_delete' ), url( - regex=r'^tools/workflow/all/launch/$', - view=ToolLaunchAllWorkflows.as_view(), - name='tool_launch_all_workflows' + regex=r'^workflow_templates/transitions/(?P\d+)/edit/$', + view=WorkflowTemplateTransitionEditView.as_view(), + name='workflow_template_transition_edit' ), +] + +urlpatterns_workflow_transition_fields = [ url( - regex=r'all/$', - view=WorkflowListView.as_view(), - name='workflow_list' + regex=r'^workflow_templates/transitions/(?P\d+)/fields/create/$', + view=WorkflowTemplateTransitionFieldCreateView.as_view(), + name='workflow_template_transition_field_create' ), url( - regex=r'^(?P\d+)/documents/$', - view=WorkflowDocumentListView.as_view(), - name='workflow_document_list' + regex=r'^workflow_templates/transitions/(?P\d+)/fields/$', + view=WorkflowTemplateTransitionFieldListView.as_view(), + name='workflow_template_transition_field_list' ), url( - regex=r'^(?P\d+)/states/$', - view=WorkflowStateListView.as_view(), - name='workflow_state_list' + regex=r'^workflow_templates/transitions/fields/(?P\d+)/delete/$', + view=WorkflowTemplateTransitionFieldDeleteView.as_view(), + name='workflow_template_transition_field_delete' ), url( - regex=r'^(?P\d+)/preview/$', - view=WorkflowPreviewView.as_view(), - name='workflow_preview' + regex=r'^workflow_templates/transitions/fields/(?P\d+)/edit/$', + view=WorkflowTemplateTransitionFieldEditView.as_view(), + name='workflow_template_transition_field_edit' ), +] + +urlpatterns = [ url( - regex=r'^state/(?P\d+)/documents/$', - view=WorkflowStateDocumentListView.as_view(), - name='workflow_state_document_list' + regex=r'^tools/workflows/launch/$', + view=ToolLaunchWorkflows.as_view(), + name='tool_launch_workflows' ), ] +urlpatterns.extend(urlpatterns_workflow_instances) +urlpatterns.extend(urlpatterns_workflow_runtime_proxies) urlpatterns.extend(urlpatterns_workflow_states) urlpatterns.extend(urlpatterns_workflow_state_actions) urlpatterns.extend(urlpatterns_workflow_templates) @@ -230,7 +243,7 @@ api_urls = [ url( - regex=r'^workflows/$', view=APIWorkflowListView.as_view(), + regex=r'^workflows/$', view=APIWorkflowRuntimeProxyListView.as_view(), name='workflow-list' ), url( @@ -286,7 +299,7 @@ ), url( regex=r'^document_types/(?P[0-9]+)/workflows/$', - view=APIDocumentTypeWorkflowListView.as_view(), + view=APIDocumentTypeWorkflowRuntimeProxyListView.as_view(), name='documenttype-workflow-list' ), ] diff --git a/mayan/apps/document_states/views/__init__.py b/mayan/apps/document_states/views/__init__.py index d6e36751b2c..8b137891791 100644 --- a/mayan/apps/document_states/views/__init__.py +++ b/mayan/apps/document_states/views/__init__.py @@ -1,3 +1 @@ -from .workflow_instance_views import * # NOQA -from .workflow_proxy_views import * # NOQA -from .workflow_views import * # NOQA + diff --git a/mayan/apps/document_states/views/workflow_instance_views.py b/mayan/apps/document_states/views/workflow_instance_views.py index bed57fdcbc4..6fcfd38fbdf 100644 --- a/mayan/apps/document_states/views/workflow_instance_views.py +++ b/mayan/apps/document_states/views/workflow_instance_views.py @@ -14,20 +14,14 @@ from mayan.apps.documents.models import Document from ..forms import WorkflowInstanceTransitionSelectForm -from ..icons import icon_workflow_instance_detail, icon_workflow_list +from ..icons import icon_workflow_instance_detail, icon_workflow_template_list from ..links import link_workflow_instance_transition from ..literals import FIELD_TYPE_MAPPING, WIDGET_CLASS_MAPPING from ..models import WorkflowInstance from ..permissions import permission_workflow_view -__all__ = ( - 'DocumentWorkflowInstanceListView', 'WorkflowInstanceDetailView', - 'WorkflowInstanceTransitionSelectView', - 'WorkflowInstanceTransitionExecuteView' -) - -class DocumentWorkflowInstanceListView(SingleObjectListView): +class WorkflowInstanceListView(SingleObjectListView): def dispatch(self, request, *args, **kwargs): AccessControlList.objects.check_access( obj=self.get_document(), permissions=(permission_workflow_view,), @@ -35,7 +29,7 @@ def dispatch(self, request, *args, **kwargs): ) return super( - DocumentWorkflowInstanceListView, self + WorkflowInstanceListView, self ).dispatch(request, *args, **kwargs) def get_document(self): @@ -44,7 +38,7 @@ def get_document(self): def get_extra_context(self): return { 'hide_link': True, - 'no_results_icon': icon_workflow_list, + 'no_results_icon': icon_workflow_template_list, 'no_results_text': _( 'Assign workflows to the document type of this document ' 'to have this document execute those workflows. ' diff --git a/mayan/apps/document_states/views/workflow_proxy_views.py b/mayan/apps/document_states/views/workflow_proxy_views.py index 21efa2bb9e2..77bd864ea45 100644 --- a/mayan/apps/document_states/views/workflow_proxy_views.py +++ b/mayan/apps/document_states/views/workflow_proxy_views.py @@ -9,18 +9,13 @@ from mayan.apps.documents.models import Document from mayan.apps.documents.views import DocumentListView -from ..icons import icon_workflow_list -from ..links import link_setup_workflow_create, link_setup_workflow_state_create +from ..icons import icon_workflow_template_list +from ..links import link_workflow_template_create, link_workflow_template_state_create from ..models import WorkflowRuntimeProxy, WorkflowStateRuntimeProxy from ..permissions import permission_workflow_view -__all__ = ( - 'WorkflowDocumentListView', 'WorkflowListView', - 'WorkflowStateDocumentListView', 'WorkflowStateListView' -) - -class WorkflowDocumentListView(DocumentListView): +class WorkflowRuntimeProxyDocumentListView(DocumentListView): def dispatch(self, request, *args, **kwargs): self.workflow = get_object_or_404( klass=WorkflowRuntimeProxy, pk=self.kwargs['pk'] @@ -32,14 +27,14 @@ def dispatch(self, request, *args, **kwargs): ) return super( - WorkflowDocumentListView, self + WorkflowRuntimeProxyDocumentListView, self ).dispatch(request, *args, **kwargs) def get_document_queryset(self): return Document.objects.filter(workflows__workflow=self.workflow) def get_extra_context(self): - context = super(WorkflowDocumentListView, self).get_extra_context() + context = super(WorkflowRuntimeProxyDocumentListView, self).get_extra_context() context.update( { 'no_results_text': _( @@ -56,14 +51,14 @@ def get_extra_context(self): return context -class WorkflowListView(SingleObjectListView): +class WorkflowRuntimeProxyListView(SingleObjectListView): object_permission = permission_workflow_view def get_extra_context(self): return { 'hide_object': True, - 'no_results_icon': icon_workflow_list, - 'no_results_main_link': link_setup_workflow_create.resolve( + 'no_results_icon': icon_workflow_template_list, + 'no_results_main_link': link_workflow_template_create.resolve( context=RequestContext(request=self.request) ), 'no_results_text': _( @@ -79,13 +74,13 @@ def get_source_queryset(self): return WorkflowRuntimeProxy.objects.all() -class WorkflowStateDocumentListView(DocumentListView): +class WorkflowRuntimeProxyStateDocumentListView(DocumentListView): def get_document_queryset(self): return self.get_workflow_state().get_documents() def get_extra_context(self): workflow_state = self.get_workflow_state() - context = super(WorkflowStateDocumentListView, self).get_extra_context() + context = super(WorkflowRuntimeProxyStateDocumentListView, self).get_extra_context() context.update( { 'object': workflow_state, @@ -118,7 +113,7 @@ def get_workflow_state(self): return workflow_state -class WorkflowStateListView(SingleObjectListView): +class WorkflowRuntimeProxyStateListView(SingleObjectListView): def dispatch(self, request, *args, **kwargs): AccessControlList.objects.check_access( obj=self.get_workflow(), permissions=(permission_workflow_view,), @@ -126,14 +121,14 @@ def dispatch(self, request, *args, **kwargs): ) return super( - WorkflowStateListView, self + WorkflowRuntimeProxyStateListView, self ).dispatch(request, *args, **kwargs) def get_extra_context(self): return { 'hide_link': True, 'hide_object': True, - 'no_results_main_link': link_setup_workflow_state_create.resolve( + 'no_results_main_link': link_workflow_template_state_create.resolve( context=RequestContext( request=self.request, dict_={'object': self.get_workflow()} ) diff --git a/mayan/apps/document_states/views/workflow_template_state_views.py b/mayan/apps/document_states/views/workflow_template_state_views.py new file mode 100644 index 00000000000..a0f1e3cec7b --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_state_views.py @@ -0,0 +1,326 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class WorkflowTemplateStateActionCreateView(SingleObjectDynamicFormCreateView): + form_class = WorkflowStateActionDynamicForm + object_permission = permission_workflow_edit + + def get_class(self): + try: + return WorkflowAction.get(name=self.kwargs['class_path']) + except KeyError: + raise Http404( + '{} class not found'.format(self.kwargs['class_path']) + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow'), + 'object': self.get_object(), + 'title': _( + 'Create a "%s" workflow action' + ) % self.get_class().label, + 'workflow': self.get_object().workflow + } + + def get_form_extra_kwargs(self): + return { + 'request': self.request, + 'action_path': self.kwargs['class_path'] + } + + def get_form_schema(self): + return self.get_class()().get_form_schema(request=self.request) + + def get_instance_extra_data(self): + return { + 'action_path': self.kwargs['class_path'], + 'state': self.get_object() + } + + def get_object(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().pk} + ) + + +class WorkflowTemplateStateActionDeleteView(SingleObjectDeleteView): + model = WorkflowStateAction + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_state', 'workflow' + ), + 'object': self.get_object(), + 'title': _('Delete workflow state action: %s') % self.get_object(), + 'workflow': self.get_object().state.workflow, + 'workflow_state': self.get_object().state, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().state.pk} + ) + + +class WorkflowTemplateStateActionEditView(SingleObjectDynamicFormEditView): + form_class = WorkflowStateActionDynamicForm + model = WorkflowStateAction + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_state', 'workflow' + ), + 'object': self.get_object(), + 'title': _('Edit workflow state action: %s') % self.get_object(), + 'workflow': self.get_object().state.workflow, + 'workflow_state': self.get_object().state, + } + + def get_form_extra_kwargs(self): + return { + 'request': self.request, + 'action_path': self.get_object().action_path, + } + + def get_form_schema(self): + return self.get_object().get_class_instance().get_form_schema( + request=self.request + ) + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_state_action_list', + kwargs={'pk': self.get_object().state.pk} + ) + + +class WorkflowTemplateStateActionListView(SingleObjectListView): + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'hide_object': True, + 'navigation_object_list': ('object', 'workflow'), + 'no_results_icon': icon_workflow_state_action, + 'no_results_main_link': link_workflow_template_state_action_selection.resolve( + context=RequestContext( + request=self.request, dict_={ + 'object': self.get_workflow_state() + } + ) + ), + 'no_results_text': _( + 'Workflow state actions are macros that get executed when ' + 'documents enters or leaves the state in which they reside.' + ), + 'no_results_title': _( + 'There are no actions for this workflow state' + ), + 'object': self.get_workflow_state(), + 'title': _( + 'Actions for workflow state: %s' + ) % self.get_workflow_state(), + 'workflow': self.get_workflow_state().workflow, + } + + def get_form_schema(self): + return {'fields': self.get_class().fields} + + def get_source_queryset(self): + return self.get_workflow_state().actions.all() + + def get_workflow_state(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + +class WorkflowTemplateStateActionSelectionView(FormView): + form_class = WorkflowActionSelectionForm + view_permission = permission_workflow_edit + + def form_valid(self, form): + klass = form.cleaned_data['klass'] + return HttpResponseRedirect( + redirect_to=reverse( + viewname='document_states:workflow_template_state_action_create', + kwargs={'pk': self.get_object().pk, 'class_path': klass} + ) + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow' + ), + 'object': self.get_object(), + 'title': _('New workflow state action selection'), + 'workflow': self.get_object().workflow, + } + + def get_object(self): + return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) + + +class WorkflowTemplateStateCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = Workflow + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowStateForm + + def get_extra_context(self): + return { + 'object': self.get_workflow(), + 'title': _( + 'Create states for workflow: %s' + ) % self.get_workflow() + } + + def get_instance_extra_data(self): + return {'workflow': self.get_workflow()} + + def get_source_queryset(self): + return self.get_workflow().states.all() + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.kwargs['pk']} + ) + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateStateDeleteView(SingleObjectDeleteView): + model = WorkflowState + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Delete workflow state: %s?' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateStateEditView(SingleObjectEditView): + form_class = WorkflowStateForm + model = WorkflowState + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Edit workflow state: %s' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_state_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateStateListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = Workflow + external_object_permission = permission_workflow_view + external_object_pk_url_kwarg = 'pk' + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_state, + 'no_results_main_link': link_workflow_template_state_create.resolve( + context=RequestContext( + self.request, {'object': self.get_workflow()} + ) + ), + 'no_results_text': _( + 'Create states and link them using transitions.' + ), + 'no_results_title': _( + 'This workflow doesn\'t have any states' + ), + 'object': self.get_workflow(), + 'title': _('States of workflow: %s') % self.get_workflow() + } + + def get_source_queryset(self): + return self.get_workflow().states.all() + + def get_workflow(self): + return self.external_object diff --git a/mayan/apps/document_states/views/workflow_template_transition_views.py b/mayan/apps/document_states/views/workflow_template_transition_views.py new file mode 100644 index 00000000000..e56a10a6259 --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_transition_views.py @@ -0,0 +1,372 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class WorkflowTemplateTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = Workflow + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowTransitionForm + + def get_extra_context(self): + return { + 'object': self.get_workflow(), + 'title': _( + 'Create transitions for workflow: %s' + ) % self.get_workflow() + } + + def get_form_kwargs(self): + kwargs = super( + WorkflowTemplateTransitionCreateView, self + ).get_form_kwargs() + kwargs['workflow'] = self.get_workflow() + return kwargs + + def get_instance_extra_data(self): + return {'workflow': self.get_workflow()} + + def get_source_queryset(self): + return self.get_workflow().transitions.all() + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.kwargs['pk']} + ) + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateTransitionDeleteView(SingleObjectDeleteView): + model = WorkflowTransition + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'object': self.get_object(), + 'navigation_object_list': ('object', 'workflow_instance'), + 'title': _( + 'Delete workflow transition: %s?' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionEditView(SingleObjectEditView): + form_class = WorkflowTransitionForm + model = WorkflowTransition + object_permission = permission_workflow_edit + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'navigation_object_list': ('object', 'workflow_instance'), + 'object': self.get_object(), + 'title': _( + 'Edit workflow transition: %s' + ) % self.object, + 'workflow_instance': self.get_object().workflow, + } + + def get_form_kwargs(self): + kwargs = super( + WorkflowTemplateTransitionEditView, self + ).get_form_kwargs() + kwargs['workflow'] = self.get_object().workflow + return kwargs + + def get_success_url(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = Workflow + external_object_permission = permission_workflow_view + external_object_pk_url_kwarg = 'pk' + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_transition, + 'no_results_main_link': link_workflow_template_transition_create.resolve( + context=RequestContext( + self.request, {'object': self.get_workflow()} + ) + ), + 'no_results_text': _( + 'Create a transition and use it to move a workflow from ' + ' one state to another.' + ), + 'no_results_title': _( + 'This workflow doesn\'t have any transitions' + ), + 'object': self.get_workflow(), + 'title': _( + 'Transitions of workflow: %s' + ) % self.get_workflow() + } + + def get_source_queryset(self): + return self.get_workflow().transitions.all() + + def get_workflow(self): + return self.external_object + + +class WorkflowTemplateTransitionTriggerEventListView(ExternalObjectMixin, FormView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + external_object_pk_url_kwarg = 'pk' + form_class = WorkflowTransitionTriggerEventRelationshipFormSet + + def dispatch(self, *args, **kwargs): + EventType.refresh() + return super( + WorkflowTemplateTransitionTriggerEventListView, self + ).dispatch(*args, **kwargs) + + def form_valid(self, form): + try: + for instance in form: + instance.save() + except Exception as exception: + messages.error( + message=_( + 'Error updating workflow transition trigger events; %s' + ) % exception, request=self.request + + ) + else: + messages.success( + message=_( + 'Workflow transition trigger events updated successfully' + ), request=self.request + ) + + return super( + WorkflowTemplateTransitionTriggerEventListView, self + ).form_valid(form=form) + + def get_extra_context(self): + return { + 'form_display_mode_table': True, + 'navigation_object_list': ('object', 'workflow'), + 'object': self.get_object(), + 'subtitle': _( + 'Triggers are events that cause this transition to execute ' + 'automatically.' + ), + 'title': _( + 'Workflow transition trigger events for: %s' + ) % self.get_object(), + 'workflow': self.get_object().workflow, + } + + def get_initial(self): + obj = self.get_object() + initial = [] + + # Return the queryset by name from the sorted list of the class + event_type_ids = [event_type.id for event_type in EventType.all()] + event_type_queryset = StoredEventType.objects.filter( + name__in=event_type_ids + ) + + # Sort queryset in Python by namespace, then by label + event_type_queryset = sorted( + event_type_queryset, key=lambda x: (x.namespace, x.label) + ) + + for event_type in event_type_queryset: + initial.append({ + 'transition': obj, + 'event_type': event_type, + }) + return initial + + def get_object(self): + return self.external_object + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_list', + kwargs={'pk': self.get_object().workflow.pk} + ) + + +class WorkflowTemplateTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCreateView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + + def get_extra_context(self): + return { + 'navigation_object_list': ('transition', 'workflow'), + 'transition': self.external_object, + 'title': _( + 'Create a field for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow + } + + def get_instance_extra_data(self): + return { + 'transition': self.external_object, + } + + def get_queryset(self): + return self.external_object.fields.all() + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.external_object.pk} + ) + + +class WorkflowTemplateTransitionFieldDeleteView(SingleObjectDeleteView): + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Delete workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class WorkflowTemplateTransitionFieldEditView(SingleObjectEditView): + fields = ( + 'name', 'label', 'field_type', 'help_text', 'required', 'widget', + 'widget_kwargs' + ) + model = WorkflowTransitionField + object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'navigation_object_list': ( + 'object', 'workflow_transition', 'workflow' + ), + 'object': self.object, + 'title': _('Edit workflow transition field: %s') % self.object, + 'workflow': self.object.transition.workflow, + 'workflow_transition': self.object.transition, + } + + def get_post_action_redirect(self): + return reverse( + viewname='document_states:workflow_template_transition_field_list', + kwargs={'pk': self.object.transition.pk} + ) + + +class WorkflowTemplateTransitionFieldListView(ExternalObjectMixin, SingleObjectListView): + external_object_class = WorkflowTransition + external_object_permission = permission_workflow_edit + + def get_extra_context(self): + return { + 'hide_object': True, + 'navigation_object_list': ('object', 'workflow'), + 'no_results_icon': icon_workflow_transition_field, + 'no_results_main_link': link_workflow_template_transition_field_create.resolve( + context=RequestContext( + request=self.request, dict_={ + 'object': self.external_object + } + ) + ), + 'no_results_text': _( + 'Workflow transition fields allow adding data to the ' + 'workflow\'s context. This additional context data can then ' + 'be used by other elements of the workflow system like the ' + 'workflow state actions.' + ), + 'no_results_title': _( + 'There are no fields for this workflow transition' + ), + 'object': self.external_object, + 'title': _( + 'Fields for workflow transition: %s' + ) % self.external_object, + 'workflow': self.external_object.workflow, + } + + def get_source_queryset(self): + return self.external_object.fields.all() diff --git a/mayan/apps/document_states/views/workflow_template_views.py b/mayan/apps/document_states/views/workflow_template_views.py new file mode 100644 index 00000000000..e755f42e339 --- /dev/null +++ b/mayan/apps/document_states/views/workflow_template_views.py @@ -0,0 +1,261 @@ +from __future__ import absolute_import, unicode_literals + +from django.contrib import messages +from django.db import transaction +from django.http import Http404, HttpResponseRedirect +from django.shortcuts import get_object_or_404 +from django.template import RequestContext +from django.urls import reverse, reverse_lazy +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.generics import ( + AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, + SingleObjectDeleteView, SingleObjectDetailView, + SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + SingleObjectEditView, SingleObjectListView +) +from mayan.apps.common.mixins import ExternalObjectMixin +from mayan.apps.documents.events import event_document_type_edited +from mayan.apps.documents.models import DocumentType +from mayan.apps.documents.permissions import permission_document_type_edit +from mayan.apps.events.classes import EventType +from mayan.apps.events.models import StoredEventType + +from ..classes import WorkflowAction +from ..events import event_workflow_edited +from ..forms import ( + WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, + WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, + WorkflowTransitionTriggerEventRelationshipFormSet +) +from ..icons import ( + icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, + icon_workflow_transition, icon_workflow_transition_field +) +from ..links import ( + link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_action_selection, + link_workflow_template_transition_create, + link_workflow_template_transition_field_create, +) +from ..models import ( + Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, + WorkflowTransitionField +) +from ..permissions import ( + permission_workflow_create, permission_workflow_delete, + permission_workflow_edit, permission_workflow_tools, + permission_workflow_view, +) +from ..tasks import task_launch_all_workflows + + +class DocumentTypeWorkflowTemplatesView(AddRemoveView): + main_object_permission = permission_document_type_edit + main_object_model = DocumentType + main_object_pk_url_kwarg = 'pk' + secondary_object_model = Workflow + secondary_object_permission = permission_workflow_edit + list_available_title = _('Available workflows') + list_added_title = _('Workflows assigned this document type') + related_field = 'workflows' + + def get_actions_extra_kwargs(self): + return {'_user': self.request.user} + + def get_extra_context(self): + return { + 'object': self.main_object, + 'subtitle': _( + 'Removing a workflow from a document type will also ' + 'remove all running instances of that workflow.' + ), + 'title': _( + 'Workflows assigned the document type: %s' + ) % self.main_object, + } + + def action_add(self, queryset, _user): + with transaction.atomic(): + event_document_type_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.workflows.add(obj) + event_workflow_edited.commit( + action_object=self.main_object, actor=_user, target=obj + ) + + def action_remove(self, queryset, _user): + with transaction.atomic(): + event_document_type_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.workflows.remove(obj) + event_workflow_edited.commit( + action_object=self.main_object, actor=_user, + target=obj + ) + obj.instances.filter( + document__document_type=self.main_object + ).delete() + + +class WorkflowTemplateListView(SingleObjectListView): + model = Workflow + object_permission = permission_workflow_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'no_results_icon': icon_workflow_template_list, + 'no_results_main_link': link_workflow_template_create.resolve( + context=RequestContext(request=self.request) + ), + 'no_results_text': _( + 'Workflows store a series of states and keep track of the ' + 'current state of a document. Transitions are used to change the ' + 'current state to a new one.' + ), + 'no_results_title': _( + 'No workflows have been defined' + ), + 'title': _('Workflows'), + } + + +class WorkflowTemplateCreateView(SingleObjectCreateView): + extra_context = {'title': _('Create workflow')} + form_class = WorkflowForm + model = Workflow + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + view_permission = permission_workflow_create + + def get_save_extra_data(self): + return {'_user': self.request.user} + + +class WorkflowTemplateDeleteView(SingleObjectDeleteView): + model = Workflow + object_permission = permission_workflow_delete + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + + def get_extra_context(self): + return { + 'title': _( + 'Delete workflow: %s?' + ) % self.object, + } + + +class WorkflowTemplateEditView(SingleObjectEditView): + form_class = WorkflowForm + model = Workflow + object_permission = permission_workflow_edit + post_action_redirect = reverse_lazy( + viewname='document_states:workflow_template_list' + ) + + def get_extra_context(self): + return { + 'title': _( + 'Edit workflow: %s' + ) % self.object, + } + + def get_save_extra_data(self): + return {'_user': self.request.user} + + +class WorkflowTemplateDocumentTypesView(AddRemoveView): + main_object_permission = permission_workflow_edit + main_object_model = Workflow + main_object_pk_url_kwarg = 'pk' + secondary_object_model = DocumentType + secondary_object_permission = permission_document_type_edit + list_available_title = _('Available document types') + list_added_title = _('Document types assigned this workflow') + related_field = 'document_types' + + def get_actions_extra_kwargs(self): + return {'_user': self.request.user} + + def get_extra_context(self): + return { + 'object': self.main_object, + 'subtitle': _( + 'Removing a document type from a workflow will also ' + 'remove all running instances of that workflow for ' + 'documents of the document type just removed.' + ), + 'title': _( + 'Document types assigned the workflow: %s' + ) % self.main_object, + } + + def action_add(self, queryset, _user): + with transaction.atomic(): + event_workflow_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.document_types.add(obj) + event_document_type_edited.commit( + action_object=self.main_object, actor=_user, target=obj + ) + + def action_remove(self, queryset, _user): + with transaction.atomic(): + event_workflow_edited.commit( + actor=_user, target=self.main_object + ) + + for obj in queryset: + self.main_object.document_types.remove(obj) + event_document_type_edited.commit( + action_object=self.main_object, actor=_user, + target=obj + ) + self.main_object.instances.filter( + document__document_type=obj + ).delete() + + +class WorkflowTemplatePreviewView(SingleObjectDetailView): + form_class = WorkflowPreviewForm + model = Workflow + object_permission = permission_workflow_view + pk_url_kwarg = 'pk' + + def get_extra_context(self): + return { + 'hide_labels': True, + 'object': self.get_object(), + 'title': _('Preview of: %s') % self.get_object() + } + + +class ToolLaunchWorkflows(ConfirmView): + extra_context = { + 'title': _('Launch all workflows?'), + 'subtitle': _( + 'This will launch all workflows created after documents have ' + 'already been uploaded.' + ) + } + view_permission = permission_workflow_tools + + def view_action(self): + task_launch_all_workflows.apply_async() + messages.success( + message=_('Workflow launch queued successfully.'), + request=self.request + ) diff --git a/mayan/apps/document_states/views/workflow_views.py b/mayan/apps/document_states/views/workflow_views.py deleted file mode 100644 index 104560a805e..00000000000 --- a/mayan/apps/document_states/views/workflow_views.py +++ /dev/null @@ -1,884 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -from django.contrib import messages -from django.db import transaction -from django.http import Http404, HttpResponseRedirect -from django.shortcuts import get_object_or_404 -from django.template import RequestContext -from django.urls import reverse, reverse_lazy -from django.utils.translation import ugettext_lazy as _ - -from mayan.apps.common.generics import ( - AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, - SingleObjectDeleteView, SingleObjectDetailView, - SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, - SingleObjectEditView, SingleObjectListView -) -from mayan.apps.common.mixins import ExternalObjectMixin -from mayan.apps.documents.events import event_document_type_edited -from mayan.apps.documents.models import DocumentType -from mayan.apps.documents.permissions import permission_document_type_edit -from mayan.apps.events.classes import EventType -from mayan.apps.events.models import StoredEventType - -from ..classes import WorkflowAction -from ..events import event_workflow_edited -from ..forms import ( - WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, - WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, - WorkflowTransitionTriggerEventRelationshipFormSet -) -from ..icons import ( - icon_workflow_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition, icon_workflow_transition_field -) -from ..links import ( - link_setup_workflow_create, link_setup_workflow_state_create, - link_setup_workflow_state_action_selection, - link_setup_workflow_transition_create, - link_setup_workflow_transition_field_create, -) -from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, - WorkflowTransitionField -) -from ..permissions import ( - permission_workflow_create, permission_workflow_delete, - permission_workflow_edit, permission_workflow_tools, - permission_workflow_view, -) -from ..tasks import task_launch_all_workflows - -__all__ = ( - 'SetupWorkflowListView', 'SetupWorkflowCreateView', 'SetupWorkflowEditView', - 'SetupWorkflowDeleteView', 'SetupWorkflowDocumentTypesView', - 'SetupWorkflowStateActionCreateView', 'SetupWorkflowStateActionDeleteView', - 'SetupWorkflowStateActionEditView', 'SetupWorkflowStateActionListView', - 'SetupWorkflowStateActionSelectionView', 'SetupWorkflowStateCreateView', - 'SetupWorkflowStateDeleteView', 'SetupWorkflowStateEditView', - 'SetupWorkflowStateListView', 'SetupWorkflowTransitionCreateView', - 'SetupWorkflowTransitionDeleteView', 'SetupWorkflowTransitionEditView', - 'SetupWorkflowTransitionListView', - 'SetupWorkflowTransitionTriggerEventListView', 'ToolLaunchAllWorkflows', - 'WorkflowPreviewView' -) - - -class SetupDocumentTypeWorkflowsView(AddRemoveView): - main_object_permission = permission_document_type_edit - main_object_model = DocumentType - main_object_pk_url_kwarg = 'pk' - secondary_object_model = Workflow - secondary_object_permission = permission_workflow_edit - list_available_title = _('Available workflows') - list_added_title = _('Workflows assigned this document type') - related_field = 'workflows' - - def get_actions_extra_kwargs(self): - return {'_user': self.request.user} - - def get_extra_context(self): - return { - 'object': self.main_object, - 'subtitle': _( - 'Removing a workflow from a document type will also ' - 'remove all running instances of that workflow.' - ), - 'title': _( - 'Workflows assigned the document type: %s' - ) % self.main_object, - } - - def action_add(self, queryset, _user): - with transaction.atomic(): - event_document_type_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.workflows.add(obj) - event_workflow_edited.commit( - action_object=self.main_object, actor=_user, target=obj - ) - - def action_remove(self, queryset, _user): - with transaction.atomic(): - event_document_type_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.workflows.remove(obj) - event_workflow_edited.commit( - action_object=self.main_object, actor=_user, - target=obj - ) - obj.instances.filter( - document__document_type=self.main_object - ).delete() - - -class SetupWorkflowListView(SingleObjectListView): - model = Workflow - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_list, - 'no_results_main_link': link_setup_workflow_create.resolve( - context=RequestContext(request=self.request) - ), - 'no_results_text': _( - 'Workflows store a series of states and keep track of the ' - 'current state of a document. Transitions are used to change the ' - 'current state to a new one.' - ), - 'no_results_title': _( - 'No workflows have been defined' - ), - 'title': _('Workflows'), - } - - -class SetupWorkflowCreateView(SingleObjectCreateView): - extra_context = {'title': _('Create workflow')} - form_class = WorkflowForm - model = Workflow - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - view_permission = permission_workflow_create - - def get_save_extra_data(self): - return {'_user': self.request.user} - - -class SetupWorkflowDeleteView(SingleObjectDeleteView): - model = Workflow - object_permission = permission_workflow_delete - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - - def get_extra_context(self): - return { - 'title': _( - 'Delete workflow: %s?' - ) % self.object, - } - - -class SetupWorkflowEditView(SingleObjectEditView): - form_class = WorkflowForm - model = Workflow - object_permission = permission_workflow_edit - post_action_redirect = reverse_lazy( - viewname='document_states:setup_workflow_list' - ) - - def get_extra_context(self): - return { - 'title': _( - 'Edit workflow: %s' - ) % self.object, - } - - def get_save_extra_data(self): - return {'_user': self.request.user} - - -class SetupWorkflowDocumentTypesView(AddRemoveView): - main_object_permission = permission_workflow_edit - main_object_model = Workflow - main_object_pk_url_kwarg = 'pk' - secondary_object_model = DocumentType - secondary_object_permission = permission_document_type_edit - list_available_title = _('Available document types') - list_added_title = _('Document types assigned this workflow') - related_field = 'document_types' - - def get_actions_extra_kwargs(self): - return {'_user': self.request.user} - - def get_extra_context(self): - return { - 'object': self.main_object, - 'subtitle': _( - 'Removing a document type from a workflow will also ' - 'remove all running instances of that workflow for ' - 'documents of the document type just removed.' - ), - 'title': _( - 'Document types assigned the workflow: %s' - ) % self.main_object, - } - - def action_add(self, queryset, _user): - with transaction.atomic(): - event_workflow_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.document_types.add(obj) - event_document_type_edited.commit( - action_object=self.main_object, actor=_user, target=obj - ) - - def action_remove(self, queryset, _user): - with transaction.atomic(): - event_workflow_edited.commit( - actor=_user, target=self.main_object - ) - - for obj in queryset: - self.main_object.document_types.remove(obj) - event_document_type_edited.commit( - action_object=self.main_object, actor=_user, - target=obj - ) - self.main_object.instances.filter( - document__document_type=obj - ).delete() - - -# Workflow state actions - - -class SetupWorkflowStateActionCreateView(SingleObjectDynamicFormCreateView): - form_class = WorkflowStateActionDynamicForm - object_permission = permission_workflow_edit - - def get_class(self): - try: - return WorkflowAction.get(name=self.kwargs['class_path']) - except KeyError: - raise Http404( - '{} class not found'.format(self.kwargs['class_path']) - ) - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow'), - 'object': self.get_object(), - 'title': _( - 'Create a "%s" workflow action' - ) % self.get_class().label, - 'workflow': self.get_object().workflow - } - - def get_form_extra_kwargs(self): - return { - 'request': self.request, - 'action_path': self.kwargs['class_path'] - } - - def get_form_schema(self): - return self.get_class()().get_form_schema(request=self.request) - - def get_instance_extra_data(self): - return { - 'action_path': self.kwargs['class_path'], - 'state': self.get_object() - } - - def get_object(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().pk} - ) - - -class SetupWorkflowStateActionDeleteView(SingleObjectDeleteView): - model = WorkflowStateAction - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_state', 'workflow' - ), - 'object': self.get_object(), - 'title': _('Delete workflow state action: %s') % self.get_object(), - 'workflow': self.get_object().state.workflow, - 'workflow_state': self.get_object().state, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().state.pk} - ) - - -class SetupWorkflowStateActionEditView(SingleObjectDynamicFormEditView): - form_class = WorkflowStateActionDynamicForm - model = WorkflowStateAction - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_state', 'workflow' - ), - 'object': self.get_object(), - 'title': _('Edit workflow state action: %s') % self.get_object(), - 'workflow': self.get_object().state.workflow, - 'workflow_state': self.get_object().state, - } - - def get_form_extra_kwargs(self): - return { - 'request': self.request, - 'action_path': self.get_object().action_path, - } - - def get_form_schema(self): - return self.get_object().get_class_instance().get_form_schema( - request=self.request - ) - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_state_action_list', - kwargs={'pk': self.get_object().state.pk} - ) - - -class SetupWorkflowStateActionListView(SingleObjectListView): - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'hide_object': True, - 'navigation_object_list': ('object', 'workflow'), - 'no_results_icon': icon_workflow_state_action, - 'no_results_main_link': link_setup_workflow_state_action_selection.resolve( - context=RequestContext( - request=self.request, dict_={ - 'object': self.get_workflow_state() - } - ) - ), - 'no_results_text': _( - 'Workflow state actions are macros that get executed when ' - 'documents enters or leaves the state in which they reside.' - ), - 'no_results_title': _( - 'There are no actions for this workflow state' - ), - 'object': self.get_workflow_state(), - 'title': _( - 'Actions for workflow state: %s' - ) % self.get_workflow_state(), - 'workflow': self.get_workflow_state().workflow, - } - - def get_form_schema(self): - return {'fields': self.get_class().fields} - - def get_source_queryset(self): - return self.get_workflow_state().actions.all() - - def get_workflow_state(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - -class SetupWorkflowStateActionSelectionView(FormView): - form_class = WorkflowActionSelectionForm - view_permission = permission_workflow_edit - - def form_valid(self, form): - klass = form.cleaned_data['klass'] - return HttpResponseRedirect( - redirect_to=reverse( - viewname='document_states:setup_workflow_state_action_create', - kwargs={'pk': self.get_object().pk, 'class_path': klass} - ) - ) - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow' - ), - 'object': self.get_object(), - 'title': _('New workflow state action selection'), - 'workflow': self.get_object().workflow, - } - - def get_object(self): - return get_object_or_404(klass=WorkflowState, pk=self.kwargs['pk']) - - -# Workflow states - - -class SetupWorkflowStateCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = Workflow - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowStateForm - - def get_extra_context(self): - return { - 'object': self.get_workflow(), - 'title': _( - 'Create states for workflow: %s' - ) % self.get_workflow() - } - - def get_instance_extra_data(self): - return {'workflow': self.get_workflow()} - - def get_source_queryset(self): - return self.get_workflow().states.all() - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.kwargs['pk']} - ) - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowStateDeleteView(SingleObjectDeleteView): - model = WorkflowState - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Delete workflow state: %s?' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowStateEditView(SingleObjectEditView): - form_class = WorkflowStateForm - model = WorkflowState - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Edit workflow state: %s' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_state_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowStateListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = Workflow - external_object_permission = permission_workflow_view - external_object_pk_url_kwarg = 'pk' - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_state, - 'no_results_main_link': link_setup_workflow_state_create.resolve( - context=RequestContext( - self.request, {'object': self.get_workflow()} - ) - ), - 'no_results_text': _( - 'Create states and link them using transitions.' - ), - 'no_results_title': _( - 'This workflow doesn\'t have any states' - ), - 'object': self.get_workflow(), - 'title': _('States of workflow: %s') % self.get_workflow() - } - - def get_source_queryset(self): - return self.get_workflow().states.all() - - def get_workflow(self): - return self.external_object - - -# Transitions - - -class SetupWorkflowTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = Workflow - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowTransitionForm - - def get_extra_context(self): - return { - 'object': self.get_workflow(), - 'title': _( - 'Create transitions for workflow: %s' - ) % self.get_workflow() - } - - def get_form_kwargs(self): - kwargs = super( - SetupWorkflowTransitionCreateView, self - ).get_form_kwargs() - kwargs['workflow'] = self.get_workflow() - return kwargs - - def get_instance_extra_data(self): - return {'workflow': self.get_workflow()} - - def get_source_queryset(self): - return self.get_workflow().transitions.all() - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.kwargs['pk']} - ) - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowTransitionDeleteView(SingleObjectDeleteView): - model = WorkflowTransition - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'object': self.get_object(), - 'navigation_object_list': ('object', 'workflow_instance'), - 'title': _( - 'Delete workflow transition: %s?' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowTransitionEditView(SingleObjectEditView): - form_class = WorkflowTransitionForm - model = WorkflowTransition - object_permission = permission_workflow_edit - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'navigation_object_list': ('object', 'workflow_instance'), - 'object': self.get_object(), - 'title': _( - 'Edit workflow transition: %s' - ) % self.object, - 'workflow_instance': self.get_object().workflow, - } - - def get_form_kwargs(self): - kwargs = super( - SetupWorkflowTransitionEditView, self - ).get_form_kwargs() - kwargs['workflow'] = self.get_object().workflow - return kwargs - - def get_success_url(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -class SetupWorkflowTransitionListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = Workflow - external_object_permission = permission_workflow_view - external_object_pk_url_kwarg = 'pk' - object_permission = permission_workflow_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'no_results_icon': icon_workflow_transition, - 'no_results_main_link': link_setup_workflow_transition_create.resolve( - context=RequestContext( - self.request, {'object': self.get_workflow()} - ) - ), - 'no_results_text': _( - 'Create a transition and use it to move a workflow from ' - ' one state to another.' - ), - 'no_results_title': _( - 'This workflow doesn\'t have any transitions' - ), - 'object': self.get_workflow(), - 'title': _( - 'Transitions of workflow: %s' - ) % self.get_workflow() - } - - def get_source_queryset(self): - return self.get_workflow().transitions.all() - - def get_workflow(self): - return self.external_object - - -class SetupWorkflowTransitionTriggerEventListView(ExternalObjectMixin, FormView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - external_object_pk_url_kwarg = 'pk' - form_class = WorkflowTransitionTriggerEventRelationshipFormSet - - def dispatch(self, *args, **kwargs): - EventType.refresh() - return super( - SetupWorkflowTransitionTriggerEventListView, self - ).dispatch(*args, **kwargs) - - def form_valid(self, form): - try: - for instance in form: - instance.save() - except Exception as exception: - messages.error( - message=_( - 'Error updating workflow transition trigger events; %s' - ) % exception, request=self.request - - ) - else: - messages.success( - message=_( - 'Workflow transition trigger events updated successfully' - ), request=self.request - ) - - return super( - SetupWorkflowTransitionTriggerEventListView, self - ).form_valid(form=form) - - def get_extra_context(self): - return { - 'form_display_mode_table': True, - 'navigation_object_list': ('object', 'workflow'), - 'object': self.get_object(), - 'subtitle': _( - 'Triggers are events that cause this transition to execute ' - 'automatically.' - ), - 'title': _( - 'Workflow transition trigger events for: %s' - ) % self.get_object(), - 'workflow': self.get_object().workflow, - } - - def get_initial(self): - obj = self.get_object() - initial = [] - - # Return the queryset by name from the sorted list of the class - event_type_ids = [event_type.id for event_type in EventType.all()] - event_type_queryset = StoredEventType.objects.filter( - name__in=event_type_ids - ) - - # Sort queryset in Python by namespace, then by label - event_type_queryset = sorted( - event_type_queryset, key=lambda x: (x.namespace, x.label) - ) - - for event_type in event_type_queryset: - initial.append({ - 'transition': obj, - 'event_type': event_type, - }) - return initial - - def get_object(self): - return self.external_object - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_list', - kwargs={'pk': self.get_object().workflow.pk} - ) - - -# Transition fields - -class SetupWorkflowTransitionFieldCreateView(ExternalObjectMixin, SingleObjectCreateView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - fields = ( - 'name', 'label', 'field_type', 'help_text', 'required', 'widget', - 'widget_kwargs' - ) - - def get_extra_context(self): - return { - 'navigation_object_list': ('transition', 'workflow'), - 'transition': self.external_object, - 'title': _( - 'Create a field for workflow transition: %s' - ) % self.external_object, - 'workflow': self.external_object.workflow - } - - def get_instance_extra_data(self): - return { - 'transition': self.external_object, - } - - def get_queryset(self): - return self.external_object.fields.all() - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.external_object.pk} - ) - - -class SetupWorkflowTransitionFieldDeleteView(SingleObjectDeleteView): - model = WorkflowTransitionField - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_transition', 'workflow' - ), - 'object': self.object, - 'title': _('Delete workflow transition field: %s') % self.object, - 'workflow': self.object.transition.workflow, - 'workflow_transition': self.object.transition, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.object.transition.pk} - ) - - -class SetupWorkflowTransitionFieldEditView(SingleObjectEditView): - fields = ( - 'name', 'label', 'field_type', 'help_text', 'required', 'widget', - 'widget_kwargs' - ) - model = WorkflowTransitionField - object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'navigation_object_list': ( - 'object', 'workflow_transition', 'workflow' - ), - 'object': self.object, - 'title': _('Edit workflow transition field: %s') % self.object, - 'workflow': self.object.transition.workflow, - 'workflow_transition': self.object.transition, - } - - def get_post_action_redirect(self): - return reverse( - viewname='document_states:setup_workflow_transition_field_list', - kwargs={'pk': self.object.transition.pk} - ) - - -class SetupWorkflowTransitionFieldListView(ExternalObjectMixin, SingleObjectListView): - external_object_class = WorkflowTransition - external_object_permission = permission_workflow_edit - - def get_extra_context(self): - return { - 'hide_object': True, - 'navigation_object_list': ('object', 'workflow'), - 'no_results_icon': icon_workflow_transition_field, - 'no_results_main_link': link_setup_workflow_transition_field_create.resolve( - context=RequestContext( - request=self.request, dict_={ - 'object': self.external_object - } - ) - ), - 'no_results_text': _( - 'Workflow transition fields allow adding data to the ' - 'workflow\'s context. This additional context data can then ' - 'be used by other elements of the workflow system like the ' - 'workflow state actions.' - ), - 'no_results_title': _( - 'There are no fields for this workflow transition' - ), - 'object': self.external_object, - 'title': _( - 'Fields for workflow transition: %s' - ) % self.external_object, - 'workflow': self.external_object.workflow, - } - - def get_source_queryset(self): - return self.external_object.fields.all() - - -class ToolLaunchAllWorkflows(ConfirmView): - extra_context = { - 'title': _('Launch all workflows?'), - 'subtitle': _( - 'This will launch all workflows created after documents have ' - 'already been uploaded.' - ) - } - view_permission = permission_workflow_tools - - def view_action(self): - task_launch_all_workflows.apply_async() - messages.success( - message=_('Workflow launch queued successfully.'), - request=self.request - ) - - -class WorkflowPreviewView(SingleObjectDetailView): - form_class = WorkflowPreviewForm - model = Workflow - object_permission = permission_workflow_view - pk_url_kwarg = 'pk' - - def get_extra_context(self): - return { - 'hide_labels': True, - 'object': self.get_object(), - 'title': _('Preview of: %s') % self.get_object() - } diff --git a/mayan/apps/mailer/tests/test_actions.py b/mayan/apps/mailer/tests/test_actions.py index eed6d9f49ad..cf21f68b0d7 100644 --- a/mayan/apps/mailer/tests/test_actions.py +++ b/mayan/apps/mailer/tests/test_actions.py @@ -139,7 +139,7 @@ def test_email_action_create_get_view(self): self._create_test_user_mailer() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', @@ -151,7 +151,7 @@ def test_email_action_create_get_view(self): def _request_email_action_create_post_view(self): return self.post( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.mailer.workflow_actions.EmailAction', diff --git a/mayan/apps/tags/tests/test_actions.py b/mayan/apps/tags/tests/test_actions.py index 0aa8c550789..ec0fe3caa5e 100644 --- a/mayan/apps/tags/tests/test_actions.py +++ b/mayan/apps/tags/tests/test_actions.py @@ -39,7 +39,7 @@ def test_tag_attach_action_create_view(self): self._create_test_workflow_state() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.tags.workflow_actions.AttachTagAction' @@ -53,7 +53,7 @@ def test_tag_remove_action_create_view(self): self._create_test_workflow_state() response = self.get( - viewname='document_states:setup_workflow_state_action_create', + viewname='document_states:workflow_template_state_action_create', kwargs={ 'pk': self.test_workflow_state.pk, 'class_path': 'mayan.apps.tags.workflow_actions.RemoveTagAction' From 8c73fda1ae98cae5402455113ce6fe7e90ebf194 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 02:35:14 -0400 Subject: [PATCH 023/402] Rename installjavascript to installdependencies Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docker/Dockerfile | 2 +- docs/releases/3.3.rst | 1 + .../commands/{installjavascript.py => installdependencies.py} | 0 4 files changed, 3 insertions(+), 1 deletion(-) rename mayan/apps/dependencies/management/commands/{installjavascript.py => installdependencies.py} (100%) diff --git a/HISTORY.rst b/HISTORY.rst index 389a9ccc5c6..0744e906848 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ - Backport workflow transitions field support. - Backport workflow email action. - Backport individual index rebuild support. +- Rename the installjavascript command to installdependencies. 3.2.5 (2019-07-05) ================== diff --git a/docker/Dockerfile b/docker/Dockerfile index 14ae97ed690..2f364989998 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -122,7 +122,7 @@ RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \ # Install the built Mayan EDMS package && pip install --no-cache-dir --no-use-pep517 dist/mayan* \ # Install the static content -&& mayan-edms.py installjavascript \ +&& mayan-edms.py installdependencies \ && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}" diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index ba872b7fdb4..e6e437790ce 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -24,6 +24,7 @@ Changes - Backport workflow transitions field support. - Backport workflow email action. - Backport individual index rebuild support. +- Rename the installjavascript command to installdependencies. Removals -------- diff --git a/mayan/apps/dependencies/management/commands/installjavascript.py b/mayan/apps/dependencies/management/commands/installdependencies.py similarity index 100% rename from mayan/apps/dependencies/management/commands/installjavascript.py rename to mayan/apps/dependencies/management/commands/installdependencies.py From 2cc35c3c61610b05470573ea595e28b4f0d10e31 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 02:37:58 -0400 Subject: [PATCH 024/402] Remove outdated contrib scripts Signed-off-by: Roberto Rosario --- contrib/scripts/install/development.sh | 72 - contrib/scripts/install/dialog.sh | 1733 ------------------------ contrib/scripts/install/production.sh | 171 --- contrib/scripts/start_gunicorn.sh | 35 - 4 files changed, 2011 deletions(-) delete mode 100644 contrib/scripts/install/development.sh delete mode 100644 contrib/scripts/install/dialog.sh delete mode 100644 contrib/scripts/install/production.sh delete mode 100644 contrib/scripts/start_gunicorn.sh diff --git a/contrib/scripts/install/development.sh b/contrib/scripts/install/development.sh deleted file mode 100644 index a8c9497c464..00000000000 --- a/contrib/scripts/install/development.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -INSTALLATION_DIRECTORY=/home/vagrant/mayan-edms/ -DB_NAME=mayan_edms -DB_PASSWORD=test123 - -cat << EOF | sudo tee -a /etc/motd.tail -**********************************sudo apt - -Mayan EDMS Vagrant Development Box - -********************************** -EOF - -# Update sources -echo -e "\n -> Running apt-get update & upgrade \n" -sudo apt-get -qq update -sudo apt-get -y upgrade - -echo -e "\n -> Installing core binaries \n" -sudo apt-get -y install git-core python-virtualenv gcc python-dev libjpeg-dev libpng-dev libtiff-dev tesseract-ocr poppler-utils libreoffice - -echo -e "\n -> Cloning development branch of repository \n" -git clone /mayan-edms-repository/ $INSTALLATION_DIRECTORY -cd $INSTALLATION_DIRECTORY -git checkout development -git reset HEAD --hard - -echo -e "\n -> Setting up virtual env \n" -virtualenv venv -source venv/bin/activate - -echo -e "\n -> Installing python dependencies \n" -pip install -r requirements.txt - -echo -e "\n -> Running Mayan EDMS initial setup \n" -./manage.py initialsetup - -echo -e "\n -> Installing Redis server \n" -sudo apt-get install -y redis-server -pip install redis - -echo -e "\n -> Installing testing software \n" -pip install coverage - -echo -e "\n -> Installing MySQL \n" -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password password '$DB_PASSWORD -sudo debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password '$DB_PASSWORD -sudo apt-get install -y mysql-server libmysqlclient-dev -# Create a passwordless root and travis users -mysql -u root -p$DB_PASSWORD -e "SET PASSWORD = PASSWORD('');" -mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';GRANT ALL PRIVILEGES ON * . * TO 'travis'@'localhost';FLUSH PRIVILEGES;" -mysql -u travis -e "CREATE DATABASE $DB_NAME;" -pip install mysql-python - -echo -e "\n -> Installing PostgreSQL \n" -sudo apt-get install -y postgresql postgresql-server-dev-all -sudo -u postgres psql -c 'create database mayan_edms;' -U postgres -sudo cat > /etc/postgresql/9.3/main/pg_hba.conf << EOF -local all postgres trust - -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all peer -# IPv4 local connections: -host all all 127.0.0.1/32 md5 -# IPv6 local connections: -host all all ::1/128 md5 -EOF - -pip install -q psycopg2 diff --git a/contrib/scripts/install/dialog.sh b/contrib/scripts/install/dialog.sh deleted file mode 100644 index e7d929639c3..00000000000 --- a/contrib/scripts/install/dialog.sh +++ /dev/null @@ -1,1733 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017 Igor Pecovnik, igor.pecovnik@gma**.com -# -# This file is licensed under the terms of the GNU General Public -# License version 2. This program is licensed "as is" without any -# warranty of any kind, whether express or implied. - -# Functions: -# check_status -# choose_webserver -# server_conf -# install_packet -# alive_port -# alive_process -# install_basic -# create_ispconfig_configuration -# install_cups -# install_samba -# install_omv -# install_tvheadend -# install_urbackup -# install_transmission -# install_transmission_seed_armbian_torrents -# install_syncthing -# install_vpn_server -# install_vpn_client -# install_DashNTP -# install_MySQL -# install_MySQLDovecot -# install_Virus -# install_hhvm -# install_phpmyadmin -# install_apache -# install_nginx -# install_PureFTPD -# install_Bind -# install_Stats -# install_Jailkit -# install_Fail2BanDovecot -# install_Fail2BanRulesDovecot -# install_ISPConfig -# check_if_installed - -# -# load functions, local first -# -if [[ -f debian-config-jobs ]]; then source debian-config-jobs; - elif [[ -f /usr/lib/armbian-config/jobs.sh ]]; then source /usr/lib/armbian-config/jobs.sh; - else exit 1; -fi - -if [[ -f debian-config-submenu ]]; then source debian-config-submenu; - elif [[ -f /usr/lib/armbian-config/submenu.sh ]]; then source /usr/lib/armbian-config/submenu.sh; - else exit 1; -fi - -if [[ -f debian-config-functions ]]; then source debian-config-functions; - elif [[ -f /usr/lib/armbian-config/functions.sh ]]; then source /usr/lib/armbian-config/functions.sh; - else exit 1; -fi - -if [[ -f debian-config-functions-network ]]; then source debian-config-functions-network; - elif [[ -f /usr/lib/armbian-config/functions-network.sh ]]; then source /usr/lib/armbian-config/functions-network.sh; - else exit 1; -fi - - - - -# -# not sure if needed -# -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - - - - -function check_status -{ -# -# Check if service is already installed and show it's status -# - -dialog --backtitle "$BACKTITLE" --title "Please wait" --infobox "\nLoading install info ... " 5 28 -LIST=() -LIST_CONST=3 - -# Samba -SAMBA_STATUS="$(check_if_installed samba && echo "on" || echo "off" )" -alive_port "Windows compatible file sharing" "445" "boolean" -LIST+=( "Samba" "$DESCRIPTION" "$SAMBA_STATUS" ) - -# CUPS -CUPS_STATUS="$(check_if_installed cups && echo "on" || echo "off" )" -alive_port "Common UNIX Printing System (CUPS)" "631" "boolean" -LIST+=( "CUPS" "$DESCRIPTION" "$CUPS_STATUS" ) - -# TV headend -TVHEADEND_STATUS="$(check_if_installed tvheadend && echo "on" || echo "off" )" -alive_port "TV streaming server" "9981" -LIST+=( "TV headend" "$DESCRIPTION" "$TVHEADEND_STATUS" ) - -# Synthing -SYNCTHING_STATUS="$([[ -d /usr/bin/syncthing ]] && echo "on" || echo "off" )" -alive_port "Personal cloud @syncthing.net" "8384" "boolean" -LIST+=( "Syncthing" "$DESCRIPTION" "$SYNCTHING_STATUS" ) - -# Mayan EDMS -MAYAN_STATUS="$( [[ -d /opt/mayan ]] && echo "on" || echo "off" )" -alive_port "Electronic Document Management System" "8000" "boolean" -LIST+=( "Mayan EDMS" "$DESCRIPTION" "$MAYAN_STATUS" ) - -# Exagear -if [[ "$(check_if_installed xserver-xorg && echo "on")" == "on" && "$family" == "Ubuntu" ]]; then - EXAGEAR_STATUS="$(check_if_installed exagear-armbian && echo "on" || echo "off" )" - LIST+=( "ExaGear" "32bit x86 Linux/Windows emulator trial" "$EXAGEAR_STATUS" ) -fi - -if [[ "$(dpkg --print-architecture)" == "armhf" || "$(dpkg --print-architecture)" == "amd64" ]]; then - LIST_CONST=2 - # vpn server - VPN_SERVER_STATUS="$([[ -d /usr/local/vpnserver ]] && echo "on" || echo "off" )" - LIST+=( "VPN server" "Softether VPN server" "$VPN_SERVER_STATUS" ) - # vpn client - VPN_CLIENT_STATUS="$([[ -d /usr/local/vpnclient ]] && echo "on" || echo "off" )" - LIST+=( "VPN client" "Softether VPN client" "$VPN_CLIENT_STATUS" ) -fi -# NCP -NCP_STATUS="$( [[ -d /var/www/nextcloud ]] && echo "on" || echo "off" )" -[[ "$family" != "Ubuntu" ]] && LIST+=( "NCP" "Nextcloud personal cloud" "$NCP_STATUS" ) -# OMV -OMV_STATUS="$(check_if_installed openmediavault && echo "on" || echo "off" )" -[[ "$family" != "Ubuntu" ]] && LIST+=( "OMV" "OpenMediaVault NAS solution" "$OMV_STATUS" ) && LIST_CONST=3 - -# Plex media server -PLEX_STATUS="$((check_if_installed plexmediaserver || check_if_installed plexmediaserver-installer) && echo "on" || echo "off" )" -alive_port "Plex media server" "32400" -LIST+=( "Plex" "$DESCRIPTION" "$PLEX_STATUS" ) - -# Radarr -RADARR_STATUS="$([[ -d /opt/Radarr ]] && echo "on" || echo "off" )" -alive_port "Movies downloading server" "7878" -LIST+=( "Radarr" "$DESCRIPTION" "$RADARR_STATUS" ) - -# Sonarr -SONARR_STATUS="$([[ -d /opt/NzbDrone ]] && echo "on" || echo "off" )" -alive_port "TV shows downloading server" "8989" -LIST+=( "Sonarr" "$DESCRIPTION" "$SONARR_STATUS" ) - -# MINIdlna -MINIDLNA_STATUS="$(check_if_installed minidlna && echo "on" || echo "off" )" -alive_port "Lightweight DLNA/UPnP-AV server" "8200" "boolean" -LIST+=( "Minidlna" "$DESCRIPTION" "$MINIDLNA_STATUS" ) - -# Pi hole -PI_HOLE_STATUS="$([[ -d /etc/pihole ]] && echo "on" || echo "off" )" -alive_process "Ad blocker" "pihole-FTL" -LIST+=( "Pi hole" "$DESCRIPTION" "$PI_HOLE_STATUS" ) - -# Transmission -TRANSMISSION_STATUS="$(check_if_installed transmission-daemon && echo "on" || echo "off" )" -alive_port "Torrent download server" "9091" "boolean" -LIST+=( "Transmission" "$DESCRIPTION" "$TRANSMISSION_STATUS" ) - - -# UrBackup -URBACKUP_STATUS="$((check_if_installed urbackup-server || check_if_installed urbackup-server-dbg) && echo "on" || echo "off" )" -alive_port "Client/server backup system" "51413" "boolean" -LIST+=( "UrBackup" "$DESCRIPTION" "$URBACKUP_STATUS" ) - - -# ISPconfig -ISPCONFIG_STATUS="$([[ -d /usr/local/ispconfig ]] && echo "on" || echo "off" )" -LIST+=( "ISPConfig" "SMTP mail, IMAP, POP3 & LAMP/LEMP web server" "$ISPCONFIG_STATUS" ) -} - - - - -function choose_webserver -{ -# -# Target web server selection -# -check_if_installed openmediavault -case $? in - 0) - # OMV installed, prevent switching from nginx to apache which would trash OMV installation - server="nginx" - ;; - *) - dialog --title "Choose a webserver" --backtitle "$BACKTITLE" --yes-label "Apache" --no-label "Nginx" \ - --yesno "\nChoose a web server which you are familiar with. They both work almost the same." 8 70 - response=$? - case $response in - 0) server="apache";; - 1) server="nginx";; - 255) exit;; - esac - ;; -esac -} - - - - -function server_conf -{ -# -# Add some reqired date for installation -# -exec 3>&1 -dialog --title "Server configuration" --separate-widget $'\n' --ok-label "Install" --backtitle "$BACKTITLE" \ ---form "\nPlease fill out this form:\n " \ -12 70 0 \ -"Your FQDN for $serverip:" 1 1 "$hostnamefqdn" 1 31 32 0 \ -"Mysql root password:" 2 1 "$mysql_pass" 2 31 32 0 \ -2>&1 1>&3 | { - -read -r hostnamefqdn -read -r mysql_pass -echo $mysql_pass > ${TEMP_DIR}/mysql_pass -echo $hostnamefqdn > ${TEMP_DIR}/hostnamefqdn -# end -} -exec 3>&- -# read variables back -read MYSQL_PASS < ${TEMP_DIR}/mysql_pass -read HOSTNAMEFQDN < ${TEMP_DIR}/hostnamefqdn -} - - - - -install_packet () -{ -# -# Install missing packets -# -i=0 -j=1 -IFS=" " -declare -a PACKETS=($1) -#skupaj=$(apt-get -s -y -qq install $1 | wc -l) -skupaj=${#PACKETS[@]} -while [[ $i -lt $skupaj ]]; do -procent=$(echo "scale=2;($j/$skupaj)*100"|bc) - x=${PACKETS[$i]} - if [ $(dpkg-query -W -f='${Status}' $x 2>/dev/null | grep -c "ok installed") -eq 0 ]; then - printf '%.0f\n' $procent | dialog \ - --backtitle "$BACKTITLE" \ - --title "Installing" \ - --gauge "\n$2\n\n$x" 10 70 - if [ "$(DEBIAN_FRONTEND=noninteractive apt-get -qq -y install $x >${TEMP_DIR}/install.log 2>&1 || echo 'Installation failed' \ - | grep 'Installation failed')" != "" ]; then - echo -e "[\e[0;31m error \x1B[0m] Installation failed" - tail ${TEMP_DIR}/install.log - exit - fi - fi - i=$[$i+1] - j=$[$j+1] -done -echo "" -} - - -alive_port () -{ -# -# Displays URL to the service $1 on port $2 or just that is active if $3 = boolean -# -DEFAULT_ADAPTER=$(ip -4 route ls | grep default | grep -Po '(?<=dev )(\S+)') -LOCALIPADD=$(ip -4 addr show dev $DEFAULT_ADAPTER | awk '/inet/ {print $2}' | cut -d'/' -f1) -if [[ -n $(netstat -lnt | awk '$6 == "LISTEN" && $4 ~ ".'$2'"') ]]; then - if [[ $3 == boolean ]]; then - DESCRIPTION="$1 is \Z1active\Z0"; - else - DESCRIPTION="Active on http://${LOCALIPADD}:\Z1$2\Z0"; - fi -else -DESCRIPTION="$1"; -fi -} - - - -alive_process () -{ -# -# check if process name $2 is running. Display it's name $1 or $1 is active if active -# -if pgrep -x "$2" > /dev/null 2>&1; then DESCRIPTION="$1 is \Z1active\Z0"; else DESCRIPTION="$1"; fi -} - - - - - - - -install_basic (){ -# -# Set hostname, FQDN, add to sources list -# -IFS=" " -set ${HOSTNAMEFQDN//./ } -HOSTNAMESHORT="$1" -cp /etc/hosts /etc/hosts.backup -cp /etc/hostname /etc/hostname.backup -# create new -echo "127.0.0.1 localhost.localdomain localhost" > /etc/hosts -echo "${serverIP} ${HOSTNAMEFQDN} ${HOSTNAMESHORT} #ispconfig " >> /etc/hosts -echo "$HOSTNAMESHORT" > /etc/hostname -/etc/init.d/hostname.sh start >/dev/null 2>&1 -hostnamectl set-hostname $HOSTNAMESHORT -if [[ $family == "Ubuntu" ]]; then - # set hostname in Ubuntu - hostnamectl set-hostname $HOSTNAMESHORT - # disable AppArmor - if [[ -n $(service apparmor status | grep -w active | grep -w running) ]]; then - service apparmor stop - update-rc.d -f apparmor remove - apt-get -y -qq remove apparmor apparmor-utils - fi -else - grep -q "contrib" /etc/apt/sources.list || sed -i 's|main|main contrib|' /etc/apt/sources.list - grep -q "non-free" /etc/apt/sources.list || sed -i 's|contrib|contrib non-free|' /etc/apt/sources.list - grep -q "deb http://ftp.debian.org/debian jessie-backports main" /etc/apt/sources.list || echo "deb http://ftp.debian.org/debian jessie-backports main" >> /etc/apt/sources.list - debconf-apt-progress -- apt-get update -fi -} - - - - -create_ispconfig_configuration (){ -# -# ISPConfig autoconfiguration -# -cat > ${TEMP_DIR}/isp.conf.php < -EOF -} - - - -install_cups () -{ -# -# Install printer system -# -debconf-apt-progress -- apt-get -y install cups lpr cups-filters -# cups-filters if jessie -sed -e 's/Listen localhost:631/Listen 631/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -sed -e 's//\nallow $SUBNET/g' -i /etc/cups/cupsd.conf -service cups restart -service samba restart | service smbd restart >/dev/null 2>&1 -} - - - - -install_samba () -{ -# -# install Samba file sharing -# -local SECTION="Samba" -SMBUSER=$(whiptail --inputbox "What is your samba username?" 8 78 $SMBUSER --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -SMBPASS=$(whiptail --inputbox "What is your samba password?" 8 78 $SMBPASS --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -SMBGROUP=$(whiptail --inputbox "What is your samba group?" 8 78 $SMBGROUP --title "$SECTION" 3>&1 1>&2 2>&3) -exitstatus=$?; if [ $exitstatus = 1 ]; then exit 1; fi -# -debconf-apt-progress -- apt-get -y install samba samba-common-bin samba-vfs-modules -useradd $SMBUSER -echo -ne "$SMBPASS\n$SMBPASS\n" | passwd $SMBUSER >/dev/null 2>&1 -echo -ne "$SMBPASS\n$SMBPASS\n" | smbpasswd -a -s $SMBUSER >/dev/null 2>&1 -service samba stop | service smbd stop >/dev/null 2>&1 -cp /etc/samba/smb.conf /etc/samba/smb.conf.stock -cat > /etc/samba/smb.conf.tmp << EOF -[global] - workgroup = SMBGROUP - server string = %h server - hosts allow = SUBNET - log file = /var/log/samba/log.%m - max log size = 1000 - syslog = 0 - panic action = /usr/share/samba/panic-action %d - load printers = yes - printing = cups - printcap name = cups - min receivefile size = 16384 - write cache size = 524288 - getwd cache = yes - socket options = TCP_NODELAY IPTOS_LOWDELAY - -[printers] - comment = All Printers - path = /var/spool/samba - browseable = no - public = yes - guest ok = yes - writable = no - printable = yes - printer admin = SMBUSER - -[print$] - comment = Printer Drivers - path = /etc/samba/drivers - browseable = yes - guest ok = no - read only = yes - write list = SMBUSER - -[ext] - comment = Storage - path = /ext - writable = yes - public = no - valid users = SMBUSER - force create mode = 0644 -EOF -sed -i "s/SMBGROUP/$SMBGROUP/" /etc/samba/smb.conf.tmp -sed -i "s/SMBUSER/$SMBUSER/" /etc/samba/smb.conf.tmp -sed -i "s/SUBNET/$SUBNET/" /etc/samba/smb.conf.tmp -dialog --backtitle "$BACKTITLE" --title "Review samba configuration" --no-collapse --editbox /etc/samba/smb.conf.tmp 30 0 2> /etc/samba/smb.conf.tmp.out -if [[ $? = 0 ]]; then - mv /etc/samba/smb.conf.tmp.out /etc/samba/smb.conf - install -m 755 -g $SMBUSER -o $SMBUSER -d /ext - service service smbd stop >/dev/null 2>&1 - sleep 3 - service service smbd start >/dev/null 2>&1 -fi -} - -install_ncp (){ - curl -sSL https://raw.githubusercontent.com/nextcloud/nextcloudpi/master/install.sh | bash -} - -install_omv (){ -# -# On Debian install OpenMediaVault 3 (Jessie) or 4 (Stretch) -# -# TODO: Some OMV packages lack authentication - -if [[ "$family" == "Ubuntu" ]]; then - dialog --backtitle "$BACKTITLE" --title "Dependencies not met" --msgbox "\nOpenMediaVault can only be installed on Debian." 7 52 - sleep 5 - exit 1 -fi - -case $distribution in - jessie) - OMV_Name="erasmus" - OMV_EXTRAS_URL="https://github.com/OpenMediaVault-Plugin-Developers/packages/raw/master/openmediavault-omvextrasorg_latest_all3.deb" - ;; - stretch) - OMV_Name="arrakis" - OMV_EXTRAS_URL="https://github.com/OpenMediaVault-Plugin-Developers/packages/raw/master/openmediavault-omvextrasorg_latest_all4.deb" - ;; -esac - -systemctl status log2ram >/dev/null 2>&1 && (systemctl stop log2ram ; systemctl disable log2ram >/dev/null 2>&1; rm /etc/cron.daily/log2ram) -export APT_LISTCHANGES_FRONTEND=none -if [ -f /etc/armbian-release ]; then - . /etc/armbian-release -else - sed -i "s/^# en_US.UTF-8/en_US.UTF-8/" /etc/locale.gen - locale-gen -fi - -# preserve cpufrequtils settings: -if [ -f /etc/default/cpufrequtils ]; then - . /etc/default/cpufrequtils -fi - -cat > /etc/apt/sources.list.d/openmediavault.list << EOF -deb https://openmediavault.github.io/packages/ ${OMV_Name} main - -## Uncomment the following line to add software from the proposed repository. -deb https://openmediavault.github.io/packages/ ${OMV_Name}-proposed main - -## This software is not part of OpenMediaVault, but is offered by third-party -## developers as a service to OpenMediaVault users. -# deb https://openmediavault.github.io/packages/ ${OMV_Name} partner -EOF - -debconf-apt-progress -- apt-get update - -read HOSTNAME /dev/null | awk -F" " '/additional disk space will be used/ {print $4}') -SPACE_AVAIL=$(df -k / | awk -F" " '/\/$/ {printf ("%0.0f",$4/1200); }') -if [ ${SPACE_AVAIL} -lt ${SPACE_NEEDED} ]; then - dialog --backtitle "$BACKTITLE" --title "No space left on device" --msgbox "\nOpenMediaVault needs ${SPACE_NEEDED} MB for installation while only ${SPACE_AVAIL} MB are available." 7 52 - exit 1 -fi -apt-get --allow-unauthenticated install openmediavault-keyring -apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 7AA630A1EDEE7D73 -debconf-apt-progress -- apt-get -y --allow-unauthenticated --fix-missing --no-install-recommends \ - -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install openmediavault postfix dirmngr -FILE="${TEMP_DIR}/omv_extras.deb"; wget "$OMV_EXTRAS_URL" -qO $FILE && dpkg -i $FILE ; rm $FILE -# /usr/sbin/omv-update -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get --yes --force-yes --fix-missing --auto-remove --allow-unauthenticated \ - --show-upgraded --option DPkg::Options::="--force-confold" dist-upgrade - -# Install flashmemory plugin and netatalk by default, use nice logo for the latter, -# disable OMV monitoring by default -. /usr/share/openmediavault/scripts/helper-functions -debconf-apt-progress -- apt-get -y --fix-missing --no-install-recommends --auto-remove install openmediavault-flashmemory openmediavault-netatalk -AFP_Options="mimic model = Macmini" -SMB_Options="min receivefile size = 16384\nwrite cache size = 524288\ngetwd cache = yes\nsocket options = TCP_NODELAY IPTOS_LOWDELAY" -xmlstarlet ed -L -u "/config/services/afp/extraoptions" -v "$(echo -e "${AFP_Options}")" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/smb/extraoptions" -v "$(echo -e "${SMB_Options}")" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/flashmemory/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/ssh/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/services/ssh/permitrootlogin" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/time/ntp/enable" -v "1" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/time/timezone" -v "${TZ}" ${OMV_CONFIG_FILE} -xmlstarlet ed -L -u "/config/system/network/dns/hostname" -v "${HOSTNAME}" ${OMV_CONFIG_FILE} -/usr/sbin/omv-rpc -u admin "perfstats" "set" '{"enable":false}' -/usr/sbin/omv-rpc -u admin "config" "applyChanges" '{ "modules": ["monit","rrdcached","collectd"],"force": true }' -sed -i 's|-j /var/lib/rrdcached/journal/ ||' /etc/init.d/rrdcached -/sbin/folder2ram -enablesystemd 2>/dev/null - -# Prevent accidentally destroying board performance by clicking around in OMV UI since -# OMV sets 'powersave' governor when touching 'Power Management' settings. -if [ ! -f /etc/default/cpufrequtils ]; then - DEFAULT_GOV="$(zgrep "^CONFIG_CPU_FREQ_DEFAULT_GOV_" /proc/config.gz 2>/dev/null | sed 's/CONFIG_CPU_FREQ_DEFAULT_GOV_//')" - if [ -n "${DEFAULT_GOV}" ]; then - GOVERNOR=$(cut -f1 -d= <<<"${DEFAULT_GOV}" | tr '[:upper:]' '[:lower:]') - else - GOVERNOR=ondemand - fi - MIN_SPEED="0" - MAX_SPEED="0" -fi -echo -e "OMV_CPUFREQUTILS_GOVERNOR=${GOVERNOR}" >>/etc/default/openmediavault -echo -e "OMV_CPUFREQUTILS_MINSPEED=${MIN_SPEED}" >>/etc/default/openmediavault -echo -e "OMV_CPUFREQUTILS_MAXSPEED=${MAX_SPEED}" >>/etc/default/openmediavault -for i in netatalk samba flashmemory ssh ntp timezone monit rrdcached collectd cpufrequtils ; do - /usr/sbin/omv-mkconf $i -done - -# Hardkernel Cloudshell 1 and 2 fixes, read the whole thread for details: -# https://forum.openmediavault.org/index.php/Thread/17855 -lsusb | grep -q -i "05e3:0735" && sed -i "/exit 0/i echo 20 > /sys/class/block/sda/queue/max_sectors_kb" /etc/rc.local -if [ "X${BOARD}" = "Xodroidxu4" ]; then - HMP_Fix='; taskset -c -p 4-7 $i ' - apt install -y i2c-tools - /usr/sbin/i2cdetect -y 1 | grep -q "60: 60" - if [ $? -eq 0 ]; then - add-apt-repository -y ppa:kyle1117/ppa - sed -i 's/jessie/xenial/' /etc/apt/sources.list.d/kyle1117-ppa-jessie.list - apt install -y -q cloudshell-lcd odroid-cloudshell cloudshell2-fan & - lsusb -v | awk -F"__" '/RANDOM_/ {print $2}' | head -n1 | while read ; do - echo "ATTRS{idVendor}==\"152d\", ATTRS{idProduct}==\"0561\", KERNEL==\"sd*\", ENV{DEVTYPE}==\"disk\", SYMLINK=\"disk/by-id/\$env{ID_BUS}-CloudShell2-${REPLY}-\$env{ID_MODEL}\"" >> /etc/udev/rules.d/99-cloudshell2.rules - echo "ATTRS{idVendor}==\"152d\", ATTRS{idProduct}==\"0561\", KERNEL==\"sd*\", ENV{DEVTYPE}==\"partition\", SYMLINK=\"disk/by-id/\$env{ID_BUS}-CloudShell2-${REPLY}-\$env{ID_MODEL}-part%n\"" >> /etc/udev/rules.d/99-cloudshell2.rules - done - fi -fi - -# Add a cron job to make NAS processes more snappy -systemctl status rsyslog >/dev/null 2>&1 -if [ $? -eq 0 ]; then - echo ':msg, contains, "do ionice -c1" ~' >/etc/rsyslog.d/omv-armbian.conf - systemctl restart rsyslog -fi -echo "* * * * * root for i in \`pgrep \"ftpd|nfsiod|smbd|afpd|cnid\"\` ; do ionice -c1 -p \$i ${HMP_Fix}; done >/dev/null 2>&1" >/etc/cron.d/make_nas_processes_faster -chmod 600 /etc/cron.d/make_nas_processes_faster - -/usr/sbin/omv-initsystem -} - - - - -install_tvheadend () -{ -# -# TVheadend https://tvheadend.org/ unofficial port https://tvheadend.org/boards/5/topics/21528 -# -if [ ! -f /etc/apt/sources.list.d/tvheadend.list ]; then - echo "deb https://dl.bintray.com/tvheadend/deb xenial release-4.2" >> /etc/apt/sources.list.d/tvheadend.list - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 >/dev/null 2>&1 -fi - -if [[ $distribution == "stretch" ]]; then - URL="http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.0.0_1.0.1t-1+deb8u8_"$(dpkg --print-architecture)".deb" - fancy_wget "$URL" "-O ${TEMP_DIR}/package.deb" - dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 - local pkglist="libssl-doc zlib1g-dev tvheadend xmltv-util" -else - local pkglist="libssl-doc libssl1.0.0 zlib1g-dev tvheadend xmltv-util" -fi - -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get -y install $pkglist -} - - - - -install_urbackup () -{ -# -# Client/server backup system https://www.urbackup.org/ -# -if [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then local arch=armhf; else local arch=$(dpkg --print-architecture); fi -PREFIX="http://hndl.urbackup.org/Server/latest/" -URL="http://hndl.urbackup.org/Server/latest/"$(wget -q $PREFIX -O - | html2text -width 120 | grep deb | awk ' { print $3 }' | grep $arch) -fancy_wget "$URL" "-O ${TEMP_DIR}/package.deb" -dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 -apt-get -yy -f install -} - - - - -install_transmission () -{ -# -# transmission -# -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading dependencies" -install_packet "transmission-cli transmission-common transmission-daemon" "Install torrent server" -# systemd workaround -# https://forum.armbian.com/index.php?/topic/4017-programs-does-not-start-automatically-at-boot/ -sed -e 's/exit 0//g' -i /etc/rc.local - cat >> /etc/rc.local <<"EOF" -service transmission-daemon restart -exit 0 -EOF -} - - - -install_transmission_seed_armbian_torrents () -{ -# -# seed our torrents -# -# adjust network buffers if necessary -rmem_recommended=4194304 -wmem_recommended=1048576 -rmem_actual=$(sysctl net.core.rmem_max | awk -F" " '{print $3}') -if [ ${rmem_actual} -lt ${rmem_recommended} ]; then - grep -q net.core.rmem_max /etc/sysctl.conf && \ - sed -i "s/net.core.rmem_max =.*/net.core.rmem_max = ${rmem_recommended}/" /etc/sysctl.conf || \ - echo "net.core.rmem_max = ${rmem_recommended}" >> /etc/sysctl.conf -fi -wmem_actual=$(sysctl net.core.wmem_max | awk -F" " '{print $3}') -if [ ${wmem_actual} -lt ${wmem_recommended} ]; then - grep -q net.core.wmem_max /etc/sysctl.conf && \ - sed -i "s/net.core.wmem_max =.*/net.core.wmem_max = ${wmem_recommended}/" /etc/sysctl.conf || \ - echo "net.core.wmem_max = ${wmem_recommended}" >> /etc/sysctl.conf -fi -/sbin/sysctl -p -# create cron job for daily sync with official Armbian torrents -cat > /etc/cron.daily/seed-armbian-torrent <<"EOF" -#!/bin/bash -# -# armbian torrents auto update -# -# download latest torrent pack -wget -qO- -O ${TEMP_DIR}/armbian-torrents.zip https://dl.armbian.com/torrent/all-torrents.zip -# test zip for corruption -unzip -t ${TEMP_DIR}/armbian-torrents.zip >/dev/null 2>&1 -[[ $? -ne 0 ]] && echo "Error in zip" && exit -# extract zip -unzip -o ${TEMP_DIR}/armbian-torrents.zip -d ${TEMP_DIR}/torrent-tmp >/dev/null 2>&1 -# create list of current active torrents -transmission-remote -n 'transmission:transmission' -l | sed '1d; $d' > ${TEMP_DIR}/torrent-tmp/active.torrents -# loop and add/update torrent files -for f in ${TEMP_DIR}/torrent-tmp/*.torrent; do - transmission-remote -n 'transmission:transmission' -a $f > /dev/null 2>&1 - # remove added from the list - pattern="${f//.torrent}"; pattern="${pattern##*/}"; - sed -i "/$pattern/d" ${TEMP_DIR}/torrent-tmp/active.torrents -done -# remove old armbian torrents -while read i; do - [[ $i == *Armbian_* ]] && transmission-remote -n 'transmission:transmission' -t $(echo "$i" | awk '{print $1}';) --remove-and-delete -done < ${TEMP_DIR}/torrent-tmp/active.torrents -# remove temporally files and direcotories -EOF -chmod +x /etc/cron.daily/seed-armbian-torrent -/etc/cron.daily/seed-armbian-torrent & -} - - - - -install_syncthing () -{ -# -# Install Personal cloud https://syncthing.net/ -# - -if [ "$(dpkg --print-architecture | grep armhf)" == "armhf" ]; then - local filename="linux-arm" -elif [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - local filename="linux-arm64" -else - local filename="linux-amd64" -fi -mkdir -p /usr/bin/syncthing -wgeturl=$(curl -s "https://api.github.com/repos/syncthing/syncthing/releases" | grep $filename | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/syncthing.tgz" -wgeturl=$(curl -s "https://api.github.com/repos/syncthing/syncthing-inotify/releases" | grep $filename | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/syncthing-inotify.tgz" -tar xf ${TEMP_DIR}/syncthing.tgz -C ${TEMP_DIR} -tar xf ${TEMP_DIR}/syncthing-inotify.tgz -C /usr/bin -cp -R ${TEMP_DIR}/syncthing-*/syncthing /usr/bin -cp ${TEMP_DIR}/syncthing-*/etc/linux-systemd/system/syncthing* /etc/systemd/system/ -cp /etc/systemd/system/syncthing@.service /etc/systemd/system/syncthing-inotify@.service - -# adjust service for inotify -sed -i "s/^Description=.*/Description=Syncthing Inotify File Watcher/" /etc/systemd/system/syncthing-inotify@.service -sed -i "s/^After=.*/After=network.target syncthing.service/" /etc/systemd/system/syncthing-inotify@.service -sed -i "s/^ExecStart=.*/ExecStart=\/usr\/bin\/syncthing-inotify -logfile=\/var\/log\/syncthing-inotify.log -logflags=3/" /etc/systemd/system/syncthing-inotify@.service -sed -i "/^\[Install\]/a Requires=syncthing.service" /etc/systemd/system/syncthing-inotify@.service - -# increase open file limit -if !(grep -qs "fs.inotify.max_user_watches=204800" "/etc/sysctl.conf");then - echo -e "fs.inotify.max_user_watches=204800" | tee -a /etc/sysctl.conf -fi -add_choose_user -systemctl enable syncthing@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl start syncthing@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl enable syncthing-inotify@${CHOSEN_USER}.service >/dev/null 2>&1 -systemctl start syncthing-inotify@${CHOSEN_USER}.service >/dev/null 2>&1 -} - - - - -install_plex_media_server () -{ -# -# Media server -# -if [ "$(dpkg --print-architecture | grep armhf)" == "armhf" ]; then - echo -e "deb [arch=armhf] http://dev2day.de/pms/ stretch main" > /etc/apt/sources.list.d/plex.list - wget -q -O - http://dev2day.de/pms/dev2day-pms.gpg.key | apt-key add - >/dev/null 2>&1 - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install plexmediaserver-installer -elif [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - echo -e "deb [arch=armhf] http://dev2day.de/pms/ stretch main" > /etc/apt/sources.list.d/plex.list - wget -q -O - http://dev2day.de/pms/dev2day-pms.gpg.key | apt-key add - >/dev/null 2>&1 - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install binutils:armhf plexmediaserver-installer:armhf -else - fancy_wget "https://downloads.plex.tv/plex-media-server/1.12.3.4973-215c28d86/plexmediaserver_1.12.3.4973-215c28d86_amd64.deb" "-O ${TEMP_DIR}/package.deb" - dpkg -i ${TEMP_DIR}/package.deb >/dev/null 2>&1 -fi -} - - - - -install_radarr () -{ -# -# Automatically downloading movies -# -debconf-apt-progress -- apt-get update -debconf-apt-progress -- apt-get -y install mono-devel mediainfo libmono-cil-dev -wgeturl=$(curl -s "https://api.github.com/repos/Radarr/Radarr/releases" | grep 'linux.tar.gz' | grep 'browser_download_url' | head -1 | cut -d \" -f 4) -fancy_wget "$wgeturl" "-O ${TEMP_DIR}/radarr.tgz" -tar xf ${TEMP_DIR}/radarr.tgz -C /opt -cat << _EOF_ > /etc/systemd/system/radarr.service -[Unit] -Description=Radarr Daemon -After=network.target -[Service] -User=root -Type=simple -ExecStart=/usr/bin/mono --debug /opt/Radarr/Radarr.exe -nobrowser -[Install] -WantedBy=multi-user.target -_EOF_ -systemctl enable radarr >/dev/null 2>&1 -systemctl start radarr -} - - - - -install_sonarr () -{ -# -# Automatically downloading TV shows -# -if [ "$(dpkg --print-architecture | grep arm64)" == "arm64" ]; then - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install mono-complete mediainfo - fancy_wget "http://update.sonarr.tv/v2/develop/mono/NzbDrone.develop.tar.gz" "-O ${TEMP_DIR}/sonarr.tgz" - tar xf ${TEMP_DIR}/sonarr.tgz -C /opt -else - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys FDA5DFFC >/dev/null 2>&1 - echo -e "deb https://apt.sonarr.tv/ develop main" > /etc/apt/sources.list.d/sonarr.list - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install nzbdrone -fi -cat << _EOF_ > /etc/systemd/system/sonarr.service -[Unit] -Description=Sonarr (NzbDrone) Daemon -After=network.target -[Service] -User=root -Type=simple -ExecStart=/usr/bin/mono --debug /opt/NzbDrone/NzbDrone.exe -nobrowser -[Install] -WantedBy=multi-user.target -_EOF_ -systemctl enable sonarr >/dev/null 2>&1 -systemctl start sonarr -} - - - - -install_vpn_server () -{ -# -# Script downloads latest stable -# -cd ${TEMP_DIR} -PREFIX="http://www.softether-download.com/files/softether/" -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading basic packages" -URL=$(wget -q $PREFIX -O - | html2text | grep rtm | awk ' { print $(NF) }' | tail -1) -SUFIX="${URL/-tree/}" -if [ "$(dpkg --print-architecture | grep armhf)" != "" ]; then -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Server/32bit_-_ARM_EABI/softether-vpnserver-$SUFIX-linux-arm_eabi-32bit.tar.gz" -else -install_packet "gcc-multilib" "Install libraries" -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Server/32bit_-_Intel_x86/softether-vpnserver-$SUFIX-linux-x86-32bit.tar.gz" -fi -wget -q $DLURL -O - | tar -xz -cd vpnserver -make i_read_and_agree_the_license_agreement | dialog --backtitle "$BACKTITLE" --title "Compiling SoftEther VPN" --progressbox $TTY_Y $TTY_X -cd .. -cp -R vpnserver /usr/local -cd /usr/local/vpnserver/ -chmod 600 * -chmod 700 vpncmd -chmod 700 vpnserver -if [[ -d /run/systemd/system/ ]]; then -cat </lib/systemd/system/ethervpn.service -[Unit] -Description=VPN service - -[Service] -Type=oneshot -ExecStart=/usr/local/vpnserver/vpnserver start -ExecStop=/usr/local/vpnserver/vpnserver stop -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target -EOT -systemctl enable ethervpn.service -service ethervpn start - -else - -cat < /etc/init.d/vpnserver -#!/bin/sh -### BEGIN INIT INFO -# Provides: vpnserver -# Required-Start: \$remote_fs \$syslog -# Required-Stop: \$remote_fs \$syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start daemon at boot time -# Description: Enable Softether by daemon. -### END INIT INFO -DAEMON=/usr/local/vpnserver/vpnserver -LOCK=/var/lock/vpnserver -test -x $DAEMON || exit 0 -case "\$1" in -start) -\$DAEMON start -touch \$LOCK -;; -stop) -\$DAEMON stop -rm \$LOCK -;; -restart) -\$DAEMON stop -sleep 3 -\$DAEMON start -;; -*) -echo "Usage: \$0 {start|stop|restart}" -exit 1 -esac -exit 0 -EOT -chmod 755 /etc/init.d/vpnserver -mkdir /var/lock/subsys -update-rc.d vpnserver defaults >> $logfile -/etc/init.d/vpnserver start -fi -} - - - - -install_vpn_client () -{ -# -# Script downloads latest stable -# -cd ${TEMP_DIR} -PREFIX="http://www.softether-download.com/files/softether/" -install_packet "debconf-utils unzip build-essential html2text apt-transport-https" "Downloading basic packages" -URL=$(wget -q $PREFIX -O - | html2text | grep rtm | awk ' { print $(NF) }' | tail -1) -SUFIX="${URL/-tree/}" -if [ "$(dpkg --print-architecture | grep armhf)" != "" ]; then -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Client/32bit_-_ARM_EABI/softether-vpnclient-$SUFIX-linux-arm_eabi-32bit.tar.gz" -else -install_packet "gcc-multilib" "Install libraries" -DLURL=$PREFIX$URL"/Linux/SoftEther_VPN_Client/32bit_-_Intel_x86/softether-vpnclient-$SUFIX-linux-x86-32bit.tar.gz" -fi -wget -q $DLURL -O - | tar -xz -cd vpnclient -make i_read_and_agree_the_license_agreement | dialog --backtitle "$BACKTITLE" --title "Compiling SoftEther VPN vpnclient" --progressbox $TTY_Y $TTY_X -cd .. -cp -R vpnclient /usr/local -cd /usr/local/vpnclient/ -chmod 600 * -chmod 700 vpncmd -chmod 700 vpnclient -} - - - - -install_DashNTP () -{ -# -# Install DASH and ntp service -# -echo "dash dash/sh boolean false" | debconf-set-selections -dpkg-reconfigure -f noninteractive dash > /dev/null 2>&1 -install_packet "ntp ntpdate" "Install DASH and ntp service" -} - - - - -install_MySQL () -{ -# -# Maria SQL -# -install_packet "mariadb-client mariadb-server" "SQL client and server" -#Allow MySQL to listen on all interfaces -cp /etc/mysql/my.cnf /etc/mysql/my.cnf.backup -[[ -f /etc/mysql/my.cnf ]] && sed -i 's|bind-address.*|#bind-address = 127.0.0.1|' /etc/mysql/my.cnf -[[ -f /etc/mysql/mariadb.conf.d/50-server.cnf ]] && sed -i 's|bind-address.*|#bind-address = 127.0.0.1|' /etc/mysql/mariadb.conf.d/50-server.cnf -SECURE_MYSQL=$(expect -c " -set timeout 3 -spawn mysql_secure_installation -expect \"Enter current password for root (enter for none):\" -send \"\r\" -expect \"root password?\" -send \"y\r\" -expect \"New password:\" -send \"$MYSQL_PASS\r\" -expect \"Re-enter new password:\" -send \"$MYSQL_PASS\r\" -expect \"Remove anonymous users?\" -send \"y\r\" -expect \"Disallow root login remotely?\" -send \"y\r\" -expect \"Remove test database and access to it?\" -send \"y\r\" -expect \"Reload privilege tables now?\" -send \"y\r\" -expect eof -") -# -# Execution mysql_secure_installation -# -echo "${SECURE_MYSQL}" >> /dev/null -# ISP config exception -mkdir -p /etc/mysql/mariadb.conf.d/ -cat > /etc/mysql/mariadb.conf.d/99-ispconfig.cnf<<"EOF" -[mysqld] -sql-mode="NO_ENGINE_SUBSTITUTION" -EOF -service mysql restart >> /dev/null -} - - - - -install_MySQLDovecot () -{ -# -# Install Postfix, Dovecot, Saslauthd, rkhunter, binutils -# -echo "postfix postfix/main_mailer_type select Internet Site" | debconf-set-selections -echo "postfix postfix/mailname string $HOSTNAMEFQDN" | debconf-set-selections -install_packet "postfix postfix-mysql postfix-doc openssl getmail4 rkhunter binutils dovecot-imapd dovecot-pop3d dovecot-mysql \ -dovecot-sieve sudo libsasl2-modules" "postfix, dovecot, saslauthd, rkhunter, binutils" -#Uncommenting some Postfix configuration files -cp /etc/postfix/master.cf /etc/postfix/master.cf.backup -sed -i 's|#submission inet n - - - - smtpd|submission inet n - - - - smtpd|' /etc/postfix/master.cf -sed -i 's|# -o syslog_name=postfix/submission| -o syslog_name=postfix/submission|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_tls_security_level=encrypt| -o smtpd_tls_security_level=encrypt|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_client_restrictions=permit_sasl_authenticated,reject| -o smtpd_client_restrictions=permit_sasl_authenticated,reject|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_sasl_auth_enable=yes| -o smtpd_sasl_auth_enable=yes|' /etc/postfix/master.cf -sed -i 's|#smtps inet n - - - - smtpd|smtps inet n - - - - smtpd|' /etc/postfix/master.cf -sed -i 's|# -o syslog_name=postfix/smtps| -o syslog_name=postfix/smtps|' /etc/postfix/master.cf -sed -i 's|# -o smtpd_tls_wrappermode=yes| -o smtpd_tls_wrappermode=yes|' /etc/postfix/master.cf -service postfix restart >> /dev/null -} - - - - -install_Virus () -{ -# -# Install Amavisd-new, SpamAssassin, And Clamav -# -install_packet "amavisd-new spamassassin clamav clamav-daemon zoo unzip bzip2 arj p7zip unrar-free ripole rpm nomarch lzop \ -cabextract apt-listchanges libnet-ldap-perl libauthen-sasl-perl clamav-docs daemon libio-string-perl libio-socket-ssl-perl \ -libnet-ident-perl zip libnet-dns-perl postgrey" "amavisd, spamassassin, clamav" -sed -i "s/^AllowSupplementaryGroups.*/AllowSupplementaryGroups true/" /etc/clamav/clamd.conf -service spamassassin stop >/dev/null 2>&1 -systemctl disable spamassassin >/dev/null 2>&1 -} - - - - -install_hhvm () -{ -# -# Install HipHop Virtual Machine -# -apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xB4112585D386EB94 >/dev/null 2>&1 -add-apt-repository https://dl.hhvm.com/"${family,,}" >/dev/null 2>&1 -debconf-apt-progress -- apt-get update -install_packet "hhvm" "HipHop Virtual Machine" -} - - - - -install_phpmyadmin () -{ -# -# Phpmyadmin unattended installation -# -if [[ "$family" != "Ubuntu" ]]; then -DEBIAN_FRONTEND=noninteractive debconf-apt-progress -- apt-get -y install phpmyadmin -else -debconf-set-selections <<< "phpmyadmin phpmyadmin/internal/skip-preseed boolean true" -debconf-set-selections <<< "phpmyadmin phpmyadmin/reconfigure-webserver multiselect true" -debconf-set-selections <<< "phpmyadmin phpmyadmin/dbconfig-install boolean false" -echo "phpmyadmin phpmyadmin/internal/skip-preseed boolean true" | debconf-set-selections -echo "phpmyadmin phpmyadmin/reconfigure-webserver multiselect" | debconf-set-selections -echo "phpmyadmin phpmyadmin/dbconfig-install boolean false" | debconf-set-selections -debconf-apt-progress -- apt-get install -y phpmyadmin -fi -} - - - - -install_apache () -{ -# -# Install Apache2, PHP5, FCGI, suExec, Pear and mcrypt -# - -local pkg="apache2 apache2-doc apache2-utils libapache2-mod-fcgid php-pear mcrypt imagemagick libruby libapache2-mod-python memcached" - -local pkg_xenial="libapache2-mod-php php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -apache2-suexec-pristine php-auth php7.0-mcrypt php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php7.0-opcache php-apcu \ -libapache2-mod-fastcgi php7.0-fpm letsencrypt" - -local pkg_stretch="libapache2-mod-php php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi libapache2-mod-fcgid \ -apache2-suexec-pristine php7.0-mcrypt libapache2-mod-python php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 \ -php7.0-tidy php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring libapache2-mod-passenger \ -php7.0-soap php7.0-fpm php7.0-opcache php-apcu certbot" - -local pkg_jessie="apache2.2-common apache2-mpm-prefork libexpat1 ssl-cert libapache2-mod-php5 php5 php5-common php5-gd php5-mysql \ -php5-imap php5-cli php5-cgi libapache2-mod-fcgid apache2-suexec php-pear php-auth php5-mcrypt mcrypt php5-imagick libapache2-mod-python \ -php5-curl php5-intl php5-memcache php5-memcached php5-pspell php5-recode php5-sqlite php5-tidy php5-xmlrpc php5-xsl \ -libapache2-mod-passenger php5-xcache libapache2-mod-fastcgi php5-fpm" - -local temp="pkg_${distribution}" -install_packet "${pkg} ${!temp}" "Apache for $family $distribution" -# fix HTTPOXY vulnerability -cat < /etc/apache2/conf-available/httpoxy.conf - - RequestHeader unset Proxy early - - -EOT - -a2enmod actions proxy_fcgi setenvif fastcgi alias httpoxy suexec rewrite ssl actions include dav_fs dav auth_digest cgi headers >/dev/null 2>&1 -a2enconf php7.0-fpm >/dev/null 2>&1 -service apache2 restart >> /dev/null -} - - - - -install_nginx () -{ -# -# Install NginX, PHP5, FCGI, suExec, Pear, And mcrypt -# -local pkg="nginx php-pear memcached fcgiwrap" - -local pkg_xenial="php7.0-fpm php7.0-opcache php7.0-fpm php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -php7.0-mcrypt mcrypt imagemagick libruby php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php-apcu" - -local pkg_stretch="php7.0-fpm php7.0-opcache php7.0-fpm php7.0 php7.0-common php7.0-gd php7.0-mysql php7.0-imap php7.0-cli php7.0-cgi \ -php7.0-mcrypt mcrypt imagemagick libruby php7.0-curl php7.0-intl php7.0-pspell php7.0-recode php7.0-sqlite3 php7.0-tidy \ -php7.0-xmlrpc php7.0-xsl php-memcache php-imagick php-gettext php7.0-zip php7.0-mbstring php-apcu" - -local pkg_jessie="php5-fpm php5-mysql php5-curl php5-gd php5-intl php5-imagick php5-imap php5-mcrypt php5-memcache \ -php5-memcached php5-pspell php5-recode php5-snmp php5-sqlite php5-tidy php5-xmlrpc php5-xsl php-apc" - -local temp="pkg_${distribution}" -install_packet "${pkg} ${!temp}" "Nginx for $family $distribution" - -phpenmod mcrypt mbstring - -if [[ -f /etc/php/7.0/fpm/php.ini ]]; then - tz=$(cat /etc/timezone | sed 's/\//\\\//g') - sed -i "s/^cgi.fix_pathinfo=.*/cgi.fix_pathinfo=0/" /etc/php/7.0/fpm/php.ini - sed -i "s/^date.timezone=.*/date.timezone=""$tz""/" /etc/php/7.0/fpm/php.ini - service php7.0-fpm reload >> /dev/null -else - debconf-apt-progress -- apt-get install -y python-certbot -t jessie-backports - service php5-fpm reload >> /dev/null -fi -} - - - - -install_PureFTPD () -{ -# -# Install PureFTPd and Quota -# -install_packet "pure-ftpd-common pure-ftpd-mysql quota quotatool" "pureFTPd and Quota" - -sed -i 's/VIRTUALCHROOT=false/VIRTUALCHROOT=true/' /etc/default/pure-ftpd-common -echo 1 > /etc/pure-ftpd/conf/TLS -mkdir -p /etc/ssl/private/ -openssl req -x509 -nodes -days 7300 -newkey rsa:2048 -subj "/C=GB/ST=GB/L=GB/O=GB/OU=GB/CN=$(hostname -f)/emailAddress=joe@joe.com" -keyout /etc/ssl/private/pure-ftpd.pem -out /etc/ssl/private/pure-ftpd.pem >/dev/null 2>&1 -chmod 600 /etc/ssl/private/pure-ftpd.pem -/etc/init.d/pure-ftpd-mysql restart >/dev/null 2>&1 -local temp=$(cat /etc/fstab | grep "/ " | tail -1 | awk '{print $4}') -sed -i "s/$temp/$temp,usrjquota=quota.user,grpjquota=quota.group,jqfmt=vfsv0/" /etc/fstab -mount -o remount / >/dev/null 2>&1 -quotacheck -avugm >/dev/null 2>&1 -quotaon -avug >/dev/null 2>&1 -} - - - - -install_Bind () -{ -# -# Install BIND DNS Server -# -install_packet "bind9 dnsutils" "Install BIND DNS Server" -} - - - - -install_Stats () -{ -# -# Install Vlogger, Webalizer, And AWstats -# -install_packet "vlogger webalizer awstats geoip-database libclass-dbi-mysql-perl" "vlogger, webalizer, awstats" -sed -i "s/*/10 * * * * www-data/#*/10 * * * * www-data/" /etc/cron.d/awstats -sed -i "s/10 03 * * * www-data/#10 03 * * * www-data/" /etc/cron.d/awstats -} - - - - -install_Jailkit() -{ -# -debconf-apt-progress -- apt-get install -y build-essential autoconf automake libtool flex bison debhelper binutils -cd ${TEMP_DIR} -wget -q http://olivier.sessink.nl/jailkit/jailkit-2.19.tar.gz -O - | tar -xz && cd jailkit-2.19 -echo 5 > debian/compat -./debian/rules binary > /dev/null 2>&1 -dpkg -i ../jailkit_2.19-1_*.deb > /dev/null 2>&1 -} - - - - -install_Fail2BanDovecot() -{ -# -# Install fail2ban -# -install_packet "fail2ban ufw" "Install fail2ban and UFW Firewall" -if [[ $distribution == "stretch" ]]; then -cat > /etc/fail2ban/jail.local <<"EOF" -[pure-ftpd] -enabled = true -port = ftp -filter = pure-ftpd -logpath = /var/log/syslog -maxretry = 3 - -[dovecot] -enabled = true -filter = dovecot -logpath = /var/log/mail.log -maxretry = 5 - -[postfix-sasl] -enabled = true -port = smtp -filter = postfix-sasl -logpath = /var/log/mail.log -maxretry = 3 -EOF -else -cat > /etc/fail2ban/jail.local <<"EOF" -[pureftpd] -enabled = true -port = ftp -filter = pureftpd -logpath = /var/log/syslog -maxretry = 3 - -[dovecot-pop3imap] -enabled = true -filter = dovecot-pop3imap -action = iptables-multiport[name=dovecot-pop3imap, port="pop3,pop3s,imap,imaps", protocol=tcp] -logpath = /var/log/mail.log -maxretry = 5 - -[sasl] -enabled = true -port = smtp -filter = postfix-sasl -logpath = /var/log/mail.log -maxretry = 3 -EOF -fi -} - - - - -install_Fail2BanRulesDovecot() -{ -# -# Dovecot rules -# -cat > /etc/fail2ban/filter.d/pureftpd.conf <<"EOF" -[Definition] -failregex = .*pure-ftpd: \(.*@\) \[WARNING\] Authentication failed for user.* -ignoreregex = -EOF - -cat > /etc/fail2ban/filter.d/dovecot-pop3imap.conf <<"EOF" -[Definition] -failregex = (?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed|Aborted login \(\d+ authentication attempts).*rip=(?P\S*),.* -ignoreregex = -EOF -# Add the missing ignoreregex line -echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf -service fail2ban restart >> /dev/null -} - - - - -install_ISPConfig (){ -#------------------------------------------------------------------------------------------------------------------------------------------ -# Install ISPConfig 3 -#------------------------------------------------------------------------------------------------------------------------------------------ -cd ${TEMP_DIR} -wget -q http://www.ispconfig.org/downloads/ISPConfig-3-stable.tar.gz -O - | tar -xz -cd ${TEMP_DIR}/ispconfig3_install/install/ -#apt-get -y install php5-cli php5-mysql -php -q install.php --autoinstall=${TEMP_DIR}/isp.conf.php -echo "Admin panel: https://$serverIP:8080" -echo "PHPmyadmin: http://$serverIP:8081/phpmyadmin" -} - - -install_mayan_edms (){ -# -# Install Mayan EDMS -# - -# Default values -MAYAN_DATABASE_PASSWORD="mayandbpass" -MAYAN_INSTALLATION_FOLDER="/opt/mayan-edms" -MAYAN_MEDIA_ROOT="/opt/mayan-edms-data" - -# User interaction -exec 3>&1 -dialog --title "Server configuration" --separate-widget $'\n' \ ---ok-label "Install" --backtitle "$BACKTITLE" \ ---form "\nPlease fill out this form:\n " 13 70 0 \ -"Ddatabase password:" 1 1 "$MAYAN_DATABASE_PASSWORD" 1 31 32 0 \ -"Installation folder:" 2 1 "$MAYAN_INSTALLATION_FOLDER" 2 31 32 0 \ -"Data folder:" 3 1 "$MAYAN_MEDIA_ROOT" 3 31 32 0 \ -2>&1 1>&3 | { -read -r MAYAN_DATABASE_PASSWORD -read -r MAYAN_MEDIA_ROOT -read -r MAYAN_INSTALLATION_FOLDER -echo $MAYAN_DATABASE_PASSWORD > ${TEMP_DIR}/MAYAN_DATABASE_PASSWORD -echo $MAYAN_MEDIA_ROOT > ${TEMP_DIR}/MAYAN_MEDIA_ROOT -echo $MAYAN_INSTALLATION_FOLDER > ${TEMP_DIR}/MAYAN_INSTALLATION_FOLDER -} -exec 3>&- -read MAYAN_DATABASE_PASSWORD < ${TEMP_DIR}/MAYAN_DATABASE_PASSWORD -read MAYAN_MEDIA_ROOT < ${TEMP_DIR}/MAYAN_MEDIA_ROOT -read MAYAN_INSTALLATION_FOLDER < ${TEMP_DIR}/MAYAN_INSTALLATION_FOLDER - -# OS dependencies -install_packet "g++ gcc ghostscript gnupg1 graphviz libffi-dev libjpeg-dev libmagic1 libpq-dev libpng-dev libreoffice libssl-dev libtiff-dev poppler-utils postgresql python-dev python-pip python-virtualenv redis-server sane-utils supervisor tesseract-ocr zlib1g-dev" "Installing dependencies" - -# Mayan OS user account -dialog --infobox "Adding Mayan EDMS user account" 3 70 -adduser mayan --disabled-password --disabled-login --no-create-home --gecos "" >/dev/null 2>&1 -sleep 1 - -# Create installtion and data folders -mkdir -p "${MAYAN_INSTALLATION_FOLDER}" -mkdir -p "${MAYAN_MEDIA_ROOT}" - -# Create the Python virtualenv to isolate Python dependencies of Mayan -dialog --infobox "Creating Python virtual environment" 3 70 -python /usr/lib/python2.7/dist-packages/virtualenv.py $MAYAN_INSTALLATION_FOLDER > /dev/null - -# Give ownership to the Mayan OS user -chown mayan:mayan "${MAYAN_INSTALLATION_FOLDER}" -R -chown mayan:mayan "${MAYAN_MEDIA_ROOT}" -R - -# Pillow can't find zlib or libjpeg on aarch64 (ODROID C2) -if [ "$(uname -m)" = "aarch64" ]; then \ - ln -s /usr/lib/aarch64-linux-gnu/libz.so /usr/lib/ && \ - ln -s /usr/lib/aarch64-linux-gnu/libjpeg.so /usr/lib/ \ -; fi - -# Pillow can't find zlib or libjpeg on armv7l (ODROID HC1) -if [ "$(uname -m)" = "armv7l" ]; then \ - ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ && \ - ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \ -; fi - -# Install Mayan from the web and all its Python dependencies -MAYAN_PIP=$MAYAN_INSTALLATION_FOLDER/bin/pip -dialog --infobox "Installing Mayan EDMS Python package (Takes several minutes)" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir mayan-edms > /dev/null 2>&1 - -# Python Postgres driver -dialog --infobox "Installing PostgreSQL database driver" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir psycopg2==2.7.3.2 > /dev/null - -# Python Redis driver -dialog --infobox "Installing Redis driver" 3 70 -sudo -u mayan $MAYAN_PIP install --no-cache-dir redis==2.10.6 > /dev/null - -# Create postgres Mayan user and database -MAYAN_BIN=$MAYAN_INSTALLATION_FOLDER/bin/mayan-edms.py -dialog --infobox "Creating and initializing database (Takes several minutes)" 3 70 -sudo -u postgres psql -c "CREATE USER mayan WITH password '$MAYAN_DATABASE_PASSWORD';" -sudo -u postgres createdb -O mayan mayan - -# Execute initialsetup command. Migrate DB, create base files, downloads Javascript libraries -sudo -u mayan \ - MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \ - MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 \ - MAYAN_MEDIA_ROOT=$MAYAN_MEDIA_ROOT \ - MAYAN_DATABASE_PASSWORD=$MAYAN_DATABASE_PASSWORD \ - $MAYAN_BIN initialsetup --force > /dev/null - -# Compress and merge Javascript, CSS for web serving -dialog --infobox "Preparing static files" 3 70 -sudo -u mayan \ - MAYAN_MEDIA_ROOT=$MAYAN_MEDIA_ROOT \ - $MAYAN_BIN preparestatic --noinput > /dev/null - -# Create supervisor file for gunicorn (frontend), 3 background workers, and the scheduler for periodic tasks -cat > /etc/supervisor/conf.d/mayan.conf <> /etc/redis/redis.conf - -# This starts all of Mayan's processes -dialog --infobox "Starting service" 3 70 -systemctl restart supervisor.service - -# Installation report -dialog --msgbox "Installation complete.\nInstallation folder: $MAYAN_INSTALLATION_FOLDER\nData folder: $MAYAN_MEDIA_ROOT\nPort: 8000" 10 70 -} - - -#------------------------------------------------------------------------------------------------------------------------------------------ -# Main choices -#------------------------------------------------------------------------------------------------------------------------------------------ - -# check for root -# -if [[ $EUID != 0 ]]; then - dialog --title "Warning" --infobox "\nThis script requires root privileges.\n\nExiting ..." 7 41 - sleep 3 - exit -fi - -# nameserver backup -if [ -d /etc/resolvconf/resolv.conf.d ]; then - echo 'nameserver 8.8.8.8' > /etc/resolvconf/resolv.conf.d/head - resolvconf -u -fi - -# Create a safe temporary directory -TEMP_DIR=$(mktemp -d || exit 1) -chmod 700 ${TEMP_DIR} -trap "rm -rf \"${TEMP_DIR}\" ; exit 0" 0 1 2 3 15 - -# Install basic stuff, we have to wait for other apt tasks to finish -# (eg unattended-upgrades) -i=0 -tput sc -while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do - case $(($i % 4)) in - 0 ) j="-" ;; - 1 ) j="\\" ;; - 2 ) j="|" ;; - 3 ) j="/" ;; - esac - tput rc - echo -en "\r[$j] Waiting for other software managers to finish..." - sleep 0.5 - ((i=i+1)) -done - -apt-get -qq -y --no-install-recommends install debconf-utils html2text apt-transport-https dialog whiptail lsb-release bc expect > /dev/null - -# gather some info -# -TTY_X=$(($(stty size | awk '{print $2}')-6)) # determine terminal width -TTY_Y=$(($(stty size | awk '{print $1}')-6)) # determine terminal height -distribution=$(lsb_release -cs) -family=$(lsb_release -is) -serverIP=$(ip route get 8.8.8.8 | awk '{ print $NF; exit }') -set ${serverIP//./ } -SUBNET="$1.$2.$3." -hostnamefqdn=$(hostname -f) -mysql_pass="" -BACKTITLE="Softy - Armbian post deployment scripts, http://www.armbian.com" -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -#check_status - -# main dialog routine -# -DIALOG_CANCEL=1 -DIALOG_ESC=255 - -while true; do - - # prepare menu items - check_status - LISTLENGHT="$((${#LIST[@]}/2))" - exec 3>&1 - selection=$(dialog --backtitle "$BACKTITLE" --title "Installing to $family $distribution" --colors --clear --cancel-label \ - "Exit" --checklist "\nChoose what you want to install:\n " $(($LISTLENGHT+$LIST_CONST)) 70 15 "${LIST[@]}" 2>&1 1>&3) - exit_status=$? - exec 3>&- - case $exit_status in - $DIALOG_ESC | $DIALOG_CANCEL) - clear - exit 1 - ;; - esac - - # cycle trought all install options - i=0 - - while [ "$i" -lt "$LISTLENGHT" ]; do - - if [[ "$selection" == *Samba* && "$SAMBA_STATUS" != "on" ]]; then - install_samba - selection=${selection//Samba/} - fi - - if [[ "$selection" == *CUPS* && "$CUPS_STATUS" != "on" ]]; then - install_cups - selection=${selection//CUPS/} - fi - - if [[ "$selection" == *headend* && "$TVHEADEND_STATUS" != "on" ]]; then - install_tvheadend - selection=${selection//\"TV headend\"/} - fi - - if [[ "$selection" == *Minidlna* && "$MINIDLNA_STATUS" != "on" ]]; then - install_packet "minidlna" "Install lightweight DLNA/UPnP-AV server" - selection=${selection//Minidlna/} - fi - - if [[ "$selection" == *ISPConfig* && "$ISPCONFIG_STATUS" != "on" ]]; then - server_conf - if [[ "$MYSQL_PASS" == "" ]]; then - dialog --msgbox "Mysql password can't be blank. Exiting..." 7 70 - exit - fi - if [[ "$(echo $HOSTNAMEFQDN | grep -P '(?=^.{1,254}$)(^(?>(?!\d+\.)[a-zA-Z0-9_\-]{1,63}\.?)+(?:[a-zA-Z]{2,})$)')" == "" ]]; then - dialog --msgbox "Invalid FQDN. Exiting..." 7 70 - exit - fi - choose_webserver; install_basic; install_DashNTP; install_MySQL; install_MySQLDovecot; install_Virus; install_$server; - install_phpmyadmin - [[ -z "$(dpkg --print-architecture | grep arm)" ]] && install_hhvm - create_ispconfig_configuration;install_PureFTPD; - install_Jailkit; install_Fail2BanDovecot; install_Fail2BanRulesDovecot; - install_ISPConfig - read -n 1 -s -p "Press any key to continue" - selection=${selection//ISPConfig/} - fi - - if [[ "$selection" == *Syncthing* && "$SYNCTHING_STATUS" != "on" ]]; then - install_syncthing - selection=${selection//Syncthing/} - fi - - if [[ "$selection" == *ExaGear* && "$EXAGEAR_STATUS" != "on" ]]; then - debconf-apt-progress -- apt-get update - debconf-apt-progress -- apt-get -y install exagear-armbian exagear-desktop exagear-dsound-server exagear-guest-ubuntu-1604 - selection=${selection//ExaGear/} - fi - - if [[ "$selection" == *server* && "$VPN_SERVER_STATUS" != "on" ]]; then - install_vpn_server - selection=${selection//\"VPN server\"/} - fi - - if [[ "$selection" == *client* && "$VPN_CLIENT_STATUS" != "on" ]]; then - install_vpn_client - selection=${selection//\"VPN client\"/} - fi - if [[ "$selection" == *NCP* && "$NCP_STATUS" != "on" ]]; then - install_ncp - selection=${selection//NCP/} - fi - - if [[ "$selection" == *OMV* && "$OMV_STATUS" != "on" ]]; then - install_omv - selection=${selection//OMV/} - fi - - if [[ "$selection" == *Plex* && "$PLEX_STATUS" != "on" ]]; then - install_plex_media_server - selection=${selection//Plex/} - fi - - if [[ "$selection" == *Radarr* && "$RADARR_STATUS" != "on" ]]; then - install_radarr - selection=${selection//Radarr/} - fi - - if [[ "$selection" == *Sonarr* && "$SONARR_STATUS" != "on" ]]; then - install_sonarr - selection=${selection//Sonarr/} - fi - - if [[ "$selection" == *hole* && "$PI_HOLE_STATUS" != "on" ]]; then - curl -L "https://install.pi-hole.net" | bash - selection=${selection//\"Pi hole\"/} - fi - - if [[ "$selection" == *Transmission* && "$TRANSMISSION_STATUS" != "on" ]]; then - install_transmission - selection=${selection//Transmission/} - dialog --title "Seed Armbian torrents" --backtitle "$BACKTITLE" --yes-label "Yes" --no-label "Cancel" --yesno "\ - \nDo you want to help community and seed armbian torrent files? It will ensure faster download for everyone.\ - \n\nWe need around 80Gb of your space." 11 44 - if [[ $? = 0 ]]; then - install_transmission_seed_armbian_torrents - fi - fi - - if [[ "$selection" == *UrBackup* && "$URBACKUP_STATUS" != "on" ]]; then - install_urbackup - selection=${selection//UrBackup/} - fi - - if [[ "$selection" == *Mayan* && "$MAYAN_STATUS" != "on" ]]; then - install_mayan_edms - selection=${selection//\"Mayan EDMS\"/} - fi - - i=$[$i+1] - done - # reread statuses - check_status -done - - - diff --git a/contrib/scripts/install/production.sh b/contrib/scripts/install/production.sh deleted file mode 100644 index 2f70812fec5..00000000000 --- a/contrib/scripts/install/production.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash - -# ====== CONFIG ====== -INSTALLATION_DIRECTORY=/usr/share/mayan-edms/ -DB_NAME=mayan_edms -DB_USERNAME=mayan -DB_PASSWORD=test123 -# ==== END CONFIG ==== - -cat << EOF | tee -a /etc/motd.tail -********************************** - -Mayan EDMS Vagrant Production Box - -********************************** -EOF - -echo -e "\n -> Running apt-get update & upgrade \n" -apt-get -qq update -apt-get -y upgrade - -echo -e "\n -> Installing core binaries \n" -apt-get install nginx supervisor redis-server postgresql libpq-dev libjpeg-dev libmagic1 libpng-dev libreoffice libtiff-dev gcc ghostscript gpgv python-dev python-virtualenv tesseract-ocr poppler-utils -y - -echo -e "\n -> Setting up virtualenv \n" -rm -f ${INSTALLATION_DIRECTORY} -virtualenv ${INSTALLATION_DIRECTORY} -source ${INSTALLATION_DIRECTORY}bin/activate - -echo -e "\n -> Installing Mayan EDMS from PyPI \n" -pip install mayan-edms - -echo -e "\n -> Installing Python client for PostgreSQL, Redis, and uWSGI \n" -pip install psycopg2 redis uwsgi - -echo -e "\n -> Creating the database for the installation \n" -echo "CREATE USER mayan WITH PASSWORD '$DB_PASSWORD';" | sudo -u postgres psql -sudo -u postgres createdb -O $DB_USERNAME $DB_NAME - -echo -e "\n -> Creating the directories for the logs \n" -mkdir /var/log/mayan - -echo -e "\n -> Making a convenience symlink \n" -cd ${INSTALLATION_DIRECTORY} -ln -s lib/python2.7/site-packages/mayan . - -echo -e "\n -> Creating an initial settings file \n" -mayan-edms.py createsettings - -echo -e "\n -> Updating the mayan/settings/local.py file \n" -cat >> mayan/settings/local.py << EOF -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': '$DB_NAME', - 'USER': '$DB_USERNAME', - 'PASSWORD': '$DB_PASSWORD', - 'HOST': 'localhost', - 'PORT': '5432', - } -} - -BROKER_URL = 'redis://127.0.0.1:6379/0' -CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0' -EOF - -echo -e "\n -> Migrating the database or initialize the project \n" -mayan-edms.py initialsetup - -echo -e "\n -> Disabling the default NGINX site \n" -rm -f /etc/nginx/sites-enabled/default - -echo -e "\n -> Creating a uwsgi.ini file \n" -cat > uwsgi.ini << EOF -[uwsgi] -chdir = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan -chmod-socket = 664 -chown-socket = www-data:www-data -env = DJANGO_SETTINGS_MODULE=mayan.settings.production -gid = www-data -logto = /var/log/uwsgi/%n.log -pythonpath = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages -master = True -max-requests = 5000 -socket = ${INSTALLATION_DIRECTORY}uwsgi.sock -uid = www-data -vacuum = True -wsgi-file = ${INSTALLATION_DIRECTORY}lib/python2.7/site-packages/mayan/wsgi.py -EOF - -echo -e "\n -> Creating the directory for the uWSGI log files \n" -mkdir -p /var/log/uwsgi - -echo -e "\n -> Creating the NGINX site file for Mayan EDMS, /etc/nginx/sites-available/mayan \n" -cat > /etc/nginx/sites-available/mayan << EOF -server { - listen 80; - server_name localhost; - - location / { - include uwsgi_params; - uwsgi_pass unix:${INSTALLATION_DIRECTORY}uwsgi.sock; - - client_max_body_size 30M; # Increse if your plan to upload bigger documents - proxy_read_timeout 30s; # Increase if your document uploads take more than 30 seconds - } - - location /static { - alias ${INSTALLATION_DIRECTORY}mayan/media/static; - expires 1h; - } - - location /favicon.ico { - alias ${INSTALLATION_DIRECTORY}mayan/media/static/appearance/images/favicon.ico; - expires 1h; - } -} -EOF - -echo -e "\n -> Enabling the NGINX site for Mayan EDMS \n" -ln -s /etc/nginx/sites-available/mayan /etc/nginx/sites-enabled/ - -echo -e "\n -> Creating the supervisor file for the uWSGI process, /etc/supervisor/conf.d/mayan-uwsgi.conf \n" -cat > /etc/supervisor/conf.d/mayan-uwsgi.conf << EOF -[program:mayan-uwsgi] -command = ${INSTALLATION_DIRECTORY}bin/uwsgi --ini ${INSTALLATION_DIRECTORY}uwsgi.ini -user = root -autostart = true -autorestart = true -redirect_stderr = true -EOF - -echo -e "\n -> Creating the supervisor file for the Celery worker, /etc/supervisor/conf.d/mayan-celery.conf \n" -cat > /etc/supervisor/conf.d/mayan-celery.conf << EOF -[program:mayan-worker] -command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production worker -Ofair -l ERROR -directory = ${INSTALLATION_DIRECTORY} -user = www-data -stdout_logfile = /var/log/mayan/worker-stdout.log -stderr_logfile = /var/log/mayan/worker-stderr.log -autostart = true -autorestart = true -startsecs = 10 -stopwaitsecs = 10 -killasgroup = true -priority = 998 - -[program:mayan-beat] -command = ${INSTALLATION_DIRECTORY}bin/python ${INSTALLATION_DIRECTORY}bin/mayan-edms.py celery --settings=mayan.settings.production beat -l ERROR -directory = ${INSTALLATION_DIRECTORY} -user = www-data -numprocs = 1 -stdout_logfile = /var/log/mayan/beat-stdout.log -stderr_logfile = /var/log/mayan/beat-stderr.log -autostart = true -autorestart = true -startsecs = 10 -stopwaitsecs = 1 -killasgroup = true -priority = 998 -EOF - -echo -e "\n -> Collecting the static files \n" -mayan-edms.py preparestatic --noinput - -echo -e "\n -> Making the installation directory readable and writable by the webserver user \n" -chown www-data:www-data ${INSTALLATION_DIRECTORY} -R - -echo -e "\n -> Restarting the services \n" -/etc/init.d/nginx restart -/etc/init.d/supervisor restart diff --git a/contrib/scripts/start_gunicorn.sh b/contrib/scripts/start_gunicorn.sh deleted file mode 100644 index 9e52a959c4d..00000000000 --- a/contrib/scripts/start_gunicorn.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -NAME="mayan-edms" -DJANGODIR=/usr/share/mayan-edms -SOCKFILE=/var/tmp/filesystem.sock -USER=www-data -GROUP=www-data -NUM_WORKERS=3 -DJANGO_SETTINGS_MODULE=mayan.settings.production -DJANGO_WSGI_MODULE=mayan.wsgi -TIMEOUT=600 - -echo "Starting $NAME as `whoami`" - -# Activate the virtual environment -cd $DJANGODIR -source bin/activate -export DJANGO_SETTINGS_MODULE=$DJANGO_SETTINGS_MODULE -export PYTHONPATH=$DJANGODIR:$PYTHONPATH - -# Create the run directory if it doesn't exist -RUNDIR=$(dirname $SOCKFILE) -test -d $RUNDIR || mkdir -p $RUNDIR - -# Start your Django Unicorn -# Programs meant to be run under supervisor should not daemonize themselves (do not use --daemon) -exec bin/gunicorn ${DJANGO_WSGI_MODULE}:application \ - --name $NAME \ - --workers $NUM_WORKERS \ - --user=$USER --group=$GROUP \ - --log-level=debug \ - --bind=unix:$SOCKFILE \ - --timeout=$TIMEOUT - - From 51f278301bc8f61cb04f5e77dd8cd10090ba2ce2 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 02:40:24 -0400 Subject: [PATCH 025/402] Sort list of apps Signed-off-by: Roberto Rosario --- contrib/scripts/process_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/scripts/process_messages.py b/contrib/scripts/process_messages.py index f473f050b6f..29b7efc4166 100755 --- a/contrib/scripts/process_messages.py +++ b/contrib/scripts/process_messages.py @@ -14,7 +14,7 @@ 'django_gpg', 'document_comments', 'document_indexing', 'document_parsing', 'document_signatures', 'document_states', 'documents', 'dynamic_search', 'events', 'file_metadata', 'linking', - 'lock_manager', 'mayan_statistics', 'mailer', 'metadata', 'mirroring', + 'lock_manager', 'mailer', 'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings', 'sources', 'storage', 'tags', 'task_manager', 'user_management' From 7faa24eb7b754e62c3b9852cd842b2311a0f38a8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sun, 7 Jul 2019 02:42:11 -0400 Subject: [PATCH 026/402] Remove database conversion command Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 1 + .../common/management/commands/convertdb.py | 107 ------------------ 3 files changed, 2 insertions(+), 107 deletions(-) delete mode 100644 mayan/apps/common/management/commands/convertdb.py diff --git a/HISTORY.rst b/HISTORY.rst index 0744e906848..c4be3eb90de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ - Backport workflow email action. - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. +- Remove database conversion command. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index e6e437790ce..e2098af3f64 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -25,6 +25,7 @@ Changes - Backport workflow email action. - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. +- Remove database conversion command. Removals -------- diff --git a/mayan/apps/common/management/commands/convertdb.py b/mayan/apps/common/management/commands/convertdb.py deleted file mode 100644 index 30bc2d0e67b..00000000000 --- a/mayan/apps/common/management/commands/convertdb.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import unicode_literals - -import errno -import os -import warnings - -from pathlib2 import Path - -from django.conf import settings -from django.core import management -from django.core.management.base import CommandError -from django.utils.encoding import force_text -from django.utils.translation import ugettext_lazy as _ - -from mayan.apps.documents.models import DocumentType -from mayan.apps.storage.utils import fs_cleanup - -from ...literals import MESSAGE_DEPRECATION_WARNING -from ...warnings import DeprecationWarning - -CONVERTDB_FOLDER = 'convertdb' -CONVERTDB_OUTPUT_FILENAME = 'migrate.json' - - -class Command(management.BaseCommand): - help = 'Convert from a database backend to another one.' - - def __init__(self, *args, **kwargs): - warnings.warn( - category=DeprecationWarning, - message=force_text(MESSAGE_DEPRECATION_WARNING) - ) - - super(Command, self).__init__(*args, **kwargs) - - def add_arguments(self, parser): - parser.add_argument( - 'args', metavar='app_label[.ModelName]', nargs='*', - help=_( - 'Restricts dumped data to the specified app_label or ' - 'app_label.ModelName.' - ) - ) - parser.add_argument( - '--from', action='store', default='default', dest='from', - help=_( - 'The database from which data will be exported. If omitted ' - 'the database named "default" will be used.' - ), - ) - parser.add_argument( - '--to', action='store', default='default', dest='to', - help=_( - 'The database to which data will be imported. If omitted ' - 'the database named "default" will be used.' - ), - ) - parser.add_argument( - '--force', action='store_true', dest='force', - help=_( - 'Force the conversion of the database even if the receiving ' - 'database is not empty.' - ), - ) - - def handle(self, *app_labels, **options): - # Create the media/convertdb folder - convertdb_folder_path = force_text( - Path( - settings.MEDIA_ROOT, CONVERTDB_FOLDER - ) - ) - - try: - os.makedirs(convertdb_folder_path) - except OSError as exception: - if exception.errno == errno.EEXIST: - pass - - convertdb_file_path = force_text( - Path( - convertdb_folder_path, CONVERTDB_OUTPUT_FILENAME - ) - ) - - management.call_command(command_name='purgeperiodictasks') - - management.call_command( - 'dumpdata', *app_labels, all=True, - database=options['from'], natural_primary=True, - natural_foreign=True, output=convertdb_file_path, - interactive=False, format='json' - ) - - if DocumentType.objects.using(options['to']).count() and not options['force']: - fs_cleanup(convertdb_file_path) - raise CommandError( - 'There is existing data in the database that will be ' - 'used for the import. If you proceed with the conversion ' - 'you might lose data. Please check your settings.' - ) - - management.call_command( - 'loaddata', convertdb_file_path, database=options['to'], interactive=False, - verbosity=3 - ) - fs_cleanup(convertdb_file_path) From 9564db398fed0134dc97c82f4f9a28d42b9a0e22 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:40:20 -0400 Subject: [PATCH 027/402] Backport configuration file improvements Remove support for quoted entried. Support unquoted entries. Support custom location for the config files. Signed-off-by: Roberto Rosario --- HISTORY.rst | 8 ++ docs/releases/3.3.rst | 38 ++++++--- mayan/apps/common/settings.py | 6 +- mayan/apps/common/storages.py | 14 +-- mayan/apps/converter/backends/python.py | 23 ++--- mayan/apps/converter/classes.py | 20 ++--- mayan/apps/converter/settings.py | 29 +++---- mayan/apps/document_signatures/settings.py | 8 +- mayan/apps/document_signatures/storages.py | 14 +-- mayan/apps/documents/settings.py | 16 ++-- mayan/apps/documents/storages.py | 21 +---- mayan/apps/file_metadata/drivers/exiftool.py | 12 +-- mayan/apps/file_metadata/settings.py | 11 +-- mayan/apps/metadata/settings.py | 2 +- mayan/apps/mirroring/settings.py | 2 +- mayan/apps/ocr/backends/tesseract.py | 14 +-- mayan/apps/ocr/runtime.py | 13 +-- mayan/apps/ocr/settings.py | 2 +- mayan/apps/smart_settings/classes.py | 26 +++++- mayan/apps/smart_settings/forms.py | 12 --- mayan/apps/smart_settings/literals.py | 36 ++++++++ mayan/apps/smart_settings/utils.py | 73 ++++++++++++++++ mayan/apps/sources/settings.py | 12 +-- mayan/apps/sources/storages.py | 13 +-- mayan/settings/base.py | 90 +++++++++----------- mayan/settings/utils.py | 39 --------- 26 files changed, 258 insertions(+), 296 deletions(-) create mode 100644 mayan/apps/smart_settings/literals.py create mode 100644 mayan/apps/smart_settings/utils.py delete mode 100644 mayan/settings/utils.py diff --git a/HISTORY.rst b/HISTORY.rst index c4be3eb90de..84551e6da12 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,14 @@ - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. - Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and + CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings apps. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index e2098af3f64..1c8fd00d20a 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -26,6 +26,14 @@ Changes - Backport individual index rebuild support. - Rename the installjavascript command to installdependencies. - Remove database conversion command. +- Remove support for quoted configuration entries. Support unquoted, + nested dictionaries in the configuration. Requires manual + update of existing config.yml files. +- Support user specified locations for the configuration file with the + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and + CONFIGURATION_LAST_GOOD_FILEPATH + (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. +- Move bootstrapped settings code to their own module in the smart_settings apps. Removals -------- @@ -41,11 +49,11 @@ If installed via Python's PIP Remove deprecated requirements:: - $ curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt | pip uninstall -r /dev/stdin + sudo -u mayan curl https://gitlab.com/mayan-edms/mayan-edms/raw/master/removals.txt -o /tmp/removals.txt && sudo -u mayan /opt/mayan-edms/bin/pip uninstall -y -r /tmp/removals.txt Type in the console:: - $ pip install mayan-edms==3.3 + /opt/mayan-edms/bin/pip install mayan-edms==3.3 the requirements will also be updated automatically. @@ -55,19 +63,19 @@ Using Git If you installed Mayan EDMS by cloning the Git repository issue the commands:: - $ git reset --hard HEAD - $ git pull + git reset --hard HEAD + git pull otherwise download the compressed archived and uncompress it overriding the existing installation. Remove deprecated requirements:: - $ pip uninstall -y -r removals.txt + pip uninstall -y -r removals.txt Next upgrade/add the new requirements:: - $ pip install --upgrade -r requirements.txt + pip install --upgrade -r requirements.txt Common steps @@ -84,9 +92,8 @@ variables values show here with your respective settings. This step will refresh the supervisord configuration file with the new queues and the latest recommended layout:: - sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf Edit the supervisord configuration file and update any setting the template @@ -96,11 +103,11 @@ generator missed:: Migrate existing database schema with:: - $ mayan-edms.py performupgrade + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py performupgrade Add new static media:: - $ mayan-edms.py preparestatic --noinput + sudo -u mayan MAYAN_MEDIA_ROOT=/opt/mayan-edms/media /opt/mayan-edms/bin/mayan-edms.py preparestatic --noinput The upgrade procedure is now complete. @@ -108,7 +115,14 @@ The upgrade procedure is now complete. Backward incompatible changes ----------------------------- -- None +- Update quoted settings to be unquoted: + + - COMMON_SHARED_STORAGE_ARGUMENTS + - CONVERTER_GRAPHICS_BACKEND_ARGUMENTS + - DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS + - DOCUMENTS_STORAGE_BACKEND_ARGUMENTS + - FILE_METADATA_DRIVERS_ARGUMENTS + - SIGNATURES_STORAGE_BACKEND_ARGUMENTS Bugs fixed or issues closed diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 841549b38f9..056e12764a6 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ import mayan -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_COMMON_HOME_VIEW @@ -95,9 +95,7 @@ ) setting_shared_storage_arguments = namespace.add_setting( global_name='COMMON_SHARED_STORAGE_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'shared_files') - ), quoted=True + default={'location': os.path.join(settings.MEDIA_ROOT, 'shared_files')} ) namespace = Namespace(label=_('Django'), name='django') diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8c..f1acfdb7ca6 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +8,4 @@ storage_sharedupload = import_string( dotted_path=setting_shared_storage.value -)( - **yaml.load( - stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_shared_storage_arguments.value) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f8794..700ad2a3810 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,11 +7,6 @@ from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -20,16 +15,14 @@ from ..classes import ConverterBase from ..exceptions import PageCountError -from ..settings import setting_graphics_backend_config +from ..settings import setting_graphics_backend_arguments from ..literals import ( DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdftoppm_path = setting_graphics_backend_arguments.value.get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,26 +32,20 @@ pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + setting_graphics_backend_arguments.value.get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) ) pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -).get( +pdfinfo_path = setting_graphics_backend_arguments.value.get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac518..30c94d663bd 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,12 +7,6 @@ from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ @@ -27,16 +21,14 @@ CONVERTER_OFFICE_FILE_MIMETYPES, DEFAULT_LIBREOFFICE_PATH, DEFAULT_PAGE_NUMBER, DEFAULT_PILLOW_FORMAT ) -from .settings import setting_graphics_backend_config +from .settings import setting_graphics_backend_arguments -logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader -) -libreoffice_path = BACKEND_CONFIG.get( +libreoffice_path = setting_graphics_backend_arguments.value.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH ) +logger = logging.getLogger(__name__) + class ConverterBase(object): def __init__(self, file_object, mime_type=None): @@ -62,9 +54,7 @@ def detect_orientation(self, page_number): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader - ).get( + output_format = output_format or setting_graphics_backend_arguments.value.get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/settings.py b/mayan/apps/converter/settings.py index 14ce07a295e..0d30648fc23 100644 --- a/mayan/apps/converter/settings.py +++ b/mayan/apps/converter/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, DEFAULT_PDFTOPPM_FORMAT, @@ -16,22 +16,15 @@ help_text=_('Graphics conversion backend to use.'), global_name='CONVERTER_GRAPHICS_BACKEND', ) -setting_graphics_backend_config = namespace.add_setting( - default=''' - {{ - libreoffice_path: {}, - pdftoppm_dpi: {}, - pdftoppm_format: {}, - pdftoppm_path: {}, - pdfinfo_path: {}, - pillow_format: {} - - }} - '''.replace('\n', '').format( - DEFAULT_LIBREOFFICE_PATH, DEFAULT_PDFTOPPM_DPI, - DEFAULT_PDFTOPPM_FORMAT, DEFAULT_PDFTOPPM_PATH, DEFAULT_PDFINFO_PATH, - DEFAULT_PILLOW_FORMAT - ), help_text=_( +setting_graphics_backend_arguments = namespace.add_setting( + default={ + 'libreoffice_path': DEFAULT_LIBREOFFICE_PATH, + 'pdftoppm_dpi': DEFAULT_PDFTOPPM_DPI, + 'pdftoppm_format': DEFAULT_PDFTOPPM_FORMAT, + 'pdftoppm_path': DEFAULT_PDFTOPPM_PATH, + 'pdfinfo_path': DEFAULT_PDFINFO_PATH, + 'pillow_format': DEFAULT_PILLOW_FORMAT, + }, help_text=_( 'Configuration options for the graphics conversion backend.' - ), global_name='CONVERTER_GRAPHICS_BACKEND_CONFIG', quoted=True + ), global_name='CONVERTER_GRAPHICS_BACKEND_ARGUMENTS' ) diff --git a/mayan/apps/document_signatures/settings.py b/mayan/apps/document_signatures/settings.py index 6e0b75ec234..7a5e9dcfe1c 100644 --- a/mayan/apps/document_signatures/settings.py +++ b/mayan/apps/document_signatures/settings.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document signatures'), name='signatures') @@ -18,9 +18,9 @@ ) setting_storage_backend_arguments = namespace.add_setting( global_name='SIGNATURES_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_signatures') - ), quoted=True, help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'document_signatures') + }, help_text=_( 'Arguments to pass to the SIGNATURE_STORAGE_BACKEND. ' ) ) diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e6..2a06b3c913d 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +8,4 @@ storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index 19c0162ca8d..a4d5014e918 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES @@ -18,15 +18,14 @@ default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'document image files.' - ), quoted=True + ) ) setting_documentimagecache_storage_arguments = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_cache') - ), help_text=_( + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_cache')}, + help_text=_( 'Arguments to pass to the DOCUMENT_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ), ) setting_disable_base_image_cache = namespace.add_setting( global_name='DOCUMENTS_DISABLE_BASE_IMAGE_CACHE', default=False, @@ -127,9 +126,8 @@ ) setting_storage_backend_arguments = namespace.add_setting( global_name='DOCUMENTS_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'document_storage') - ), help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') + default={'location': os.path.join(settings.MEDIA_ROOT, 'document_storage')}, + help_text=_('Arguments to pass to the DOCUMENT_STORAGE_BACKEND.') ) setting_thumbnail_height = namespace.add_setting( global_name='DOCUMENTS_THUMBNAIL_HEIGHT', default='', help_text=_( diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e92420..a6cec362d65 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,12 +1,5 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -17,18 +10,8 @@ storage_documentversion = import_string( dotted_path=setting_storage_backend.value -)( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_storage_backend_arguments.value) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value -)( - **yaml.load( - stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_documentimagecache_storage_arguments.value) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916a..cff210335e5 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,12 +4,6 @@ import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ @@ -57,11 +51,7 @@ def _process(self, document_version): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader - ) - - self.exiftool_path = driver_arguments.get( + self.exiftool_path = setting_drivers_arguments.value.get( 'exif_driver', {} ).get('exiftool_path', DEFAULT_EXIF_PATH) diff --git a/mayan/apps/file_metadata/settings.py b/mayan/apps/file_metadata/settings.py index 247f0b72ece..b0f8c190648 100644 --- a/mayan/apps/file_metadata/settings.py +++ b/mayan/apps/file_metadata/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EXIF_PATH @@ -16,12 +16,7 @@ ) ) setting_drivers_arguments = namespace.add_setting( - default=''' - {{ - exif_driver: {{exiftool_path: {}}}, - - }} - '''.replace('\n', '').format(DEFAULT_EXIF_PATH), help_text=_( + default={'exif_driver': {'exiftool_path': DEFAULT_EXIF_PATH}}, help_text=_( 'Arguments to pass to the drivers.' - ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS', quoted=True + ), global_name='FILE_METADATA_DRIVERS_ARGUMENTS' ) diff --git a/mayan/apps/metadata/settings.py b/mayan/apps/metadata/settings.py index ce7aa6de352..0387f45309b 100644 --- a/mayan/apps/metadata/settings.py +++ b/mayan/apps/metadata/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .parsers import MetadataParser from .validators import MetadataValidator diff --git a/mayan/apps/mirroring/settings.py b/mayan/apps/mirroring/settings.py index aa16aa1335f..41f9b372a59 100644 --- a/mayan/apps/mirroring/settings.py +++ b/mayan/apps/mirroring/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Mirroring'), name='mirroring') diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4da..0a8f87cf2e1 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,11 +4,6 @@ import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ @@ -115,15 +110,10 @@ def initialize(self): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, - stream=setting_ocr_backend_arguments.value or '{}', - ) - - self.tesseract_binary_path = backend_arguments.get( + self.tesseract_binary_path = setting_ocr_backend_arguments.value.get( 'tesseract_path', DEFAULT_TESSERACT_BINARY_PATH ) - self.command_timeout = backend_arguments.get( + self.command_timeout = setting_ocr_backend_arguments.value.get( 'timeout', DEFAULT_TESSERACT_TIMEOUT ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b6..55e6a36a35d 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value -)( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader - ) -) +)(**setting_ocr_backend_arguments.value) diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index f2aa1052a31..4293c3ac241 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('OCR'), name='ocr') diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608e..8a79ba87576 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -21,6 +21,8 @@ force_bytes, force_text, python_2_unicode_compatible ) +from .utils import read_configuration_file + logger = logging.getLogger(__name__) @@ -82,6 +84,7 @@ def settings(self): class Setting(object): _registry = {} _cache_hash = None + _config_file_cache = None @staticmethod def deserialize_value(value): @@ -103,6 +106,7 @@ def express_promises(value): def serialize_value(value): result = yaml.dump( data=Setting.express_promises(value), allow_unicode=True, + default_flow_style=False, Dumper=SafeDumper ) # safe_dump returns bytestrings @@ -140,6 +144,16 @@ def get(cls, global_name): def get_all(cls): return sorted(cls._registry.values(), key=lambda x: x.global_name) + @classmethod + def get_config_file_content(cls): + # Cache content of config file to speed up initial boot up + if not cls._config_file_cache: + cls._config_file_cache = read_configuration_file( + path=settings.CONFIGURATION_FILEPATH + ) + + return cls._config_file_cache + @classmethod def get_hash(cls): return force_text( @@ -167,13 +181,12 @@ def save_last_known_good(cls): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False, quoted=False): + def __init__(self, namespace, global_name, default, help_text=None, is_path=False): self.global_name = global_name self.default = default self.help_text = help_text self.loaded = False self.namespace = namespace - self.quoted = quoted self.environment_variable = False namespace._settings.append(self) self.__class__._registry[global_name] = self @@ -186,7 +199,7 @@ def cache_value(self): if environment_value: self.environment_variable = True try: - self.raw_value = environment_value + self.raw_value = yaml.load(stream=environment_value, Loader=SafeLoader) except yaml.YAMLError as exception: raise type(exception)( 'Error interpreting environment variable: {} with ' @@ -195,7 +208,12 @@ def cache_value(self): ) ) else: - self.raw_value = getattr(settings, self.global_name, self.default) + self.raw_value = self.get_config_file_content().get( + self.global_name, getattr( + settings, self.global_name, self.default + ) + ) + self.yaml = Setting.serialize_value(self.raw_value) self.loaded = True diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316df..25f86875976 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -25,18 +25,6 @@ def __init__(self, *args, **kwargs): self.fields['value'].initial = self.setting.serialized_value def clean(self): - quotes = ['"', "'"] - - if self.setting.quoted: - stripped = self.cleaned_data['value'].strip() - - if stripped[0] not in quotes or stripped[-1] not in quotes: - raise ValidationError( - _( - 'Value must be properly quoted.' - ) - ) - try: yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) except yaml.YAMLError: diff --git a/mayan/apps/smart_settings/literals.py b/mayan/apps/smart_settings/literals.py new file mode 100644 index 00000000000..6db3f2ee9c5 --- /dev/null +++ b/mayan/apps/smart_settings/literals.py @@ -0,0 +1,36 @@ +from __future__ import unicode_literals + +# Default in YAML format +BOOTSTRAP_SETTING_LIST = ( + {'name': 'ALLOWED_HOSTS', 'default': "['127.0.0.1', 'localhost', '[::1]']"}, + {'name': 'APPEND_SLASH'}, + {'name': 'AUTH_PASSWORD_VALIDATORS'}, + {'name': 'COMMON_DISABLED_APPS'}, + {'name': 'COMMON_EXTRA_APPS'}, + {'name': 'DATA_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'DATABASES'}, + {'name': 'DEBUG', 'default': 'false'}, + {'name': 'DEFAULT_FROM_EMAIL'}, + {'name': 'DISALLOWED_USER_AGENTS'}, + {'name': 'EMAIL_BACKEND'}, + {'name': 'EMAIL_HOST'}, + {'name': 'EMAIL_HOST_PASSWORD'}, + {'name': 'EMAIL_HOST_USER'}, + {'name': 'EMAIL_PORT'}, + {'name': 'EMAIL_TIMEOUT'}, + {'name': 'EMAIL_USE_SSL'}, + {'name': 'EMAIL_USE_TLS'}, + {'name': 'FILE_UPLOAD_MAX_MEMORY_SIZE'}, + {'name': 'HOME_VIEW'}, + {'name': 'INSTALLED_APPS'}, + {'name': 'INTERNAL_IPS', 'default': "['127.0.0.1']"}, + {'name': 'LANGUAGES'}, + {'name': 'LANGUAGE_CODE'}, + {'name': 'LOGIN_REDIRECT_URL', 'default': 'common:home'}, + {'name': 'LOGIN_URL', 'default': 'authentication:login_view'}, + {'name': 'LOGOUT_REDIRECT_URL', 'default': 'authentication:login_view'}, + {'name': 'STATIC_URL'}, + {'name': 'STATICFILES_STORAGE'}, + {'name': 'TIME_ZONE'}, + {'name': 'WSGI_APPLICATION'} +) diff --git a/mayan/apps/smart_settings/utils.py b/mayan/apps/smart_settings/utils.py new file mode 100644 index 00000000000..4282d2c38d7 --- /dev/null +++ b/mayan/apps/smart_settings/utils.py @@ -0,0 +1,73 @@ +from __future__ import unicode_literals + +import errno +import os + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader + +from .literals import BOOTSTRAP_SETTING_LIST + + +def get_default(name, fallback_default=None): + for item in BOOTSTRAP_SETTING_LIST: + if item['name'] == name: + return item.get('default', fallback_default) + + return fallback_default + + +def get_environment_variables(): + result = {} + + for setting in BOOTSTRAP_SETTING_LIST: + environment_value = os.environ.get('MAYAN_{}'.format(setting['name'])) + if environment_value: + environment_value = yaml.load(stream=environment_value, Loader=SafeLoader) + result[setting['name']] = environment_value + + return result + + +def get_environment_setting(name, fallback_default=None): + value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default)) + + if value: + return yaml.load(stream=value, Loader=SafeLoader) + + +def read_configuration_file(path): + try: + with open(path) as file_object: + file_object.seek(0, os.SEEK_END) + if file_object.tell(): + file_object.seek(0) + try: + return yaml.load(stream=file_object, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + 'Error loading configuration file: {}; {}'.format( + path, exception + ) + ) + except IOError as exception: + if exception.errno == errno.ENOENT: + pass + else: + raise + + +def yaml_loads(data, error_message=None): + if not error_message: + error_message = 'Error loading: {}; {}' + + try: + return yaml.load(stream=data, Loader=SafeLoader) + except yaml.YAMLError as exception: + exit( + error_message.format(data, exception) + ) diff --git a/mayan/apps/sources/settings.py b/mayan/apps/sources/settings.py index 8f0156d427e..aef20ad07d1 100644 --- a/mayan/apps/sources/settings.py +++ b/mayan/apps/sources/settings.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Sources'), name='sources') @@ -21,13 +21,13 @@ default='django.core.files.storage.FileSystemStorage', help_text=_( 'Path to the Storage subclass to use when storing the cached ' 'staging_file image files.' - ), quoted=True + ) ) setting_staging_file_image_cache_storage_arguments = namespace.add_setting( global_name='SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND_ARGUMENTS', - default='{{location: {}}}'.format( - os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') - ), help_text=_( + default={ + 'location': os.path.join(settings.MEDIA_ROOT, 'staging_file_cache') + }, help_text=_( 'Arguments to pass to the SOURCES_STAGING_FILE_CACHE_STORAGE_BACKEND.' - ), quoted=True, + ) ) diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5f..2849d8e9150 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,11 +1,5 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string from .settings import ( @@ -15,9 +9,4 @@ storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value -)( - **yaml.load( - stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader - ) -) +)(**setting_staging_file_image_cache_storage_arguments.value) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 3ec88a53227..6292ecd168c 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -17,15 +17,15 @@ from django.core.exceptions import ImproperlyConfigured from django.utils.translation import ugettext_lazy as _ -import environ +from mayan.apps.smart_settings.literals import BOOTSTRAP_SETTING_LIST +from mayan.apps.smart_settings.utils import ( + get_environment_setting, get_environment_variables, read_configuration_file +) from .literals import ( CONFIGURATION_FILENAME, CONFIGURATION_LAST_GOOD_FILENAME, DEFAULT_SECRET_KEY, SECRET_KEY_FILENAME, SYSTEM_DIR ) -from .utils import yaml_loads, read_configuration_file - -env = environ.Env() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -34,8 +34,8 @@ # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ -MEDIA_ROOT = os.environ.get( - 'MAYAN_MEDIA_ROOT', os.path.join(BASE_DIR, 'media') +MEDIA_ROOT = get_environment_setting( + name='MAYAN_MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media') ) # SECURITY WARNING: keep the secret key used in production secret! @@ -50,13 +50,9 @@ SECRET_KEY = DEFAULT_SECRET_KEY # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool('MAYAN_DEBUG', default=False) +DEBUG = get_environment_setting(name='DEBUG') -ALLOWED_HOSTS = yaml_loads( - env( - 'MAYAN_ALLOWED_HOSTS', default="['127.0.0.1', 'localhost', '[::1]']" - ) -) +ALLOWED_HOSTS = get_environment_setting(name='ALLOWED_HOSTS') # Application definition @@ -257,12 +253,10 @@ # --------- Django ------------------- -LOGIN_URL = env('MAYAN_LOGIN_URL', default='authentication:login_view') -LOGIN_REDIRECT_URL = env('MAYAN_LOGIN_REDIRECT_URL', default='common:root') -LOGOUT_REDIRECT_URL = env( - 'MAYAN_LOGOUT_REDIRECT_URL', default='authentication:login_view' -) -INTERNAL_IPS = ('127.0.0.1',) +LOGIN_URL = get_environment_setting(name='LOGIN_URL') +LOGIN_REDIRECT_URL = get_environment_setting(name='LOGIN_REDIRECT_URL') +LOGOUT_REDIRECT_URL = get_environment_setting(name='LOGOUT_REDIRECT_URL') +INTERNAL_IPS = get_environment_setting(name='INTERNAL_IPS') # ---------- Django REST framework ----------- @@ -325,51 +319,43 @@ # ----- Celery ----- -BROKER_URL = os.environ.get('MAYAN_BROKER_URL') -CELERY_ALWAYS_EAGER = env.bool('MAYAN_CELERY_ALWAYS_EAGER', default=True) -CELERY_RESULT_BACKEND = os.environ.get('MAYAN_CELERY_RESULT_BACKEND') +BROKER_URL = get_environment_setting(name='BROKER_URL') +CELERY_ALWAYS_EAGER = get_environment_setting(name='CELERY_ALWAYS_EAGER') +CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND') # ----- Database ----- -environment_database_engine = os.environ.get('MAYAN_DATABASE_ENGINE') - -if environment_database_engine: - environment_database_conn_max_age = os.environ.get('MAYAN_DATABASE_CONN_MAX_AGE', 0) - if environment_database_conn_max_age: - environment_database_conn_max_age = int(environment_database_conn_max_age) - - DATABASES = { - 'default': { - 'ENGINE': environment_database_engine, - 'NAME': os.environ['MAYAN_DATABASE_NAME'], - 'USER': os.environ['MAYAN_DATABASE_USER'], - 'PASSWORD': os.environ['MAYAN_DATABASE_PASSWORD'], - 'HOST': os.environ.get('MAYAN_DATABASE_HOST', None), - 'PORT': os.environ.get('MAYAN_DATABASE_PORT', None), - 'CONN_MAX_AGE': environment_database_conn_max_age, - } - } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } - +} BASE_INSTALLED_APPS = INSTALLED_APPS COMMON_EXTRA_APPS = () COMMON_DISABLED_APPS = () -CONFIGURATION_FILEPATH = os.path.join(MEDIA_ROOT, CONFIGURATION_FILENAME) -CONFIGURATION_LAST_GOOD_FILEPATH = os.path.join( - MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME +CONFIGURATION_FILEPATH = get_environment_setting( + name='CONFIGURATION_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_FILENAME + ) +) + +CONFIGURATION_LAST_GOOD_FILEPATH = get_environment_setting( + name='CONFIGURATION_LAST_GOOD_FILEPATH', fallback_default=os.path.join( + MEDIA_ROOT, CONFIGURATION_LAST_GOOD_FILENAME + ) ) if 'revertsettings' not in sys.argv: - result = read_configuration_file(CONFIGURATION_FILEPATH) - if result: - globals().update(result) + configuration_result = read_configuration_file(CONFIGURATION_FILEPATH) + environment_result = get_environment_variables() + + for setting in BOOTSTRAP_SETTING_LIST: + if setting['name'] in configuration_result: + globals().update({setting['name']: configuration_result[setting['name']]}) + elif setting['name'] in environment_result: + globals().update({setting['name']: environment_result[setting['name']]}) for app in INSTALLED_APPS: diff --git a/mayan/settings/utils.py b/mayan/settings/utils.py deleted file mode 100644 index bb2a8da489c..00000000000 --- a/mayan/settings/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import unicode_literals - -import errno -import os - -import yaml - - -def read_configuration_file(path): - try: - with open(path) as file_object: - file_object.seek(0, os.SEEK_END) - if file_object.tell(): - file_object.seek(0) - try: - return yaml.safe_load(file_object) - except yaml.YAMLError as exception: - exit( - 'Error loading configuration file: {}; {}'.format( - path, exception - ) - ) - except IOError as exception: - if exception.errno == errno.ENOENT: - pass - else: - raise - - -def yaml_loads(data, error_message=None): - if not error_message: - error_message = 'Error loading: {}; {}' - - try: - return yaml.safe_load(data) - except yaml.YAMLError as exception: - exit( - error_message.format(data, exception) - ) From 7a01a77c431563f8cb21625497bc409aa37155a1 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:42:57 -0400 Subject: [PATCH 028/402] Remove smart_settings * import Signed-off-by: Roberto Rosario --- mayan/apps/appearance/settings.py | 2 +- mayan/apps/authentication/settings.py | 2 +- mayan/apps/autoadmin/settings.py | 2 +- mayan/apps/django_gpg/settings.py | 2 +- mayan/apps/document_parsing/settings.py | 2 +- mayan/apps/document_states/settings.py | 2 +- mayan/apps/lock_manager/settings.py | 2 +- mayan/apps/smart_settings/__init__.py | 2 -- mayan/apps/storage/settings.py | 2 +- 9 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mayan/apps/appearance/settings.py b/mayan/apps/appearance/settings.py index 5b93d435255..ad0927530ad 100644 --- a/mayan/apps/appearance/settings.py +++ b/mayan/apps/appearance/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_MAXIMUM_TITLE_LENGTH diff --git a/mayan/apps/authentication/settings.py b/mayan/apps/authentication/settings.py index de693442d55..ec264c7933b 100644 --- a/mayan/apps/authentication/settings.py +++ b/mayan/apps/authentication/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_LOGIN_METHOD, DEFAULT_MAXIMUM_SESSION_LENGTH diff --git a/mayan/apps/autoadmin/settings.py b/mayan/apps/autoadmin/settings.py index d74b9b0d74e..3c21d273397 100644 --- a/mayan/apps/autoadmin/settings.py +++ b/mayan/apps/autoadmin/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_EMAIL, DEFAULT_PASSWORD, DEFAULT_USERNAME diff --git a/mayan/apps/django_gpg/settings.py b/mayan/apps/django_gpg/settings.py index b7c763f45f6..651f54bcbc3 100644 --- a/mayan/apps/django_gpg/settings.py +++ b/mayan/apps/django_gpg/settings.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Signatures'), name='django_gpg') diff --git a/mayan/apps/document_parsing/settings.py b/mayan/apps/document_parsing/settings.py index c6626fbbd10..7bf2b78d3a2 100644 --- a/mayan/apps/document_parsing/settings.py +++ b/mayan/apps/document_parsing/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Document parsing'), name='document_parsing') diff --git a/mayan/apps/document_states/settings.py b/mayan/apps/document_states/settings.py index b90ae5e3821..26298fbdf92 100644 --- a/mayan/apps/document_states/settings.py +++ b/mayan/apps/document_states/settings.py @@ -5,7 +5,7 @@ from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Workflows'), name='document_states') diff --git a/mayan/apps/lock_manager/settings.py b/mayan/apps/lock_manager/settings.py index d401b4c462b..30448fa1fb3 100644 --- a/mayan/apps/lock_manager/settings.py +++ b/mayan/apps/lock_manager/settings.py @@ -2,7 +2,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import DEFAULT_BACKEND, DEFAULT_LOCK_TIMEOUT_VALUE diff --git a/mayan/apps/smart_settings/__init__.py b/mayan/apps/smart_settings/__init__.py index 143550fa7b6..54f25908f0f 100644 --- a/mayan/apps/smart_settings/__init__.py +++ b/mayan/apps/smart_settings/__init__.py @@ -1,5 +1,3 @@ from __future__ import unicode_literals -from .classes import Namespace, Setting # NOQA - default_app_config = 'mayan.apps.smart_settings.apps.SmartSettingsApp' diff --git a/mayan/apps/storage/settings.py b/mayan/apps/storage/settings.py index e17acbd76cd..362aa9e7026 100644 --- a/mayan/apps/storage/settings.py +++ b/mayan/apps/storage/settings.py @@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(label=_('Storage'), name='storage') From c9668d62e5d228ba69f0dd0d501e96e2a9f87a33 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:43:15 -0400 Subject: [PATCH 029/402] Move mailer defaults to the literals module Signed-off-by: Roberto Rosario --- mayan/apps/mailer/literals.py | 4 ++-- mayan/apps/mailer/settings.py | 31 ++++++++++++++++--------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/mayan/apps/mailer/literals.py b/mayan/apps/mailer/literals.py index b22405cee30..c76b51f7d3f 100644 --- a/mayan/apps/mailer/literals.py +++ b/mayan/apps/mailer/literals.py @@ -8,11 +8,11 @@ '--------\n ' 'This email has been sent from %(project_title)s (%(project_website)s)' ) - +DEFAULT_DOCUMENT_SUBJECT_TEMPLATE = _('Document: {{ document }}') DEFAULT_LINK_BODY_TEMPLATE = _( 'To access this document click on the following link: ' '{{ link }}\n\n--------\n ' 'This email has been sent from %(project_title)s (%(project_website)s)' ) - +DEFAULT_LINK_SUBJECT_TEMPLATE = _('Link for document: {{ document }}') EMAIL_SEPARATORS = (',', ';') diff --git a/mayan/apps/mailer/settings.py b/mayan/apps/mailer/settings.py index ed689b60a3a..7fbf7f6b8b3 100644 --- a/mayan/apps/mailer/settings.py +++ b/mayan/apps/mailer/settings.py @@ -2,31 +2,32 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace from .literals import ( - DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_LINK_BODY_TEMPLATE + DEFAULT_DOCUMENT_BODY_TEMPLATE, DEFAULT_DOCUMENT_SUBJECT_TEMPLATE, + DEFAULT_LINK_BODY_TEMPLATE, DEFAULT_LINK_SUBJECT_TEMPLATE ) namespace = Namespace(label=_('Mailing'), name='mailer') -setting_link_subject_template = namespace.add_setting( - default=_('Link for document: {{ document }}'), - help_text=_('Template for the document link email form subject line.'), - global_name='MAILER_LINK_SUBJECT_TEMPLATE', quoted=True -) -setting_link_body_template = namespace.add_setting( - default=DEFAULT_LINK_BODY_TEMPLATE, - help_text=_('Template for the document link email form body text. Can include HTML.'), - global_name='MAILER_LINK_BODY_TEMPLATE', quoted=True -) setting_document_subject_template = namespace.add_setting( - default=_('Document: {{ document }}'), + default=DEFAULT_DOCUMENT_SUBJECT_TEMPLATE, help_text=_('Template for the document email form subject line.'), - global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_SUBJECT_TEMPLATE' ) setting_document_body_template = namespace.add_setting( default=DEFAULT_DOCUMENT_BODY_TEMPLATE, help_text=_('Template for the document email form body text. Can include HTML.'), - global_name='MAILER_DOCUMENT_BODY_TEMPLATE', quoted=True + global_name='MAILER_DOCUMENT_BODY_TEMPLATE' +) +setting_link_subject_template = namespace.add_setting( + default=DEFAULT_LINK_SUBJECT_TEMPLATE, + help_text=_('Template for the document link email form subject line.'), + global_name='MAILER_LINK_SUBJECT_TEMPLATE' +) +setting_link_body_template = namespace.add_setting( + default=DEFAULT_LINK_BODY_TEMPLATE, + help_text=_('Template for the document link email form body text. Can include HTML.'), + global_name='MAILER_LINK_BODY_TEMPLATE' ) From 22da1e0a781887fdc13b39b154b85df9cfe0ca1a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:43:39 -0400 Subject: [PATCH 030/402] Update import Signed-off-by: Roberto Rosario --- mayan/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/conf.py b/mayan/conf.py index 212322bd5da..898ccdd7dac 100644 --- a/mayan/conf.py +++ b/mayan/conf.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.smart_settings import Namespace +from mayan.apps.smart_settings.classes import Namespace namespace = Namespace(name='mayan', label=_('Mayan')) From 8a54deba3d4d5421b3b53e91930064f036576b46 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:45:30 -0400 Subject: [PATCH 031/402] Unify individual database configuration options All database configuration is now done using MAYAN_DATABASES to mirror Django way of doing database setup. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 ++ docs/chapters/deploying.rst | 10 ++++------ docs/releases/3.3.rst | 2 ++ mayan/apps/platform/classes.py | 25 +++---------------------- 4 files changed, 11 insertions(+), 28 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 84551e6da12..3ff767725a6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,8 @@ CONFIGURATION_LAST_GOOD_FILEPATH (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. - Move bootstrapped settings code to their own module in the smart_settings apps. +- Remove individual database configuration options. All database configuration + is now done using MAYAN_DATABASES to mirror Django way of doing database setup. 3.2.5 (2019-07-05) ================== diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index 425f3274f9b..d3016b98a32 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -127,9 +127,8 @@ For another setup that offers more performance and scalability refer to the :: - sudo -u mayan MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo -u mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py initialsetup @@ -148,9 +147,8 @@ For another setup that offers more performance and scalability refer to the ------------------------------------------------------------------------ :: - sudo MAYAN_DATABASE_ENGINE=django.db.backends.postgresql MAYAN_DATABASE_NAME=mayan \ - MAYAN_DATABASE_PASSWORD=mayanuserpass MAYAN_DATABASE_USER=mayan \ - MAYAN_DATABASE_HOST=127.0.0.1 MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ + sudo mayan MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}" \ + MAYAN_MEDIA_ROOT=/opt/mayan-edms/media \ /opt/mayan-edms/bin/mayan-edms.py platformtemplate supervisord > /etc/supervisor/conf.d/mayan.conf diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 1c8fd00d20a..74d0e402af0 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -34,6 +34,8 @@ Changes CONFIGURATION_LAST_GOOD_FILEPATH (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. - Move bootstrapped settings code to their own module in the smart_settings apps. +- Remove individual database configuration options. All database configuration + is now done using MAYAN_DATABASES to mirror Django way of doing database setup. Removals -------- diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 1d0152442f1..9f897fbf7f7 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -125,28 +125,9 @@ class PlatformTemplateSupervisord(PlatformTemplate): environment_name='MAYAN_GUNICORN_TIMEOUT' ), Variable( - name='DATABASE_CONN_MAX_AGE', default=0, - environment_name='MAYAN_DATABASE_CONN_MAX_AGE' - ), - Variable( - name='DATABASE_ENGINE', default='django.db.backends.postgresql', - environment_name='MAYAN_DATABASE_ENGINE' - ), - Variable( - name='DATABASE_HOST', default='127.0.0.1', - environment_name='MAYAN_DATABASE_HOST' - ), - Variable( - name='DATABASE_NAME', default='mayan', - environment_name='MAYAN_DATABASE_NAME' - ), - Variable( - name='DATABASE_PASSWORD', default='mayanuserpass', - environment_name='MAYAN_DATABASE_PASSWORD' - ), - Variable( - name='DATABASE_USER', default='mayan', - environment_name='MAYAN_DATABASE_USER' + name='DATABASES', + default="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}", + environment_name='MAYAN_DATABASES' ), Variable( name='INSTALLATION_PATH', default='/opt/mayan-edms', From 91b0b2d9c38cafb237e4b723b517c0d388baf2b9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 9 Jul 2019 15:46:09 -0400 Subject: [PATCH 032/402] Update smart setting's app URLs for uniformity Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/urls.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mayan/apps/smart_settings/urls.py b/mayan/apps/smart_settings/urls.py index 5640901c1ac..56d05c5e5a2 100644 --- a/mayan/apps/smart_settings/urls.py +++ b/mayan/apps/smart_settings/urls.py @@ -6,15 +6,15 @@ urlpatterns = [ url( - regex=r'^namespace/all/$', view=NamespaceListView.as_view(), + regex=r'^namespaces/$', view=NamespaceListView.as_view(), name='namespace_list' ), url( - regex=r'^namespace/(?P\w+)/$', + regex=r'^namespaces/(?P\w+)/$', view=NamespaceDetailView.as_view(), name='namespace_detail' ), url( - regex=r'^edit/(?P\w+)/$', + regex=r'^namespaces/settings/(?P\w+)/edit/$', view=SettingEditView.as_view(), name='setting_edit_view' ), ] From 78a0189e1c68b6a2df96d82f66fab1c84e6633d8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 00:34:09 -0400 Subject: [PATCH 033/402] Add YAML env variables support to platform app Signed-off-by: Roberto Rosario --- HISTORY.rst | 14 +++-- docs/releases/3.3.rst | 2 + mayan/apps/platform/classes.py | 55 ++++++++++++++----- .../templates/platform/supervisord.tmpl | 15 ++--- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3ff767725a6..825274d4541 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,12 +18,16 @@ nested dictionaries in the configuration. Requires manual update of existing config.yml files. - Support user specified locations for the configuration file with the - CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), and - CONFIGURATION_LAST_GOOD_FILEPATH + CONFIGURATION_FILEPATH (MAYAN_CONFIGURATION_FILEPATH environment variable), + and CONFIGURATION_LAST_GOOD_FILEPATH (MAYAN_CONFIGURATION_LAST_GOOD_FILEPATH environment variable) settings. -- Move bootstrapped settings code to their own module in the smart_settings apps. -- Remove individual database configuration options. All database configuration - is now done using MAYAN_DATABASES to mirror Django way of doing database setup. +- Move bootstrapped settings code to their own module in the smart_settings + apps. +- Remove individual database configuration options. All database + configuration is now done using MAYAN_DATABASES to mirror Django way of + doing atabase etup. +- Added support for YAML encoded environment variables to the platform + templates apps. 3.2.5 (2019-07-05) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 74d0e402af0..c15eb651bb6 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -36,6 +36,8 @@ Changes - Move bootstrapped settings code to their own module in the smart_settings apps. - Remove individual database configuration options. All database configuration is now done using MAYAN_DATABASES to mirror Django way of doing database setup. +- Added support for YAML encoded environment variables to the platform + templates apps. Removals -------- diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 9f897fbf7f7..f9f08196a66 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -4,11 +4,12 @@ import yaml try: - from yaml import CSafeLoader as SafeLoader + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper except ImportError: - from yaml import SafeLoader + from yaml import SafeLoader, SafeDumper from django.template import loader +from django.utils.html import mark_safe from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ @@ -24,9 +25,25 @@ def __init__(self, name, default, environment_name): self.default = default self.environment_name = environment_name - def get_value(self): + def _get_value(self): return os.environ.get(self.environment_name, self.default) + def get_value(self): + return mark_safe(self._get_value()) + + +class YAMLVariable(Variable): + def _get_value(self): + value = os.environ.get(self.environment_name) + if value: + value = yaml.load(stream=value, Loader=SafeLoader) + else: + value = self.default + + return yaml.dump( + data=value, allow_unicode=True, default_flow_style=True, width=999, Dumper=SafeDumper + ).replace('...\n', '').replace('\n', '') + @python_2_unicode_compatible class PlatformTemplate(object): @@ -106,10 +123,6 @@ def render(self, context_string=None): class PlatformTemplateSupervisord(PlatformTemplate): - context_defaults = { - 'BROKER_URL': 'redis://127.0.0.1:6379/0', - 'CELERY_RESULT_BACKEND': 'redis://127.0.0.1:6379/0', - } label = _('Template for Supervisord.') name = 'supervisord' settings = ( @@ -124,16 +137,32 @@ class PlatformTemplateSupervisord(PlatformTemplate): name='GUNICORN_TIMEOUT', default=120, environment_name='MAYAN_GUNICORN_TIMEOUT' ), - Variable( - name='DATABASES', - default="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}", - environment_name='MAYAN_DATABASES' - ), Variable( name='INSTALLATION_PATH', default='/opt/mayan-edms', environment_name='MAYAN_INSTALLATION_PATH' ), - Variable( + YAMLVariable( + name='ALLOWED_HOSTS', + default=['*'], + environment_name='MAYAN_ALLOWED_HOSTS' + ), + YAMLVariable( + name='BROKER_URL', + default='redis://127.0.0.1:6379/0', + environment_name='MAYAN_BROKER_URL' + ), + YAMLVariable( + name='CELERY_RESULT_BACKEND', + default='redis://127.0.0.1:6379/0', + environment_name='MAYAN_CELERY_RESULT_BACKEND' + ), + YAMLVariable( + name='DATABASES', + default={'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'127.0.0.1'}}, + environment_name='MAYAN_DATABASES' + ), + YAMLVariable + ( name='MEDIA_ROOT', default='/opt/mayan-edms/media', environment_name='MAYAN_MEDIA_ROOT' ), diff --git a/mayan/apps/platform/templates/platform/supervisord.tmpl b/mayan/apps/platform/templates/platform/supervisord.tmpl index b87686c8278..f6623d12deb 100644 --- a/mayan/apps/platform/templates/platform/supervisord.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord.tmpl @@ -1,17 +1,12 @@ [supervisord] environment= - MAYAN_ALLOWED_HOSTS='["*"]', # Allow access to other network hosts other than localhost + PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, + DJANGO_SETTINGS_MODULE=mayan.settings.production, + MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}", + MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}", MAYAN_CELERY_RESULT_BACKEND="{{ CELERY_RESULT_BACKEND }}", MAYAN_BROKER_URL="{{ BROKER_URL }}", - PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, - MAYAN_MEDIA_ROOT={{ MEDIA_ROOT }}, - MAYAN_DATABASE_ENGINE={{ DATABASE_ENGINE }}, - MAYAN_DATABASE_HOST={{ DATABASE_HOST }}, - MAYAN_DATABASE_NAME={{ DATABASE_NAME }}, - MAYAN_DATABASE_PASSWORD={{ DATABASE_PASSWORD }}, - MAYAN_DATABASE_USER={{ DATABASE_USER }}, - MAYAN_DATABASE_CONN_MAX_AGE={{ DATABASE_CONN_MAX_AGE }}, - DJANGO_SETTINGS_MODULE=mayan.settings.production + MAYAN_DATABASES="{{ DATABASES }}" [program:mayan-gunicorn] autorestart = true From 37e85590e8b56ad58663a7c9a26e92d2503b38e3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 19:02:22 -0400 Subject: [PATCH 034/402] Move Django and Celery settings Django settings now reside in the smart settings app. Celery settings now reside in the task manager app. Signed-off-by: Roberto Rosario --- mayan/apps/common/settings.py | 316 -------------------------- mayan/apps/smart_settings/apps.py | 1 + mayan/apps/smart_settings/settings.py | 302 ++++++++++++++++++++++++ mayan/apps/task_manager/apps.py | 1 + mayan/apps/task_manager/settings.py | 29 +++ 5 files changed, 333 insertions(+), 316 deletions(-) create mode 100644 mayan/apps/smart_settings/settings.py create mode 100644 mayan/apps/task_manager/settings.py diff --git a/mayan/apps/common/settings.py b/mayan/apps/common/settings.py index 84009fe1707..a7653a833c7 100644 --- a/mayan/apps/common/settings.py +++ b/mayan/apps/common/settings.py @@ -10,7 +10,6 @@ from .literals import DEFAULT_COMMON_HOME_VIEW - namespace = Namespace(label=_('Common'), name='common') setting_auto_logging = namespace.add_setting( @@ -99,318 +98,3 @@ os.path.join(settings.MEDIA_ROOT, 'shared_files') ), quoted=True ) - -namespace = Namespace(label=_('Django'), name='django') - -setting_django_allowed_hosts = namespace.add_setting( - global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS, - help_text=_( - 'A list of strings representing the host/domain names that this site ' - 'can serve. This is a security measure to prevent HTTP Host header ' - 'attacks, which are possible even under many seemingly-safe web ' - 'server configurations. Values in this list can be ' - 'fully qualified names (e.g. \'www.example.com\'), in which case ' - 'they will be matched against the request\'s Host header exactly ' - '(case-insensitive, not including port). A value beginning with a ' - 'period can be used as a subdomain wildcard: \'.example.com\' will ' - 'match example.com, www.example.com, and any other subdomain of ' - 'example.com. A value of \'*\' will match anything; in this case you ' - 'are responsible to provide your own validation of the Host header ' - '(perhaps in a middleware; if so this middleware must be listed ' - 'first in MIDDLEWARE).' - ), -) -setting_django_append_slash = namespace.add_setting( - global_name='APPEND_SLASH', default=settings.APPEND_SLASH, - help_text=_( - 'When set to True, if the request URL does not match any of the ' - 'patterns in the URLconf and it doesn\'t end in a slash, an HTTP ' - 'redirect is issued to the same URL with a slash appended. Note ' - 'that the redirect may cause any data submitted in a POST request ' - 'to be lost. The APPEND_SLASH setting is only used if ' - 'CommonMiddleware is installed (see Middleware). See also ' - 'PREPEND_WWW.' - ) -) -setting_django_auth_password_validators = namespace.add_setting( - global_name='AUTH_PASSWORD_VALIDATORS', - default=settings.AUTH_PASSWORD_VALIDATORS, - help_text=_( - 'The list of validators that are used to check the strength of ' - 'user\'s passwords.' - ) -) -setting_django_databases = namespace.add_setting( - global_name='DATABASES', default=settings.DATABASES, - help_text=_( - 'A dictionary containing the settings for all databases to be used ' - 'with Django. It is a nested dictionary whose contents map a ' - 'database alias to a dictionary containing the options for an ' - 'individual database. The DATABASES setting must configure a ' - 'default database; any number of additional databases may also ' - 'be specified.' - ), -) -setting_django_data_upload_max_memory_size = namespace.add_setting( - global_name='DATA_UPLOAD_MAX_MEMORY_SIZE', - default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE, - help_text=_( - 'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a ' - 'request body may be before a SuspiciousOperation ' - '(RequestDataTooBig) is raised. The check is done when accessing ' - 'request.body or request.POST and is calculated against the total ' - 'request size excluding any file upload data. You can set this to ' - 'None to disable the check. Applications that are expected to ' - 'receive unusually large form posts should tune this setting. The ' - 'amount of request data is correlated to the amount of memory ' - 'needed to process the request and populate the GET and POST ' - 'dictionaries. Large requests could be used as a ' - 'denial-of-service attack vector if left unchecked. Since web ' - 'servers don\'t typically perform deep request inspection, it\'s ' - 'not possible to perform a similar check at that level. See also ' - 'FILE_UPLOAD_MAX_MEMORY_SIZE.' - ), -) -setting_django_default_from_email = namespace.add_setting( - global_name='DEFAULT_FROM_EMAIL', - default=settings.DEFAULT_FROM_EMAIL, - help_text=_( - 'Default: \'webmaster@localhost\' ' - 'Default email address to use for various automated correspondence ' - 'from the site manager(s). This doesn\'t include error messages sent ' - 'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.' - ), -) -setting_django_disallowed_user_agents = namespace.add_setting( - global_name='DISALLOWED_USER_AGENTS', - default=settings.DISALLOWED_USER_AGENTS, - help_text=_( - 'Default: [] (Empty list). List of compiled regular expression ' - 'objects representing User-Agent strings that are not allowed to ' - 'visit any page, systemwide. Use this for bad robots/crawlers. ' - 'This is only used if CommonMiddleware is installed ' - '(see Middleware).' - ), -) -setting_django_email_backend = namespace.add_setting( - global_name='EMAIL_BACKEND', - default=settings.EMAIL_BACKEND, - help_text=_( - 'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The ' - 'backend to use for sending emails.' - ), -) -setting_django_email_host = namespace.add_setting( - global_name='EMAIL_HOST', - default=settings.EMAIL_HOST, - help_text=_( - 'Default: \'localhost\'. The host to use for sending email.' - ), -) -setting_django_email_host_password = namespace.add_setting( - global_name='EMAIL_HOST_PASSWORD', - default=settings.EMAIL_HOST_PASSWORD, - help_text=_( - 'Default: \'\' (Empty string). Password to use for the SMTP ' - 'server defined in EMAIL_HOST. This setting is used in ' - 'conjunction with EMAIL_HOST_USER when authenticating to the ' - 'SMTP server. If either of these settings is empty, ' - 'Django won\'t attempt authentication.' - ), -) -setting_django_email_host_user = namespace.add_setting( - global_name='EMAIL_HOST_USER', - default=settings.EMAIL_HOST_USER, - help_text=_( - 'Default: \'\' (Empty string). Username to use for the SMTP ' - 'server defined in EMAIL_HOST. If empty, Django won\'t attempt ' - 'authentication.' - ), -) -setting_django_email_port = namespace.add_setting( - global_name='EMAIL_PORT', - default=settings.EMAIL_PORT, - help_text=_( - 'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.' - ), -) -setting_django_email_timeout = namespace.add_setting( - global_name='EMAIL_TIMEOUT', - default=settings.EMAIL_TIMEOUT, - help_text=_( - 'Default: None. Specifies a timeout in seconds for blocking ' - 'operations like the connection attempt.' - ), -) -setting_django_email_user_tls = namespace.add_setting( - global_name='EMAIL_USE_TLS', - default=settings.EMAIL_USE_TLS, - help_text=_( - 'Default: False. Whether to use a TLS (secure) connection when ' - 'talking to the SMTP server. This is used for explicit TLS ' - 'connections, generally on port 587. If you are experiencing ' - 'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.' - ), -) -setting_django_email_user_ssl = namespace.add_setting( - global_name='EMAIL_USE_SSL', - default=settings.EMAIL_USE_SSL, - help_text=_( - 'Default: False. Whether to use an implicit TLS (secure) connection ' - 'when talking to the SMTP server. In most email documentation this ' - 'type of TLS connection is referred to as SSL. It is generally used ' - 'on port 465. If you are experiencing problems, see the explicit ' - 'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL ' - 'are mutually exclusive, so only set one of those settings to True.' - ), -) -setting_django_file_upload_max_memory_size = namespace.add_setting( - global_name='FILE_UPLOAD_MAX_MEMORY_SIZE', - default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, - help_text=_( - 'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) ' - 'that an upload will be before it gets streamed to the file ' - 'system. See Managing files for details. See also ' - 'DATA_UPLOAD_MAX_MEMORY_SIZE.' - ), -) -setting_django_login_url = namespace.add_setting( - global_name='LOGIN_URL', - default=settings.LOGIN_URL, - help_text=_( - 'Default: \'/accounts/login/\' The URL where requests are ' - 'redirected for login, especially when using the login_required() ' - 'decorator. This setting also accepts named URL patterns which ' - 'can be used to reduce configuration duplication since you ' - 'don\'t have to define the URL in two places (settings ' - 'and URLconf).' - ) -) -setting_django_login_redirect_url = namespace.add_setting( - global_name='LOGIN_REDIRECT_URL', - default=settings.LOGIN_REDIRECT_URL, - help_text=_( - 'Default: \'/accounts/profile/\' The URL where requests are ' - 'redirected after login when the contrib.auth.login view gets no ' - 'next parameter. This is used by the login_required() decorator, ' - 'for example. This setting also accepts named URL patterns which ' - 'can be used to reduce configuration duplication since you don\'t ' - 'have to define the URL in two places (settings and URLconf).' - ), -) -setting_django_logout_redirect_url = namespace.add_setting( - global_name='LOGOUT_REDIRECT_URL', - default=settings.LOGOUT_REDIRECT_URL, - help_text=_( - 'Default: None. The URL where requests are redirected after a user ' - 'logs out using LogoutView (if the view doesn\'t get a next_page ' - 'argument). If None, no redirect will be performed and the logout ' - 'view will be rendered. This setting also accepts named URL ' - 'patterns which can be used to reduce configuration duplication ' - 'since you don\'t have to define the URL in two places (settings ' - 'and URLconf).' - ) -) -setting_django_internal_ips = namespace.add_setting( - global_name='INTERNAL_IPS', - default=settings.INTERNAL_IPS, - help_text=_( - 'A list of IP addresses, as strings, that: Allow the debug() ' - 'context processor to add some variables to the template context. ' - 'Can use the admindocs bookmarklets even if not logged in as a ' - 'staff user. Are marked as "internal" (as opposed to "EXTERNAL") ' - 'in AdminEmailHandler emails.' - ), -) -setting_django_languages = namespace.add_setting( - global_name='LANGUAGES', - default=settings.LANGUAGES, - help_text=_( - 'A list of all available languages. The list is a list of ' - 'two-tuples in the format (language code, language name) ' - 'for example, (\'ja\', \'Japanese\'). This specifies which ' - 'languages are available for language selection. ' - 'Generally, the default value should suffice. Only set this ' - 'setting if you want to restrict language selection to a ' - 'subset of the Django-provided languages. ' - ), -) -setting_django_language_code = namespace.add_setting( - global_name='LANGUAGE_CODE', - default=settings.LANGUAGE_CODE, - help_text=_( - 'A string representing the language code for this installation. ' - 'This should be in standard language ID format. For example, U.S. ' - 'English is "en-us". It serves two purposes: If the locale ' - 'middleware isn\'t in use, it decides which translation is served ' - 'to all users. If the locale middleware is active, it provides a ' - 'fallback language in case the user\'s preferred language can\'t ' - 'be determined or is not supported by the website. It also provides ' - 'the fallback translation when a translation for a given literal ' - 'doesn\'t exist for the user\'s preferred language.' - ), -) -setting_django_static_url = namespace.add_setting( - global_name='STATIC_URL', - default=settings.STATIC_URL, - help_text=_( - 'URL to use when referring to static files located in STATIC_ROOT. ' - 'Example: "/static/" or "http://static.example.com/" ' - 'If not None, this will be used as the base path for asset ' - 'definitions (the Media class) and the staticfiles app. ' - 'It must end in a slash if set to a non-empty value.' - ), -) -setting_django_staticfiles_storage = namespace.add_setting( - global_name='STATICFILES_STORAGE', - default=settings.STATICFILES_STORAGE, - help_text=_( - 'The file storage engine to use when collecting static files with ' - 'the collectstatic management command. A ready-to-use instance of ' - 'the storage backend defined in this setting can be found at ' - 'django.contrib.staticfiles.storage.staticfiles_storage.' - ), -) -setting_django_time_zone = namespace.add_setting( - global_name='TIME_ZONE', - default=settings.TIME_ZONE, - help_text=_( - 'A string representing the time zone for this installation. ' - 'Note that this isn\'t necessarily the time zone of the server. ' - 'For example, one server may serve multiple Django-powered sites, ' - 'each with a separate time zone setting.' - ), -) -setting_django_wsgi_application = namespace.add_setting( - global_name='WSGI_APPLICATION', - default=settings.WSGI_APPLICATION, - help_text=_( - 'The full Python path of the WSGI application object that Django\'s ' - 'built-in servers (e.g. runserver) will use. The django-admin ' - 'startproject management command will create a simple wsgi.py ' - 'file with an application callable in it, and point this setting ' - 'to that application.' - ), -) - -namespace = Namespace(label=_('Celery'), name='celery') - -setting_celery_broker_url = namespace.add_setting( - global_name='BROKER_URL', default=settings.BROKER_URL, - help_text=_( - 'Default: "amqp://". Default broker URL. This must be a URL in ' - 'the form of: transport://userid:password@hostname:port/virtual_host ' - 'Only the scheme part (transport://) is required, the rest is ' - 'optional, and defaults to the specific transports default values.' - ), -) -setting_celery_result_backend = namespace.add_setting( - global_name='CELERY_RESULT_BACKEND', - default=settings.CELERY_RESULT_BACKEND, - help_text=_( - 'Default: No result backend enabled by default. The backend used ' - 'to store task results (tombstones). Refer to ' - 'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.' - 'html#result-backend' - ) -) diff --git a/mayan/apps/smart_settings/apps.py b/mayan/apps/smart_settings/apps.py index 32d2a699501..d84e9dedf8b 100644 --- a/mayan/apps/smart_settings/apps.py +++ b/mayan/apps/smart_settings/apps.py @@ -11,6 +11,7 @@ link_namespace_detail, link_namespace_list, link_namespace_root_list, link_setting_edit ) +from .settings import * # NOQA from .widgets import setting_widget diff --git a/mayan/apps/smart_settings/settings.py b/mayan/apps/smart_settings/settings.py new file mode 100644 index 00000000000..8ee2f848792 --- /dev/null +++ b/mayan/apps/smart_settings/settings.py @@ -0,0 +1,302 @@ +from __future__ import unicode_literals + +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings.classes import Namespace + +# Don't import anything on start import, we just want to make it easy +# for apps.py to activate the settings in this module. +__all__ = () +namespace = Namespace(label=_('Django'), name='django') + +setting_django_allowed_hosts = namespace.add_setting( + global_name='ALLOWED_HOSTS', default=settings.ALLOWED_HOSTS, + help_text=_( + 'A list of strings representing the host/domain names that this site ' + 'can serve. This is a security measure to prevent HTTP Host header ' + 'attacks, which are possible even under many seemingly-safe web ' + 'server configurations. Values in this list can be ' + 'fully qualified names (e.g. \'www.example.com\'), in which case ' + 'they will be matched against the request\'s Host header exactly ' + '(case-insensitive, not including port). A value beginning with a ' + 'period can be used as a subdomain wildcard: \'.example.com\' will ' + 'match example.com, www.example.com, and any other subdomain of ' + 'example.com. A value of \'*\' will match anything; in this case you ' + 'are responsible to provide your own validation of the Host header ' + '(perhaps in a middleware; if so this middleware must be listed ' + 'first in MIDDLEWARE).' + ), +) +setting_django_append_slash = namespace.add_setting( + global_name='APPEND_SLASH', default=settings.APPEND_SLASH, + help_text=_( + 'When set to True, if the request URL does not match any of the ' + 'patterns in the URLconf and it doesn\'t end in a slash, an HTTP ' + 'redirect is issued to the same URL with a slash appended. Note ' + 'that the redirect may cause any data submitted in a POST request ' + 'to be lost. The APPEND_SLASH setting is only used if ' + 'CommonMiddleware is installed (see Middleware). See also ' + 'PREPEND_WWW.' + ) +) +setting_django_auth_password_validators = namespace.add_setting( + global_name='AUTH_PASSWORD_VALIDATORS', + default=settings.AUTH_PASSWORD_VALIDATORS, + help_text=_( + 'The list of validators that are used to check the strength of ' + 'user\'s passwords.' + ) +) +setting_django_databases = namespace.add_setting( + global_name='DATABASES', default=settings.DATABASES, + help_text=_( + 'A dictionary containing the settings for all databases to be used ' + 'with Django. It is a nested dictionary whose contents map a ' + 'database alias to a dictionary containing the options for an ' + 'individual database. The DATABASES setting must configure a ' + 'default database; any number of additional databases may also ' + 'be specified.' + ), +) +setting_django_data_upload_max_memory_size = namespace.add_setting( + global_name='DATA_UPLOAD_MAX_MEMORY_SIZE', + default=settings.DATA_UPLOAD_MAX_MEMORY_SIZE, + help_text=_( + 'Default: 2621440 (i.e. 2.5 MB). The maximum size in bytes that a ' + 'request body may be before a SuspiciousOperation ' + '(RequestDataTooBig) is raised. The check is done when accessing ' + 'request.body or request.POST and is calculated against the total ' + 'request size excluding any file upload data. You can set this to ' + 'None to disable the check. Applications that are expected to ' + 'receive unusually large form posts should tune this setting. The ' + 'amount of request data is correlated to the amount of memory ' + 'needed to process the request and populate the GET and POST ' + 'dictionaries. Large requests could be used as a ' + 'denial-of-service attack vector if left unchecked. Since web ' + 'servers don\'t typically perform deep request inspection, it\'s ' + 'not possible to perform a similar check at that level. See also ' + 'FILE_UPLOAD_MAX_MEMORY_SIZE.' + ), +) +setting_django_default_from_email = namespace.add_setting( + global_name='DEFAULT_FROM_EMAIL', + default=settings.DEFAULT_FROM_EMAIL, + help_text=_( + 'Default: \'webmaster@localhost\' ' + 'Default email address to use for various automated correspondence ' + 'from the site manager(s). This doesn\'t include error messages sent ' + 'to ADMINS and MANAGERS; for that, see SERVER_EMAIL.' + ), +) +setting_django_disallowed_user_agents = namespace.add_setting( + global_name='DISALLOWED_USER_AGENTS', + default=settings.DISALLOWED_USER_AGENTS, + help_text=_( + 'Default: [] (Empty list). List of compiled regular expression ' + 'objects representing User-Agent strings that are not allowed to ' + 'visit any page, systemwide. Use this for bad robots/crawlers. ' + 'This is only used if CommonMiddleware is installed ' + '(see Middleware).' + ), +) +setting_django_email_backend = namespace.add_setting( + global_name='EMAIL_BACKEND', + default=settings.EMAIL_BACKEND, + help_text=_( + 'Default: \'django.core.mail.backends.smtp.EmailBackend\'. The ' + 'backend to use for sending emails.' + ), +) +setting_django_email_host = namespace.add_setting( + global_name='EMAIL_HOST', + default=settings.EMAIL_HOST, + help_text=_( + 'Default: \'localhost\'. The host to use for sending email.' + ), +) +setting_django_email_host_password = namespace.add_setting( + global_name='EMAIL_HOST_PASSWORD', + default=settings.EMAIL_HOST_PASSWORD, + help_text=_( + 'Default: \'\' (Empty string). Password to use for the SMTP ' + 'server defined in EMAIL_HOST. This setting is used in ' + 'conjunction with EMAIL_HOST_USER when authenticating to the ' + 'SMTP server. If either of these settings is empty, ' + 'Django won\'t attempt authentication.' + ), +) +setting_django_email_host_user = namespace.add_setting( + global_name='EMAIL_HOST_USER', + default=settings.EMAIL_HOST_USER, + help_text=_( + 'Default: \'\' (Empty string). Username to use for the SMTP ' + 'server defined in EMAIL_HOST. If empty, Django won\'t attempt ' + 'authentication.' + ), +) +setting_django_email_port = namespace.add_setting( + global_name='EMAIL_PORT', + default=settings.EMAIL_PORT, + help_text=_( + 'Default: 25. Port to use for the SMTP server defined in EMAIL_HOST.' + ), +) +setting_django_email_timeout = namespace.add_setting( + global_name='EMAIL_TIMEOUT', + default=settings.EMAIL_TIMEOUT, + help_text=_( + 'Default: None. Specifies a timeout in seconds for blocking ' + 'operations like the connection attempt.' + ), +) +setting_django_email_user_tls = namespace.add_setting( + global_name='EMAIL_USE_TLS', + default=settings.EMAIL_USE_TLS, + help_text=_( + 'Default: False. Whether to use a TLS (secure) connection when ' + 'talking to the SMTP server. This is used for explicit TLS ' + 'connections, generally on port 587. If you are experiencing ' + 'hanging connections, see the implicit TLS setting EMAIL_USE_SSL.' + ), +) +setting_django_email_user_ssl = namespace.add_setting( + global_name='EMAIL_USE_SSL', + default=settings.EMAIL_USE_SSL, + help_text=_( + 'Default: False. Whether to use an implicit TLS (secure) connection ' + 'when talking to the SMTP server. In most email documentation this ' + 'type of TLS connection is referred to as SSL. It is generally used ' + 'on port 465. If you are experiencing problems, see the explicit ' + 'TLS setting EMAIL_USE_TLS. Note that EMAIL_USE_TLS/EMAIL_USE_SSL ' + 'are mutually exclusive, so only set one of those settings to True.' + ), +) +setting_django_file_upload_max_memory_size = namespace.add_setting( + global_name='FILE_UPLOAD_MAX_MEMORY_SIZE', + default=settings.FILE_UPLOAD_MAX_MEMORY_SIZE, + help_text=_( + 'Default: 2621440 (i.e. 2.5 MB). The maximum size (in bytes) ' + 'that an upload will be before it gets streamed to the file ' + 'system. See Managing files for details. See also ' + 'DATA_UPLOAD_MAX_MEMORY_SIZE.' + ), +) +setting_django_login_url = namespace.add_setting( + global_name='LOGIN_URL', + default=settings.LOGIN_URL, + help_text=_( + 'Default: \'/accounts/login/\' The URL where requests are ' + 'redirected for login, especially when using the login_required() ' + 'decorator. This setting also accepts named URL patterns which ' + 'can be used to reduce configuration duplication since you ' + 'don\'t have to define the URL in two places (settings ' + 'and URLconf).' + ) +) +setting_django_login_redirect_url = namespace.add_setting( + global_name='LOGIN_REDIRECT_URL', + default=settings.LOGIN_REDIRECT_URL, + help_text=_( + 'Default: \'/accounts/profile/\' The URL where requests are ' + 'redirected after login when the contrib.auth.login view gets no ' + 'next parameter. This is used by the login_required() decorator, ' + 'for example. This setting also accepts named URL patterns which ' + 'can be used to reduce configuration duplication since you don\'t ' + 'have to define the URL in two places (settings and URLconf).' + ), +) +setting_django_logout_redirect_url = namespace.add_setting( + global_name='LOGOUT_REDIRECT_URL', + default=settings.LOGOUT_REDIRECT_URL, + help_text=_( + 'Default: None. The URL where requests are redirected after a user ' + 'logs out using LogoutView (if the view doesn\'t get a next_page ' + 'argument). If None, no redirect will be performed and the logout ' + 'view will be rendered. This setting also accepts named URL ' + 'patterns which can be used to reduce configuration duplication ' + 'since you don\'t have to define the URL in two places (settings ' + 'and URLconf).' + ) +) +setting_django_internal_ips = namespace.add_setting( + global_name='INTERNAL_IPS', + default=settings.INTERNAL_IPS, + help_text=_( + 'A list of IP addresses, as strings, that: Allow the debug() ' + 'context processor to add some variables to the template context. ' + 'Can use the admindocs bookmarklets even if not logged in as a ' + 'staff user. Are marked as "internal" (as opposed to "EXTERNAL") ' + 'in AdminEmailHandler emails.' + ), +) +setting_django_languages = namespace.add_setting( + global_name='LANGUAGES', + default=settings.LANGUAGES, + help_text=_( + 'A list of all available languages. The list is a list of ' + 'two-tuples in the format (language code, language name) ' + 'for example, (\'ja\', \'Japanese\'). This specifies which ' + 'languages are available for language selection. ' + 'Generally, the default value should suffice. Only set this ' + 'setting if you want to restrict language selection to a ' + 'subset of the Django-provided languages. ' + ), +) +setting_django_language_code = namespace.add_setting( + global_name='LANGUAGE_CODE', + default=settings.LANGUAGE_CODE, + help_text=_( + 'A string representing the language code for this installation. ' + 'This should be in standard language ID format. For example, U.S. ' + 'English is "en-us". It serves two purposes: If the locale ' + 'middleware isn\'t in use, it decides which translation is served ' + 'to all users. If the locale middleware is active, it provides a ' + 'fallback language in case the user\'s preferred language can\'t ' + 'be determined or is not supported by the website. It also provides ' + 'the fallback translation when a translation for a given literal ' + 'doesn\'t exist for the user\'s preferred language.' + ), +) +setting_django_static_url = namespace.add_setting( + global_name='STATIC_URL', + default=settings.STATIC_URL, + help_text=_( + 'URL to use when referring to static files located in STATIC_ROOT. ' + 'Example: "/static/" or "http://static.example.com/" ' + 'If not None, this will be used as the base path for asset ' + 'definitions (the Media class) and the staticfiles app. ' + 'It must end in a slash if set to a non-empty value.' + ), +) +setting_django_staticfiles_storage = namespace.add_setting( + global_name='STATICFILES_STORAGE', + default=settings.STATICFILES_STORAGE, + help_text=_( + 'The file storage engine to use when collecting static files with ' + 'the collectstatic management command. A ready-to-use instance of ' + 'the storage backend defined in this setting can be found at ' + 'django.contrib.staticfiles.storage.staticfiles_storage.' + ), +) +setting_django_time_zone = namespace.add_setting( + global_name='TIME_ZONE', + default=settings.TIME_ZONE, + help_text=_( + 'A string representing the time zone for this installation. ' + 'Note that this isn\'t necessarily the time zone of the server. ' + 'For example, one server may serve multiple Django-powered sites, ' + 'each with a separate time zone setting.' + ), +) +setting_django_wsgi_application = namespace.add_setting( + global_name='WSGI_APPLICATION', + default=settings.WSGI_APPLICATION, + help_text=_( + 'The full Python path of the WSGI application object that Django\'s ' + 'built-in servers (e.g. runserver) will use. The django-admin ' + 'startproject management command will create a simple wsgi.py ' + 'file with an application callable in it, and point this setting ' + 'to that application.' + ), +) diff --git a/mayan/apps/task_manager/apps.py b/mayan/apps/task_manager/apps.py index 00996c65280..781632b9270 100644 --- a/mayan/apps/task_manager/apps.py +++ b/mayan/apps/task_manager/apps.py @@ -13,6 +13,7 @@ link_queue_scheduled_task_list, link_queue_reserved_task_list, link_task_manager ) +from .settings import * # NOQA class TaskManagerApp(MayanAppConfig): diff --git a/mayan/apps/task_manager/settings.py b/mayan/apps/task_manager/settings.py new file mode 100644 index 00000000000..9074439afe4 --- /dev/null +++ b/mayan/apps/task_manager/settings.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.smart_settings.classes import Namespace + +# Don't import anything on start import, we just want to make it easy +# for apps.py to activate the settings in this module. +__all__ = () +namespace = Namespace(label=_('Celery'), name='celery') + +setting_celery_broker_url = namespace.add_setting( + global_name='BROKER_URL', default=None, + help_text=_( + 'Default: "amqp://". Default broker URL. This must be a URL in ' + 'the form of: transport://userid:password@hostname:port/virtual_host ' + 'Only the scheme part (transport://) is required, the rest is ' + 'optional, and defaults to the specific transports default values.' + ), +) +setting_celery_result_backend = namespace.add_setting( + global_name='CELERY_RESULT_BACKEND', default=None, + help_text=_( + 'Default: No result backend enabled by default. The backend used ' + 'to store task results (tombstones). Refer to ' + 'http://docs.celeryproject.org/en/v4.1.0/userguide/configuration.' + 'html#result-backend' + ) +) From 8bc4b6a95e43d33a4ede40dd41a1523117dd34e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 10 Jul 2019 19:35:42 -0400 Subject: [PATCH 035/402] Move YAML code to its own module Code now resides in common.serialization in the form of two new functions: yaml_load and yaml_dump. Signed-off-by: Roberto Rosario --- mayan/apps/common/serialization.py | 22 +++++++++++++++++++ mayan/apps/common/storages.py | 12 +++------- mayan/apps/converter/backends/python.py | 23 +++++++++----------- mayan/apps/converter/classes.py | 15 +++++-------- mayan/apps/converter/forms.py | 9 +++----- mayan/apps/converter/managers.py | 16 +++++--------- mayan/apps/converter/validators.py | 9 +++----- mayan/apps/document_signatures/storages.py | 14 ++++-------- mayan/apps/documents/storages.py | 15 ++++--------- mayan/apps/file_metadata/drivers/exiftool.py | 11 +++------- mayan/apps/ocr/backends/tesseract.py | 9 ++------ mayan/apps/ocr/runtime.py | 13 ++++------- mayan/apps/platform/classes.py | 11 ++-------- mayan/apps/smart_settings/classes.py | 16 +++++--------- mayan/apps/smart_settings/forms.py | 9 +++----- mayan/apps/sources/models/email_sources.py | 11 +++------- mayan/apps/sources/storages.py | 11 +++------- mayan/apps/sources/tests/test_models.py | 10 +++------ 18 files changed, 88 insertions(+), 148 deletions(-) create mode 100644 mayan/apps/common/serialization.py diff --git a/mayan/apps/common/serialization.py b/mayan/apps/common/serialization.py new file mode 100644 index 00000000000..359500a6229 --- /dev/null +++ b/mayan/apps/common/serialization.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +import yaml + +try: + from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper +except ImportError: + from yaml import SafeLoader, SafeDumper + + +def yaml_dump(*args, **kwargs): + defaults = {'Dumper': SafeDumper} + defaults.update(kwargs) + + return yaml.dump(*args, **defaults) + + +def yaml_load(*args, **kwargs): + defaults = {'Loader': SafeLoader} + defaults.update(kwargs) + + return yaml.load(*args, **defaults) diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index ded073d4e8c..44a97a08f6d 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_shared_storage, setting_shared_storage_arguments ) @@ -16,8 +11,7 @@ storage_sharedupload = import_string( dotted_path=setting_shared_storage.value )( - **yaml.load( + **yaml_load( stream=setting_shared_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/converter/backends/python.py b/mayan/apps/converter/backends/python.py index 672997f8794..35cb0b31dc7 100644 --- a/mayan/apps/converter/backends/python.py +++ b/mayan/apps/converter/backends/python.py @@ -7,15 +7,12 @@ from PIL import Image import PyPDF2 import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader + from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..classes import ConverterBase @@ -27,8 +24,8 @@ DEFAULT_PDFINFO_PATH ) -pdftoppm_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdftoppm_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_path', DEFAULT_PDFTOPPM_PATH ) @@ -39,16 +36,16 @@ pdftoppm = None else: pdftoppm_format = '-{}'.format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_format', DEFAULT_PDFTOPPM_FORMAT ) ) pdftoppm_dpi = format( - yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdftoppm_dpi', DEFAULT_PDFTOPPM_DPI ) @@ -56,8 +53,8 @@ pdftoppm = pdftoppm.bake(pdftoppm_format, '-r', pdftoppm_dpi) -pdfinfo_path = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +pdfinfo_path = yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pdfinfo_path', DEFAULT_PDFINFO_PATH ) diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index 517b04ac518..f248240561a 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -7,15 +7,10 @@ from PIL import Image import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.mimetype.api import get_mimetype from mayan.apps.storage.settings import setting_temporary_directory from mayan.apps.storage.utils import ( @@ -30,8 +25,8 @@ from .settings import setting_graphics_backend_config logger = logging.getLogger(__name__) -BACKEND_CONFIG = yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader +BACKEND_CONFIG = yaml_load( + stream=setting_graphics_backend_config.value ) libreoffice_path = BACKEND_CONFIG.get( 'libreoffice_path', DEFAULT_LIBREOFFICE_PATH @@ -62,8 +57,8 @@ def detect_orientation(self, page_number): pass def get_page(self, output_format=None): - output_format = output_format or yaml.load( - stream=setting_graphics_backend_config.value, Loader=SafeLoader + output_format = output_format or yaml_load( + stream=setting_graphics_backend_config.value ).get( 'pillow_format', DEFAULT_PILLOW_FORMAT ) diff --git a/mayan/apps/converter/forms.py b/mayan/apps/converter/forms.py index 73749d2897a..3137f0dc800 100644 --- a/mayan/apps/converter/forms.py +++ b/mayan/apps/converter/forms.py @@ -2,15 +2,12 @@ import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + from .models import Transformation @@ -21,7 +18,7 @@ class Meta: def clean(self): try: - yaml.load(stream=self.cleaned_data['arguments'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['arguments']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/converter/managers.py b/mayan/apps/converter/managers.py index 58562312032..45c4d02a107 100644 --- a/mayan/apps/converter/managers.py +++ b/mayan/apps/converter/managers.py @@ -2,16 +2,11 @@ import logging -import yaml - -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.contrib.contenttypes.models import ContentType from django.db import models, transaction +from mayan.apps.common.serialization import yaml_dump, yaml_load + from .transformations import BaseTransformation logger = logging.getLogger(__name__) @@ -23,8 +18,8 @@ def add_to_object(self, obj, transformation, arguments=None): self.create( content_type=content_type, object_id=obj.pk, - name=transformation.name, arguments=yaml.dump( - data=arguments, Dumper=SafeDumper + name=transformation.name, arguments=yaml_dump( + data=arguments ) ) @@ -96,9 +91,8 @@ def get_for_object(self, obj, as_classes=False): # Some transformations don't require arguments # return an empty dictionary as ** doesn't allow None if transformation.arguments: - kwargs = yaml.load( + kwargs = yaml_load( stream=transformation.arguments, - Loader=SafeLoader ) else: kwargs = {} diff --git a/mayan/apps/converter/validators.py b/mayan/apps/converter/validators.py index 49c45d0cae1..dd8562a945e 100644 --- a/mayan/apps/converter/validators.py +++ b/mayan/apps/converter/validators.py @@ -2,15 +2,12 @@ import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + @deconstructible class YAMLValidator(object): @@ -20,7 +17,7 @@ class YAMLValidator(object): def __call__(self, value): value = value.strip() try: - yaml.load(stream=value, Loader=SafeLoader) + yaml_load(stream=value) except yaml.error.YAMLError: raise ValidationError( _('Enter a valid YAML value.'), diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 45d5d63d5e6..6ec00d087d6 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) @@ -16,8 +11,7 @@ storage_detachedsignature = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( - stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader + **yaml_load( + stream=setting_storage_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index 95405e92420..5d2ca25472a 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,14 +1,9 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_documentimagecache_storage, setting_documentimagecache_storage_arguments, @@ -18,17 +13,15 @@ storage_documentversion = import_string( dotted_path=setting_storage_backend.value )( - **yaml.load( + **yaml_load( stream=setting_storage_backend_arguments.value or '{}', - Loader=SafeLoader ) ) storage_documentimagecache = import_string( dotted_path=setting_documentimagecache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_documentimagecache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index 79a0993916a..1c4df94ed80 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -4,15 +4,10 @@ import logging import sh -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import NamedTemporaryFile from ..literals import DEFAULT_EXIF_PATH @@ -57,8 +52,8 @@ def _process(self, document_version): ) def read_settings(self): - driver_arguments = yaml.load( - stream=setting_drivers_arguments.value, Loader=SafeLoader + driver_arguments = yaml_load( + stream=setting_drivers_arguments.value ) self.exiftool_path = driver_arguments.get( diff --git a/mayan/apps/ocr/backends/tesseract.py b/mayan/apps/ocr/backends/tesseract.py index 3444198e4da..6a84510d431 100644 --- a/mayan/apps/ocr/backends/tesseract.py +++ b/mayan/apps/ocr/backends/tesseract.py @@ -4,15 +4,11 @@ import shutil import sh -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.storage.utils import TemporaryFile from ..classes import OCRBackendBase @@ -115,8 +111,7 @@ def initialize(self): logger.debug('Available languages: %s', ', '.join(self.languages)) def read_settings(self): - backend_arguments = yaml.load( - Loader=SafeLoader, + backend_arguments = yaml_load( stream=setting_ocr_backend_arguments.value or '{}', ) diff --git a/mayan/apps/ocr/runtime.py b/mayan/apps/ocr/runtime.py index 1d8643819b6..6e2c3ed95c1 100644 --- a/mayan/apps/ocr/runtime.py +++ b/mayan/apps/ocr/runtime.py @@ -1,20 +1,15 @@ from __future__ import unicode_literals -import yaml - -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import setting_ocr_backend, setting_ocr_backend_arguments ocr_backend = import_string( dotted_path=setting_ocr_backend.value )( - **yaml.load( - stream=setting_ocr_backend_arguments.value or '{}', Loader=SafeLoader + **yaml_load( + stream=setting_ocr_backend_arguments.value or '{}' ) ) diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 1d0152442f1..e50cb0a4776 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -2,16 +2,11 @@ import os -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.template import loader from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.common.settings import ( setting_celery_broker_url, setting_celery_result_backend ) @@ -95,9 +90,7 @@ def render(self, context_string=None): if context_string: context.update( - yaml.load( - stream=context_string, Loader=SafeLoader - ) + yaml_load(stream=context_string) ) return loader.render_to_string( template_name=self.get_template_name(), diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index 9f35323608e..85b040feaaa 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -9,11 +9,6 @@ import yaml -try: - from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeLoader, SafeDumper - from django.apps import apps from django.conf import settings from django.utils.functional import Promise @@ -21,6 +16,8 @@ force_bytes, force_text, python_2_unicode_compatible ) +from mayan.apps.common.serialization import yaml_dump, yaml_load + logger = logging.getLogger(__name__) @@ -85,7 +82,7 @@ class Setting(object): @staticmethod def deserialize_value(value): - return yaml.load(stream=value, Loader=SafeLoader) + return yaml_load(stream=value) @staticmethod def express_promises(value): @@ -101,9 +98,8 @@ def express_promises(value): @staticmethod def serialize_value(value): - result = yaml.dump( + result = yaml_dump( data=Setting.express_promises(value), allow_unicode=True, - Dumper=SafeDumper ) # safe_dump returns bytestrings # Disregard the last 3 dots that mark the end of the YAML document @@ -128,8 +124,8 @@ def dump_data(cls, filter_term=None, namespace=None): if (filter_term and filter_term.lower() in setting.global_name.lower()) or not filter_term: dictionary[setting.global_name] = Setting.express_promises(setting.value) - return yaml.dump( - data=dictionary, default_flow_style=False, Dumper=SafeDumper + return yaml_dump( + data=dictionary, default_flow_style=False ) @classmethod diff --git a/mayan/apps/smart_settings/forms.py b/mayan/apps/smart_settings/forms.py index d72548316df..d00aa2a6f23 100644 --- a/mayan/apps/smart_settings/forms.py +++ b/mayan/apps/smart_settings/forms.py @@ -2,15 +2,12 @@ import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django import forms from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load + class SettingForm(forms.Form): value = forms.CharField( @@ -38,7 +35,7 @@ def clean(self): ) try: - yaml.load(stream=self.cleaned_data['value'], Loader=SafeLoader) + yaml_load(stream=self.cleaned_data['value']) except yaml.YAMLError: raise ValidationError( _( diff --git a/mayan/apps/sources/models/email_sources.py b/mayan/apps/sources/models/email_sources.py index 4c072190888..fa8ba913f8c 100644 --- a/mayan/apps/sources/models/email_sources.py +++ b/mayan/apps/sources/models/email_sources.py @@ -4,18 +4,13 @@ import logging import poplib -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.db import models from django.utils.encoding import force_bytes from django.utils.translation import ugettext_lazy as _ +from mayan.apps.common.serialization import yaml_load from mayan.apps.documents.models import Document from mayan.apps.metadata.api import set_bulk_metadata from mayan.apps.metadata.models import MetadataType @@ -142,8 +137,8 @@ def _process_message(source, message): with ContentFile(content=message.body, name=label) as file_object: if label == source.metadata_attachment_name: - metadata_dictionary = yaml.load( - stream=file_object.read(), Loader=SafeLoader + metadata_dictionary = yaml_load( + stream=file_object.read() ) logger.debug( 'Got metadata dictionary: %s', diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 30e23ab7c5f..791d4b12bce 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader - from django.utils.module_loading import import_string +from mayan.apps.common.serialization import yaml_load + from .settings import ( setting_staging_file_image_cache_storage, setting_staging_file_image_cache_storage_arguments, @@ -16,8 +12,7 @@ storage_staging_file_image_cache = import_string( dotted_path=setting_staging_file_image_cache_storage.value )( - **yaml.load( + **yaml_load( stream=setting_staging_file_image_cache_storage_arguments.value or '{}', - Loader=SafeLoader ) ) diff --git a/mayan/apps/sources/tests/test_models.py b/mayan/apps/sources/tests/test_models.py index 9288ff57bee..a49b00ce424 100644 --- a/mayan/apps/sources/tests/test_models.py +++ b/mayan/apps/sources/tests/test_models.py @@ -6,15 +6,11 @@ import mock from pathlib2 import Path -import yaml -try: - from yaml import CSafeDumper as SafeDumper -except ImportError: - from yaml import SafeDumper from django.core import mail from django.utils.encoding import force_text +from mayan.apps.common.serialization import yaml_dump from mayan.apps.documents.models import Document from mayan.apps.documents.tests import ( GenericDocumentTestCase, TEST_COMPRESSED_DOCUMENT_PATH, @@ -213,8 +209,8 @@ def test_metadata_yaml_attachment(self): metadata_type=test_metadata_type_2 ) - test_metadata_yaml = yaml.dump( - Dumper=SafeDumper, data={ + test_metadata_yaml = yaml_dump( + data={ test_metadata_type_1.name: TEST_METADATA_VALUE_1, test_metadata_type_2.name: TEST_METADATA_VALUE_2, } From 516c3aeb2c39d878a3c5cf139b286ec987d04c2c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 01:31:05 -0400 Subject: [PATCH 036/402] Add default for OCR backend argument setting Signed-off-by: Roberto Rosario --- mayan/apps/ocr/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/ocr/settings.py b/mayan/apps/ocr/settings.py index 4293c3ac241..51b92149aa5 100644 --- a/mayan/apps/ocr/settings.py +++ b/mayan/apps/ocr/settings.py @@ -13,7 +13,7 @@ ) setting_ocr_backend_arguments = namespace.add_setting( global_name='OCR_BACKEND_ARGUMENTS', - default='' + default={} ) setting_auto_ocr = namespace.add_setting( global_name='OCR_AUTO_OCR', default=True, From 3fab5c14276fcec6d2b91339d107c2154dbc173a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 01:31:37 -0400 Subject: [PATCH 037/402] Return empty dict if there is no config file Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/smart_settings/utils.py b/mayan/apps/smart_settings/utils.py index 4282d2c38d7..77ce325e0c2 100644 --- a/mayan/apps/smart_settings/utils.py +++ b/mayan/apps/smart_settings/utils.py @@ -56,7 +56,7 @@ def read_configuration_file(path): ) except IOError as exception: if exception.errno == errno.ENOENT: - pass + return {} # No config file, return empty dictionary else: raise From 1ab7b7b9b184e1eb4e4fcdb19a89ed3aab412ffa Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 01:56:06 -0400 Subject: [PATCH 038/402] Backport FakeStorageSubclass from versions/next Signed-off-by: Roberto Rosario --- HISTORY.rst | 3 ++ docs/releases/3.3.rst | 8 +++++ .../migrations/0012_auto_20190711_0548.py | 32 +++++++++++++++++++ mayan/apps/common/storages.py | 4 +-- .../migrations/0009_auto_20190711_0544.py | 22 +++++++++++++ mayan/apps/document_signatures/storages.py | 4 +-- mayan/apps/document_states/storages.py | 4 +-- .../migrations/0048_auto_20190711_0544.py | 22 +++++++++++++ mayan/apps/documents/storages.py | 6 ++-- mayan/apps/sources/storages.py | 4 +-- mayan/apps/storage/classes.py | 10 ++++++ mayan/apps/storage/utils.py | 23 +++++++++++++ 12 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 mayan/apps/common/migrations/0012_auto_20190711_0548.py create mode 100644 mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py create mode 100644 mayan/apps/documents/migrations/0048_auto_20190711_0544.py create mode 100644 mayan/apps/storage/classes.py diff --git a/HISTORY.rst b/HISTORY.rst index adb93fb809f..a3915dae27c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,9 @@ doing atabase etup. - Added support for YAML encoded environment variables to the platform templates apps. +- Move YAML code to its own module. +- Move Django and Celery settings. +- Backport FakeStorageSubclass from versions/next. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index c15eb651bb6..eca213bfda9 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -38,6 +38,14 @@ Changes is now done using MAYAN_DATABASES to mirror Django way of doing database setup. - Added support for YAML encoded environment variables to the platform templates apps. +- Move YAML code to its own module. Code now resides in common.serialization + in the form of two new functions: yaml_load and yaml_dump. +- Move Django and Celery settings. Django settings now reside in the smart + settings app. Celery settings now reside in the task manager app. +- Backport FakeStorageSubclass from versions/next. Placeholder class to allow + serializing the real storage subclass to support migrations. + Used by all configurable storages. + Removals -------- diff --git a/mayan/apps/common/migrations/0012_auto_20190711_0548.py b/mayan/apps/common/migrations/0012_auto_20190711_0548.py new file mode 100644 index 00000000000..9915d05e2c9 --- /dev/null +++ b/mayan/apps/common/migrations/0012_auto_20190711_0548.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:48 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.common.models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0011_auto_20180429_0758'), + ] + + operations = [ + migrations.AlterField( + model_name='shareduploadedfile', + name='file', + field=models.FileField(storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.common.models.upload_to, verbose_name='File'), + ), + migrations.AlterField( + model_name='userlocaleprofile', + name='language', + field=models.CharField(choices=[('ar', 'Arabic'), ('bg', 'Bulgarian'), ('bs', 'Bosnian'), ('cs', 'Czech'), ('da', 'Danish'), ('de', 'German'), ('el', 'Greek'), ('en', 'English'), ('es', 'Spanish'), ('fa', 'Persian'), ('fr', 'French'), ('hu', 'Hungarian'), ('id', 'Indonesian'), ('it', 'Italian'), ('lv', 'Latvian'), ('nl', 'Dutch'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('pt-br', 'Portuguese (Brazil)'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sl', 'Slovenian'), ('tr', 'Turkish'), ('vi', 'Vietnamese'), ('zh', 'Chinese')], max_length=8, verbose_name='Language'), + ), + migrations.AlterField( + model_name='userlocaleprofile', + name='timezone', + field=models.CharField(choices=[(b'Africa/Abidjan', b'Africa/Abidjan'), (b'Africa/Accra', b'Africa/Accra'), (b'Africa/Addis_Ababa', b'Africa/Addis_Ababa'), (b'Africa/Algiers', b'Africa/Algiers'), (b'Africa/Asmara', b'Africa/Asmara'), (b'Africa/Bamako', b'Africa/Bamako'), (b'Africa/Bangui', b'Africa/Bangui'), (b'Africa/Banjul', b'Africa/Banjul'), (b'Africa/Bissau', b'Africa/Bissau'), (b'Africa/Blantyre', b'Africa/Blantyre'), (b'Africa/Brazzaville', b'Africa/Brazzaville'), (b'Africa/Bujumbura', b'Africa/Bujumbura'), (b'Africa/Cairo', b'Africa/Cairo'), (b'Africa/Casablanca', b'Africa/Casablanca'), (b'Africa/Ceuta', b'Africa/Ceuta'), (b'Africa/Conakry', b'Africa/Conakry'), (b'Africa/Dakar', b'Africa/Dakar'), (b'Africa/Dar_es_Salaam', b'Africa/Dar_es_Salaam'), (b'Africa/Djibouti', b'Africa/Djibouti'), (b'Africa/Douala', b'Africa/Douala'), (b'Africa/El_Aaiun', b'Africa/El_Aaiun'), (b'Africa/Freetown', b'Africa/Freetown'), (b'Africa/Gaborone', b'Africa/Gaborone'), (b'Africa/Harare', b'Africa/Harare'), (b'Africa/Johannesburg', b'Africa/Johannesburg'), (b'Africa/Juba', b'Africa/Juba'), (b'Africa/Kampala', b'Africa/Kampala'), (b'Africa/Khartoum', b'Africa/Khartoum'), (b'Africa/Kigali', b'Africa/Kigali'), (b'Africa/Kinshasa', b'Africa/Kinshasa'), (b'Africa/Lagos', b'Africa/Lagos'), (b'Africa/Libreville', b'Africa/Libreville'), (b'Africa/Lome', b'Africa/Lome'), (b'Africa/Luanda', b'Africa/Luanda'), (b'Africa/Lubumbashi', b'Africa/Lubumbashi'), (b'Africa/Lusaka', b'Africa/Lusaka'), (b'Africa/Malabo', b'Africa/Malabo'), (b'Africa/Maputo', b'Africa/Maputo'), (b'Africa/Maseru', b'Africa/Maseru'), (b'Africa/Mbabane', b'Africa/Mbabane'), (b'Africa/Mogadishu', b'Africa/Mogadishu'), (b'Africa/Monrovia', b'Africa/Monrovia'), (b'Africa/Nairobi', b'Africa/Nairobi'), (b'Africa/Ndjamena', b'Africa/Ndjamena'), (b'Africa/Niamey', b'Africa/Niamey'), (b'Africa/Nouakchott', b'Africa/Nouakchott'), (b'Africa/Ouagadougou', b'Africa/Ouagadougou'), (b'Africa/Porto-Novo', b'Africa/Porto-Novo'), (b'Africa/Sao_Tome', b'Africa/Sao_Tome'), (b'Africa/Tripoli', b'Africa/Tripoli'), (b'Africa/Tunis', b'Africa/Tunis'), (b'Africa/Windhoek', b'Africa/Windhoek'), (b'America/Adak', b'America/Adak'), (b'America/Anchorage', b'America/Anchorage'), (b'America/Anguilla', b'America/Anguilla'), (b'America/Antigua', b'America/Antigua'), (b'America/Araguaina', b'America/Araguaina'), (b'America/Argentina/Buenos_Aires', b'America/Argentina/Buenos_Aires'), (b'America/Argentina/Catamarca', b'America/Argentina/Catamarca'), (b'America/Argentina/Cordoba', b'America/Argentina/Cordoba'), (b'America/Argentina/Jujuy', b'America/Argentina/Jujuy'), (b'America/Argentina/La_Rioja', b'America/Argentina/La_Rioja'), (b'America/Argentina/Mendoza', b'America/Argentina/Mendoza'), (b'America/Argentina/Rio_Gallegos', b'America/Argentina/Rio_Gallegos'), (b'America/Argentina/Salta', b'America/Argentina/Salta'), (b'America/Argentina/San_Juan', b'America/Argentina/San_Juan'), (b'America/Argentina/San_Luis', b'America/Argentina/San_Luis'), (b'America/Argentina/Tucuman', b'America/Argentina/Tucuman'), (b'America/Argentina/Ushuaia', b'America/Argentina/Ushuaia'), (b'America/Aruba', b'America/Aruba'), (b'America/Asuncion', b'America/Asuncion'), (b'America/Atikokan', b'America/Atikokan'), (b'America/Bahia', b'America/Bahia'), (b'America/Bahia_Banderas', b'America/Bahia_Banderas'), (b'America/Barbados', b'America/Barbados'), (b'America/Belem', b'America/Belem'), (b'America/Belize', b'America/Belize'), (b'America/Blanc-Sablon', b'America/Blanc-Sablon'), (b'America/Boa_Vista', b'America/Boa_Vista'), (b'America/Bogota', b'America/Bogota'), (b'America/Boise', b'America/Boise'), (b'America/Cambridge_Bay', b'America/Cambridge_Bay'), (b'America/Campo_Grande', b'America/Campo_Grande'), (b'America/Cancun', b'America/Cancun'), (b'America/Caracas', b'America/Caracas'), (b'America/Cayenne', b'America/Cayenne'), (b'America/Cayman', b'America/Cayman'), (b'America/Chicago', b'America/Chicago'), (b'America/Chihuahua', b'America/Chihuahua'), (b'America/Costa_Rica', b'America/Costa_Rica'), (b'America/Creston', b'America/Creston'), (b'America/Cuiaba', b'America/Cuiaba'), (b'America/Curacao', b'America/Curacao'), (b'America/Danmarkshavn', b'America/Danmarkshavn'), (b'America/Dawson', b'America/Dawson'), (b'America/Dawson_Creek', b'America/Dawson_Creek'), (b'America/Denver', b'America/Denver'), (b'America/Detroit', b'America/Detroit'), (b'America/Dominica', b'America/Dominica'), (b'America/Edmonton', b'America/Edmonton'), (b'America/Eirunepe', b'America/Eirunepe'), (b'America/El_Salvador', b'America/El_Salvador'), (b'America/Fort_Nelson', b'America/Fort_Nelson'), (b'America/Fortaleza', b'America/Fortaleza'), (b'America/Glace_Bay', b'America/Glace_Bay'), (b'America/Godthab', b'America/Godthab'), (b'America/Goose_Bay', b'America/Goose_Bay'), (b'America/Grand_Turk', b'America/Grand_Turk'), (b'America/Grenada', b'America/Grenada'), (b'America/Guadeloupe', b'America/Guadeloupe'), (b'America/Guatemala', b'America/Guatemala'), (b'America/Guayaquil', b'America/Guayaquil'), (b'America/Guyana', b'America/Guyana'), (b'America/Halifax', b'America/Halifax'), (b'America/Havana', b'America/Havana'), (b'America/Hermosillo', b'America/Hermosillo'), (b'America/Indiana/Indianapolis', b'America/Indiana/Indianapolis'), (b'America/Indiana/Knox', b'America/Indiana/Knox'), (b'America/Indiana/Marengo', b'America/Indiana/Marengo'), (b'America/Indiana/Petersburg', b'America/Indiana/Petersburg'), (b'America/Indiana/Tell_City', b'America/Indiana/Tell_City'), (b'America/Indiana/Vevay', b'America/Indiana/Vevay'), (b'America/Indiana/Vincennes', b'America/Indiana/Vincennes'), (b'America/Indiana/Winamac', b'America/Indiana/Winamac'), (b'America/Inuvik', b'America/Inuvik'), (b'America/Iqaluit', b'America/Iqaluit'), (b'America/Jamaica', b'America/Jamaica'), (b'America/Juneau', b'America/Juneau'), (b'America/Kentucky/Louisville', b'America/Kentucky/Louisville'), (b'America/Kentucky/Monticello', b'America/Kentucky/Monticello'), (b'America/Kralendijk', b'America/Kralendijk'), (b'America/La_Paz', b'America/La_Paz'), (b'America/Lima', b'America/Lima'), (b'America/Los_Angeles', b'America/Los_Angeles'), (b'America/Lower_Princes', b'America/Lower_Princes'), (b'America/Maceio', b'America/Maceio'), (b'America/Managua', b'America/Managua'), (b'America/Manaus', b'America/Manaus'), (b'America/Marigot', b'America/Marigot'), (b'America/Martinique', b'America/Martinique'), (b'America/Matamoros', b'America/Matamoros'), (b'America/Mazatlan', b'America/Mazatlan'), (b'America/Menominee', b'America/Menominee'), (b'America/Merida', b'America/Merida'), (b'America/Metlakatla', b'America/Metlakatla'), (b'America/Mexico_City', b'America/Mexico_City'), (b'America/Miquelon', b'America/Miquelon'), (b'America/Moncton', b'America/Moncton'), (b'America/Monterrey', b'America/Monterrey'), (b'America/Montevideo', b'America/Montevideo'), (b'America/Montserrat', b'America/Montserrat'), (b'America/Nassau', b'America/Nassau'), (b'America/New_York', b'America/New_York'), (b'America/Nipigon', b'America/Nipigon'), (b'America/Nome', b'America/Nome'), (b'America/Noronha', b'America/Noronha'), (b'America/North_Dakota/Beulah', b'America/North_Dakota/Beulah'), (b'America/North_Dakota/Center', b'America/North_Dakota/Center'), (b'America/North_Dakota/New_Salem', b'America/North_Dakota/New_Salem'), (b'America/Ojinaga', b'America/Ojinaga'), (b'America/Panama', b'America/Panama'), (b'America/Pangnirtung', b'America/Pangnirtung'), (b'America/Paramaribo', b'America/Paramaribo'), (b'America/Phoenix', b'America/Phoenix'), (b'America/Port-au-Prince', b'America/Port-au-Prince'), (b'America/Port_of_Spain', b'America/Port_of_Spain'), (b'America/Porto_Velho', b'America/Porto_Velho'), (b'America/Puerto_Rico', b'America/Puerto_Rico'), (b'America/Punta_Arenas', b'America/Punta_Arenas'), (b'America/Rainy_River', b'America/Rainy_River'), (b'America/Rankin_Inlet', b'America/Rankin_Inlet'), (b'America/Recife', b'America/Recife'), (b'America/Regina', b'America/Regina'), (b'America/Resolute', b'America/Resolute'), (b'America/Rio_Branco', b'America/Rio_Branco'), (b'America/Santarem', b'America/Santarem'), (b'America/Santiago', b'America/Santiago'), (b'America/Santo_Domingo', b'America/Santo_Domingo'), (b'America/Sao_Paulo', b'America/Sao_Paulo'), (b'America/Scoresbysund', b'America/Scoresbysund'), (b'America/Sitka', b'America/Sitka'), (b'America/St_Barthelemy', b'America/St_Barthelemy'), (b'America/St_Johns', b'America/St_Johns'), (b'America/St_Kitts', b'America/St_Kitts'), (b'America/St_Lucia', b'America/St_Lucia'), (b'America/St_Thomas', b'America/St_Thomas'), (b'America/St_Vincent', b'America/St_Vincent'), (b'America/Swift_Current', b'America/Swift_Current'), (b'America/Tegucigalpa', b'America/Tegucigalpa'), (b'America/Thule', b'America/Thule'), (b'America/Thunder_Bay', b'America/Thunder_Bay'), (b'America/Tijuana', b'America/Tijuana'), (b'America/Toronto', b'America/Toronto'), (b'America/Tortola', b'America/Tortola'), (b'America/Vancouver', b'America/Vancouver'), (b'America/Whitehorse', b'America/Whitehorse'), (b'America/Winnipeg', b'America/Winnipeg'), (b'America/Yakutat', b'America/Yakutat'), (b'America/Yellowknife', b'America/Yellowknife'), (b'Antarctica/Casey', b'Antarctica/Casey'), (b'Antarctica/Davis', b'Antarctica/Davis'), (b'Antarctica/DumontDUrville', b'Antarctica/DumontDUrville'), (b'Antarctica/Macquarie', b'Antarctica/Macquarie'), (b'Antarctica/Mawson', b'Antarctica/Mawson'), (b'Antarctica/McMurdo', b'Antarctica/McMurdo'), (b'Antarctica/Palmer', b'Antarctica/Palmer'), (b'Antarctica/Rothera', b'Antarctica/Rothera'), (b'Antarctica/Syowa', b'Antarctica/Syowa'), (b'Antarctica/Troll', b'Antarctica/Troll'), (b'Antarctica/Vostok', b'Antarctica/Vostok'), (b'Arctic/Longyearbyen', b'Arctic/Longyearbyen'), (b'Asia/Aden', b'Asia/Aden'), (b'Asia/Almaty', b'Asia/Almaty'), (b'Asia/Amman', b'Asia/Amman'), (b'Asia/Anadyr', b'Asia/Anadyr'), (b'Asia/Aqtau', b'Asia/Aqtau'), (b'Asia/Aqtobe', b'Asia/Aqtobe'), (b'Asia/Ashgabat', b'Asia/Ashgabat'), (b'Asia/Atyrau', b'Asia/Atyrau'), (b'Asia/Baghdad', b'Asia/Baghdad'), (b'Asia/Bahrain', b'Asia/Bahrain'), (b'Asia/Baku', b'Asia/Baku'), (b'Asia/Bangkok', b'Asia/Bangkok'), (b'Asia/Barnaul', b'Asia/Barnaul'), (b'Asia/Beirut', b'Asia/Beirut'), (b'Asia/Bishkek', b'Asia/Bishkek'), (b'Asia/Brunei', b'Asia/Brunei'), (b'Asia/Chita', b'Asia/Chita'), (b'Asia/Choibalsan', b'Asia/Choibalsan'), (b'Asia/Colombo', b'Asia/Colombo'), (b'Asia/Damascus', b'Asia/Damascus'), (b'Asia/Dhaka', b'Asia/Dhaka'), (b'Asia/Dili', b'Asia/Dili'), (b'Asia/Dubai', b'Asia/Dubai'), (b'Asia/Dushanbe', b'Asia/Dushanbe'), (b'Asia/Famagusta', b'Asia/Famagusta'), (b'Asia/Gaza', b'Asia/Gaza'), (b'Asia/Hebron', b'Asia/Hebron'), (b'Asia/Ho_Chi_Minh', b'Asia/Ho_Chi_Minh'), (b'Asia/Hong_Kong', b'Asia/Hong_Kong'), (b'Asia/Hovd', b'Asia/Hovd'), (b'Asia/Irkutsk', b'Asia/Irkutsk'), (b'Asia/Jakarta', b'Asia/Jakarta'), (b'Asia/Jayapura', b'Asia/Jayapura'), (b'Asia/Jerusalem', b'Asia/Jerusalem'), (b'Asia/Kabul', b'Asia/Kabul'), (b'Asia/Kamchatka', b'Asia/Kamchatka'), (b'Asia/Karachi', b'Asia/Karachi'), (b'Asia/Kathmandu', b'Asia/Kathmandu'), (b'Asia/Khandyga', b'Asia/Khandyga'), (b'Asia/Kolkata', b'Asia/Kolkata'), (b'Asia/Krasnoyarsk', b'Asia/Krasnoyarsk'), (b'Asia/Kuala_Lumpur', b'Asia/Kuala_Lumpur'), (b'Asia/Kuching', b'Asia/Kuching'), (b'Asia/Kuwait', b'Asia/Kuwait'), (b'Asia/Macau', b'Asia/Macau'), (b'Asia/Magadan', b'Asia/Magadan'), (b'Asia/Makassar', b'Asia/Makassar'), (b'Asia/Manila', b'Asia/Manila'), (b'Asia/Muscat', b'Asia/Muscat'), (b'Asia/Nicosia', b'Asia/Nicosia'), (b'Asia/Novokuznetsk', b'Asia/Novokuznetsk'), (b'Asia/Novosibirsk', b'Asia/Novosibirsk'), (b'Asia/Omsk', b'Asia/Omsk'), (b'Asia/Oral', b'Asia/Oral'), (b'Asia/Phnom_Penh', b'Asia/Phnom_Penh'), (b'Asia/Pontianak', b'Asia/Pontianak'), (b'Asia/Pyongyang', b'Asia/Pyongyang'), (b'Asia/Qatar', b'Asia/Qatar'), (b'Asia/Qostanay', b'Asia/Qostanay'), (b'Asia/Qyzylorda', b'Asia/Qyzylorda'), (b'Asia/Riyadh', b'Asia/Riyadh'), (b'Asia/Sakhalin', b'Asia/Sakhalin'), (b'Asia/Samarkand', b'Asia/Samarkand'), (b'Asia/Seoul', b'Asia/Seoul'), (b'Asia/Shanghai', b'Asia/Shanghai'), (b'Asia/Singapore', b'Asia/Singapore'), (b'Asia/Srednekolymsk', b'Asia/Srednekolymsk'), (b'Asia/Taipei', b'Asia/Taipei'), (b'Asia/Tashkent', b'Asia/Tashkent'), (b'Asia/Tbilisi', b'Asia/Tbilisi'), (b'Asia/Tehran', b'Asia/Tehran'), (b'Asia/Thimphu', b'Asia/Thimphu'), (b'Asia/Tokyo', b'Asia/Tokyo'), (b'Asia/Tomsk', b'Asia/Tomsk'), (b'Asia/Ulaanbaatar', b'Asia/Ulaanbaatar'), (b'Asia/Urumqi', b'Asia/Urumqi'), (b'Asia/Ust-Nera', b'Asia/Ust-Nera'), (b'Asia/Vientiane', b'Asia/Vientiane'), (b'Asia/Vladivostok', b'Asia/Vladivostok'), (b'Asia/Yakutsk', b'Asia/Yakutsk'), (b'Asia/Yangon', b'Asia/Yangon'), (b'Asia/Yekaterinburg', b'Asia/Yekaterinburg'), (b'Asia/Yerevan', b'Asia/Yerevan'), (b'Atlantic/Azores', b'Atlantic/Azores'), (b'Atlantic/Bermuda', b'Atlantic/Bermuda'), (b'Atlantic/Canary', b'Atlantic/Canary'), (b'Atlantic/Cape_Verde', b'Atlantic/Cape_Verde'), (b'Atlantic/Faroe', b'Atlantic/Faroe'), (b'Atlantic/Madeira', b'Atlantic/Madeira'), (b'Atlantic/Reykjavik', b'Atlantic/Reykjavik'), (b'Atlantic/South_Georgia', b'Atlantic/South_Georgia'), (b'Atlantic/St_Helena', b'Atlantic/St_Helena'), (b'Atlantic/Stanley', b'Atlantic/Stanley'), (b'Australia/Adelaide', b'Australia/Adelaide'), (b'Australia/Brisbane', b'Australia/Brisbane'), (b'Australia/Broken_Hill', b'Australia/Broken_Hill'), (b'Australia/Currie', b'Australia/Currie'), (b'Australia/Darwin', b'Australia/Darwin'), (b'Australia/Eucla', b'Australia/Eucla'), (b'Australia/Hobart', b'Australia/Hobart'), (b'Australia/Lindeman', b'Australia/Lindeman'), (b'Australia/Lord_Howe', b'Australia/Lord_Howe'), (b'Australia/Melbourne', b'Australia/Melbourne'), (b'Australia/Perth', b'Australia/Perth'), (b'Australia/Sydney', b'Australia/Sydney'), (b'Canada/Atlantic', b'Canada/Atlantic'), (b'Canada/Central', b'Canada/Central'), (b'Canada/Eastern', b'Canada/Eastern'), (b'Canada/Mountain', b'Canada/Mountain'), (b'Canada/Newfoundland', b'Canada/Newfoundland'), (b'Canada/Pacific', b'Canada/Pacific'), (b'Europe/Amsterdam', b'Europe/Amsterdam'), (b'Europe/Andorra', b'Europe/Andorra'), (b'Europe/Astrakhan', b'Europe/Astrakhan'), (b'Europe/Athens', b'Europe/Athens'), (b'Europe/Belgrade', b'Europe/Belgrade'), (b'Europe/Berlin', b'Europe/Berlin'), (b'Europe/Bratislava', b'Europe/Bratislava'), (b'Europe/Brussels', b'Europe/Brussels'), (b'Europe/Bucharest', b'Europe/Bucharest'), (b'Europe/Budapest', b'Europe/Budapest'), (b'Europe/Busingen', b'Europe/Busingen'), (b'Europe/Chisinau', b'Europe/Chisinau'), (b'Europe/Copenhagen', b'Europe/Copenhagen'), (b'Europe/Dublin', b'Europe/Dublin'), (b'Europe/Gibraltar', b'Europe/Gibraltar'), (b'Europe/Guernsey', b'Europe/Guernsey'), (b'Europe/Helsinki', b'Europe/Helsinki'), (b'Europe/Isle_of_Man', b'Europe/Isle_of_Man'), (b'Europe/Istanbul', b'Europe/Istanbul'), (b'Europe/Jersey', b'Europe/Jersey'), (b'Europe/Kaliningrad', b'Europe/Kaliningrad'), (b'Europe/Kiev', b'Europe/Kiev'), (b'Europe/Kirov', b'Europe/Kirov'), (b'Europe/Lisbon', b'Europe/Lisbon'), (b'Europe/Ljubljana', b'Europe/Ljubljana'), (b'Europe/London', b'Europe/London'), (b'Europe/Luxembourg', b'Europe/Luxembourg'), (b'Europe/Madrid', b'Europe/Madrid'), (b'Europe/Malta', b'Europe/Malta'), (b'Europe/Mariehamn', b'Europe/Mariehamn'), (b'Europe/Minsk', b'Europe/Minsk'), (b'Europe/Monaco', b'Europe/Monaco'), (b'Europe/Moscow', b'Europe/Moscow'), (b'Europe/Oslo', b'Europe/Oslo'), (b'Europe/Paris', b'Europe/Paris'), (b'Europe/Podgorica', b'Europe/Podgorica'), (b'Europe/Prague', b'Europe/Prague'), (b'Europe/Riga', b'Europe/Riga'), (b'Europe/Rome', b'Europe/Rome'), (b'Europe/Samara', b'Europe/Samara'), (b'Europe/San_Marino', b'Europe/San_Marino'), (b'Europe/Sarajevo', b'Europe/Sarajevo'), (b'Europe/Saratov', b'Europe/Saratov'), (b'Europe/Simferopol', b'Europe/Simferopol'), (b'Europe/Skopje', b'Europe/Skopje'), (b'Europe/Sofia', b'Europe/Sofia'), (b'Europe/Stockholm', b'Europe/Stockholm'), (b'Europe/Tallinn', b'Europe/Tallinn'), (b'Europe/Tirane', b'Europe/Tirane'), (b'Europe/Ulyanovsk', b'Europe/Ulyanovsk'), (b'Europe/Uzhgorod', b'Europe/Uzhgorod'), (b'Europe/Vaduz', b'Europe/Vaduz'), (b'Europe/Vatican', b'Europe/Vatican'), (b'Europe/Vienna', b'Europe/Vienna'), (b'Europe/Vilnius', b'Europe/Vilnius'), (b'Europe/Volgograd', b'Europe/Volgograd'), (b'Europe/Warsaw', b'Europe/Warsaw'), (b'Europe/Zagreb', b'Europe/Zagreb'), (b'Europe/Zaporozhye', b'Europe/Zaporozhye'), (b'Europe/Zurich', b'Europe/Zurich'), (b'GMT', b'GMT'), (b'Indian/Antananarivo', b'Indian/Antananarivo'), (b'Indian/Chagos', b'Indian/Chagos'), (b'Indian/Christmas', b'Indian/Christmas'), (b'Indian/Cocos', b'Indian/Cocos'), (b'Indian/Comoro', b'Indian/Comoro'), (b'Indian/Kerguelen', b'Indian/Kerguelen'), (b'Indian/Mahe', b'Indian/Mahe'), (b'Indian/Maldives', b'Indian/Maldives'), (b'Indian/Mauritius', b'Indian/Mauritius'), (b'Indian/Mayotte', b'Indian/Mayotte'), (b'Indian/Reunion', b'Indian/Reunion'), (b'Pacific/Apia', b'Pacific/Apia'), (b'Pacific/Auckland', b'Pacific/Auckland'), (b'Pacific/Bougainville', b'Pacific/Bougainville'), (b'Pacific/Chatham', b'Pacific/Chatham'), (b'Pacific/Chuuk', b'Pacific/Chuuk'), (b'Pacific/Easter', b'Pacific/Easter'), (b'Pacific/Efate', b'Pacific/Efate'), (b'Pacific/Enderbury', b'Pacific/Enderbury'), (b'Pacific/Fakaofo', b'Pacific/Fakaofo'), (b'Pacific/Fiji', b'Pacific/Fiji'), (b'Pacific/Funafuti', b'Pacific/Funafuti'), (b'Pacific/Galapagos', b'Pacific/Galapagos'), (b'Pacific/Gambier', b'Pacific/Gambier'), (b'Pacific/Guadalcanal', b'Pacific/Guadalcanal'), (b'Pacific/Guam', b'Pacific/Guam'), (b'Pacific/Honolulu', b'Pacific/Honolulu'), (b'Pacific/Kiritimati', b'Pacific/Kiritimati'), (b'Pacific/Kosrae', b'Pacific/Kosrae'), (b'Pacific/Kwajalein', b'Pacific/Kwajalein'), (b'Pacific/Majuro', b'Pacific/Majuro'), (b'Pacific/Marquesas', b'Pacific/Marquesas'), (b'Pacific/Midway', b'Pacific/Midway'), (b'Pacific/Nauru', b'Pacific/Nauru'), (b'Pacific/Niue', b'Pacific/Niue'), (b'Pacific/Norfolk', b'Pacific/Norfolk'), (b'Pacific/Noumea', b'Pacific/Noumea'), (b'Pacific/Pago_Pago', b'Pacific/Pago_Pago'), (b'Pacific/Palau', b'Pacific/Palau'), (b'Pacific/Pitcairn', b'Pacific/Pitcairn'), (b'Pacific/Pohnpei', b'Pacific/Pohnpei'), (b'Pacific/Port_Moresby', b'Pacific/Port_Moresby'), (b'Pacific/Rarotonga', b'Pacific/Rarotonga'), (b'Pacific/Saipan', b'Pacific/Saipan'), (b'Pacific/Tahiti', b'Pacific/Tahiti'), (b'Pacific/Tarawa', b'Pacific/Tarawa'), (b'Pacific/Tongatapu', b'Pacific/Tongatapu'), (b'Pacific/Wake', b'Pacific/Wake'), (b'Pacific/Wallis', b'Pacific/Wallis'), (b'US/Alaska', b'US/Alaska'), (b'US/Arizona', b'US/Arizona'), (b'US/Central', b'US/Central'), (b'US/Eastern', b'US/Eastern'), (b'US/Hawaii', b'US/Hawaii'), (b'US/Mountain', b'US/Mountain'), (b'US/Pacific', b'US/Pacific'), (b'UTC', b'UTC')], max_length=48, verbose_name='Timezone'), + ), + ] diff --git a/mayan/apps/common/storages.py b/mayan/apps/common/storages.py index f1acfdb7ca6..422b20682a1 100644 --- a/mayan/apps/common/storages.py +++ b/mayan/apps/common/storages.py @@ -1,11 +1,11 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_shared_storage, setting_shared_storage_arguments ) -storage_sharedupload = import_string( +storage_sharedupload = get_storage_subclass( dotted_path=setting_shared_storage.value )(**setting_shared_storage_arguments.value) diff --git a/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py b/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py new file mode 100644 index 00000000000..b77b3ae66ee --- /dev/null +++ b/mayan/apps/document_signatures/migrations/0009_auto_20190711_0544.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.document_signatures.models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('document_signatures', '0008_auto_20180429_0759'), + ] + + operations = [ + migrations.AlterField( + model_name='detachedsignature', + name='signature_file', + field=models.FileField(blank=True, null=True, storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.document_signatures.models.upload_to, verbose_name='Signature file'), + ), + ] diff --git a/mayan/apps/document_signatures/storages.py b/mayan/apps/document_signatures/storages.py index 2a06b3c913d..423f11c1e50 100644 --- a/mayan/apps/document_signatures/storages.py +++ b/mayan/apps/document_signatures/storages.py @@ -1,11 +1,11 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_storage_backend, setting_storage_backend_arguments ) -storage_detachedsignature = import_string( +storage_detachedsignature = get_storage_subclass( dotted_path=setting_storage_backend.value )(**setting_storage_backend_arguments.value) diff --git a/mayan/apps/document_states/storages.py b/mayan/apps/document_states/storages.py index 8a689634c70..d4ab8de5248 100644 --- a/mayan/apps/document_states/storages.py +++ b/mayan/apps/document_states/storages.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_workflowimagecache_storage, setting_workflowimagecache_storage_arguments ) -storage_workflowimagecache = import_string( +storage_workflowimagecache = get_storage_subclass( dotted_path=setting_workflowimagecache_storage.value )(**setting_workflowimagecache_storage_arguments.value) diff --git a/mayan/apps/documents/migrations/0048_auto_20190711_0544.py b/mayan/apps/documents/migrations/0048_auto_20190711_0544.py new file mode 100644 index 00000000000..627329af902 --- /dev/null +++ b/mayan/apps/documents/migrations/0048_auto_20190711_0544.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-07-11 05:44 +from __future__ import unicode_literals + +from django.db import migrations, models +import mayan.apps.documents.models.document_version_models +import mayan.apps.storage.classes + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0047_auto_20180917_0737'), + ] + + operations = [ + migrations.AlterField( + model_name='documentversion', + name='file', + field=models.FileField(storage=mayan.apps.storage.classes.FakeStorageSubclass(), upload_to=mayan.apps.documents.models.document_version_models.UUID_FUNCTION, verbose_name='File'), + ), + ] diff --git a/mayan/apps/documents/storages.py b/mayan/apps/documents/storages.py index a6cec362d65..a4e9132fde7 100644 --- a/mayan/apps/documents/storages.py +++ b/mayan/apps/documents/storages.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_documentimagecache_storage, @@ -8,10 +8,10 @@ setting_storage_backend, setting_storage_backend_arguments ) -storage_documentversion = import_string( +storage_documentversion = get_storage_subclass( dotted_path=setting_storage_backend.value )(**setting_storage_backend_arguments.value) -storage_documentimagecache = import_string( +storage_documentimagecache = get_storage_subclass( dotted_path=setting_documentimagecache_storage.value )(**setting_documentimagecache_storage_arguments.value) diff --git a/mayan/apps/sources/storages.py b/mayan/apps/sources/storages.py index 2849d8e9150..faa863ae21c 100644 --- a/mayan/apps/sources/storages.py +++ b/mayan/apps/sources/storages.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals -from django.utils.module_loading import import_string +from mayan.apps.storage.utils import get_storage_subclass from .settings import ( setting_staging_file_image_cache_storage, setting_staging_file_image_cache_storage_arguments, ) -storage_staging_file_image_cache = import_string( +storage_staging_file_image_cache = get_storage_subclass( dotted_path=setting_staging_file_image_cache_storage.value )(**setting_staging_file_image_cache_storage_arguments.value) diff --git a/mayan/apps/storage/classes.py b/mayan/apps/storage/classes.py new file mode 100644 index 00000000000..0c4bdd2cd5a --- /dev/null +++ b/mayan/apps/storage/classes.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + + +class FakeStorageSubclass(object): + """ + Placeholder class to allow serializing the real storage subclass to + support migrations. + """ + def __eq__(self, other): + return True diff --git a/mayan/apps/storage/utils.py b/mayan/apps/storage/utils.py index 90eab5bae5c..5cc27d0d821 100644 --- a/mayan/apps/storage/utils.py +++ b/mayan/apps/storage/utils.py @@ -5,6 +5,8 @@ import shutil import tempfile +from django.utils.module_loading import import_string + from .settings import setting_temporary_directory logger = logging.getLogger(__name__) @@ -39,6 +41,27 @@ def fs_cleanup(filename, file_descriptor=None, suppress_exceptions=True): raise +def get_storage_subclass(dotted_path): + """ + Import a storage class and return a subclass that will always return eq + True to avoid creating a new migration when for runtime storage class + changes. + """ + imported_storage_class = import_string(dotted_path=dotted_path) + + class StorageSubclass(imported_storage_class): + def __init__(self, *args, **kwargs): + return super(StorageSubclass, self).__init__(*args, **kwargs) + + def __eq__(self, other): + return True + + def deconstruct(self): + return ('mayan.apps.storage.classes.FakeStorageSubclass', (), {}) + + return StorageSubclass + + def mkdtemp(*args, **kwargs): kwargs.update({'dir': setting_temporary_directory.value}) return tempfile.mkdtemp(*args, **kwargs) From 3e3b1f75a0358a8e7007a0d1cc5cb5b4dc9c5876 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 02:02:45 -0400 Subject: [PATCH 039/402] Remove django-environ Work done in 9564db398fed0134dc97c82f4f9a28d42b9a0e22 Signed-off-by: Roberto Rosario --- HISTORY.rst | 1 + docs/releases/3.3.rst | 4 ++-- removals.txt | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a3915dae27c..59ccbc7e980 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,6 +31,7 @@ - Move YAML code to its own module. - Move Django and Celery settings. - Backport FakeStorageSubclass from versions/next. +- Remove django-environ. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index eca213bfda9..848fb9bd610 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -36,7 +36,7 @@ Changes - Move bootstrapped settings code to their own module in the smart_settings apps. - Remove individual database configuration options. All database configuration is now done using MAYAN_DATABASES to mirror Django way of doing database setup. -- Added support for YAML encoded environment variables to the platform +- Added support for YAML encoded environment variables to the platform templates apps. - Move YAML code to its own module. Code now resides in common.serialization in the form of two new functions: yaml_load and yaml_dump. @@ -50,7 +50,7 @@ Changes Removals -------- -- None +- Django environ Upgrading from a previous version diff --git a/removals.txt b/removals.txt index 6a40aee5354..2163f9ee5d2 100644 --- a/removals.txt +++ b/removals.txt @@ -1,6 +1,7 @@ # Packages to be remove during upgrades cssmin django-autoadmin +django-environ django-suit django-compressor django-filetransfers From 8a7da6a103ea5de97f1a4585528ca7df2ded4edd Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 02:26:24 -0400 Subject: [PATCH 040/402] Update release notes closed issues Signed-off-by: Roberto Rosario --- docs/releases/3.3.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 848fb9bd610..58396a0f027 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -141,5 +141,6 @@ Bugs fixed or issues closed --------------------------- - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified +- :gitlab-issue:`634` Failing docker entrypoint when using secret config .. _PyPI: https://pypi.python.org/pypi/mayan-edms/ From c44090aca6cf4ac436393317d9d77a7a9f7493d3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 11 Jul 2019 20:00:17 -0400 Subject: [PATCH 041/402] Initial commit to support multidocument checkouts Signed-off-by: Roberto Rosario --- mayan/apps/checkouts/apps.py | 15 ++- mayan/apps/checkouts/links.py | 16 ++- mayan/apps/checkouts/tests/mixins.py | 39 ++++++ mayan/apps/checkouts/tests/test_api.py | 6 +- mayan/apps/checkouts/tests/test_views.py | 92 +++++--------- mayan/apps/checkouts/urls.py | 25 ++-- mayan/apps/checkouts/views.py | 147 +++++++++++++++++++---- mayan/apps/checkouts/widgets.py | 4 +- 8 files changed, 241 insertions(+), 103 deletions(-) diff --git a/mayan/apps/checkouts/apps.py b/mayan/apps/checkouts/apps.py index 6f9e5b7c00c..56041bdb257 100644 --- a/mayan/apps/checkouts/apps.py +++ b/mayan/apps/checkouts/apps.py @@ -6,7 +6,9 @@ from mayan.apps.acls.classes import ModelPermission from mayan.apps.common.apps import MayanAppConfig -from mayan.apps.common.menus import menu_facet, menu_main, menu_secondary +from mayan.apps.common.menus import ( + menu_facet, menu_main, menu_multi_item, menu_secondary +) from mayan.apps.dashboards.dashboards import dashboard_main from mayan.apps.events.classes import ModelEventType @@ -17,8 +19,9 @@ ) from .handlers import handler_check_new_version_creation from .links import ( - link_check_in_document, link_check_out_document, link_check_out_info, - link_check_out_list + link_check_in_document, link_check_in_document_multiple, + link_check_out_document, link_check_out_document_multiple, + link_check_out_info, link_check_out_list ) from .methods import ( method_check_in, method_get_check_out_info, method_get_check_out_state, @@ -85,6 +88,12 @@ def ready(self): links=(link_check_out_info,), sources=(Document,) ) menu_main.bind_links(links=(link_check_out_list,), position=98) + menu_multi_item.bind_links( + links=( + link_check_in_document_multiple, + link_check_out_document_multiple + ), sources=(Document,) + ) menu_secondary.bind_links( links=(link_check_out_document, link_check_in_document), sources=( diff --git a/mayan/apps/checkouts/links.py b/mayan/apps/checkouts/links.py index d03d54c23d1..c8012653c9a 100644 --- a/mayan/apps/checkouts/links.py +++ b/mayan/apps/checkouts/links.py @@ -38,16 +38,26 @@ def is_not_checked_out(context): args='object.pk', condition=is_not_checked_out, icon_class=icon_check_out_document, permissions=(permission_document_check_out,), - text=_('Check out document'), view='checkouts:check_out_document', + text=_('Check out document'), view='checkouts:check_out_document' +) +link_check_out_document_multiple = Link( + icon_class=icon_check_out_document, + permissions=(permission_document_check_out,), text=_('Check out'), + view='checkouts:check_out_document_multiple' ) link_check_in_document = Link( args='object.pk', icon_class=icon_check_in_document, condition=is_checked_out, permissions=( permission_document_check_in, permission_document_check_in_override - ), text=_('Check in document'), view='checkouts:check_in_document', + ), text=_('Check in document'), view='checkouts:check_in_document' +) +link_check_in_document_multiple = Link( + icon_class=icon_check_in_document, + permissions=(permission_document_check_in,), text=_('Check in'), + view='checkouts:check_in_document_multiple' ) link_check_out_info = Link( args='resolved_object.pk', icon_class=icon_check_out_info, permissions=( permission_document_check_out_detail_view, - ), text=_('Check in/out'), view='checkouts:check_out_info', + ), text=_('Check in/out'), view='checkouts:check_out_info' ) diff --git a/mayan/apps/checkouts/tests/mixins.py b/mayan/apps/checkouts/tests/mixins.py index fa8d601a4bf..2bf2041ab8f 100644 --- a/mayan/apps/checkouts/tests/mixins.py +++ b/mayan/apps/checkouts/tests/mixins.py @@ -4,6 +4,8 @@ from django.utils.timezone import now +from mayan.apps.common.literals import TIME_DELTA_UNIT_DAYS + from ..models import DocumentCheckout @@ -23,3 +25,40 @@ def _check_out_test_document(self, user=None): expiration_datetime=self._check_out_expiration_datetime, user=user ) + + +class DocumentCheckoutViewTestMixin(object): + def _request_test_document_check_in_get_view(self): + return self.get( + viewname='checkouts:check_in_document', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_check_in_post_view(self): + return self.post( + viewname='checkouts:check_in_document', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_check_out_view(self): + return self.post( + viewname='checkouts:check_out_document', kwargs={ + 'pk': self.test_document.pk + }, data={ + 'block_new_version': True, + 'expiration_datetime_0': TIME_DELTA_UNIT_DAYS, + 'expiration_datetime_1': 2 + } + ) + + def _request_test_document_check_out_detail_view(self): + return self.get( + viewname='checkouts:check_out_info', kwargs={ + 'pk': self.test_document.pk + } + ) + + def _request_test_document_check_out_list_view(self): + return self.get(viewname='checkouts:check_out_list') diff --git a/mayan/apps/checkouts/tests/test_api.py b/mayan/apps/checkouts/tests/test_api.py index 71ec5ea5360..d87bae3c172 100644 --- a/mayan/apps/checkouts/tests/test_api.py +++ b/mayan/apps/checkouts/tests/test_api.py @@ -65,7 +65,7 @@ def test_checkedout_document_view_with_access(self): force_text(self.test_document.uuid) ) - def _request_document_checkout_view(self): + def _request_test_document_check_out_view(self): return self.post( viewname='rest_api:checkout-document-list', data={ 'document_pk': self.test_document.pk, @@ -74,7 +74,7 @@ def _request_document_checkout_view(self): ) def test_document_checkout_no_access(self): - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(DocumentCheckout.objects.count(), 0) @@ -82,7 +82,7 @@ def test_document_checkout_no_access(self): def test_document_checkout_with_access(self): self.grant_access(permission=permission_document_check_out, obj=self.test_document) - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual( diff --git a/mayan/apps/checkouts/tests/test_views.py b/mayan/apps/checkouts/tests/test_views.py index 6b4648145fe..ccfbef95af4 100644 --- a/mayan/apps/checkouts/tests/test_views.py +++ b/mayan/apps/checkouts/tests/test_views.py @@ -12,64 +12,53 @@ permission_document_check_out, permission_document_check_out_detail_view ) -from .mixins import DocumentCheckoutTestMixin +from .mixins import DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin -class DocumentCheckoutViewTestCase(DocumentCheckoutTestMixin, GenericDocumentViewTestCase): - def _request_document_check_in_get_view(self): - return self.get( - viewname='checkouts:check_in_document', kwargs={ - 'pk': self.test_document.pk - } - ) - - def test_check_in_document_get_view_no_permission(self): +class DocumentCheckoutViewTestCase( + DocumentCheckoutTestMixin, DocumentCheckoutViewTestMixin, + GenericDocumentViewTestCase +): + def test_document_check_in_get_view_no_permission(self): self._check_out_test_document() - response = self._request_document_check_in_get_view() + response = self._request_test_document_check_in_get_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) self.assertTrue(self.test_document.is_checked_out()) - def test_check_in_document_get_view_with_access(self): + def test_document_check_in_get_view_with_access(self): self._check_out_test_document() self.grant_access( obj=self.test_document, permission=permission_document_check_in ) - response = self._request_document_check_in_get_view() + response = self._request_test_document_check_in_get_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) self.assertTrue(self.test_document.is_checked_out()) - def _request_document_check_in_post_view(self): - return self.post( - viewname='checkouts:check_in_document', kwargs={ - 'pk': self.test_document.pk - } - ) - - def test_check_in_document_post_view_no_permission(self): + def test_document_check_in_post_view_no_permission(self): self._check_out_test_document() - response = self._request_document_check_in_post_view() - self.assertEqual(response.status_code, 403) + response = self._request_test_document_check_in_post_view() + self.assertEqual(response.status_code, 404) self.assertTrue(self.test_document.is_checked_out()) - def test_check_in_document_post_view_with_access(self): + def test_document_check_in_post_view_with_access(self): self._check_out_test_document() self.grant_access( obj=self.test_document, permission=permission_document_check_in ) - response = self._request_document_check_in_post_view() + response = self._request_test_document_check_in_post_view() self.assertEqual(response.status_code, 302) self.assertFalse(self.test_document.is_checked_out()) @@ -79,24 +68,13 @@ def test_check_in_document_post_view_with_access(self): ) ) - def _request_document_checkout_view(self): - return self.post( - viewname='checkouts:check_out_document', kwargs={ - 'pk': self.test_document.pk - }, data={ - 'expiration_datetime_0': 2, - 'expiration_datetime_1': TIME_DELTA_UNIT_DAYS, - 'block_new_version': True - } - ) - - def test_check_out_document_view_no_permission(self): - response = self._request_document_checkout_view() - self.assertEqual(response.status_code, 403) + def test_document_check_out_view_no_permission(self): + response = self._request_test_document_check_out_view() + self.assertEqual(response.status_code, 404) self.assertFalse(self.test_document.is_checked_out()) - def test_check_out_document_view_with_access(self): + def test_document_check_out_view_with_access(self): self.grant_access( obj=self.test_document, permission=permission_document_check_out ) @@ -105,28 +83,21 @@ def test_check_out_document_view_with_access(self): permission=permission_document_check_out_detail_view ) - response = self._request_document_checkout_view() + response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, 302) self.assertTrue(self.test_document.is_checked_out()) - def _request_check_out_detail_view(self): - return self.get( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.test_document.pk - } - ) - - def test_checkout_detail_view_no_permission(self): + def test_document_check_out_detail_view_no_permission(self): self._check_out_test_document() - response = self._request_check_out_detail_view() + response = self._request_test_document_check_out_detail_view() self.assertNotContains( response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=404 ) - def test_checkout_detail_view_with_access(self): + def test_document_check_out_detail_view_with_access(self): self._check_out_test_document() self.grant_access( @@ -134,15 +105,12 @@ def test_checkout_detail_view_with_access(self): permission=permission_document_check_out_detail_view ) - response = self._request_check_out_detail_view() + response = self._request_test_document_check_out_detail_view() self.assertContains( response, text=STATE_LABELS[STATE_CHECKED_OUT], status_code=200 ) - def _request_check_out_list_view(self): - return self.get(viewname='checkouts:check_out_list') - - def test_checkout_list_view_no_permission(self): + def test_document_checkout_list_view_no_permission(self): self._check_out_test_document() self.grant_access( @@ -150,12 +118,12 @@ def test_checkout_list_view_no_permission(self): permission=permission_document_view ) - response = self._request_check_out_list_view() + response = self._request_test_document_check_out_list_view() self.assertNotContains( response=response, text=self.test_document.label, status_code=200 ) - def test_checkout_list_view_with_access(self): + def test_document_checkout_list_view_with_access(self): self._check_out_test_document() self.grant_access( @@ -167,12 +135,12 @@ def test_checkout_list_view_with_access(self): permission=permission_document_view ) - response = self._request_check_out_list_view() + response = self._request_test_document_check_out_list_view() self.assertContains( response=response, text=self.test_document.label, status_code=200 ) - def test_document_new_version_after_check_out(self): + def test_document_check_out_new_version(self): """ Gitlab issue #231 User shown option to upload new version of a document even though it @@ -209,7 +177,7 @@ def test_document_new_version_after_check_out(self): self.assertEqual(resolved_link, None) - def test_forcefull_check_in_document_view_no_permission(self): + def test_document_forcefull_check_in_view_no_permission(self): # Gitlab issue #237 # Forcefully checking in a document by a user without adequate # permissions throws out an error @@ -232,7 +200,7 @@ def test_forcefull_check_in_document_view_no_permission(self): self.assertTrue(self.test_document.is_checked_out()) - def test_forcefull_check_in_document_view_with_permission(self): + def test_document_forcefull_check_in_view_with_permission(self): self._create_test_case_superuser() self._check_out_test_document(user=self._test_case_superuser) diff --git a/mayan/apps/checkouts/urls.py b/mayan/apps/checkouts/urls.py index 994d18edde4..0e30989cf4f 100644 --- a/mayan/apps/checkouts/urls.py +++ b/mayan/apps/checkouts/urls.py @@ -4,25 +4,34 @@ from .api_views import APICheckedoutDocumentListView, APICheckedoutDocumentView from .views import ( - CheckoutDocumentView, CheckoutDetailView, CheckoutListView, - DocumentCheckinView + DocumentCheckinView, DocumentCheckoutDetailView, DocumentCheckoutView, + DocumentCheckoutListView ) urlpatterns = [ url( - regex=r'^list/$', view=CheckoutListView.as_view(), name='check_out_list' + regex=r'^documents/$', view=DocumentCheckoutListView.as_view(), + name='check_out_list' ), url( - regex=r'^(?P\d+)/check/out/$', view=CheckoutDocumentView.as_view(), + regex=r'^documents/(?P\d+)/check_in/$', view=DocumentCheckinView.as_view(), + name='check_in_document' + ), + url( + regex=r'^documents/multiple/check_in/$', + name='check_in_document_multiple', view=DocumentCheckinView.as_view() + ), + url( + regex=r'^documents/(?P\d+)/check_out/$', view=DocumentCheckoutView.as_view(), name='check_out_document' ), url( - regex=r'^(?P\d+)/check/in/$', view=DocumentCheckinView.as_view(), - name='check_in_document' + regex=r'^documents/multiple/check_out/$', + name='check_out_document_multiple', view=DocumentCheckoutView.as_view() ), url( - regex=r'^(?P\d+)/check/info/$', view=CheckoutDetailView.as_view(), - name='check_out_info' + regex=r'^documents/(?P\d+)/checkout/info/$', + view=DocumentCheckoutDetailView.as_view(), name='check_out_info' ), ] diff --git a/mayan/apps/checkouts/views.py b/mayan/apps/checkouts/views.py index 65063d2cfef..93f54a8190a 100644 --- a/mayan/apps/checkouts/views.py +++ b/mayan/apps/checkouts/views.py @@ -4,11 +4,12 @@ from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.acls.models import AccessControlList from mayan.apps.common.generics import ( - ConfirmView, SingleObjectCreateView, SingleObjectDetailView + ConfirmView, MultipleObjectConfirmActionView, MultipleObjectFormActionView, + SingleObjectCreateView, SingleObjectDetailView ) from mayan.apps.common.utils import encapsulate from mayan.apps.documents.models import Document @@ -24,6 +25,7 @@ ) +""" class DocumentCheckinView(ConfirmView): def get_extra_context(self): document = self.get_object() @@ -80,8 +82,59 @@ def view_action(self): 'Document "%s" checked in successfully.' ) % document, request=self.request ) +""" + +class DocumentCheckinView(MultipleObjectConfirmActionView): + error_message = 'Unable to check in document "%(instance)s". %(exception)s' + model = Document + object_permission = permission_document_check_in + pk_url_kwarg = 'pk' + success_message_singular = '%(count)d document checked in.' + success_message_plural = '%(count)d documents checked in.' + + def get_extra_context(self): + queryset = self.get_object_list() + + result = { + 'title': ungettext( + singular='Check in %(count)d document', + plural='Check in %(count)d documents', + number=queryset.count() + ) % { + 'count': queryset.count(), + } + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Check in document: %s' + ) % queryset.first() + } + ) + + return result + + def get_post_object_action_url(self): + if self.action_count == 1: + return reverse( + viewname='checkouts:document_checkout_info', + kwargs={'pk': self.action_id_list[0]} + ) + else: + super(DocumentCheckinView, self).get_post_action_redirect() + + def object_action(self, form, instance): + DocumentCheckout.objects.check_in_document( + document=instance, user=self.request.user + ) + + +""" class CheckoutDocumentView(SingleObjectCreateView): form_class = DocumentCheckoutForm @@ -129,9 +182,74 @@ def get_post_action_redirect(self): 'pk': self.document.pk } ) +""" +class DocumentCheckoutView(MultipleObjectFormActionView): + error_message = 'Unable to checkout document "%(instance)s". %(exception)s' + form_class = DocumentCheckoutForm + model = Document + object_permission = permission_document_check_out + pk_url_kwarg = 'pk' + success_message_singular = '%(count)d document checked out.' + success_message_plural = '%(count)d documents checked out.' + + def get_extra_context(self): + queryset = self.get_object_list() + + result = { + 'title': ungettext( + singular='Checkout %(count)d document', + plural='Checkout %(count)d documents', + number=queryset.count() + ) % { + 'count': queryset.count(), + } + } + + if queryset.count() == 1: + result.update( + { + 'object': queryset.first(), + 'title': _( + 'Check out document: %s' + ) % queryset.first() + } + ) + + return result + + def get_post_object_action_url(self): + if self.action_count == 1: + return reverse( + viewname='checkouts:document_checkout_info', + kwargs={'pk': self.action_id_list[0]} + ) + else: + super(DocumentCheckoutView, self).get_post_action_redirect() + + def object_action(self, form, instance): + DocumentCheckout.objects.check_out_document( + block_new_version=form.cleaned_data['block_new_version'], + document=instance, + expiration_datetime=form.cleaned_data['expiration_datetime'], + user=self.request.user, + ) + + +class DocumentCheckoutDetailView(SingleObjectDetailView): + form_class = DocumentCheckoutDefailForm + model = Document + object_permission = permission_document_check_out_detail_view + def get_extra_context(self): + return { + 'object': self.object, + 'title': _( + 'Check out details for document: %s' + ) % self.object + } -class CheckoutListView(DocumentListView): + +class DocumentCheckoutListView(DocumentListView): def get_document_queryset(self): return AccessControlList.objects.restrict_queryset( permission=permission_document_check_out_detail_view, @@ -140,7 +258,7 @@ def get_document_queryset(self): ) def get_extra_context(self): - context = super(CheckoutListView, self).get_extra_context() + context = super(DocumentCheckoutListView, self).get_extra_context() context.update( { 'extra_columns': ( @@ -165,26 +283,11 @@ def get_extra_context(self): ), 'no_results_icon': icon_check_out_info, 'no_results_text': _( - 'Checking out a document blocks certain document ' - 'operations for a predetermined amount of ' - 'time.' + 'Checking out a document, blocks certain operations ' + 'for a predetermined amount of time.' ), 'no_results_title': _('No documents have been checked out'), - 'title': _('Documents checked out'), + 'title': _('Checked out documents'), } ) return context - - -class CheckoutDetailView(SingleObjectDetailView): - form_class = DocumentCheckoutDefailForm - model = Document - object_permission = permission_document_check_out_detail_view - - def get_extra_context(self): - return { - 'object': self.object, - 'title': _( - 'Check out details for document: %s' - ) % self.object - } diff --git a/mayan/apps/checkouts/widgets.py b/mayan/apps/checkouts/widgets.py index 72497c8fba9..5206cc662e8 100644 --- a/mayan/apps/checkouts/widgets.py +++ b/mayan/apps/checkouts/widgets.py @@ -32,8 +32,8 @@ def decompress(self, value): return (None, None) def value_from_datadict(self, querydict, files, name): - unit = querydict.get('{}_1'.format(name)) - period = querydict.get('{}_0'.format(name)) + unit = querydict.get('{}_0'.format(name)) + period = querydict.get('{}_1'.format(name)) if not unit or not period: return now() From ed227b411187fe0ae42bacea6257999b52a927ad Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:38:06 -0400 Subject: [PATCH 042/402] Emphasize source column labels Use the same CSS style as the view's extra_columns. Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 4 ++++ .../templates/appearance/generic_list_items_subtemplate.html | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index b63d146d709..0cb7944b434 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -216,6 +216,10 @@ a i { font-weight: bold; } +.source-column-label { + font-weight: bold; +} + /* Content */ @media (min-width:1200px) { .container-fluid { diff --git a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html index 991162c0c05..e0c4166cefb 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html @@ -86,7 +86,7 @@

{% if not hide_columns %} {% navigation_get_source_columns source=object exclude_identifier=True as source_columns %} {% for column in source_columns %} -
{% navigation_source_column_resolve column=column as column_value %}{% if column_value != '' %}{% if column.include_label %}{{ column.label }}: {% endif %}{{ column_value }}{% endif %}
+
{% navigation_source_column_resolve column=column as column_value %}{% if column_value != '' %}{% if column.include_label %}{{ column.label }}: {% endif %}{{ column_value }}{% endif %}
{% endfor %} {% endif %} From 119c1bde76daf6e1d6311cf798c957037334cc30 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:39:18 -0400 Subject: [PATCH 043/402] Add user test mixin to base test class Allow tests to create test users. Signed-off-by: Roberto Rosario --- mayan/apps/common/tests/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mayan/apps/common/tests/base.py b/mayan/apps/common/tests/base.py index 5c08bffd579..3c352df9df1 100644 --- a/mayan/apps/common/tests/base.py +++ b/mayan/apps/common/tests/base.py @@ -7,6 +7,7 @@ from mayan.apps.acls.tests.mixins import ACLTestCaseMixin from mayan.apps.permissions.classes import Permission from mayan.apps.smart_settings.classes import Namespace +from mayan.apps.user_management.tests.mixins import UserTestMixin from .mixins import ( ClientMethodsTestCaseMixin, ConnectionsCheckTestCaseMixin, @@ -21,7 +22,7 @@ class BaseTestCase( SilenceLoggerTestCaseMixin, ConnectionsCheckTestCaseMixin, RandomPrimaryKeyModelMonkeyPatchMixin, ACLTestCaseMixin, ModelTestCaseMixin, OpenFileCheckTestCaseMixin, - TempfileCheckTestCasekMixin, TestCase + TempfileCheckTestCasekMixin, UserTestMixin, TestCase ): """ This is the most basic test case class any test in the project should use. From 44652d49fbf1a66445d9ade04deb75463d199c38 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:39:48 -0400 Subject: [PATCH 044/402] Add test utility to return an id_list Makes creating an id_list for testing from a list test instances easier. Signed-off-by: Roberto Rosario --- mayan/apps/common/tests/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mayan/apps/common/tests/utils.py b/mayan/apps/common/tests/utils.py index 85daf0c5243..5b7aac4993d 100644 --- a/mayan/apps/common/tests/utils.py +++ b/mayan/apps/common/tests/utils.py @@ -1,6 +1,10 @@ +from __future__ import absolute_import, unicode_literals + from contextlib import contextmanager import sys +from django.utils.encoding import force_text + class NullFile(object): def write(self, string): @@ -13,3 +17,9 @@ def mute_stdout(): sys.stdout = NullFile() yield sys.stdout = stdout_old + + +def as_id_list(items): + return ','.join( + [force_text(item.pk) for item in items] + ) From 1ddd5f26b12c00bce429214769e1498f1410dc17 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:40:48 -0400 Subject: [PATCH 045/402] Support menu inheritance Proxy models will now inherit the menus from their parents. Added to allow checked out documents to show multi item links of their parents. Signed-off-by: Roberto Rosario --- mayan/apps/navigation/classes.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index a070bafa27f..e671884a5f8 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -369,10 +369,24 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): for resolved_navigation_object in resolved_navigation_object_list: resolved_links = [] + # List of resolved links source links used for deduplication + resolved_links_links = [] + for bound_source, links in self.bound_links.items(): try: if inspect.isclass(bound_source): if type(resolved_navigation_object) == bound_source: + # Check to see if object is a proxy model. If it is, add its parent model + # menu links too. + parent_model = resolved_navigation_object._meta.proxy_for_model + if parent_model: + parent_instance = parent_model.objects.filter(pk=resolved_navigation_object.pk) + if parent_instance: + for link_set in self.resolve(context=context, source=parent_instance.first()): + for link in link_set['links']: + if link.link not in self.unbound_links.get(bound_source, ()): + resolved_links.append(link) + for link in links: resolved_link = link.resolve( context=context, @@ -395,10 +409,22 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): resolved_links.append(resolved_link) # No need for further content object match testing break + except TypeError: # When source is a dictionary pass + # Remove duplicated resolved link by using their source link + # instance as reference. The actual resolved link can't be used + # since a single source link can produce multiple resolved links. + # Since dictionaries keys can't have duplicates, we use that as a + # native deduplicator. + resolved_links_dict = {} + for resolved_link in resolved_links: + resolved_links_dict[resolved_link.link] = resolved_link + + resolved_links = resolved_links_dict.values() + if resolved_links: result.append( { @@ -407,6 +433,7 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): } ) + resolved_links = [] # View links for link in self.bound_links.get(current_view_name, []): From 058e36b4a99597f7389ae7ca7feee1765decb284 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:48:00 -0400 Subject: [PATCH 046/402] Introspect proxy's parent only it is a model Signed-off-by: Roberto Rosario --- mayan/apps/navigation/classes.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index e671884a5f8..73bb0488c8a 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -378,14 +378,15 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): if type(resolved_navigation_object) == bound_source: # Check to see if object is a proxy model. If it is, add its parent model # menu links too. - parent_model = resolved_navigation_object._meta.proxy_for_model - if parent_model: - parent_instance = parent_model.objects.filter(pk=resolved_navigation_object.pk) - if parent_instance: - for link_set in self.resolve(context=context, source=parent_instance.first()): - for link in link_set['links']: - if link.link not in self.unbound_links.get(bound_source, ()): - resolved_links.append(link) + if hasattr(resolved_navigation_object, '_meta'): + parent_model = resolved_navigation_object._meta.proxy_for_model + if parent_model: + parent_instance = parent_model.objects.filter(pk=resolved_navigation_object.pk) + if parent_instance: + for link_set in self.resolve(context=context, source=parent_instance.first()): + for link in link_set['links']: + if link.link not in self.unbound_links.get(bound_source, ()): + resolved_links.append(link) for link in links: resolved_link = link.resolve( From d4f7e2cd164b4fb5f08d5a0e88c76ecc065c7335 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:49:09 -0400 Subject: [PATCH 047/402] Support creating multiple test users Signed-off-by: Roberto Rosario --- mayan/apps/user_management/tests/mixins.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mayan/apps/user_management/tests/mixins.py b/mayan/apps/user_management/tests/mixins.py index 548df197e8f..a37792fd551 100644 --- a/mayan/apps/user_management/tests/mixins.py +++ b/mayan/apps/user_management/tests/mixins.py @@ -249,11 +249,19 @@ def _create_test_superuser(self): self.test_superuser.cleartext_password = TEST_USER_PASSWORD def _create_test_user(self): + total_test_users = len(self.test_users) + username = '{}_{}'.format(TEST_USER_USERNAME, total_test_users) + self.test_user = get_user_model().objects.create_user( - username=TEST_USER_USERNAME, email=TEST_USER_EMAIL, + username=username, email=TEST_USER_EMAIL, password=TEST_USER_PASSWORD ) self.test_user.cleartext_password = TEST_USER_PASSWORD + self.test_users.append(self.test_user) + + def setUp(self): + super(UserTestMixin, self).setUp() + self.test_users = [] class UserViewTestMixin(object): From e2f2181ebba49cba208b3ecacd1fb95f83a4b3b3 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:49:39 -0400 Subject: [PATCH 048/402] Complete multiple check in/out support Signed-off-by: Roberto Rosario --- mayan/apps/checkouts/apps.py | 31 +++- mayan/apps/checkouts/managers.py | 78 +++++--- mayan/apps/checkouts/methods.py | 2 +- mayan/apps/checkouts/models.py | 33 +++- mayan/apps/checkouts/tests/mixins.py | 25 ++- mayan/apps/checkouts/tests/test_models.py | 7 +- mayan/apps/checkouts/tests/test_views.py | 205 ++++++++++++++++++++-- mayan/apps/checkouts/views.py | 154 ++-------------- 8 files changed, 343 insertions(+), 192 deletions(-) diff --git a/mayan/apps/checkouts/apps.py b/mayan/apps/checkouts/apps.py index 56041bdb257..7c067edb0bb 100644 --- a/mayan/apps/checkouts/apps.py +++ b/mayan/apps/checkouts/apps.py @@ -11,6 +11,7 @@ ) from mayan.apps.dashboards.dashboards import dashboard_main from mayan.apps.events.classes import ModelEventType +from mayan.apps.navigation.classes import SourceColumn from .dashboard_widgets import DashboardWidgetTotalCheckouts from .events import ( @@ -46,6 +47,8 @@ class CheckoutsApp(MayanAppConfig): def ready(self): super(CheckoutsApp, self).ready() + CheckedOutDocument = self.get_model(model_name='CheckedOutDocument') + DocumentCheckout = self.get_model(model_name='DocumentCheckout') Document = apps.get_model( app_label='documents', model_name='Document' ) @@ -79,6 +82,22 @@ def ready(self): permission_document_check_out_detail_view ) ) + ModelPermission.register_inheritance( + model=DocumentCheckout, related='document' + ) + + SourceColumn( + attribute='get_user_display', include_label=True, order=99, + source=CheckedOutDocument + ) + SourceColumn( + attribute='get_checkout_datetime', include_label=True, order=99, + source=CheckedOutDocument + ) + SourceColumn( + attribute='get_checkout_expiration', include_label=True, order=99, + source=CheckedOutDocument + ) dashboard_main.add_widget( widget=DashboardWidgetTotalCheckouts, order=-1 @@ -91,9 +110,19 @@ def ready(self): menu_multi_item.bind_links( links=( link_check_in_document_multiple, - link_check_out_document_multiple + ), sources=(CheckedOutDocument,) + ) + menu_multi_item.bind_links( + links=( + link_check_in_document_multiple, + link_check_out_document_multiple, ), sources=(Document,) ) + menu_multi_item.unbind_links( + links=( + link_check_out_document_multiple, + ), sources=(CheckedOutDocument,) + ) menu_secondary.bind_links( links=(link_check_out_document, link_check_in_document), sources=( diff --git a/mayan/apps/checkouts/managers.py b/mayan/apps/checkouts/managers.py index 674a77257e4..7b6b77635d0 100644 --- a/mayan/apps/checkouts/managers.py +++ b/mayan/apps/checkouts/managers.py @@ -6,6 +6,7 @@ from django.db import models, transaction from django.utils.timezone import now +from mayan.apps.acls.models import AccessControlList from mayan.apps.documents.models import Document from .events import ( @@ -14,10 +15,53 @@ ) from .exceptions import DocumentNotCheckedOut from .literals import STATE_CHECKED_OUT, STATE_CHECKED_IN +from .permissions import ( + permission_document_check_in, permission_document_check_in_override +) logger = logging.getLogger(__name__) +class DocumentCheckoutBusinessLogicManager(models.Manager): + def check_in_document(self, document, user=None): + queryset = document._meta.default_manager.filter(pk=document.pk) + return self.check_in_documents(queryset=queryset, user=user) + + def check_in_documents(self, queryset, user=None): + if user: + user_document_checkouts = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in, + queryset=self.filter(user_id=user.pk, document__in=queryset), + user=user + ) + + others_document_checkouts = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in_override, + queryset=self.exclude(user_id=user.pk, document__in=queryset), + user=user + ) + + with transaction.atomic(): + if user: + for checkout in user_document_checkouts: + event_document_check_in.commit( + actor=user, target=checkout.document + ) + checkout.delete() + + for checkout in others_document_checkouts: + event_document_forceful_check_in.commit( + actor=user, target=checkout.document + ) + checkout.delete() + else: + for checkout in self.filter(document__in=queryset): + event_document_auto_check_in.commit( + target=checkout.document + ) + checkout.delete() + + class DocumentCheckoutManager(models.Manager): def are_document_new_versions_allowed(self, document, user=None): try: @@ -27,25 +71,6 @@ def are_document_new_versions_allowed(self, document, user=None): else: return not check_out_info.block_new_version - def check_in_document(self, document, user=None): - try: - document_check_out = self.model.objects.get(document=document) - except self.model.DoesNotExist: - raise DocumentNotCheckedOut - else: - with transaction.atomic(): - if user: - if self.get_check_out_info(document=document).user != user: - event_document_forceful_check_in.commit( - actor=user, target=document - ) - else: - event_document_check_in.commit(actor=user, target=document) - else: - event_document_auto_check_in.commit(target=document) - - document_check_out.delete() - def check_in_expired_check_outs(self): for document in self.expired_check_outs(): document.check_in() @@ -57,7 +82,11 @@ def check_out_document(self, document, expiration_datetime, user, block_new_vers ) def checked_out_documents(self): - return Document.objects.filter( + CheckedOutDocument = apps.get_model( + app_label='checkouts', model_name='CheckedOutDocument' + ) + + return CheckedOutDocument.objects.filter( pk__in=self.model.objects.values('document__id') ) @@ -74,7 +103,11 @@ def get_check_out_state(self, document): return STATE_CHECKED_IN def expired_check_outs(self): - expired_list = Document.objects.filter( + CheckedOutDocument = apps.get_model( + app_label='checkouts', model_name='CheckedOutDocument' + ) + + expired_list = CheckedOutDocument.objects.filter( pk__in=self.model.objects.filter( expiration_datetime__lte=now() ).values_list('document__pk', flat=True) @@ -83,9 +116,6 @@ def expired_check_outs(self): return expired_list def get_by_natural_key(self, document_natural_key): - Document = apps.get_model( - app_label='documents', model_name='Document' - ) try: document = Document.objects.get_by_natural_key(document_natural_key) except Document.DoesNotExist: diff --git a/mayan/apps/checkouts/methods.py b/mayan/apps/checkouts/methods.py index 680f6cf4c78..3972807a8f7 100644 --- a/mayan/apps/checkouts/methods.py +++ b/mayan/apps/checkouts/methods.py @@ -8,7 +8,7 @@ def method_check_in(self, user=None): app_label='checkouts', model_name='DocumentCheckout' ) - return DocumentCheckout.objects.check_in_document( + return DocumentCheckout.business_logic.check_in_document( document=self, user=user ) diff --git a/mayan/apps/checkouts/models.py b/mayan/apps/checkouts/models.py index c724e21db8d..5efc322dffe 100644 --- a/mayan/apps/checkouts/models.py +++ b/mayan/apps/checkouts/models.py @@ -14,7 +14,10 @@ from .events import event_document_check_out from .exceptions import DocumentAlreadyCheckedOut -from .managers import DocumentCheckoutManager, NewVersionBlockManager +from .managers import ( + DocumentCheckoutBusinessLogicManager, DocumentCheckoutManager, + NewVersionBlockManager +) logger = logging.getLogger(__name__) @@ -49,6 +52,7 @@ class DocumentCheckout(models.Model): ) objects = DocumentCheckoutManager() + business_logic = DocumentCheckoutBusinessLogicManager() class Meta: ordering = ('pk',) @@ -81,13 +85,13 @@ def natural_key(self): natural_key.dependencies = ['documents.Document'] def save(self, *args, **kwargs): - new_checkout = not self.pk - if not new_checkout or self.document.is_checked_out(): + is_new = not self.pk + if not is_new or self.document.is_checked_out(): raise DocumentAlreadyCheckedOut with transaction.atomic(): result = super(DocumentCheckout, self).save(*args, **kwargs) - if new_checkout: + if is_new: event_document_check_out.commit( actor=self.user, target=self.document ) @@ -119,3 +123,24 @@ class Meta: def natural_key(self): return self.document.natural_key() natural_key.dependencies = ['documents.Document'] + + +class CheckedOutDocument(Document): + class Meta: + proxy = True + + def get_user_display(self): + check_out_info = self.get_check_out_info() + return check_out_info.user.get_full_name() or check_out_info.user + + get_user_display.short_description = _('User') + + def get_checkout_datetime(self): + return self.get_check_out_info().checkout_datetime + + get_checkout_datetime.short_description = _('Checkout time and date') + + def get_checkout_expiration(self): + return self.get_check_out_info().expiration_datetime + + get_checkout_expiration.short_description = _('Checkout expiration') diff --git a/mayan/apps/checkouts/tests/mixins.py b/mayan/apps/checkouts/tests/mixins.py index 2bf2041ab8f..840fee34bf4 100644 --- a/mayan/apps/checkouts/tests/mixins.py +++ b/mayan/apps/checkouts/tests/mixins.py @@ -5,6 +5,7 @@ from django.utils.timezone import now from mayan.apps.common.literals import TIME_DELTA_UNIT_DAYS +from mayan.apps.common.tests.utils import as_id_list from ..models import DocumentCheckout @@ -12,7 +13,10 @@ class DocumentCheckoutTestMixin(object): _test_document_check_out_seconds = 0.1 - def _check_out_test_document(self, user=None): + def _check_out_test_document(self, document=None, user=None): + if not document: + document = self.test_document + if not user: user = self._test_case_user @@ -21,7 +25,7 @@ def _check_out_test_document(self, user=None): ) self.test_check_out = DocumentCheckout.objects.check_out_document( - block_new_version=True, document=self.test_document, + block_new_version=True, document=document, expiration_datetime=self._check_out_expiration_datetime, user=user ) @@ -42,6 +46,13 @@ def _request_test_document_check_in_post_view(self): } ) + def _request_test_document_multiple_check_in_post_view(self): + return self.post( + viewname='checkouts:check_in_document_multiple', data={ + 'id_list': as_id_list(items=self.test_documents) + } + ) + def _request_test_document_check_out_view(self): return self.post( viewname='checkouts:check_out_document', kwargs={ @@ -53,6 +64,16 @@ def _request_test_document_check_out_view(self): } ) + def _request_test_document_multiple_check_out_post_view(self): + return self.post( + viewname='checkouts:check_out_document_multiple', data={ + 'block_new_version': True, + 'expiration_datetime_0': TIME_DELTA_UNIT_DAYS, + 'expiration_datetime_1': 2, + 'id_list': as_id_list(items=self.test_documents) + } + ) + def _request_test_document_check_out_detail_view(self): return self.get( viewname='checkouts:check_out_info', kwargs={ diff --git a/mayan/apps/checkouts/tests/test_models.py b/mayan/apps/checkouts/tests/test_models.py index cd2ea0202fa..dbfdd64d508 100644 --- a/mayan/apps/checkouts/tests/test_models.py +++ b/mayan/apps/checkouts/tests/test_models.py @@ -7,8 +7,7 @@ from mayan.apps.documents.tests.literals import TEST_SMALL_DOCUMENT_PATH from ..exceptions import ( - DocumentAlreadyCheckedOut, DocumentNotCheckedOut, - NewDocumentVersionNotAllowed + DocumentAlreadyCheckedOut, NewDocumentVersionNotAllowed ) from ..models import DocumentCheckout, NewVersionBlock @@ -49,10 +48,6 @@ def test_double_check_out(self): block_new_version=True ) - def test_checkin_without_checkout(self): - with self.assertRaises(DocumentNotCheckedOut): - self.test_document.check_in() - def test_auto_check_in(self): self._check_out_test_document() diff --git a/mayan/apps/checkouts/tests/test_views.py b/mayan/apps/checkouts/tests/test_views.py index ccfbef95af4..3910dedd98b 100644 --- a/mayan/apps/checkouts/tests/test_views.py +++ b/mayan/apps/checkouts/tests/test_views.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from mayan.apps.common.literals import TIME_DELTA_UNIT_DAYS from mayan.apps.documents.permissions import permission_document_view from mayan.apps.documents.tests import GenericDocumentViewTestCase from mayan.apps.sources.links import link_document_version_upload @@ -23,8 +22,8 @@ def test_document_check_in_get_view_no_permission(self): self._check_out_test_document() response = self._request_test_document_check_in_get_view() - self.assertContains( - response=response, text=self.test_document.label, status_code=200 + self.assertNotContains( + response=response, text=self.test_document.label, status_code=404 ) self.assertTrue(self.test_document.is_checked_out()) @@ -68,6 +67,86 @@ def test_document_check_in_post_view_with_access(self): ) ) + def test_document_multiple_check_in_post_view_no_permission(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 404) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_in_post_view_with_document_0_access(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_in + ) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 302) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_in_post_view_with_access(self): + # Upload second document + self.upload_document() + + self._check_out_test_document(document=self.test_documents[0]) + self._check_out_test_document(document=self.test_documents[1]) + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_in + ) + self.grant_access( + obj=self.test_documents[1], permission=permission_document_check_in + ) + + response = self._request_test_document_multiple_check_in_post_view() + self.assertEqual(response.status_code, 302) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + def test_document_check_out_view_no_permission(self): response = self._request_test_document_check_out_view() self.assertEqual(response.status_code, 404) @@ -88,6 +167,102 @@ def test_document_check_out_view_with_access(self): self.assertTrue(self.test_document.is_checked_out()) + def test_document_multiple_check_out_post_view_no_permission(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view + ) + + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 404) + + self.assertFalse(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_out_post_view_with_document_0_access(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view + ) + + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 302) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertFalse(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertFalse( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + + def test_document_multiple_check_out_post_view_with_access(self): + # Upload second document + self.upload_document() + + self.grant_access( + obj=self.test_documents[0], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[1], permission=permission_document_check_out + ) + self.grant_access( + obj=self.test_documents[0], + permission=permission_document_check_out_detail_view + ) + self.grant_access( + obj=self.test_documents[1], + permission=permission_document_check_out_detail_view + ) + + response = self._request_test_document_multiple_check_out_post_view() + self.assertEqual(response.status_code, 302) + + self.assertTrue(self.test_documents[0].is_checked_out()) + self.assertTrue(self.test_documents[1].is_checked_out()) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[0] + ) + ) + self.assertTrue( + DocumentCheckout.objects.is_checked_out( + document=self.test_documents[1] + ) + ) + def test_document_check_out_detail_view_no_permission(self): self._check_out_test_document() @@ -177,13 +352,13 @@ def test_document_check_out_new_version(self): self.assertEqual(resolved_link, None) - def test_document_forcefull_check_in_view_no_permission(self): + def test_document_check_in_forcefull_view_no_permission(self): # Gitlab issue #237 # Forcefully checking in a document by a user without adequate # permissions throws out an error - self._create_test_case_superuser() - self._check_out_test_document(user=self._test_case_superuser) + self._create_test_user() + self._check_out_test_document(user=self.test_user) self.grant_access( obj=self.test_document, permission=permission_document_check_in @@ -194,21 +369,16 @@ def test_document_forcefull_check_in_view_no_permission(self): 'pk': self.test_document.pk } ) - self.assertContains( - response=response, text='Insufficient permissions', status_code=403 - ) - + self.assertEqual(response.status_code, 302) self.assertTrue(self.test_document.is_checked_out()) - def test_document_forcefull_check_in_view_with_permission(self): - self._create_test_case_superuser() - self._check_out_test_document(user=self._test_case_superuser) + def test_document_check_in_forcefull_view_with_access(self): + self._create_test_user() + self._check_out_test_document(user=self.test_user) self.grant_access( - obj=self.test_document, permission=permission_document_check_in - ) - self.grant_access( - obj=self.test_document, permission=permission_document_check_in_override + obj=self.test_document, + permission=permission_document_check_in_override ) response = self.post( @@ -217,5 +387,4 @@ def test_document_forcefull_check_in_view_with_permission(self): } ) self.assertEqual(response.status_code, 302) - self.assertFalse(self.test_document.is_checked_out()) diff --git a/mayan/apps/checkouts/views.py b/mayan/apps/checkouts/views.py index 93f54a8190a..440c73a5f2b 100644 --- a/mayan/apps/checkouts/views.py +++ b/mayan/apps/checkouts/views.py @@ -1,21 +1,16 @@ from __future__ import absolute_import, unicode_literals -from django.contrib import messages -from django.http import HttpResponseRedirect -from django.shortcuts import get_object_or_404 from django.urls import reverse from django.utils.translation import ugettext_lazy as _, ungettext from mayan.apps.acls.models import AccessControlList from mayan.apps.common.generics import ( - ConfirmView, MultipleObjectConfirmActionView, MultipleObjectFormActionView, - SingleObjectCreateView, SingleObjectDetailView + MultipleObjectConfirmActionView, MultipleObjectFormActionView, + SingleObjectDetailView ) -from mayan.apps.common.utils import encapsulate from mayan.apps.documents.models import Document from mayan.apps.documents.views import DocumentListView -from .exceptions import DocumentAlreadyCheckedOut, DocumentNotCheckedOut from .forms import DocumentCheckoutForm, DocumentCheckoutDefailForm from .icons import icon_check_out_info from .models import DocumentCheckout @@ -25,69 +20,9 @@ ) -""" -class DocumentCheckinView(ConfirmView): - def get_extra_context(self): - document = self.get_object() - - context = { - 'object': document, - } - - if document.get_check_out_info().user != self.request.user: - context['title'] = _( - 'You didn\'t originally checked out this document. ' - 'Forcefully check in the document: %s?' - ) % document - else: - context['title'] = _('Check in the document: %s?') % document - - return context - - def get_object(self): - return get_object_or_404(klass=Document, pk=self.kwargs['pk']) - - def get_post_action_redirect(self): - return reverse( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.get_object().pk - } - ) - - def view_action(self): - document = self.get_object() - - if document.get_check_out_info().user == self.request.user: - AccessControlList.objects.check_access( - obj=document, permissions=(permission_document_check_in,), - user=self.request.user - ) - else: - AccessControlList.objects.check_access( - obj=document, - permissions=(permission_document_check_in_override,), - user=self.request.user - ) - - try: - document.check_in(user=self.request.user) - except DocumentNotCheckedOut: - messages.error( - message=_('Document has not been checked out.'), - request=self.request - ) - else: - messages.success( - message=_( - 'Document "%s" checked in successfully.' - ) % document, request=self.request - ) -""" - class DocumentCheckinView(MultipleObjectConfirmActionView): error_message = 'Unable to check in document "%(instance)s". %(exception)s' model = Document - object_permission = permission_document_check_in pk_url_kwarg = 'pk' success_message_singular = '%(count)d document checked in.' success_message_plural = '%(count)d documents checked in.' @@ -126,63 +61,30 @@ def get_post_object_action_url(self): else: super(DocumentCheckinView, self).get_post_action_redirect() - def object_action(self, form, instance): - DocumentCheckout.objects.check_in_document( - document=instance, user=self.request.user - ) - + def get_source_queryset(self): + # object_permission is None to disable restricting queryset mixin + # and restrict the queryset ourselves from two permissions + source_queryset = super(DocumentCheckinView, self).get_source_queryset() + check_in_queryset = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in, queryset=source_queryset, + user=self.request.user + ) -""" -class CheckoutDocumentView(SingleObjectCreateView): - form_class = DocumentCheckoutForm - - def dispatch(self, request, *args, **kwargs): - self.document = get_object_or_404(klass=Document, pk=self.kwargs['pk']) - - AccessControlList.objects.check_access( - obj=self.document, permissions=(permission_document_check_out,), - user=request.user + check_in_override_queryset = AccessControlList.objects.restrict_queryset( + permission=permission_document_check_in_override, + queryset=source_queryset, user=self.request.user ) - return super( - CheckoutDocumentView, self - ).dispatch(request, *args, **kwargs) + return check_in_queryset | check_in_override_queryset - def form_valid(self, form): - try: - instance = form.save(commit=False) - instance.user = self.request.user - instance.document = self.document - instance.save() - except DocumentAlreadyCheckedOut: - messages.error( - message=_('Document already checked out.'), - request=self.request - ) - else: - messages.success( - message=_( - 'Document "%s" checked out successfully.' - ) % self.document, request=self.request - ) - - return HttpResponseRedirect(redirect_to=self.get_success_url()) + def object_action(self, form, instance): + DocumentCheckout.business_logic.check_in_document( + document=instance, user=self.request.user + ) - def get_extra_context(self): - return { - 'object': self.document, - 'title': _('Check out document: %s') % self.document - } - def get_post_action_redirect(self): - return reverse( - viewname='checkouts:check_out_info', kwargs={ - 'pk': self.document.pk - } - ) -""" class DocumentCheckoutView(MultipleObjectFormActionView): error_message = 'Unable to checkout document "%(instance)s". %(exception)s' form_class = DocumentCheckoutForm @@ -261,26 +163,6 @@ def get_extra_context(self): context = super(DocumentCheckoutListView, self).get_extra_context() context.update( { - 'extra_columns': ( - { - 'name': _('User'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().user.get_full_name() or document.get_check_out_info().user - ) - }, - { - 'name': _('Checkout time and date'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().checkout_datetime - ) - }, - { - 'name': _('Checkout expiration'), - 'attribute': encapsulate( - lambda document: document.get_check_out_info().expiration_datetime - ) - }, - ), 'no_results_icon': icon_check_out_info, 'no_results_text': _( 'Checking out a document, blocks certain operations ' From 4363bba0fe6921a4acca4d4ca7016962544d838a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:50:37 -0400 Subject: [PATCH 049/402] Remove encapsulate Signed-off-by: Roberto Rosario --- mayan/apps/common/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mayan/apps/common/utils.py b/mayan/apps/common/utils.py index 2d76f17d43f..85447e28136 100644 --- a/mayan/apps/common/utils.py +++ b/mayan/apps/common/utils.py @@ -21,14 +21,6 @@ def check_for_sqlite(): return settings.DATABASES['default']['ENGINE'] == DJANGO_SQLITE_BACKEND and settings.DEBUG is False -def encapsulate(function): - # Workaround Django ticket 15791 - # Changeset 16045 - # http://stackoverflow.com/questions/6861601/ - # cannot-resolve-callable-context-variable/6955045#6955045 - return lambda: function - - def get_related_field(model, related_field_name): try: local_field_name, remaining_field_path = related_field_name.split( From 7ef610287607bd844dd5e5b908cdab892275e058 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 04:52:34 -0400 Subject: [PATCH 050/402] Update release notes Signed-off-by: Roberto Rosario --- HISTORY.rst | 4 ++++ docs/releases/3.3.rst | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 59ccbc7e980..500bf6bc570 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,6 +32,10 @@ - Move Django and Celery settings. - Backport FakeStorageSubclass from versions/next. - Remove django-environ. +- Support checking in and out multiple documents. +- Remove encapsulate helper. +- Add support for menu inheritance. +- Emphasize source column labels. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 58396a0f027..f951bbd8e8e 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -45,7 +45,10 @@ Changes - Backport FakeStorageSubclass from versions/next. Placeholder class to allow serializing the real storage subclass to support migrations. Used by all configurable storages. - +- Support checking in and out multiple documents. +- Remove encapsulate helper. +- Add support for menu inheritance. +- Emphasize source column labels. Removals -------- From 6bcf35bef5cc3cb6b5771d3f5fdb34e127601ed4 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 12 Jul 2019 05:17:15 -0400 Subject: [PATCH 051/402] Add database conversion removal explanation Signed-off-by: Roberto Rosario --- docs/releases/3.3.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index f951bbd8e8e..6bc9685c3c0 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -53,6 +53,23 @@ Changes Removals -------- +- Database conversion. Reason for removal. The database conversions support + provided by this feature (SQLite to PostgreSQL) was being confused with + database migrations and upgrades. + + Database upgrades are the responsibility of the app and the framework. + Database conversions however are not the responsibility of the app (Mayan), + they are the responsibility of the framework. + + Database conversion is outside the scope of what Mayan does but we added + the code, management command, instructions and testing setup to provide + this to our users until the framework (Django) decided to add this + themselves (like they did with migrations). + + Continued confusion about the purpose of the feature and confusion about + how errors with this feature were a reflexion of the code quality of + Mayannecessitated the removal of the database conversion feature. + - Django environ From 3c7a23a5a95dcdf65b7ae1e2878d8d97282c06c2 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 15 Jul 2019 01:24:22 -0400 Subject: [PATCH 052/402] Add support for setting post update callbacks Signed-off-by: Roberto Rosario --- mayan/apps/smart_settings/classes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mayan/apps/smart_settings/classes.py b/mayan/apps/smart_settings/classes.py index be064d0fa8d..4ea8ae11dd0 100644 --- a/mayan/apps/smart_settings/classes.py +++ b/mayan/apps/smart_settings/classes.py @@ -177,13 +177,14 @@ def save_last_known_good(cls): path=settings.CONFIGURATION_LAST_GOOD_FILEPATH ) - def __init__(self, namespace, global_name, default, help_text=None, is_path=False): + def __init__(self, namespace, global_name, default, help_text=None, is_path=False, post_edit_function=None): self.global_name = global_name self.default = default self.help_text = help_text self.loaded = False self.namespace = namespace self.environment_variable = False + self.post_edit_function = post_edit_function namespace._settings.append(self) self.__class__._registry[global_name] = self @@ -239,3 +240,5 @@ def value(self, value): # value is in YAML format self.yaml = value self.raw_value = Setting.deserialize_value(value) + if self.post_edit_function: + self.post_edit_function(setting=self) From 8c064c953ad80875406a14a527ffd31fd2bb8c4b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Mon, 15 Jul 2019 01:33:32 -0400 Subject: [PATCH 053/402] Add file caching app Convert document image cache to use file cache manager app. Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. Signed-off-by: Roberto Rosario --- .tx/config | 6 + HISTORY.rst | 4 + contrib/scripts/process_messages.py | 10 +- docs/releases/3.3.rst | 4 + mayan/apps/documents/api_views.py | 7 +- mayan/apps/documents/apps.py | 10 +- mayan/apps/documents/handlers.py | 18 +- mayan/apps/documents/literals.py | 3 + .../migrations/0049_auto_20190715_0454.py | 37 ++++ .../documents/models/document_page_models.py | 91 +++------ .../models/document_version_models.py | 72 +++---- mayan/apps/documents/settings.py | 13 +- mayan/apps/documents/utils.py | 12 +- mayan/apps/file_caching/__init__.py | 3 + mayan/apps/file_caching/admin.py | 10 + mayan/apps/file_caching/apps.py | 8 + .../locale/ar/LC_MESSAGES/django.po | 72 +++++++ .../locale/bg/LC_MESSAGES/django.po | 71 +++++++ .../locale/bs_BA/LC_MESSAGES/django.po | 70 +++++++ .../locale/da/LC_MESSAGES/django.po | 71 +++++++ .../locale/de_DE/LC_MESSAGES/django.po | 70 +++++++ .../locale/en/LC_MESSAGES/django.po | 70 +++++++ .../locale/es/LC_MESSAGES/django.po | 71 +++++++ .../locale/fa/LC_MESSAGES/django.po | 71 +++++++ .../locale/fr/LC_MESSAGES/django.po | 71 +++++++ .../locale/hu/LC_MESSAGES/django.po | 71 +++++++ .../locale/id/LC_MESSAGES/django.po | 71 +++++++ .../locale/it/LC_MESSAGES/django.po | 71 +++++++ .../locale/nl_NL/LC_MESSAGES/django.po | 70 +++++++ .../locale/pl/LC_MESSAGES/django.po | 73 +++++++ .../locale/pt/LC_MESSAGES/django.po | 71 +++++++ .../locale/pt_BR/LC_MESSAGES/django.po | 71 +++++++ .../locale/ro_RO/LC_MESSAGES/django.po | 70 +++++++ .../locale/ru/LC_MESSAGES/django.po | 73 +++++++ .../locale/sl_SI/LC_MESSAGES/django.po | 70 +++++++ .../locale/tr_TR/LC_MESSAGES/django.po | 70 +++++++ .../locale/vi_VN/LC_MESSAGES/django.po | 70 +++++++ .../locale/zh_CN/LC_MESSAGES/django.po | 70 +++++++ .../file_caching/migrations/0001_initial.py | 64 ++++++ .../apps/file_caching/migrations/__init__.py | 0 mayan/apps/file_caching/models.py | 184 ++++++++++++++++++ mayan/settings/base.py | 1 + 42 files changed, 2001 insertions(+), 114 deletions(-) create mode 100644 mayan/apps/documents/migrations/0049_auto_20190715_0454.py create mode 100644 mayan/apps/file_caching/__init__.py create mode 100644 mayan/apps/file_caching/admin.py create mode 100644 mayan/apps/file_caching/apps.py create mode 100644 mayan/apps/file_caching/locale/ar/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/bg/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/bs_BA/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/da/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/de_DE/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/es/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/fa/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/fr/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/hu/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/id/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/it/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/nl_NL/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/pl/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/pt/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/pt_BR/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/ro_RO/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/ru/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/sl_SI/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/tr_TR/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/vi_VN/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/locale/zh_CN/LC_MESSAGES/django.po create mode 100644 mayan/apps/file_caching/migrations/0001_initial.py create mode 100644 mayan/apps/file_caching/migrations/__init__.py create mode 100644 mayan/apps/file_caching/models.py diff --git a/.tx/config b/.tx/config index 158957b39c2..5f5d77b3af8 100644 --- a/.tx/config +++ b/.tx/config @@ -115,6 +115,12 @@ source_lang = en source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po type = PO +[mayan-edms.file_caching-3-0] +file_filter = mayan/apps/file_caching/locale//LC_MESSAGES/django.po +source_lang = en +source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po +type = PO + [mayan-edms.file_metadata-3-0] file_filter = mayan/apps/file_metadata/locale//LC_MESSAGES/django.po source_lang = en diff --git a/HISTORY.rst b/HISTORY.rst index 500bf6bc570..aded82fb4fc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,10 @@ - Remove encapsulate helper. - Add support for menu inheritance. - Emphasize source column labels. +- Backport file cache manager app. +- Convert document image cache to use file cache manager app. + Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. + 3.2.6 (2019-07-10) ================== diff --git a/contrib/scripts/process_messages.py b/contrib/scripts/process_messages.py index 29b7efc4166..5c43e5846e9 100755 --- a/contrib/scripts/process_messages.py +++ b/contrib/scripts/process_messages.py @@ -13,11 +13,11 @@ 'checkouts', 'common', 'converter', 'dashboards', 'dependencies', 'django_gpg', 'document_comments', 'document_indexing', 'document_parsing', 'document_signatures', 'document_states', - 'documents', 'dynamic_search', 'events', 'file_metadata', 'linking', - 'lock_manager', 'mailer', 'mayan_statistics', 'metadata', 'mirroring', - 'motd', 'navigation', 'ocr', 'permissions', 'platform', 'rest_api', - 'smart_settings', 'sources', 'storage', 'tags', 'task_manager', - 'user_management' + 'documents', 'dynamic_search', 'events', 'file_caching', + 'file_metadata', 'linking', 'lock_manager', 'mailer', + 'mayan_statistics', 'metadata', 'mirroring', 'motd', 'navigation', + 'ocr', 'permissions', 'platform', 'rest_api', 'smart_settings', + 'sources', 'storage', 'tags', 'task_manager', 'user_management' ) LANGUAGE_LIST = ( diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 6bc9685c3c0..d49868c4683 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -49,6 +49,10 @@ Changes - Remove encapsulate helper. - Add support for menu inheritance. - Emphasize source column labels. +- Backport file cache manager app. +- Convert document image cache to use file cache manager app. + Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. + Removals -------- diff --git a/mayan/apps/documents/api_views.py b/mayan/apps/documents/api_views.py index c42a8d50c2e..f80f5ec67d3 100644 --- a/mayan/apps/documents/api_views.py +++ b/mayan/apps/documents/api_views.py @@ -36,7 +36,6 @@ WritableDocumentTypeSerializer, WritableDocumentVersionSerializer ) from .settings import settings_document_page_image_cache_time -from .storages import storage_documentimagecache from .tasks import task_generate_document_page_image logger = logging.getLogger(__name__) @@ -205,11 +204,13 @@ def retrieve(self, request, *args, **kwargs): ) cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) - with storage_documentimagecache.open(cache_filename) as file_object: + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: response = HttpResponse(file_object.read(), content_type='image') if '_hash' in request.GET: patch_cache_control( - response, max_age=settings_document_page_image_cache_time.value + response=response, + max_age=settings_document_page_image_cache_time.value ) return response diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 0032127a92a..25a943294b0 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals -from django.db.models.signals import post_delete +from django.db.models.signals import post_delete, post_migrate from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.classes import ModelPermission @@ -43,8 +43,8 @@ event_document_view ) from .handlers import ( - handler_create_default_document_type, handler_remove_empty_duplicates_lists, - handler_scan_duplicates_for, + handler_create_default_document_type, handler_create_document_cache, + handler_remove_empty_duplicates_lists, handler_scan_duplicates_for ) from .links import ( link_clear_image_cache, link_document_clear_transformations, @@ -527,6 +527,10 @@ def ready(self): dispatch_uid='handler_create_default_document_type', receiver=handler_create_default_document_type ) + post_migrate.connect( + dispatch_uid='documents_handler_create_document_cache', + receiver=handler_create_document_cache, + ) post_version_upload.connect( dispatch_uid='handler_scan_duplicates_for', receiver=handler_scan_duplicates_for diff --git a/mayan/apps/documents/handlers.py b/mayan/apps/documents/handlers.py index 597a38c40d0..a033ec609f2 100644 --- a/mayan/apps/documents/handlers.py +++ b/mayan/apps/documents/handlers.py @@ -1,8 +1,13 @@ from __future__ import unicode_literals from django.apps import apps +from django.utils.translation import ugettext_lazy as _ -from .literals import DEFAULT_DOCUMENT_TYPE_LABEL +from .literals import ( + DEFAULT_DOCUMENT_TYPE_LABEL, DOCUMENT_CACHE_STORAGE_INSTANCE_PATH, + DOCUMENT_IMAGES_CACHE_NAME +) +from .settings import setting_document_cache_maximum_size from .signals import post_initial_document_type from .tasks import task_clean_empty_duplicate_lists, task_scan_duplicates_for @@ -21,6 +26,17 @@ def handler_create_default_document_type(sender, **kwargs): ) +def handler_create_document_cache(sender, **kwargs): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + Cache.objects.update_or_create( + defaults={ + 'label': _('Document images'), + 'storage_instance_path': DOCUMENT_CACHE_STORAGE_INSTANCE_PATH, + 'maximum_size': setting_document_cache_maximum_size.value, + }, name=DOCUMENT_IMAGES_CACHE_NAME, + ) + + def handler_scan_duplicates_for(sender, instance, **kwargs): task_scan_duplicates_for.apply_async( kwargs={'document_id': instance.document.pk} diff --git a/mayan/apps/documents/literals.py b/mayan/apps/documents/literals.py index c56f92d39b3..e29ed17e8da 100644 --- a/mayan/apps/documents/literals.py +++ b/mayan/apps/documents/literals.py @@ -9,6 +9,7 @@ DELETE_STALE_STUBS_INTERVAL = 60 * 10 # 10 minutes DEFAULT_DELETE_PERIOD = 30 DEFAULT_DELETE_TIME_UNIT = TIME_DELTA_UNIT_DAYS +DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE = 500 * 2 ** 20 # 500 Megabytes DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE = 65535 DEFAULT_LANGUAGE = 'eng' DEFAULT_LANGUAGE_CODES = ( @@ -30,6 +31,8 @@ DEFAULT_ZIP_FILENAME = 'document_bundle.zip' DEFAULT_DOCUMENT_TYPE_LABEL = _('Default') DOCUMENT_IMAGE_TASK_TIMEOUT = 120 +DOCUMENT_IMAGES_CACHE_NAME = 'document_images' +DOCUMENT_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.documents.storages.storage_documentimagecache' STUB_EXPIRATION_INTERVAL = 60 * 60 * 24 # 24 hours UPDATE_PAGE_COUNT_RETRY_DELAY = 10 UPLOAD_NEW_VERSION_RETRY_DELAY = 10 diff --git a/mayan/apps/documents/migrations/0049_auto_20190715_0454.py b/mayan/apps/documents/migrations/0049_auto_20190715_0454.py new file mode 100644 index 00000000000..889d244bfe5 --- /dev/null +++ b/mayan/apps/documents/migrations/0049_auto_20190715_0454.py @@ -0,0 +1,37 @@ +from __future__ import unicode_literals + +from django.db import migrations + +from ..storages import storage_documentimagecache + + +def operation_clear_old_cache(apps, schema_editor): + DocumentPageCachedImage = apps.get_model( + 'documents', 'DocumentPageCachedImage' + ) + + for cached_image in DocumentPageCachedImage.objects.using(schema_editor.connection.alias).all(): + # Delete each cached image directly since the model doesn't exists and + # will not trigger the physical deletion of the stored file + storage_documentimagecache.delete(cached_image.filename) + cached_image.delete() + + +class Migration(migrations.Migration): + dependencies = [ + ('documents', '0048_auto_20190711_0544'), + ] + + operations = [ + migrations.RunPython( + code=operation_clear_old_cache, + reverse_code=migrations.RunPython.noop + ), + migrations.RemoveField( + model_name='documentpagecachedimage', + name='document_page', + ), + migrations.DeleteModel( + name='DocumentPageCachedImage', + ), + ] diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index afbf34cef3a..8aa9119445f 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -4,13 +4,14 @@ from furl import furl -from django.core.files.base import ContentFile from django.db import models from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.literals import DEFAULT_ZOOM_LEVEL, DEFAULT_ROTATION + from mayan.apps.converter.models import Transformation from mayan.apps.converter.transformations import ( BaseTransformation, TransformationResize, TransformationRotate, @@ -24,7 +25,6 @@ setting_display_width, setting_display_height, setting_zoom_max_level, setting_zoom_min_level ) -from ..storages import storage_documentimagecache from .document_version_models import DocumentVersion @@ -56,9 +56,12 @@ class Meta: def __str__(self): return self.get_label() - @property - def cache_filename(self): - return 'page-cache-{}'.format(self.uuid) + @cached_property + def cache_partition(self): + partition, created = self.document_version.cache.partitions.get_or_create( + name=self.uuid + ) + return partition def delete(self, *args, **kwargs): self.invalidate_cache() @@ -80,29 +83,24 @@ def document(self): def generate_image(self, *args, **kwargs): transformation_list = self.get_combined_transformation_list(*args, **kwargs) - - cache_filename = '{}-{}'.format( - self.cache_filename, BaseTransformation.combine(transformation_list) - ) + combined_cache_filename = BaseTransformation.combine(transformation_list) # Check is transformed image is available - logger.debug('transformations cache filename: %s', cache_filename) + logger.debug('transformations cache filename: %s', combined_cache_filename) - if not setting_disable_transformed_image_cache.value and storage_documentimagecache.exists(cache_filename): + if not setting_disable_transformed_image_cache.value and self.cache_partition.get_file(filename=combined_cache_filename): logger.debug( - 'transformations cache file "%s" found', cache_filename + 'transformations cache file "%s" found', combined_cache_filename ) else: logger.debug( - 'transformations cache file "%s" not found', cache_filename + 'transformations cache file "%s" not found', combined_cache_filename ) image = self.get_image(transformations=transformation_list) - with storage_documentimagecache.open(cache_filename, 'wb+') as file_object: + with self.cache_partition.create_file(filename=combined_cache_filename) as file_object: file_object.write(image.getvalue()) - self.cached_images.create(filename=cache_filename) - - return cache_filename + return combined_cache_filename def get_absolute_url(self): return reverse( @@ -159,7 +157,6 @@ def get_combined_transformation_list(self, *args, **kwargs): zoom_level = setting_zoom_max_level.value # Generate transformation hash - transformation_list = [] # Stored transformations first @@ -186,13 +183,15 @@ def get_combined_transformation_list(self, *args, **kwargs): return transformation_list def get_image(self, transformations=None): - cache_filename = self.cache_filename + cache_filename = 'base_image' logger.debug('Page cache filename: %s', cache_filename) - if not setting_disable_base_image_cache.value and storage_documentimagecache.exists(cache_filename): + cache_file = self.cache_partition.get_file(filename=cache_filename) + + if not setting_disable_base_image_cache.value and cache_file: logger.debug('Page cache file "%s" found', cache_filename) - with storage_documentimagecache.open(cache_filename) as file_object: + with cache_file.open as file_object: converter = get_converter_class()( file_object=file_object ) @@ -200,8 +199,8 @@ def get_image(self, transformations=None): converter.seek_page(page_number=0) # This code is also repeated below to allow using a context - # manager with storage_documentimagecache.open and close it - # automatically. + # manager with cache_file.open and close it automatically. + # Apply runtime transformations for transformation in transformations: converter.transform(transformation=transformation) @@ -218,14 +217,11 @@ def get_image(self, transformations=None): page_image = converter.get_page() - # Since open "wb+" doesn't create files, check if the file - # exists, if not then create it - if not storage_documentimagecache.exists(cache_filename): - storage_documentimagecache.save(name=cache_filename, content=ContentFile(content='')) - - with storage_documentimagecache.open(cache_filename, 'wb+') as file_object: + # Since open "wb+" doesn't create files, create it explicitly + with self.cache_partition.create_file(filename=cache_filename) as file_object: file_object.write(page_image.getvalue()) + # Apply runtime transformations for transformation in transformations: converter.transform(transformation=transformation) @@ -236,13 +232,10 @@ def get_image(self, transformations=None): 'Error creating page cache file "%s"; %s', cache_filename, exception ) - storage_documentimagecache.delete(cache_filename) raise def invalidate_cache(self): - storage_documentimagecache.delete(self.cache_filename) - for cached_image in self.cached_images.all(): - cached_image.delete() + self.cache_partition.purge() @property def is_in_trash(self): @@ -277,38 +270,6 @@ def uuid(self): return '{}-{}'.format(self.document_version.uuid, self.pk) -class DocumentPageCachedImage(models.Model): - document_page = models.ForeignKey( - on_delete=models.CASCADE, related_name='cached_images', - to=DocumentPage, verbose_name=_('Document page') - ) - datetime = models.DateTimeField( - auto_now_add=True, db_index=True, verbose_name=_('Date time') - ) - filename = models.CharField(max_length=128, verbose_name=_('Filename')) - file_size = models.PositiveIntegerField( - db_index=True, default=0, verbose_name=_('File size') - ) - - objects = DocumentPageCachedImage() - - class Meta: - verbose_name = _('Document page cached image') - verbose_name_plural = _('Document page cached images') - - def delete(self, *args, **kwargs): - storage_documentimagecache.delete(self.filename) - return super(DocumentPageCachedImage, self).delete(*args, **kwargs) - - def natural_key(self): - return (self.filename, self.document_page.natural_key()) - natural_key.dependencies = ['documents.DocumentPage'] - - def save(self, *args, **kwargs): - self.file_size = storage_documentimagecache.size(self.filename) - return super(DocumentPageCachedImage, self).save(*args, **kwargs) - - class DocumentPageResult(DocumentPage): class Meta: ordering = ('document_version__document', 'page_number') diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 78b837860d9..1428f9e6156 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -7,11 +7,11 @@ import uuid from django.apps import apps -from django.core.files.base import ContentFile from django.db import models, transaction from django.template import Template, Context from django.urls import reverse from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from mayan.apps.converter.exceptions import InvalidOfficeFormat, PageCountError @@ -21,10 +21,11 @@ from mayan.apps.mimetype.api import get_mimetype from ..events import event_document_new_version, event_document_version_revert +from ..literals import DOCUMENT_IMAGES_CACHE_NAME from ..managers import DocumentVersionManager from ..settings import setting_fix_orientation, setting_hash_block_size from ..signals import post_document_created, post_version_upload -from ..storages import storage_documentversion, storage_documentimagecache +from ..storages import storage_documentversion from .document_models import Document @@ -61,14 +62,6 @@ class DocumentVersion(models.Model): _pre_open_hooks = {} _post_save_hooks = {} - @classmethod - def register_pre_open_hook(cls, order, func): - cls._pre_open_hooks[order] = func - - @classmethod - def register_post_save_hook(cls, order, func): - cls._post_save_hooks[order] = func - document = models.ForeignKey( on_delete=models.CASCADE, related_name='versions', to=Document, verbose_name=_('Document') @@ -118,12 +111,28 @@ class Meta: objects = DocumentVersionManager() + @classmethod + def register_pre_open_hook(cls, order, func): + cls._pre_open_hooks[order] = func + + @classmethod + def register_post_save_hook(cls, order, func): + cls._post_save_hooks[order] = func + def __str__(self): return self.get_rendered_string() - @property - def cache_filename(self): - return 'document-version-{}'.format(self.uuid) + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='version-{}'.format(self.uuid) + ) + return partition def delete(self, *args, **kwargs): for page in self.pages.all(): @@ -164,43 +173,36 @@ def get_api_image_url(self, *args, **kwargs): return first_page.get_api_image_url(*args, **kwargs) def get_intermediate_file(self): - cache_filename = self.cache_filename - logger.debug('Intermidiate filename: %s', cache_filename) - - if storage_documentimagecache.exists(cache_filename): - logger.debug('Intermidiate file "%s" found.', cache_filename) - - return storage_documentimagecache.open(cache_filename) + cache_filename = 'intermediate_file' + cache_file = self.cache_partition.get_file(filename=cache_filename) + if cache_file: + logger.debug('Intermidiate file found.') + return cache_file.open() else: - logger.debug('Intermidiate file "%s" not found.', cache_filename) + logger.debug('Intermidiate file not found.') try: with self.open() as version_file_object: - converter = get_converter_class()(file_object=version_file_object) + converter = get_converter_class()( + file_object=version_file_object + ) with converter.to_pdf() as pdf_file_object: - - # Since open "wb+" doesn't create files, check if the file - # exists, if not then create it - if not storage_documentimagecache.exists(cache_filename): - storage_documentimagecache.save( - name=cache_filename, content=ContentFile(content='') - ) - - with storage_documentimagecache.open(cache_filename, mode='wb+') as file_object: + with self.cache_partition.create_file(filename=cache_filename) as file_object: shutil.copyfileobj( fsrc=pdf_file_object, fdst=file_object ) - return storage_documentimagecache.open(cache_filename) + return self.cache_partition.get_file(filename=cache_filename).open() except InvalidOfficeFormat: return self.open() except Exception as exception: - # Cleanup in case of error logger.error( 'Error creating intermediate file "%s"; %s.', cache_filename, exception ) - storage_documentimagecache.delete(cache_filename) + cache_file = self.cache_partition.get_file(filename=cache_filename) + if cache_file: + cache_file.delete() raise def get_rendered_string(self, preserve_extension=False): @@ -224,7 +226,7 @@ def natural_key(self): natural_key.dependencies = ['documents.Document'] def invalidate_cache(self): - storage_documentimagecache.delete(self.cache_filename) + self.cache_partition.purge() for page in self.pages.all(): page.invalidate_cache() diff --git a/mayan/apps/documents/settings.py b/mayan/apps/documents/settings.py index a4d5014e918..8a666c8f11d 100644 --- a/mayan/apps/documents/settings.py +++ b/mayan/apps/documents/settings.py @@ -8,11 +8,22 @@ from mayan.apps.smart_settings.classes import Namespace from .literals import ( - DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES + DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, DEFAULT_DOCUMENTS_HASH_BLOCK_SIZE, + DEFAULT_LANGUAGE, DEFAULT_LANGUAGE_CODES ) +from .utils import callback_update_cache_size namespace = Namespace(label=_('Documents'), name='documents') +setting_document_cache_maximum_size = namespace.add_setting( + global_name='DOCUMENTS_CACHE_MAXIMUM_SIZE', + default=DEFAULT_DOCUMENTS_CACHE_MAXIMUM_SIZE, + help_text=_( + 'The threshold at which the DOCUMENT_CACHE_STORAGE_BACKEND will start ' + 'deleting the oldest document image cache files. Specify the size in ' + 'bytes.' + ), post_edit_function=callback_update_cache_size +) setting_documentimagecache_storage = namespace.add_setting( global_name='DOCUMENTS_CACHE_STORAGE_BACKEND', default='django.core.files.storage.FileSystemStorage', help_text=_( diff --git a/mayan/apps/documents/utils.py b/mayan/apps/documents/utils.py index 21b2df21071..7a7e16d4554 100644 --- a/mayan/apps/documents/utils.py +++ b/mayan/apps/documents/utils.py @@ -2,9 +2,17 @@ import pycountry +from django.apps import apps from django.utils.translation import ugettext_lazy as _ -from .settings import setting_language_codes +from .literals import DOCUMENT_IMAGES_CACHE_NAME + + +def callback_update_cache_size(setting): + Cache = apps.get_model(app_label='common', model_name='Cache') + cache = Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) + cache.maximum_size = setting.value + cache.save() def get_language(language_code): @@ -19,6 +27,8 @@ def get_language(language_code): def get_language_choices(): + from .settings import setting_language_codes + return sorted( [ ( diff --git a/mayan/apps/file_caching/__init__.py b/mayan/apps/file_caching/__init__.py new file mode 100644 index 00000000000..606c594dcf0 --- /dev/null +++ b/mayan/apps/file_caching/__init__.py @@ -0,0 +1,3 @@ +from __future__ import unicode_literals + +default_app_config = 'mayan.apps.file_caching.apps.FileCachingConfig' diff --git a/mayan/apps/file_caching/admin.py b/mayan/apps/file_caching/admin.py new file mode 100644 index 00000000000..a807f197c93 --- /dev/null +++ b/mayan/apps/file_caching/admin.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals + +from django.contrib import admin + +from .models import Cache + + +@admin.register(Cache) +class CacheAdmin(admin.ModelAdmin): + list_display = ('name', 'label', 'storage_instance_path', 'maximum_size') diff --git a/mayan/apps/file_caching/apps.py b/mayan/apps/file_caching/apps.py new file mode 100644 index 00000000000..bb333be6e03 --- /dev/null +++ b/mayan/apps/file_caching/apps.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +from mayan.apps.common.apps import MayanAppConfig + + +class FileCachingConfig(MayanAppConfig): + has_tests = False + name = 'mayan.apps.file_caching' diff --git a/mayan/apps/file_caching/locale/ar/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/ar/LC_MESSAGES/django.po new file mode 100644 index 00000000000..173a9d5ee23 --- /dev/null +++ b/mayan/apps/file_caching/locale/ar/LC_MESSAGES/django.po @@ -0,0 +1,72 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 " +"&& n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5;\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/bg/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/bg/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/bg/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/bs_BA/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/bs_BA/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/bs_BA/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/da/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/da/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/da/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/de_DE/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/de_DE/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/de_DE/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/es/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/es/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/es/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/fa/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/fa/LC_MESSAGES/django.po new file mode 100644 index 00000000000..09b38b103d3 --- /dev/null +++ b/mayan/apps/file_caching/locale/fa/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=1; plural=0;\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/fr/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 00000000000..c65665a79d5 --- /dev/null +++ b/mayan/apps/file_caching/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/hu/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/hu/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/hu/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/id/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/id/LC_MESSAGES/django.po new file mode 100644 index 00000000000..09b38b103d3 --- /dev/null +++ b/mayan/apps/file_caching/locale/id/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=1; plural=0;\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/it/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/it/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/it/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/nl_NL/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/nl_NL/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/nl_NL/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/pl/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/pl/LC_MESSAGES/django.po new file mode 100644 index 00000000000..2f8fc8f37b1 --- /dev/null +++ b/mayan/apps/file_caching/locale/pl/LC_MESSAGES/django.po @@ -0,0 +1,73 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=4; plural=(n==1 ? 0 : (n%10>=2 && n%10<=4) && (n" +"%100<12 || n%100>=14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n" +"%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/pt/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/pt/LC_MESSAGES/django.po new file mode 100644 index 00000000000..4812952c387 --- /dev/null +++ b/mayan/apps/file_caching/locale/pt/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/pt_BR/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/pt_BR/LC_MESSAGES/django.po new file mode 100644 index 00000000000..c65665a79d5 --- /dev/null +++ b/mayan/apps/file_caching/locale/pt_BR/LC_MESSAGES/django.po @@ -0,0 +1,71 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/ro_RO/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/ro_RO/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/ro_RO/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/ru/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/ru/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b73a01bbe4a --- /dev/null +++ b/mayan/apps/file_caching/locale/ru/LC_MESSAGES/django.po @@ -0,0 +1,73 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" +"%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || (n" +"%100>=11 && n%100<=14)? 2 : 3);\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/sl_SI/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/sl_SI/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/sl_SI/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/tr_TR/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/tr_TR/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/tr_TR/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/vi_VN/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/vi_VN/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/vi_VN/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/locale/zh_CN/LC_MESSAGES/django.po b/mayan/apps/file_caching/locale/zh_CN/LC_MESSAGES/django.po new file mode 100644 index 00000000000..b3bd609bb36 --- /dev/null +++ b/mayan/apps/file_caching/locale/zh_CN/LC_MESSAGES/django.po @@ -0,0 +1,70 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2018-12-07 21:32-0400\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: models.py:23 models.py:70 +msgid "Name" +msgstr "" + +#: models.py:25 +msgid "Label" +msgstr "" + +#: models.py:26 +msgid "Maximum size" +msgstr "" + +#: models.py:28 +msgid "Storage instance path" +msgstr "" + +#: models.py:32 models.py:67 +msgid "Cache" +msgstr "" + +#: models.py:33 +msgid "Caches" +msgstr "" + +#: models.py:75 models.py:141 +msgid "Cache partition" +msgstr "" + +#: models.py:76 +msgid "Cache partitions" +msgstr "" + +#: models.py:144 +msgid "Date time" +msgstr "" + +#: models.py:146 +msgid "Filename" +msgstr "" + +#: models.py:148 +msgid "File size" +msgstr "" + +#: models.py:154 +msgid "Cache partition file" +msgstr "" + +#: models.py:155 +msgid "Cache partition files" +msgstr "" diff --git a/mayan/apps/file_caching/migrations/0001_initial.py b/mayan/apps/file_caching/migrations/0001_initial.py new file mode 100644 index 00000000000..46c20637ab9 --- /dev/null +++ b/mayan/apps/file_caching/migrations/0001_initial.py @@ -0,0 +1,64 @@ +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Cache', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, unique=True, verbose_name='Name')), + ('label', models.CharField(max_length=128, verbose_name='Label')), + ('maximum_size', models.PositiveIntegerField(verbose_name='Maximum size')), + ('storage_instance_path', models.CharField(max_length=255, unique=True, verbose_name='Storage instance path')), + ], + options={ + 'verbose_name': 'Cache', + 'verbose_name_plural': 'Caches', + }, + ), + migrations.CreateModel( + name='CachePartition', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('cache', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='partitions', to='file_caching.Cache', verbose_name='Cache')), + ], + options={ + 'verbose_name': 'Cache partition', + 'verbose_name_plural': 'Cache partitions', + }, + ), + migrations.CreateModel( + name='CachePartitionFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('datetime', models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date time')), + ('filename', models.CharField(max_length=255, verbose_name='Filename')), + ('file_size', models.PositiveIntegerField(default=0, verbose_name='File size')), + ('partition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='file_caching.CachePartition', verbose_name='Cache partition')), + ], + options={ + 'get_latest_by': 'datetime', + 'verbose_name': 'Cache partition file', + 'verbose_name_plural': 'Cache partition files', + }, + ), + migrations.AlterUniqueTogether( + name='cachepartitionfile', + unique_together=set([('partition', 'filename')]), + ), + migrations.AlterUniqueTogether( + name='cachepartition', + unique_together=set([('cache', 'name')]), + ), + ] diff --git a/mayan/apps/file_caching/migrations/__init__.py b/mayan/apps/file_caching/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mayan/apps/file_caching/models.py b/mayan/apps/file_caching/models.py new file mode 100644 index 00000000000..6b8e6bb3ccc --- /dev/null +++ b/mayan/apps/file_caching/models.py @@ -0,0 +1,184 @@ +from __future__ import unicode_literals + +from contextlib import contextmanager +import logging + +from django.core.files.base import ContentFile +from django.db import models, transaction +from django.db.models import Sum +from django.utils.encoding import python_2_unicode_compatible +from django.utils.functional import cached_property +from django.utils.module_loading import import_string +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.lock_manager.exceptions import LockError +from mayan.apps.lock_manager.runtime import locking_backend + +logger = logging.getLogger(__name__) + + +@python_2_unicode_compatible +class Cache(models.Model): + name = models.CharField( + max_length=128, unique=True, verbose_name=_('Name') + ) + label = models.CharField(max_length=128, verbose_name=_('Label')) + maximum_size = models.PositiveIntegerField(verbose_name=_('Maximum size')) + storage_instance_path = models.CharField( + max_length=255, unique=True, verbose_name=_('Storage instance path') + ) + + class Meta: + verbose_name = _('Cache') + verbose_name_plural = _('Caches') + + def __str__(self): + return self.label + + def get_files(self): + return CachePartitionFile.objects.filter(partition__cache__id=self.pk) + + def get_total_size(self): + return self.get_files().aggregate( + file_size__sum=Sum('file_size') + )['file_size__sum'] or 0 + + def prune(self): + while self.get_total_size() > self.maximum_size: + self.get_files().earliest().delete() + + def purge(self): + for partition in self.partitions.all(): + partition.purge() + + def save(self, *args, **kwargs): + result = super(Cache, self).save(*args, **kwargs) + self.prune() + return result + + @cached_property + def storage(self): + return import_string(self.storage_instance_path) + + +class CachePartition(models.Model): + cache = models.ForeignKey( + on_delete=models.CASCADE, related_name='partitions', + to=Cache, verbose_name=_('Cache') + ) + name = models.CharField( + max_length=128, verbose_name=_('Name') + ) + + class Meta: + unique_together = ('cache', 'name') + verbose_name = _('Cache partition') + verbose_name_plural = _('Cache partitions') + + @staticmethod + def get_combined_filename(parent, filename): + return '{}-{}'.format(parent, filename) + + @contextmanager + def create_file(self, filename): + lock_id = 'cache_partition-create_file-{}-{}'.format(self.pk, filename) + try: + logger.debug('trying to acquire lock: %s', lock_id) + lock = locking_backend.acquire_lock(lock_id) + logger.debug('acquired lock: %s', lock_id) + try: + self.cache.prune() + + # Since open "wb+" doesn't create files force the creation of an + # empty file. + self.cache.storage.delete( + name=self.get_full_filename(filename=filename) + ) + self.cache.storage.save( + name=self.get_full_filename(filename=filename), + content=ContentFile(content='') + ) + + try: + with transaction.atomic(): + partition_file = self.files.create(filename=filename) + yield partition_file.open(mode='wb') + partition_file.update_size() + except Exception as exception: + logger.error( + 'Unexpected exception while trying to save new ' + 'cache file; %s', exception + ) + self.cache.storage.delete( + name=self.get_full_filename(filename=filename) + ) + raise + finally: + lock.release() + except LockError: + logger.debug('unable to obtain lock: %s' % lock_id) + raise + + def get_file(self, filename): + try: + return self.files.get(filename=filename) + except self.files.model.DoesNotExist: + return None + + def get_full_filename(self, filename): + return CachePartition.get_combined_filename( + parent=self.name, filename=filename + ) + + def purge(self): + for parition_file in self.files.all(): + parition_file.delete() + + +class CachePartitionFile(models.Model): + partition = models.ForeignKey( + on_delete=models.CASCADE, related_name='files', + to=CachePartition, verbose_name=_('Cache partition') + ) + datetime = models.DateTimeField( + auto_now_add=True, db_index=True, verbose_name=_('Date time') + ) + filename = models.CharField(max_length=255, verbose_name=_('Filename')) + file_size = models.PositiveIntegerField( + default=0, verbose_name=_('File size') + ) + + class Meta: + get_latest_by = 'datetime' + unique_together = ('partition', 'filename') + verbose_name = _('Cache partition file') + verbose_name_plural = _('Cache partition files') + + def delete(self, *args, **kwargs): + self.partition.cache.storage.delete(name=self.full_filename) + return super(CachePartitionFile, self).delete(*args, **kwargs) + + @cached_property + def full_filename(self): + return CachePartition.get_combined_filename( + parent=self.partition.name, filename=self.filename + ) + + def open(self, mode='rb'): + # Open the file for reading. If the file is written to, the + # .update_size() must be called. + try: + return self.partition.cache.storage.open( + name=self.full_filename, mode=mode + ) + except Exception as exception: + logger.error( + 'Unexpected exception opening the cache file; %s', exception + ) + raise + + def update_size(self): + self.file_size = self.partition.cache.storage.size( + name=self.full_filename + ) + self.save() diff --git a/mayan/settings/base.py b/mayan/settings/base.py index 6292ecd168c..c18ae759513 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -95,6 +95,7 @@ 'mayan.apps.django_gpg', 'mayan.apps.dynamic_search', 'mayan.apps.events', + 'mayan.apps.file_caching', 'mayan.apps.lock_manager', 'mayan.apps.mimetype', 'mayan.apps.navigation', From 49a16acdf5bb3e651ac470c934adddc8352f8cf9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 01:24:57 -0400 Subject: [PATCH 054/402] Backport panel selection and panel toolbar Support selection by panel body clicking. Styling changes for highlighted panel. Self-display multiple item action list. New select all button. Fix fancybox click area on thumbnail display. Remove the multi item form processing view. Signed-off-by: Roberto Rosario --- .../appearance/static/appearance/css/base.css | 28 ++-- .../appearance/static/appearance/js/base.js | 5 +- .../static/appearance/js/mayan_app.js | 124 +++++++++++++----- .../generic_list_items_subtemplate.html | 31 ++--- .../appearance/generic_list_subtemplate.html | 33 +++-- .../templates/appearance/list_toolbar.html | 49 +++++++ mayan/apps/common/urls.py | 6 +- mayan/apps/common/views.py | 64 --------- 8 files changed, 194 insertions(+), 146 deletions(-) create mode 100644 mayan/apps/appearance/templates/appearance/list_toolbar.html diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 0cb7944b434..3c70bd2e8c6 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -220,6 +220,18 @@ a i { font-weight: bold; } +.panel-highlighted { + box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000; +} + +.panel-highlighted:hover { + box-shadow: 0px 0px 3px #18bc9c, 10px 10px 20px #000000, 0px 0px 8px #000000; +} + +.panel-item:not(.panel-highlighted):hover { + box-shadow: 0px 0px 8px #000000; +} + /* Content */ @media (min-width:1200px) { .container-fluid { @@ -249,14 +261,6 @@ a i { margin: auto; } -.thin_border { - border: 1px solid black; - display: block; - margin-left: auto; - margin-right: auto; -} - - .thin_border-thumbnail { display: block; max-width: 100%; @@ -266,6 +270,14 @@ a i { margin: auto; } +/* Must go after .thin_border-thumbnail */ +.thin_border { + border: 1px solid black; + display: inline; + margin-left: 0px; + margin-right: 0px; +} + #ajax-spinner { position: fixed; top: 16px; diff --git a/mayan/apps/appearance/static/appearance/js/base.js b/mayan/apps/appearance/static/appearance/js/base.js index 244ac0298ad..ff87e82871c 100644 --- a/mayan/apps/appearance/static/appearance/js/base.js +++ b/mayan/apps/appearance/static/appearance/js/base.js @@ -6,7 +6,8 @@ var MayanAppClass = MayanApp; var partialNavigation = new PartialNavigation({ initialURL: initialURL, - disabledAnchorClasses: ['disabled'], + disabledAnchorClasses: [ + 'btn-multi-item-action', 'disabled', 'pagination-disabled' + ], excludeAnchorClasses: ['fancybox', 'new_window', 'non-ajax'], - formBeforeSerializeCallbacks: [MayanApp.MultiObjectFormProcess], }); diff --git a/mayan/apps/appearance/static/appearance/js/mayan_app.js b/mayan/apps/appearance/static/appearance/js/mayan_app.js index d96c7f4c9b5..acbeeb12837 100644 --- a/mayan/apps/appearance/static/appearance/js/mayan_app.js +++ b/mayan/apps/appearance/static/appearance/js/mayan_app.js @@ -17,30 +17,36 @@ class MayanApp { // Class methods and variables - static MultiObjectFormProcess ($form, options) { - /* - * ajaxForm callback to add the external item checkboxes to the - * submitted form - */ - - if ($form.hasClass('form-multi-object-action')) { - // Turn form data into an object - var formArray = $form.serializeArray().reduce(function (obj, item) { - obj[item.name] = item.value; - return obj; - }, {}); - - // Add all checked checkboxes to the form data - $('.form-multi-object-action-checkbox:checked').each(function() { - var $this = $(this); - formArray[$this.attr('name')] = $this.attr('value'); - }); - - // Set the form data as the data to send - options.data = formArray; + static countChecked() { + var checkCount = $('.check-all-slave:checked').length; + + if (checkCount) { + $('#multi-item-title').hide(); + $('#multi-item-actions').show(); + } else { + $('#multi-item-title').show(); + $('#multi-item-actions').hide(); } } + static setupMultiItemActions () { + $('body').on('change', '.check-all-slave', function () { + MayanApp.countChecked(); + }); + + $('body').on('click', '.btn-multi-item-action', function (event) { + var id_list = []; + $('.check-all-slave:checked').each(function (index, value) { + //Split the name (ie:"pk_200") and extract only the ID + id_list.push(value.name.split('_')[1]); + }); + event.preventDefault(); + partialNavigation.setLocation( + $(this).attr('href') + '?id_list=' + id_list.join(',') + ); + }); + } + static setupNavBarState () { $('body').on('click', '.a-main-menu-accordion-link', function (event) { console.log('ad'); @@ -166,10 +172,10 @@ class MayanApp { var self = this; this.setupAJAXSpinner(); - this.setupAutoSubmit(); this.setupFormHotkeys(); this.setupFullHeightResizing(); this.setupItemsSelector(); + MayanApp.setupMultiItemActions(); this.setupNavbarCollapse(); MayanApp.setupNavBarState(); this.setupNewWindowAnchor(); @@ -177,6 +183,7 @@ class MayanApp { value.app = self; app.doRefreshAJAXMenu(value); }); + this.setupPanelSelection(); partialNavigation.initialize(); } @@ -200,14 +207,6 @@ class MayanApp { }); } - setupAutoSubmit () { - $('body').on('change', '.select-auto-submit', function () { - if ($(this).val()) { - $(this.form).trigger('submit'); - } - }); - } - setupFormHotkeys () { $('body').on('keypress', '.form-hotkey-enter', function (e) { if ((e.which && e.which == 13) || (e.keyCode && e.keyCode == 13)) { @@ -238,9 +237,22 @@ class MayanApp { app.lastChecked = null; $('body').on('click', '.check-all', function (event) { + var $this = $(this); var checked = $(event.target).prop('checked'); var $checkBoxes = $('.check-all-slave'); + if (checked === undefined) { + checked = $this.data('checked'); + checked = !checked; + $this.data('checked', checked); + + if (checked) { + $this.find('[data-fa-i2svg]').addClass($this.data('icon-checked')).removeClass($this.data('icon-unchecked')); + } else { + $this.find('[data-fa-i2svg]').addClass($this.data('icon-unchecked')).removeClass($this.data('icon-checked')); + } + } + $checkBoxes.prop('checked', checked); $checkBoxes.trigger('change'); }); @@ -286,6 +298,58 @@ class MayanApp { }); } + setupPanelSelection () { + var app = this; + + // Setup panel highlighting on check + $('body').on('change', '.check-all-slave', function (event) { + var checked = $(event.target).prop('checked'); + if (checked) { + $(this).closest('.panel-item').addClass('panel-highlighted'); + } else { + $(this).closest('.panel-item').removeClass('panel-highlighted'); + } + }); + + $('body').on('click', '.panel-item', function (event) { + var $this = $(this); + var targetSrc = $(event.target).prop('src'); + var targetHref = $(event.target).prop('href'); + var targetIsButton = event.target.tagName === 'BUTTON'; + var lastChecked = null; + + if ((targetSrc === undefined) && (targetHref === undefined) && (targetIsButton === false)) { + var $checkbox = $this.find('.check-all-slave'); + var checked = $checkbox.prop('checked'); + + if (checked) { + $checkbox.prop('checked', ''); + $checkbox.trigger('change'); + } else { + $checkbox.prop('checked', 'checked'); + $checkbox.trigger('change'); + } + + if(!app.lastChecked) { + app.lastChecked = $checkbox; + } + + if (event.shiftKey) { + var $checkBoxes = $('.check-all-slave'); + + var start = $checkBoxes.index($checkbox); + var end = $checkBoxes.index(app.lastChecked); + + $checkBoxes.slice( + Math.min(start, end), Math.max(start, end) + 1 + ).prop('checked', app.lastChecked.prop('checked')).trigger('change'); + } + app.lastChecked = $checkbox; + window.getSelection().removeAllRanges(); + } + }); + } + setupScrollView () { $('.scrollable').scrollview(); } diff --git a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html index e0c4166cefb..63ddc71018d 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html @@ -23,26 +23,20 @@

{% endif %}

+
+ + {% if object_list %} + {% if not hide_multi_item_actions %} + {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %} + {% endif %} + {% endif %} +
-
-
- {% if object_list %} - {% if not hide_multi_item_actions %} - {% get_multi_item_links_form object_list %} - {% endif %} - {% if multi_item_actions %} -
-   - {{ multi_item_form }} -
- {% endif %} - {% endif %} -
-
+ {% include 'appearance/list_toolbar.html' %}
- {% if object_list %} + {% if links_multi_menus_results %}
{% endif %} @@ -54,8 +48,8 @@

- {% if not hide_columns %} {% navigation_get_source_columns source=object exclude_identifier=True as source_columns %} {% for column in source_columns %} diff --git a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html index d0aa3d6b88a..e3e03e4d5a7 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html @@ -25,30 +25,27 @@

+ {% if object_list %} + {% if not hide_multi_item_actions %} + {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %} + {% endif %} + {% endif %} +
-
-
- {% if object_list %} - {% if not hide_multi_item_actions %} - {% get_multi_item_links_form object_list %} - {% endif %} - {% if multi_item_actions %} -
- {{ multi_item_form }} -
- {% endif %} - {% endif %} -
-
+ {% include 'appearance/list_toolbar.html' %}
+ {% if links_multi_menus_results %} +
+ {% endif %} +
{% if not hide_header %} - {% if multi_item_actions %} - + {% if links_multi_menus_results %} + {% endif %} {% if not hide_object %} @@ -99,9 +96,9 @@

{% for object in object_list %}

- {% if multi_item_actions %} + {% if links_multi_menus_results %} {% endif %} diff --git a/mayan/apps/appearance/templates/appearance/list_toolbar.html b/mayan/apps/appearance/templates/appearance/list_toolbar.html new file mode 100644 index 00000000000..e94eedb3f79 --- /dev/null +++ b/mayan/apps/appearance/templates/appearance/list_toolbar.html @@ -0,0 +1,49 @@ +{% load i18n %} + +{% load common_tags %} +{% load navigation_tags %} + +
+ +
+ + +{% if links_multi_menus_results %} +

{% trans 'Select items to activate bulk actions. Use Shift + click to select many.' %}

+ + +
+{% endif %} diff --git a/mayan/apps/common/urls.py b/mayan/apps/common/urls.py index 13fd7e3fac5..205c90b0669 100644 --- a/mayan/apps/common/urls.py +++ b/mayan/apps/common/urls.py @@ -10,7 +10,7 @@ AboutView, CurrentUserLocaleProfileDetailsView, CurrentUserLocaleProfileEditView, FaviconRedirectView, HomeView, LicenseView, ObjectErrorLogEntryListClearView, ObjectErrorLogEntryListView, - RootView, SetupListView, ToolsListView, multi_object_action_view + RootView, SetupListView, ToolsListView ) urlpatterns = [ @@ -18,10 +18,6 @@ url(regex=r'^home/$', view=HomeView.as_view(), name='home'), url(regex=r'^about/$', view=AboutView.as_view(), name='about_view'), url(regex=r'^license/$', view=LicenseView.as_view(), name='license_view'), - url( - regex=r'^object/multiple/action/$', view=multi_object_action_view, - name='multi_object_action_view' - ), url(regex=r'^setup/$', view=SetupListView.as_view(), name='setup_list'), url(regex=r'^tools/$', view=ToolsListView.as_view(), name='tools_list'), url( diff --git a/mayan/apps/common/views.py b/mayan/apps/common/views.py index 30f16cc0f47..b97cccf9e4b 100644 --- a/mayan/apps/common/views.py +++ b/mayan/apps/common/views.py @@ -220,67 +220,3 @@ def get_extra_context(self): 'These modules are used to do system maintenance.' ) } - - -def multi_object_action_view(request): - """ - Proxy view called first when using a multi object action, which - then redirects to the appropriate specialized view - """ - next = request.POST.get( - 'next', request.GET.get( - 'next', request.META.get( - 'HTTP_REFERER', reverse(setting_home_view.value) - ) - ) - ) - - action = request.GET.get('action', None) - id_list = ','.join( - [key[3:] for key in request.GET.keys() if key.startswith('pk_')] - ) - items_property_list = [ - (key[11:]) for key in request.GET.keys() if key.startswith('properties_') - ] - - if not action: - messages.error( - message=_('No action selected.'), request=request - ) - return HttpResponseRedirect( - redirect_to=request.META.get( - 'HTTP_REFERER', reverse(setting_home_view.value) - ) - ) - - if not id_list and not items_property_list: - messages.error( - message=_('Must select at least one item.'), - request=request - ) - return HttpResponseRedirect( - redirect_to=request.META.get( - 'HTTP_REFERER', reverse(setting_home_view.value) - ) - ) - - # Separate redirects to keep backwards compatibility with older - # functions that don't expect a properties_list parameter - if items_property_list: - return HttpResponseRedirect( - redirect_to='%s?%s' % ( - action, - urlencode( - { - 'items_property_list': dumps(items_property_list), - 'next': next - } - ) - ) - ) - else: - return HttpResponseRedirect( - redirect_to='%s?%s' % ( - action, urlencode({'id_list': id_list, 'next': next}) - ) - ) From daebf2ddccb9831d92aea22538736be3fce175df Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 01:27:44 -0400 Subject: [PATCH 055/402] Don't react on paginator current page click Signed-off-by: Roberto Rosario --- mayan/apps/appearance/templates/pagination/pagination.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/appearance/templates/pagination/pagination.html b/mayan/apps/appearance/templates/pagination/pagination.html index 9791e484ca5..829a6f958b9 100644 --- a/mayan/apps/appearance/templates/pagination/pagination.html +++ b/mayan/apps/appearance/templates/pagination/pagination.html @@ -11,7 +11,7 @@ {% if page %} {% ifequal page page_obj.number %} -
  • {{ page }}
    • +
  • {{ page }}
    • {% else %}
  • {{ page }}
    • {% endifequal %} From ff86c4c518d0ecbbc4628451c5e956c7cf709057 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 01:28:11 -0400 Subject: [PATCH 056/402] Unbind non applicable workflow runtime proxy links Signed-off-by: Roberto Rosario --- mayan/apps/document_states/apps.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index 065105db199..dcbcf3e574b 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -328,6 +328,17 @@ def ready(self): link_workflow_template_preview ), sources=(Workflow,) ) + + menu_list_facet.unbind_links( + links=( + link_acl_list, link_events_for_object, + link_object_event_types_user_subcriptions_list, + link_workflow_template_document_types, + link_workflow_template_state_list, link_workflow_template_transition_list, + link_workflow_template_preview + ), sources=(WorkflowRuntimeProxy,) + ) + menu_list_facet.bind_links( links=( link_document_type_workflow_templates, From ec4644b5c99f52d7567ef4dbab4dfeae76ffda58 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 01:28:55 -0400 Subject: [PATCH 057/402] Fix typo on open method Signed-off-by: Roberto Rosario --- mayan/apps/documents/models/document_page_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 8aa9119445f..3461d1f1527 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -191,7 +191,7 @@ def get_image(self, transformations=None): if not setting_disable_base_image_cache.value and cache_file: logger.debug('Page cache file "%s" found', cache_filename) - with cache_file.open as file_object: + with cache_file.open() as file_object: converter = get_converter_class()( file_object=file_object ) From 917ec55adabdc8c49192eed41cff227cbd3ebd8c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 16:18:36 -0400 Subject: [PATCH 058/402] Style tweaks Enable dashboard widget icon shadows. Make block button text shadow more pronounced. Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 12 ++---------- .../templates/dashboards/numeric_widget.html | 4 +++- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index 3c70bd2e8c6..e5b933d8bf4 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -98,14 +98,10 @@ hr { min-height: 120px; padding-bottom: 1px; padding-top: 20px; - text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); + text-shadow: 1px 1px 3px rgba(0, 0, 0, 1); white-space: normal; } -.btn-block .fa { - text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); -} - .radio ul li { list-style-type:none; } @@ -115,14 +111,10 @@ a i { } .dashboard-widget { - box-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); + box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.7); border: 1px solid black; } -.dashboard-widget .panel-heading i { - text-shadow: 1px 1px 1px rgba(0, 0, 0, 0.3); -} - .dashboard-widget-icon { font-size: 200%; } diff --git a/mayan/apps/dashboards/templates/dashboards/numeric_widget.html b/mayan/apps/dashboards/templates/dashboards/numeric_widget.html index 9a41cce3012..4aaf8481dda 100644 --- a/mayan/apps/dashboards/templates/dashboards/numeric_widget.html +++ b/mayan/apps/dashboards/templates/dashboards/numeric_widget.html @@ -1,5 +1,7 @@ {% load i18n %} +{% load appearance_tags %} +
    @@ -9,7 +11,7 @@ {% elif icon_class %}
    - {{ icon_class.render }} + {% appearance_icon_render icon_class enable_shadow=True %}
    {% endif %}
    From fd08a23339e9735bf2f0d8e4aba27cd3666cecc5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 16 Jul 2019 16:21:24 -0400 Subject: [PATCH 059/402] Soften top bar shadow Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index e5b933d8bf4..e66fc88c0fe 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -540,5 +540,5 @@ a i { } .navbar-fixed-top { - box-shadow: 0px 3px 5px rgba(0, 0, 0, 0.5); + box-shadow: 0px 3px 3px rgba(0, 0, 0, 0.4); } From b99bb88008ef9c7b1091a2c41a546cce807ed306 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 00:47:28 -0400 Subject: [PATCH 060/402] Update OCR manager to use document cache Signed-off-by: Roberto Rosario --- mayan/apps/ocr/managers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index d03b0dc0712..f6a0b689789 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -8,7 +8,6 @@ from django.conf import settings from django.db import models -from mayan.apps.documents.storages import storage_documentimagecache from mayan.apps.documents.literals import DOCUMENT_IMAGE_TASK_TIMEOUT from mayan.apps.documents.tasks import task_generate_document_page_image @@ -38,7 +37,7 @@ def process_document_page(self, document_page): cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) - with storage_documentimagecache.open(cache_filename) as file_object: + with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( document_page=document_page ) From 649571ebb13be389ef87404ef7ddffacf4a71eb8 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 00:48:22 -0400 Subject: [PATCH 061/402] Add kwargs Signed-off-by: Roberto Rosario --- mayan/apps/ocr/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/ocr/tasks.py b/mayan/apps/ocr/tasks.py index 1ef2ffedf1e..a6d9c82efb4 100644 --- a/mayan/apps/ocr/tasks.py +++ b/mayan/apps/ocr/tasks.py @@ -28,7 +28,7 @@ def task_do_ocr(self, document_version_pk): logger.debug('trying to acquire lock: %s', lock_id) # Acquire lock to avoid doing OCR on the same document version more # than once concurrently - lock = locking_backend.acquire_lock(lock_id, LOCK_EXPIRE) + lock = locking_backend.acquire_lock(name=lock_id, timeout=LOCK_EXPIRE) logger.debug('acquired lock: %s', lock_id) document_version = None try: From 030ddd68e006e74104c813c270d3b12f4721b48e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 01:13:00 -0400 Subject: [PATCH 062/402] PEP8 cleanups Signed-off-by: Roberto Rosario --- mayan/apps/common/views.py | 6 +-- mayan/apps/converter/classes.py | 1 - mayan/apps/document_states/views/__init__.py | 1 - .../views/workflow_template_state_views.py | 39 ++++--------------- .../workflow_template_transition_views.py | 37 +++--------------- .../views/workflow_template_views.py | 37 ++++-------------- mayan/apps/navigation/classes.py | 4 -- mayan/apps/platform/classes.py | 4 +- 8 files changed, 24 insertions(+), 105 deletions(-) diff --git a/mayan/apps/common/views.py b/mayan/apps/common/views.py index b97cccf9e4b..b78a0483c89 100644 --- a/mayan/apps/common/views.py +++ b/mayan/apps/common/views.py @@ -1,15 +1,11 @@ from __future__ import absolute_import, unicode_literals -from json import dumps - from django.conf import settings from django.contrib import messages from django.contrib.contenttypes.models import ContentType -from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.urls import reverse, reverse_lazy +from django.urls import reverse_lazy from django.utils import timezone, translation -from django.utils.http import urlencode from django.utils.translation import ugettext_lazy as _ from django.views.generic import RedirectView diff --git a/mayan/apps/converter/classes.py b/mayan/apps/converter/classes.py index cd6a99ce2bf..30c94d663bd 100644 --- a/mayan/apps/converter/classes.py +++ b/mayan/apps/converter/classes.py @@ -10,7 +10,6 @@ from django.utils.translation import ugettext_lazy as _ -from mayan.apps.common.serialization import yaml_load from mayan.apps.mimetype.api import get_mimetype from mayan.apps.storage.settings import setting_temporary_directory from mayan.apps.storage.utils import ( diff --git a/mayan/apps/document_states/views/__init__.py b/mayan/apps/document_states/views/__init__.py index 8b137891791..e69de29bb2d 100644 --- a/mayan/apps/document_states/views/__init__.py +++ b/mayan/apps/document_states/views/__init__.py @@ -1 +0,0 @@ - diff --git a/mayan/apps/document_states/views/workflow_template_state_views.py b/mayan/apps/document_states/views/workflow_template_state_views.py index a0f1e3cec7b..ab846c15b2c 100644 --- a/mayan/apps/document_states/views/workflow_template_state_views.py +++ b/mayan/apps/document_states/views/workflow_template_state_views.py @@ -1,53 +1,30 @@ from __future__ import absolute_import, unicode_literals -from django.contrib import messages -from django.db import transaction from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.template import RequestContext -from django.urls import reverse, reverse_lazy +from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.generics import ( - AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, - SingleObjectDeleteView, SingleObjectDetailView, + FormView, SingleObjectCreateView, SingleObjectDeleteView, SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, SingleObjectEditView, SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin -from mayan.apps.documents.events import event_document_type_edited -from mayan.apps.documents.models import DocumentType -from mayan.apps.documents.permissions import permission_document_type_edit -from mayan.apps.events.classes import EventType -from mayan.apps.events.models import StoredEventType from ..classes import WorkflowAction -from ..events import event_workflow_edited from ..forms import ( - WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, - WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, - WorkflowTransitionTriggerEventRelationshipFormSet -) -from ..icons import ( - icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition, icon_workflow_transition_field + WorkflowActionSelectionForm, WorkflowStateActionDynamicForm, + WorkflowStateForm ) +from ..icons import icon_workflow_state, icon_workflow_state_action from ..links import ( - link_workflow_template_create, link_workflow_template_state_create, + link_workflow_template_state_create, link_workflow_template_state_action_selection, - link_workflow_template_transition_create, - link_workflow_template_transition_field_create, -) -from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, - WorkflowTransitionField -) -from ..permissions import ( - permission_workflow_create, permission_workflow_delete, - permission_workflow_edit, permission_workflow_tools, - permission_workflow_view, ) -from ..tasks import task_launch_all_workflows +from ..models import Workflow, WorkflowState, WorkflowStateAction +from ..permissions import permission_workflow_edit, permission_workflow_view class WorkflowTemplateStateActionCreateView(SingleObjectDynamicFormCreateView): diff --git a/mayan/apps/document_states/views/workflow_template_transition_views.py b/mayan/apps/document_states/views/workflow_template_transition_views.py index e56a10a6259..f33444751e2 100644 --- a/mayan/apps/document_states/views/workflow_template_transition_views.py +++ b/mayan/apps/document_states/views/workflow_template_transition_views.py @@ -1,53 +1,28 @@ from __future__ import absolute_import, unicode_literals from django.contrib import messages -from django.db import transaction -from django.http import Http404, HttpResponseRedirect -from django.shortcuts import get_object_or_404 from django.template import RequestContext -from django.urls import reverse, reverse_lazy +from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.generics import ( - AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, - SingleObjectDeleteView, SingleObjectDetailView, - SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, + FormView, SingleObjectCreateView, SingleObjectDeleteView, SingleObjectEditView, SingleObjectListView ) from mayan.apps.common.mixins import ExternalObjectMixin -from mayan.apps.documents.events import event_document_type_edited -from mayan.apps.documents.models import DocumentType -from mayan.apps.documents.permissions import permission_document_type_edit from mayan.apps.events.classes import EventType from mayan.apps.events.models import StoredEventType -from ..classes import WorkflowAction -from ..events import event_workflow_edited from ..forms import ( - WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, - WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, - WorkflowTransitionTriggerEventRelationshipFormSet -) -from ..icons import ( - icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition, icon_workflow_transition_field + WorkflowTransitionForm, WorkflowTransitionTriggerEventRelationshipFormSet ) +from ..icons import icon_workflow_transition, icon_workflow_transition_field from ..links import ( - link_workflow_template_create, link_workflow_template_state_create, - link_workflow_template_state_action_selection, link_workflow_template_transition_create, link_workflow_template_transition_field_create, ) -from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, - WorkflowTransitionField -) -from ..permissions import ( - permission_workflow_create, permission_workflow_delete, - permission_workflow_edit, permission_workflow_tools, - permission_workflow_view, -) -from ..tasks import task_launch_all_workflows +from ..models import Workflow, WorkflowTransition, WorkflowTransitionField +from ..permissions import permission_workflow_edit, permission_workflow_view class WorkflowTemplateTransitionCreateView(ExternalObjectMixin, SingleObjectCreateView): diff --git a/mayan/apps/document_states/views/workflow_template_views.py b/mayan/apps/document_states/views/workflow_template_views.py index e755f42e339..71f61245d99 100644 --- a/mayan/apps/document_states/views/workflow_template_views.py +++ b/mayan/apps/document_states/views/workflow_template_views.py @@ -2,46 +2,23 @@ from django.contrib import messages from django.db import transaction -from django.http import Http404, HttpResponseRedirect -from django.shortcuts import get_object_or_404 from django.template import RequestContext -from django.urls import reverse, reverse_lazy +from django.urls import reverse_lazy from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.generics import ( - AddRemoveView, ConfirmView, FormView, SingleObjectCreateView, - SingleObjectDeleteView, SingleObjectDetailView, - SingleObjectDynamicFormCreateView, SingleObjectDynamicFormEditView, - SingleObjectEditView, SingleObjectListView + AddRemoveView, ConfirmView, SingleObjectCreateView, SingleObjectDeleteView, + SingleObjectDetailView, SingleObjectEditView, SingleObjectListView ) -from mayan.apps.common.mixins import ExternalObjectMixin from mayan.apps.documents.events import event_document_type_edited from mayan.apps.documents.models import DocumentType from mayan.apps.documents.permissions import permission_document_type_edit -from mayan.apps.events.classes import EventType -from mayan.apps.events.models import StoredEventType -from ..classes import WorkflowAction from ..events import event_workflow_edited -from ..forms import ( - WorkflowActionSelectionForm, WorkflowForm, WorkflowPreviewForm, - WorkflowStateActionDynamicForm, WorkflowStateForm, WorkflowTransitionForm, - WorkflowTransitionTriggerEventRelationshipFormSet -) -from ..icons import ( - icon_workflow_template_list, icon_workflow_state, icon_workflow_state_action, - icon_workflow_transition, icon_workflow_transition_field -) -from ..links import ( - link_workflow_template_create, link_workflow_template_state_create, - link_workflow_template_state_action_selection, - link_workflow_template_transition_create, - link_workflow_template_transition_field_create, -) -from ..models import ( - Workflow, WorkflowState, WorkflowStateAction, WorkflowTransition, - WorkflowTransitionField -) +from ..forms import WorkflowForm, WorkflowPreviewForm +from ..icons import icon_workflow_template_list +from ..links import link_workflow_template_create +from ..models import Workflow from ..permissions import ( permission_workflow_create, permission_workflow_delete, permission_workflow_edit, permission_workflow_tools, diff --git a/mayan/apps/navigation/classes.py b/mayan/apps/navigation/classes.py index 73bb0488c8a..78062aaf051 100644 --- a/mayan/apps/navigation/classes.py +++ b/mayan/apps/navigation/classes.py @@ -369,9 +369,6 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): for resolved_navigation_object in resolved_navigation_object_list: resolved_links = [] - # List of resolved links source links used for deduplication - resolved_links_links = [] - for bound_source, links in self.bound_links.items(): try: if inspect.isclass(bound_source): @@ -434,7 +431,6 @@ def resolve(self, context=None, request=None, source=None, sort_results=False): } ) - resolved_links = [] # View links for link in self.bound_links.get(current_view_name, []): diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index f7160298063..db3ab9c7866 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -154,8 +154,8 @@ class PlatformTemplateSupervisord(PlatformTemplate): default={ 'default': { 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'mayan', 'PASSWORD':'mayanuserpass', - 'USER': 'mayan', 'HOST':'127.0.0.1' + 'NAME': 'mayan', 'PASSWORD': 'mayanuserpass', + 'USER': 'mayan', 'HOST': '127.0.0.1' } }, environment_name='MAYAN_DATABASES' From 0b42567179efdaafd6365f53f5a56eaad05dedf2 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 02:41:00 -0400 Subject: [PATCH 063/402] Remove direct Celery queue update Queue updates are handled by the task manager app. Signed-off-by: Roberto Rosario --- mayan/apps/file_metadata/apps.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/mayan/apps/file_metadata/apps.py b/mayan/apps/file_metadata/apps.py index 10796b6cf5a..106780adbaf 100644 --- a/mayan/apps/file_metadata/apps.py +++ b/mayan/apps/file_metadata/apps.py @@ -18,7 +18,6 @@ from mayan.apps.documents.signals import post_version_upload from mayan.apps.events.classes import ModelEventType from mayan.apps.navigation.classes import SourceColumn -from mayan.celery import app from .classes import FileMetadataHelper from .dependencies import * # NOQA @@ -150,21 +149,6 @@ def ready(self): attribute='get_attribute_count', source=DocumentVersionDriverEntry ) - app.conf.CELERY_QUEUES.append( - Queue( - 'file_metadata', Exchange('file_metadata'), - routing_key='file_metadata' - ), - ) - - app.conf.CELERY_ROUTES.update( - { - 'mayan.apps.file_metadata.tasks.task_process_document_version': { - 'queue': 'file_metadata' - }, - } - ) - document_search.add_model_field( field='versions__file_metadata_drivers__entries__key', label=_('File metadata key') From ab601f918000b17b14ba11ef522a79d02bbe169d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 04:30:11 -0400 Subject: [PATCH 064/402] Initial commit to support Celery 4.3.0 Merges 55e9b2263cbdb9b449361412fd18d8ee0a442dd3 from versions/next with code from GitLab issue #594 and GitLab merge request !55. Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling) for much of the research and code updates. Signed-off-by: Roberto Rosario --- Makefile | 2 +- docker/Dockerfile | 26 ++++----- docker/rootfs/usr/local/bin/entrypoint.sh | 4 +- docs/chapters/deploying.rst | 4 +- docs/chapters/docker.rst | 6 +- docs/chapters/scaling_up.rst | 4 +- docs/releases/3.3.rst | 4 +- mayan/apps/common/dependencies.py | 56 ++++++++++++++----- mayan/apps/mayan_statistics/classes.py | 4 +- mayan/apps/platform/classes.py | 4 +- .../templates/platform/supervisord.tmpl | 8 +-- .../platform/supervisord_docker.tmpl | 10 +++- mayan/apps/smart_settings/utils.py | 6 +- mayan/apps/sources/models/base.py | 2 +- mayan/apps/task_manager/classes.py | 8 +-- mayan/apps/task_manager/settings.py | 2 +- mayan/celery.py | 3 +- mayan/settings/base.py | 27 ++++----- mayan/settings/development.py | 9 ++- mayan/settings/production.py | 2 +- mayan/settings/staging/docker.py | 2 +- mayan/settings/testing/base.py | 4 +- removals.txt | 1 + 23 files changed, 119 insertions(+), 79 deletions(-) diff --git a/Makefile b/Makefile index bffe667715b..5723b6631c3 100644 --- a/Makefile +++ b/Makefile @@ -258,7 +258,7 @@ test-with-docker-frontend: ## Launch a front end instance that uses the producti ./manage.py runserver --settings=mayan.settings.staging.docker test-with-docker-worker: ## Launch a worker instance that uses the production-like services. - ./manage.py celery worker --settings=mayan.settings.staging.docker -B -l INFO -O fair + DJANGO_SETTINGS_MODULE=mayan.settings.staging.docker ./manage.py celery worker -A mayan -B -l INFO -O fair docker-mysql-on: ## Launch and initialize a MySQL Docker container. docker run -d --name mysql -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=True -e MYSQL_DATABASE=mayan_edms mysql diff --git a/docker/Dockerfile b/docker/Dockerfile index 2f364989998..32eea72b812 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ # BASE_IMAGE - Bare bones image with the base packages needed to run Mayan EDMS #### -FROM debian:9.8-slim as BASE_IMAGE +FROM debian:10.0-slim as BASE_IMAGE LABEL maintainer="Roberto Rosario roberto.rosario@mayan-edms.com" @@ -29,7 +29,7 @@ apt-get update \ graphviz \ libfuse2 \ libmagic1 \ - libmariadbclient18 \ + libmmariadb3 \ libreoffice \ libpq5 \ poppler-utils \ @@ -96,31 +96,31 @@ apt-get install -y --no-install-recommends \ libssl-dev \ g++ \ gcc \ - python-dev \ - python-virtualenv \ + python3-dev \ + python3-venv \ && mkdir -p "${PROJECT_INSTALL_DIR}" \ && chown -R mayan:mayan "${PROJECT_INSTALL_DIR}" \ && chown -R mayan:mayan /src USER mayan -RUN python -m virtualenv "${PROJECT_INSTALL_DIR}" \ +RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \ && . "${PROJECT_INSTALL_DIR}/bin/activate" \ -&& pip install --no-cache-dir --no-use-pep517 \ - librabbitmq==1.6.1 \ - mysql-python==1.2.5 \ - psycopg2==2.7.3.2 \ - redis==2.10.6 \ +&& pip install --no-cache-dir \ + librabbitmq==2.0.0 \ + mysqlclient==1.4.2.post1 \ + psycopg2==2.8.3 \ + redis==3.2.1 \ # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \ - pip install --no-cache-dir --no-use-pep517 \ + pip install --no-cache-dir \ psutil==5.6.2 \ ; fi \ # Install the Python packages needed to build Mayan EDMS -&& pip install --no-cache-dir --no-use-pep517 -r /src/requirements/build.txt \ +&& pip install --no-cache-dir -r /src/requirements/build.txt \ # Build Mayan EDMS && python setup.py sdist \ # Install the built Mayan EDMS package -&& pip install --no-cache-dir --no-use-pep517 dist/mayan* \ +&& pip install --no-cache-dir dist/mayan* \ # Install the static content && mayan-edms.py installdependencies \ && MAYAN_STATIC_ROOT=${PROJECT_INSTALL_DIR}/static mayan-edms.py preparestatic --link --noinput diff --git a/docker/rootfs/usr/local/bin/entrypoint.sh b/docker/rootfs/usr/local/bin/entrypoint.sh index 733bd5c7635..2da0f768b71 100755 --- a/docker/rootfs/usr/local/bin/entrypoint.sh +++ b/docker/rootfs/usr/local/bin/entrypoint.sh @@ -8,12 +8,12 @@ CONCURRENCY_ARGUMENT=--concurrency= DEFAULT_USER_UID=1000 DEFAULT_USER_GUID=1000 -export MAYAN_DEFAULT_BROKER_URL=redis://127.0.0.1:6379/0 +export MAYAN_DEFAULT_CELERY_BROKER_URL=redis://127.0.0.1:6379/0 export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 export MAYAN_ALLOWED_HOSTS='["*"]' export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py -export MAYAN_BROKER_URL=${MAYAN_BROKER_URL:-${MAYAN_DEFAULT_BROKER_URL}} +export MAYAN_CELERY_BROKER_URL=${MAYAN_CELERY_BROKER_URL:-${MAYAN_DEFAULT_CELERY_BROKER_URL}} export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}} export MAYAN_INSTALL_DIR=/opt/mayan-edms export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/ diff --git a/docs/chapters/deploying.rst b/docs/chapters/deploying.rst index d3016b98a32..7bbc889a176 100644 --- a/docs/chapters/deploying.rst +++ b/docs/chapters/deploying.rst @@ -220,11 +220,11 @@ of a restart or power failure. The Gunicorn workers are increased to 3. --------------------------------------------------------------------- Replace (paying attention to the comma at the end):: - MAYAN_BROKER_URL="redis://127.0.0.1:6379/0", + MAYAN_CELERY_BROKER_URL="redis://127.0.0.1:6379/0", with:: - MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", + MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", increase the number of Gunicorn workers to 3 in the line (``-w 2`` section):: diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index c99c7249a67..4910f49e563 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -190,7 +190,7 @@ the default port. Not used with SQLite. For more information read the pertinent Django documentation page: :django-docs:`Settings, PORT ` -``MAYAN_BROKER_URL`` +``MAYAN_CELERY_BROKER_URL`` This optional environment variable determines the broker that Celery will use to relay task messages between the frontend code and the background workers. @@ -200,7 +200,7 @@ For more information read the pertinent Celery Kombu documentation page: `Broker This Docker image supports using Redis and RabbitMQ as brokers. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. @@ -215,7 +215,7 @@ code. For more information read the pertinent Celery Kombu documentation page: This Docker image supports using Redis and RabbitMQ as result backends. -Caveat: If the `MAYAN_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment +Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment variables are specified, the built-in Redis server inside the container will be disabled. diff --git a/docs/chapters/scaling_up.rst b/docs/chapters/scaling_up.rst index 01a59cecb3f..de03468b830 100644 --- a/docs/chapters/scaling_up.rst +++ b/docs/chapters/scaling_up.rst @@ -94,11 +94,11 @@ For the Docker image, launch a separate RabbitMQ container docker run -d --name mayan-edms-rabbitmq -e RABBITMQ_DEFAULT_USER=mayan -e RABBITMQ_DEFAULT_PASS=mayanrabbitmqpassword -e RABBITMQ_DEFAULT_VHOST=mayan rabbitmq:3 -Pass the MAYAN_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) +Pass the MAYAN_CELERY_BROKER_URL environment variable (https://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls) to the Mayan EDMS container so that it uses the RabbitMQ container the message broker:: - -e MAYAN_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", + -e MAYAN_CELERY_BROKER_URL="amqp://mayan:mayanrabbitmqpassword@localhost:5672/mayan", When tasks finish, they leave behind a return status or the result of a calculation, these are stored for a while so that whoever requested the diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index d49868c4683..b71a273c841 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -52,7 +52,9 @@ Changes - Backport file cache manager app. - Convert document image cache to use file cache manager app. Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. - +- Update Celery to version 4.3.0. Settings changed: + MAYAN_BROKER_URL to MAYAN_CELERY_BROKER_URL, + MAYAN_CELERY_ALWAYS_EAGER to MAYAN_CELERY_TASK_ALWAYS_EAGER. Removals -------- diff --git a/mayan/apps/common/dependencies.py b/mayan/apps/common/dependencies.py index c4e064da4ef..26aa3fd351c 100644 --- a/mayan/apps/common/dependencies.py +++ b/mayan/apps/common/dependencies.py @@ -117,31 +117,39 @@ Celery under the GPL license. The BSD license, unlike the GPL, let you distribute a modified version without making your changes open source. - ''', module=__name__, name='celery', version_string='==3.1.24' + ''', module=__name__, name='celery', version_string='==4.3.0' ) PythonDependency( copyright_text=''' - Copyright (c) 2012-2013 GoPivotal, Inc. All Rights Reserved. + Copyright (c) 2015-2016 Ask Solem. All Rights Reserved. + Copyright (c) 2012-2014 GoPivotal, Inc. All Rights Reserved. Copyright (c) 2009-2012 Ask Solem. All Rights Reserved. - All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: + django-celery-beat is licensed under The BSD License (3 Clause, also known as + the new BSD license). The license is an OSI approved Open Source + license and is GPL-compatible(1). - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. + The license text can also be found here: + http://www.opensource.org/licenses/BSD-3-Clause + + License + ======= - Neither the name of Ask Solem nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Ask Solem nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -149,7 +157,25 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ''', module=__name__, name='django-celery', version_string='==3.2.1' + + Documentation License + ===================== + + The documentation portion of django-celery-beat (the rendered contents of the + "docs" directory of a software distribution or checkout) is supplied + under the "Creative Commons Attribution-ShareAlike 4.0 + International" (CC BY-SA 4.0) License as described by + http://creativecommons.org/licenses/by-sa/4.0/ + + Footnotes + ========= + (1) A GPL-compatible license makes it possible to + combine django-celery-beat with other software that is released + under the GPL, it does not mean that we're distributing + django-celery-beat under the GPL license. The BSD license, unlike the GPL, + let you distribute a modified version without making your + changes open source. + ''', module=__name__, name='django-celery-beat', version_string='==1.5.0' ) PythonDependency( module=__name__, name='django-downloadview', version_string='==1.9' diff --git a/mayan/apps/mayan_statistics/classes.py b/mayan/apps/mayan_statistics/classes.py index e4296d095f9..7e5be5a6cfc 100644 --- a/mayan/apps/mayan_statistics/classes.py +++ b/mayan/apps/mayan_statistics/classes.py @@ -107,7 +107,7 @@ def __init__(self, slug, label, func, minute='*', hour='*', day_of_week='*', day day_of_month=day_of_month, month_of_year=month_of_year, ) - app.conf.CELERYBEAT_SCHEDULE.update( + app.conf.beat_schedule.update( { self.get_task_name(): { 'task': task_execute_statistic.dotted_path, @@ -117,7 +117,7 @@ def __init__(self, slug, label, func, minute='*', hour='*', day_of_week='*', day } ) - app.conf.CELERY_ROUTES.update( + app.conf.task_routes.update( { self.get_task_name(): { 'queue': queue_statistics.name diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index db3ab9c7866..723243a904d 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -140,9 +140,9 @@ class PlatformTemplateSupervisord(PlatformTemplate): environment_name='MAYAN_ALLOWED_HOSTS' ), YAMLVariable( - name='BROKER_URL', + name='CELERY_BROKER_URL', default='redis://127.0.0.1:6379/0', - environment_name='MAYAN_BROKER_URL' + environment_name='MAYAN_CELERY_BROKER_URL' ), YAMLVariable( name='CELERY_RESULT_BACKEND', diff --git a/mayan/apps/platform/templates/platform/supervisord.tmpl b/mayan/apps/platform/templates/platform/supervisord.tmpl index f6623d12deb..513a03581a5 100644 --- a/mayan/apps/platform/templates/platform/supervisord.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord.tmpl @@ -1,11 +1,11 @@ [supervisord] environment= - PYTHONPATH={{ INSTALLATION_PATH }}/lib/python2.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, + PYTHONPATH={{ INSTALLATION_PATH }}/lib/python3.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, DJANGO_SETTINGS_MODULE=mayan.settings.production, MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}", MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}", MAYAN_CELERY_RESULT_BACKEND="{{ CELERY_RESULT_BACKEND }}", - MAYAN_BROKER_URL="{{ BROKER_URL }}", + MAYAN_CELERY_BROKER_URL="{{ CELERY_BROKER_URL }}", MAYAN_DATABASES="{{ DATABASES }}" [program:mayan-gunicorn] @@ -18,7 +18,7 @@ user = mayan [program:mayan-worker-{{ worker.name }}] autorestart = true autostart = true -command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1 +command = nice -n {{ worker.nice_level }} {{ INSTALLATION_PATH }}/bin/celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h --concurrency=1 killasgroup = true numprocs = 1 priority = 998 @@ -30,7 +30,7 @@ user = mayan [program:mayan-celery-beat] autorestart = true autostart = true -command = nice -n 1 {{ INSTALLATION_PATH }}/bin/mayan-edms.py celery beat --pidfile= -l ERROR +command = nice -n 1 {{ INSTALLATION_PATH }}/bin/celery beat -A mayan --pidfile= -l ERROR killasgroup = true numprocs = 1 priority = 998 diff --git a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl index 7cad0e0c3b9..3448c3b6185 100644 --- a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl @@ -1,3 +1,7 @@ +[supervisord] +environment= + DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s" + [program:mayan-gunicorn] autorestart = false autostart = true @@ -12,7 +16,7 @@ user = mayan [program:redis] autorestart = false autostart = true -command = /bin/bash -c "if [ ${MAYAN_BROKER_URL} == ${MAYAN_DEFAULT_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" +command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" stderr_logfile = /dev/fd/2 stderr_logfile_maxbytes = 0 stdout_logfile = /dev/fd/1 @@ -23,7 +27,7 @@ user = root [program:mayan-worker-{{ worker.name }}] autorestart = false autostart = true -command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} worker -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}" +command = nice -n {{ worker.nice_level }} /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery worker -A mayan -Ofair -l ERROR -Q {% for queue in worker.queues %}{{ queue.name }}{% if not forloop.last %},{% endif %}{% endfor %} -n mayan-worker-{{ worker.name }}.%%h ${MAYAN_WORKER_{{ worker.name|upper }}_CONCURRENCY}" killasgroup = true numprocs = 1 priority = 998 @@ -39,7 +43,7 @@ user = mayan [program:mayan-celery-beat] autorestart = false autostart = true -command = nice -n 1 /bin/bash -c "${MAYAN_BIN} celery --settings=${MAYAN_SETTINGS_MODULE} beat --pidfile= -l ERROR" +command = nice -n 1 /bin/bash -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan beat --pidfile= -l ERROR" killasgroup = true numprocs = 1 priority = 998 diff --git a/mayan/apps/smart_settings/utils.py b/mayan/apps/smart_settings/utils.py index 77ce325e0c2..9d9d6dbc916 100644 --- a/mayan/apps/smart_settings/utils.py +++ b/mayan/apps/smart_settings/utils.py @@ -34,7 +34,11 @@ def get_environment_variables(): def get_environment_setting(name, fallback_default=None): - value = os.environ.get('MAYAN_{}'.format(name), get_default(name=name, fallback_default=fallback_default)) + value = os.environ.get( + 'MAYAN_{}'.format(name), get_default( + name=name, fallback_default=fallback_default + ) + ) if value: return yaml.load(stream=value, Loader=SafeLoader) diff --git a/mayan/apps/sources/models/base.py b/mayan/apps/sources/models/base.py index bcca6082567..9c18d9edbc1 100644 --- a/mayan/apps/sources/models/base.py +++ b/mayan/apps/sources/models/base.py @@ -7,7 +7,7 @@ from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ -from djcelery.models import PeriodicTask, IntervalSchedule +from django_celery_beat.models import PeriodicTask, IntervalSchedule from model_utils.managers import InheritanceManager from mayan.apps.common.compressed_files import Archive diff --git a/mayan/apps/task_manager/classes.py b/mayan/apps/task_manager/classes.py index 90770f91528..b075acfb914 100644 --- a/mayan/apps/task_manager/classes.py +++ b/mayan/apps/task_manager/classes.py @@ -156,13 +156,13 @@ def _update_celery(self): if self.transient: kwargs['delivery_mode'] = 1 - celery_app.conf.CELERY_QUEUES.append(Queue(**kwargs)) + celery_app.conf.task_queues.append(Queue(**kwargs)) if self.default_queue: - celery_app.conf.CELERY_DEFAULT_QUEUE = self.name + celery_app.conf.task_default_queue = self.name for task_type in self.task_types: - celery_app.conf.CELERY_ROUTES.update( + celery_app.conf.task_routes.update( { task_type.dotted_path: { 'queue': self.name @@ -171,7 +171,7 @@ def _update_celery(self): ) if task_type.schedule: - celery_app.conf.CELERYBEAT_SCHEDULE.update( + celery_app.conf.beat_schedule.update( { task_type.name: { 'task': task_type.dotted_path, diff --git a/mayan/apps/task_manager/settings.py b/mayan/apps/task_manager/settings.py index 9074439afe4..bcf273262f6 100644 --- a/mayan/apps/task_manager/settings.py +++ b/mayan/apps/task_manager/settings.py @@ -10,7 +10,7 @@ namespace = Namespace(label=_('Celery'), name='celery') setting_celery_broker_url = namespace.add_setting( - global_name='BROKER_URL', default=None, + global_name='CELERY_BROKER_URL', default=None, help_text=_( 'Default: "amqp://". Default broker URL. This must be a URL in ' 'the form of: transport://userid:password@hostname:port/virtual_host ' diff --git a/mayan/celery.py b/mayan/celery.py index ca8fc9cdadc..a9265d05680 100644 --- a/mayan/celery.py +++ b/mayan/celery.py @@ -9,6 +9,5 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings.production') app = celery_class('mayan') - -app.config_from_object('django.conf:settings') +app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index c18ae759513..a743c751a4e 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -74,7 +74,7 @@ 'actstream', 'colorful', 'corsheaders', - 'djcelery', + 'django_celery_beat', 'formtools', 'mathfilters', 'mptt', @@ -280,18 +280,25 @@ # ----------- Celery ---------- +CELERY_BROKER_URL = get_environment_setting(name='CELERY_BROKER_URL') +CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND') +CELERY_TASK_ALWAYS_EAGER = get_environment_setting( + name='CELERY_TASK_ALWAYS_EAGER' +) + CELERY_ACCEPT_CONTENT = ('json',) -CELERY_ALWAYS_EAGER = False -CELERY_CREATE_MISSING_QUEUES = False +CELERY_BEAT_SCHEDULE = {} +CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers.DatabaseScheduler' CELERY_DISABLE_RATE_LIMITS = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = True CELERY_ENABLE_UTC = True -CELERY_QUEUES = [] CELERY_RESULT_SERIALIZER = 'json' -CELERY_ROUTES = {} +CELERY_TASK_ALWAYS_EAGER = False +CELERY_TASK_CREATE_MISSING_QUEUES = False +CELERY_TASK_EAGER_PROPAGATES = True +CELERY_TASK_QUEUES = [] +CELERY_TASK_ROUTES = {} CELERY_TASK_SERIALIZER = 'json' CELERY_TIMEZONE = 'UTC' -CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' # ------------ CORS ------------ @@ -318,12 +325,6 @@ AJAX_REDIRECT_CODE = 278 -# ----- Celery ----- - -BROKER_URL = get_environment_setting(name='BROKER_URL') -CELERY_ALWAYS_EAGER = get_environment_setting(name='CELERY_ALWAYS_EAGER') -CELERY_RESULT_BACKEND = get_environment_setting(name='CELERY_RESULT_BACKEND') - # ----- Database ----- DATABASES = { 'default': { diff --git a/mayan/settings/development.py b/mayan/settings/development.py index 77961253a91..8668627db31 100644 --- a/mayan/settings/development.py +++ b/mayan/settings/development.py @@ -1,17 +1,20 @@ from __future__ import absolute_import, unicode_literals +from mayan.apps.smart_settings.utils import get_environment_setting + from . import * # NOQA ALLOWED_HOSTS = ['*'] DEBUG = True -CELERY_ALWAYS_EAGER = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = CELERY_ALWAYS_EAGER +CELERY_TASK_ALWAYS_EAGER = get_environment_setting( + name='CELERY_TASK_ALWAYS_EAGER', fallback_default='true' +) +CELERY_TASK_EAGER_PROPAGATES = CELERY_TASK_ALWAYS_EAGER EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' - if 'rosetta' not in INSTALLED_APPS: try: import rosetta diff --git a/mayan/settings/production.py b/mayan/settings/production.py index f5cc3f6af57..fe7ba1a037b 100644 --- a/mayan/settings/production.py +++ b/mayan/settings/production.py @@ -2,7 +2,7 @@ from . import * # NOQA -CELERY_ALWAYS_EAGER = False +CELERY_TASK_ALWAYS_EAGER = False TEMPLATES[0]['OPTIONS']['loaders'] = ( ( diff --git a/mayan/settings/staging/docker.py b/mayan/settings/staging/docker.py index 464daabcf2f..640a1b2a966 100644 --- a/mayan/settings/staging/docker.py +++ b/mayan/settings/staging/docker.py @@ -12,6 +12,6 @@ } } -BROKER_URL = 'redis://127.0.0.1:6379/0' +CELERY_BROKER_URL = 'redis://127.0.0.1:6379/0' CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/0' DEBUG = True diff --git a/mayan/settings/testing/base.py b/mayan/settings/testing/base.py index d576514d98b..4eb37026c17 100644 --- a/mayan/settings/testing/base.py +++ b/mayan/settings/testing/base.py @@ -2,8 +2,8 @@ from .. import * # NOQA -CELERY_ALWAYS_EAGER = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = True +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True BROKER_BACKEND = 'memory' COMMON_PRODUCTION_ERROR_LOG_PATH = '/tmp/mayan-errors.log' diff --git a/removals.txt b/removals.txt index 2163f9ee5d2..5d3132d41e8 100644 --- a/removals.txt +++ b/removals.txt @@ -1,6 +1,7 @@ # Packages to be remove during upgrades cssmin django-autoadmin +django-celery django-environ django-suit django-compressor From 6ca2845d197afaeec563e613f4c5ce8b775c60b9 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 04:44:00 -0400 Subject: [PATCH 065/402] Update requirement files Signed-off-by: Roberto Rosario --- mayan/__init__.py | 2 +- requirements/base.txt | 4 ++-- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mayan/__init__.py b/mayan/__init__.py index 5c740e43d99..9ca2eb1dcc5 100644 --- a/mayan/__init__.py +++ b/mayan/__init__.py @@ -3,7 +3,7 @@ __title__ = 'Mayan EDMS' __version__ = '3.2.6' __build__ = 0x030206 -__build_string__ = 'v3.2.6_Wed Jul 10 03:18:15 2019 -0400' +__build_string__ = 'v3.2.6-68-gab601f9180_Wed Jul 17 04:30:11 2019 -0400' __django_version__ = '1.11' __author__ = 'Roberto Rosario' __author_email__ = 'roberto.rosario@mayan-edms.com' diff --git a/requirements/base.txt b/requirements/base.txt index 7919f51384e..e74cc93d45a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,9 +1,9 @@ Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==3.1.24 +celery==4.3.0 django-activity-stream==0.7.0 -django-celery==3.2.1 +django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 diff --git a/setup.py b/setup.py index c0b4a2dc3db..8b51255c126 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,9 @@ def find_packages(directory): Pillow==6.0.0 PyPDF2==1.26.0 PyYAML==5.1.1 -celery==3.1.24 +celery==4.3.0 django-activity-stream==0.7.0 -django-celery==3.2.1 +django-celery-beat==1.5.0 django-colorful==1.3 django-cors-headers==2.5.2 django-downloadview==1.9 From cf43ef2f732e381347e5272d96ef5b0f9ec7400d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 17 Jul 2019 05:19:40 -0400 Subject: [PATCH 066/402] Fix setting import Signed-off-by: Roberto Rosario --- mayan/apps/platform/classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 723243a904d..8db3ebdb370 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -8,10 +8,10 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.common.serialization import yaml_dump, yaml_load -from mayan.apps.common.settings import ( +from mayan.apps.task_manager.classes import Worker +from mayan.apps.task_manager.settings import ( setting_celery_broker_url, setting_celery_result_backend ) -from mayan.apps.task_manager.classes import Worker class Variable(object): From b6565effb535e177e00f743fc9c1abbe21c80e43 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 19 Jul 2019 01:04:04 -0400 Subject: [PATCH 067/402] Support wildcard MIME type associations Signed-off-by: Roberto Rosario --- mayan/apps/file_metadata/classes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mayan/apps/file_metadata/classes.py b/mayan/apps/file_metadata/classes.py index 6af9b7a79de..60e9510b327 100644 --- a/mayan/apps/file_metadata/classes.py +++ b/mayan/apps/file_metadata/classes.py @@ -31,7 +31,12 @@ class FileMetadataDriver(object): @classmethod def process_document_version(cls, document_version): - for driver_class in cls._registry.get(document_version.mimetype, ()): + # Get list of drivers for the document's MIME type + driver_classes = cls._registry.get(document_version.mimetype, ()) + # Add wilcard drivers, drivers meant to be executed for all MIME types. + driver_classes = driver_classes + tuple(cls._registry.get('*', ())) + + for driver_class in driver_classes: try: driver = driver_class() From d3a53fb53a0bdf5ab9ac32d861afe2af0c36b8f5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 19 Jul 2019 20:02:00 -0400 Subject: [PATCH 068/402] Remove unused SETTING_FILE_TEMPLATE Signed-off-by: Roberto Rosario --- mayan/apps/common/management/commands/literals.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mayan/apps/common/management/commands/literals.py b/mayan/apps/common/management/commands/literals.py index 7b3864fbac7..e69de29bb2d 100644 --- a/mayan/apps/common/management/commands/literals.py +++ b/mayan/apps/common/management/commands/literals.py @@ -1,10 +0,0 @@ -from __future__ import unicode_literals - -SETTING_FILE_TEMPLATE = ''' -from __future__ import absolute_import, unicode_literals - -from .base import * - -SECRET_KEY = '{0}' - -''' From e889021f433cfcec7256c486f8a95702d3f0736c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 19 Jul 2019 20:02:40 -0400 Subject: [PATCH 069/402] Update command options Match the rename of the installjavascript command rename to installdependencies. Signed-off-by: Roberto Rosario --- mayan/apps/common/management/commands/initialsetup.py | 8 ++++---- mayan/apps/common/management/commands/performupgrade.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mayan/apps/common/management/commands/initialsetup.py b/mayan/apps/common/management/commands/initialsetup.py index 0b9755f1450..1b68b737f55 100644 --- a/mayan/apps/common/management/commands/initialsetup.py +++ b/mayan/apps/common/management/commands/initialsetup.py @@ -28,8 +28,8 @@ def add_arguments(self, parser): ) parser.add_argument( - '--no-javascript', action='store_true', dest='no_javascript', - help='Don\'t download the JavaScript dependencies.', + '--no-dependencies', action='store_true', dest='no_dependencies', + help='Don\'t download dependencies.', ) def initialize_system(self, force=False): @@ -88,9 +88,9 @@ def handle(self, *args, **options): self.initialize_system(force=options.get('force', False)) pre_initial_setup.send(sender=self) - if not options.get('no_javascript', False): + if not options.get('no_dependencies', False): management.call_command( - command_name='installjavascript', interactive=False + command_name='installdependencies', interactive=False ) management.call_command( diff --git a/mayan/apps/common/management/commands/performupgrade.py b/mayan/apps/common/management/commands/performupgrade.py index a0240b2bdb3..21ec2b37f8a 100644 --- a/mayan/apps/common/management/commands/performupgrade.py +++ b/mayan/apps/common/management/commands/performupgrade.py @@ -11,8 +11,8 @@ class Command(management.BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--no-javascript', action='store_true', dest='no_javascript', - help='Don\'t download the JavaScript dependencies.', + '--no-dependencies', action='store_true', dest='no_dependencies', + help='Don\'t download dependencies.', ) def handle(self, *args, **options): @@ -25,9 +25,9 @@ def handle(self, *args, **options): ) ) - if not options.get('no_javascript', False): + if not options.get('no_dependencies', False): management.call_command( - command_name='installjavascript', interactive=False + command_name='installdependencies', interactive=False ) try: From 82d7339a64fb2ad339ee27832e73a499c7573f9d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 19 Jul 2019 20:04:21 -0400 Subject: [PATCH 070/402] Update documentation Docker chapter Update to show the new MAYAN_DATABASES setting. Remove settings that are not Docker exclusive. Signed-off-by: Roberto Rosario --- docs/chapters/docker.rst | 118 +++++---------------------------------- 1 file changed, 15 insertions(+), 103 deletions(-) diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index 4910f49e563..fb5b377255f 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -49,12 +49,7 @@ Finally create and run a Mayan EDMS container:: --name mayan-edms \ --restart=always \ -p 80:8000 \ - -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \ - -e MAYAN_DATABASE_HOST=172.17.0.1 \ - -e MAYAN_DATABASE_NAME=mayan \ - -e MAYAN_DATABASE_PASSWORD=mayanuserpass \ - -e MAYAN_DATABASE_USER=mayan \ - -e MAYAN_DATABASE_CONN_MAX_AGE=0 \ + -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'172.17.0.1'}}" \ -v /docker-volumes/mayan-edms/media:/var/lib/mayan \ mayanedms/mayanedms: @@ -108,12 +103,7 @@ instead of the IP address of the Docker host (``172.17.0.1``):: --network=mayan \ --restart=always \ -p 80:8000 \ - -e MAYAN_DATABASE_ENGINE=django.db.backends.postgresql \ - -e MAYAN_DATABASE_HOST=mayan-edms-postgres \ - -e MAYAN_DATABASE_NAME=mayan \ - -e MAYAN_DATABASE_PASSWORD=mayanuserpass \ - -e MAYAN_DATABASE_USER=mayan \ - -e MAYAN_DATABASE_CONN_MAX_AGE=0 \ + -e MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayanuserpass','USER':'mayan','HOST':'mayan-edms-postgres'}}" \ -v /docker-volumes/mayan-edms/media:/var/lib/mayan \ mayanedms/mayanedms: @@ -137,102 +127,14 @@ To start the container again:: Environment Variables --------------------- -The Mayan EDMS image can be configure via environment variables. - -``MAYAN_DATABASE_ENGINE`` - -Defaults to ``None``. This environment variable configures the database -backend to use. If left unset, SQLite will be used. The database backends -supported by this Docker image are: - -- ``'django.db.backends.postgresql'`` -- ``'django.db.backends.mysql'`` -- ``'django.db.backends.sqlite3'`` - -When using the SQLite backend, the database file will be saved in the Docker -volume. The SQLite database as used by Mayan EDMS is meant only for development -or testing, never use it in production. - -``MAYAN_DATABASE_NAME`` - -Defaults to 'mayan'. This optional environment variable can be used to define -the database name that Mayan EDMS will connect to. For more information read -the pertinent Django documentation page: -:django-docs:`Connecting to the database ` - -``MAYAN_DATABASE_USER`` - -Defaults to 'mayan'. This optional environment variable is used to set the -username that will be used to connect to the database. For more information -read the pertinent Django documentation page: -:django-docs:`Settings, USER ` - -``MAYAN_DATABASE_PASSWORD`` - -Defaults to ''. This optional environment variable is used to set the -password that will be used to connect to the database. For more information -read the pertinent Django documentation page: -:django-docs:`Settings, PASSWORD ` - -``MAYAN_DATABASE_HOST`` - -Defaults to `None`. This optional environment variable is used to set the -hostname that will be used to connect to the database. This can be the -hostname of another container or an IP address. For more information read -the pertinent Django documentation page: -:django-docs:`Settings, HOST ` - -``MAYAN_DATABASE_PORT`` - -Defaults to `None`. This optional environment variable is used to set the -port number to use when connecting to the database. An empty string means -the default port. Not used with SQLite. For more information read the -pertinent Django documentation page: -:django-docs:`Settings, PORT ` - -``MAYAN_CELERY_BROKER_URL`` - -This optional environment variable determines the broker that Celery will use -to relay task messages between the frontend code and the background workers. -For more information read the pertinent Celery Kombu documentation page: `Broker URL`_ - -.. _Broker URL: http://kombu.readthedocs.io/en/latest/userguide/connections.html#connection-urls - -This Docker image supports using Redis and RabbitMQ as brokers. - -Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment -variables are specified, the built-in Redis server inside the container will -be disabled. - -``MAYAN_CELERY_RESULT_BACKEND`` - -This optional environment variable determines the results backend that Celery -will use to relay result messages from the background workers to the frontend -code. For more information read the pertinent Celery Kombu documentation page: -`Task result backend settings`_ - -.. _Task result backend settings: http://docs.celeryproject.org/en/3.1/configuration.html#celery-result-backend - -This Docker image supports using Redis and RabbitMQ as result backends. - -Caveat: If the `MAYAN_CELERY_BROKER_URL` and `MAYAN_CELERY_RESULT_BACKEND` environment -variables are specified, the built-in Redis server inside the container will -be disabled. +The common set of settings can also be modified via environment variables when +using the Docker image. In addition to the common set of settings, some Docker +image specific environment variables are available. ``MAYAN_SETTINGS_MODULE`` Optional. Allows loading an alternate settings file. -``MAYAN_DATABASE_CONN_MAX_AGE`` - -Amount in seconds to keep a database connection alive. Allow reuse of database -connections. For more information read the pertinent Django documentation -page: :django-docs:`Settings, CONN_MAX_AGE ` -According to new information Gunicorn's microthreads don't share connections -and will exhaust the available Postgres connections available if a number -other than 0 is used. Reference: https://serverfault.com/questions/635100/django-conn-max-age-persists-connections-but-doesnt-reuse-them-with-postgresq -and https://github.com/benoitc/gunicorn/issues/996 - ``MAYAN_GUNICORN_WORKERS`` Optional. This environment variable controls the number of frontend workers @@ -275,6 +177,15 @@ Optional. Changes the GUID of the ``mayan`` user internal to the Docker container. Defaults to 1000. +Included drivers +---------------- + +The Docker image supports using Redis and RabbitMQ as result backends. For +databases, the image includes support for PostgreSQL and MySQL/MariaDB. +Support for additional brokers or databases may be added using the +``MAYAN_APT_INSTALL`` environment variable. + + .. _docker-accessing-outside-data: Accessing outside data @@ -442,6 +353,7 @@ These are: Nightly images ============== + The continuous integration pipeline used for testing development builds also produces a resulting Docker image. These are build automatically and their stability is not guaranteed. They should never be used in production. From 41a7d00e9e9eaf361d442cb196dd64a6ae57007a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 19 Jul 2019 20:05:12 -0400 Subject: [PATCH 071/402] Fix setting typo Signed-off-by: Roberto Rosario --- mayan/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/settings/base.py b/mayan/settings/base.py index a743c751a4e..a0ca343f340 100644 --- a/mayan/settings/base.py +++ b/mayan/settings/base.py @@ -35,7 +35,7 @@ # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ MEDIA_ROOT = get_environment_setting( - name='MAYAN_MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media') + name='MEDIA_ROOT', fallback_default=os.path.join(BASE_DIR, 'media') ) # SECURITY WARNING: keep the secret key used in production secret! From cb7d5bf82a05c88398f9a95f5c7744972f70af2c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Sat, 20 Jul 2019 00:15:19 -0400 Subject: [PATCH 072/402] Update djcelery imports Signed-off-by: Roberto Rosario --- mayan/apps/common/management/commands/purgeperiodictasks.py | 2 +- mayan/apps/mayan_statistics/classes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mayan/apps/common/management/commands/purgeperiodictasks.py b/mayan/apps/common/management/commands/purgeperiodictasks.py index 835e47440d6..e750cf2d47a 100644 --- a/mayan/apps/common/management/commands/purgeperiodictasks.py +++ b/mayan/apps/common/management/commands/purgeperiodictasks.py @@ -2,7 +2,7 @@ from django.core import management -from djcelery.models import IntervalSchedule, PeriodicTask +from django_celery_beat.models import IntervalSchedule, PeriodicTask class Command(management.BaseCommand): diff --git a/mayan/apps/mayan_statistics/classes.py b/mayan/apps/mayan_statistics/classes.py index 7e5be5a6cfc..671dab799e9 100644 --- a/mayan/apps/mayan_statistics/classes.py +++ b/mayan/apps/mayan_statistics/classes.py @@ -61,7 +61,7 @@ def evaluate(data): @staticmethod def purge_schedules(): PeriodicTask = apps.get_model( - app_label='djcelery', model_name='PeriodicTask' + app_label='django_celery_beat', model_name='PeriodicTask' ) StatisticResult = apps.get_model( app_label='mayan_statistics', model_name='StatisticResult' From 5352c6ac6f78183ffb3e188a8a1a494bc918d561 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:12:11 -0400 Subject: [PATCH 073/402] Update Docker image - Remove Redis from the Docker image. - Add Celery flower. - Add Python 3 packages needed for in-container pip installs. - Fix typos. - Allow PIP proxying. Signed-off-by: Roberto Rosario --- docker/Dockerfile | 26 +++---- docker/docker-compose-development.yml | 72 ------------------- .../local/bin/{run-tests.sh => run_tests.sh} | 0 3 files changed, 13 insertions(+), 85 deletions(-) delete mode 100755 docker/docker-compose-development.yml rename docker/rootfs/usr/local/bin/{run-tests.sh => run_tests.sh} (100%) diff --git a/docker/Dockerfile b/docker/Dockerfile index 32eea72b812..c9fa36c6092 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,7 @@ RUN set -x \ && DEBIAN_FRONTEND=noninteractive \ apt-get update \ && apt-get install -y --no-install-recommends \ + ca-certificates \ exiftool \ ghostscript \ gpgv \ @@ -29,11 +30,11 @@ apt-get update \ graphviz \ libfuse2 \ libmagic1 \ - libmmariadb3 \ + libmariadb3 \ libreoffice \ libpq5 \ poppler-utils \ - redis-server \ + python3-distutils \ sane-utils \ sudo \ supervisor \ @@ -52,22 +53,20 @@ apt-get update \ && if [ "$(uname -m)" = "armv7l" ]; then \ ln -s /usr/lib/arm-linux-gnueabihf/libz.so /usr/lib/ \ && ln -s /usr/lib/arm-linux-gnueabihf/libjpeg.so /usr/lib/ \ -; fi \ -# Discard data when Redis runs out of memory -&& echo "maxmemory-policy allkeys-lru" >> /etc/redis/redis.conf \ -# Disable saving the Redis database -echo "save \"\"" >> /etc/redis/redis.conf \ -# Only provision 1 database -&& echo "databases 1" >> /etc/redis/redis.conf - +; fi #### -# BUILDER_IMAGE - This image buildS the Python package and is discarded afterwards +# BUILDER_IMAGE - This image builds the Python package and is discarded afterwards +# only the build artifact is carried over to the next image. #### # Reuse image FROM BASE_IMAGE as BUILDER_IMAGE +# Python libraries caching +ARG PIP_INDEX_URL +ARG PIP_TRUSTED_HOST + WORKDIR /src # Copy the source files needed to build the Python package @@ -110,6 +109,7 @@ RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \ mysqlclient==1.4.2.post1 \ psycopg2==2.8.3 \ redis==3.2.1 \ + flower==0.9.3 \ # psutil is needed by ARM builds otherwise gevent and gunicorn fail to start && UNAME=`uname -m` && if [ "${UNAME#*arm}" != $UNAME ]; then \ pip install --no-cache-dir \ @@ -118,7 +118,7 @@ RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \ # Install the Python packages needed to build Mayan EDMS && pip install --no-cache-dir -r /src/requirements/build.txt \ # Build Mayan EDMS -&& python setup.py sdist \ +&& python3 setup.py sdist \ # Install the built Mayan EDMS package && pip install --no-cache-dir dist/mayan* \ # Install the static content @@ -128,7 +128,7 @@ RUN python3 -m venv "${PROJECT_INSTALL_DIR}" \ COPY --chown=mayan:mayan requirements/testing-base.txt "${PROJECT_INSTALL_DIR}" #### -# Final image - BASE_IMAGE + Mayan install directory from the builder image +# Final image - BASE_IMAGE + BUILDER_IMAGE artifact (Mayan install directory) #### FROM BASE_IMAGE diff --git a/docker/docker-compose-development.yml b/docker/docker-compose-development.yml deleted file mode 100755 index 48c71afe269..00000000000 --- a/docker/docker-compose-development.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: '2.1' - -volumes: - broker: - driver: local - app: - driver: local - db: - driver: local - results: - driver: local - -services: - broker: - container_name: mayan-edms-broker - image: healthcheck/rabbitmq - environment: - RABBITMQ_DEFAULT_USER: mayan - RABBITMQ_DEFAULT_PASS: mayan - RABBITMQ_DEFAULT_VHOST: mayan - volumes: - - broker:/var/lib/rabbitmq - results: - container_name: mayan-edms-results - image: healthcheck/redis - volumes: - - results:/data - #db: - # container_name: mayan-edms-db - # image: healthcheck/mysql - # environment: - # MYSQL_DATABASE: mayan - # MYSQL_PASSWORD: mayan-password - # MYSQL_ROOT_PASSWORD: root-password - # MYSQL_USER: mayan - # volumes: - # - db:/var/lib/mysql - db: - container_name: mayan-edms-db - image: healthcheck/postgres - environment: - POSTGRES_DB: mayan - POSTGRES_PASSWORD: mayan-password - POSTGRES_USER: mayan - volumes: - - db:/var/lib/postgresql/data - mayan-edms: - container_name: mayan-edms-app - image: mayan-edms/next - build: - context: ./ - args: - - APT_PROXY=172.18.0.1:3142 - depends_on: - broker: - condition: service_healthy - db: - condition: service_healthy - results: - condition: service_healthy - environment: - MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan - MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0 - MAYAN_DATABASE_ENGINE: django.db.backends.postgresql - MAYAN_DATABASE_HOST: db - MAYAN_DATABASE_NAME: mayan - MAYAN_DATABASE_PASSWORD: mayan-password - MAYAN_DATABASE_USER: mayan - ports: - - "80:80" - volumes: - - app:/var/lib/mayan diff --git a/docker/rootfs/usr/local/bin/run-tests.sh b/docker/rootfs/usr/local/bin/run_tests.sh similarity index 100% rename from docker/rootfs/usr/local/bin/run-tests.sh rename to docker/rootfs/usr/local/bin/run_tests.sh From d65bbb718a0720bd0aee10ff09c9a8a6f5b0544d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:15:12 -0400 Subject: [PATCH 074/402] Update Docker entrypoint - Use bash instead of sh/dash to support argument slicing. - Default Celery worker concurrency to 0 (auto). - Set DJANGO_SETTINGS_MODULE environment variable to make it available to sub processes. - Add entrypoint commands to run single workers, single gunicorn or single celery commands like "flower". - Update Gunicorn to use sync workers. - Add platform template to return queues for a worker. Signed-off-by: Roberto Rosario --- docker/rootfs/usr/local/bin/entrypoint.sh | 113 +++++++++++++--------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/docker/rootfs/usr/local/bin/entrypoint.sh b/docker/rootfs/usr/local/bin/entrypoint.sh index 2da0f768b71..0d6c3619c49 100755 --- a/docker/rootfs/usr/local/bin/entrypoint.sh +++ b/docker/rootfs/usr/local/bin/entrypoint.sh @@ -1,4 +1,7 @@ -#!/bin/sh +#!/bin/bash + +# Use bash and not sh to support argument slicing "${@:2}" +# sh defaults to dash instead of bash. set -e echo "mayan: starting entrypoint.sh" @@ -6,19 +9,15 @@ INSTALL_FLAG=/var/lib/mayan/system/SECRET_KEY CONCURRENCY_ARGUMENT=--concurrency= DEFAULT_USER_UID=1000 -DEFAULT_USER_GUID=1000 - -export MAYAN_DEFAULT_CELERY_BROKER_URL=redis://127.0.0.1:6379/0 -export MAYAN_DEFAULT_CELERY_RESULT_BACKEND=redis://127.0.0.1:6379/0 +DEFAULT_USER_GID=1000 export MAYAN_ALLOWED_HOSTS='["*"]' export MAYAN_BIN=/opt/mayan-edms/bin/mayan-edms.py -export MAYAN_CELERY_BROKER_URL=${MAYAN_CELERY_BROKER_URL:-${MAYAN_DEFAULT_CELERY_BROKER_URL}} -export MAYAN_CELERY_RESULT_BACKEND=${MAYAN_CELERY_RESULT_BACKEND:-${MAYAN_DEFAULT_CELERY_RESULT_BACKEND}} export MAYAN_INSTALL_DIR=/opt/mayan-edms export MAYAN_PYTHON_BIN_DIR=/opt/mayan-edms/bin/ export MAYAN_MEDIA_ROOT=/var/lib/mayan export MAYAN_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE:-mayan.settings.production} +export DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE} export MAYAN_GUNICORN_BIN=${MAYAN_PYTHON_BIN_DIR}gunicorn export MAYAN_GUNICORN_WORKERS=${MAYAN_GUNICORN_WORKERS:-2} @@ -26,13 +25,9 @@ export MAYAN_GUNICORN_TIMEOUT=${MAYAN_GUNICORN_TIMEOUT:-120} export MAYAN_PIP_BIN=${MAYAN_PYTHON_BIN_DIR}pip export MAYAN_STATIC_ROOT=${MAYAN_INSTALL_DIR}/static -MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-1} -MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-1} -MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-1} - -echo "mayan: changing uid/guid" -usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}} -groupmod mayan -g ${MAYAN_USER_GUID:-${DEFAULT_USER_GUID}} +MAYAN_WORKER_FAST_CONCURRENCY=${MAYAN_WORKER_FAST_CONCURRENCY:-0} +MAYAN_WORKER_MEDIUM_CONCURRENCY=${MAYAN_WORKER_MEDIUM_CONCURRENCY:-0} +MAYAN_WORKER_SLOW_CONCURRENCY=${MAYAN_WORKER_SLOW_CONCURRENCY:-0} if [ "$MAYAN_WORKER_FAST_CONCURRENCY" -eq 0 ]; then MAYAN_WORKER_FAST_CONCURRENCY= @@ -55,11 +50,9 @@ else fi export MAYAN_WORKER_SLOW_CONCURRENCY -export CELERY_ALWAYS_EAGER=False +# Allow importing of user setting modules export PYTHONPATH=$PYTHONPATH:$MAYAN_MEDIA_ROOT -chown mayan:mayan /var/lib/mayan -R - apt_get_install() { apt-get -q update apt-get install -y --force-yes --no-install-recommends --auto-remove "$@" @@ -67,9 +60,9 @@ apt_get_install() { rm -rf /var/lib/apt/lists/* } -initialize() { - echo "mayan: initialize()" - su mayan -c "${MAYAN_BIN} initialsetup --force --no-javascript" +initialsetup() { + echo "mayan: initialsetup()" + su mayan -c "${MAYAN_BIN} initialsetup --force --no-dependencies" } os_package_installs() { @@ -86,43 +79,71 @@ pip_installs() { fi } -start() { +run_all() { echo "mayan: start()" rm -rf /var/run/supervisor.sock exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf } -upgrade() { - echo "mayan: upgrade()" - su mayan -c "${MAYAN_BIN} performupgrade --no-javascript" +performupgrade() { + echo "mayan: performupgrade()" + su mayan -c "${MAYAN_BIN} performupgrade --no-dependencies" +} + +make_ready() { + # Check if this is a new install, otherwise try to upgrade the existing + # installation on subsequent starts + if [ ! -f $INSTALL_FLAG ]; then + initialsetup + else + performupgrade + fi +} + +set_uid_guid() { + echo "mayan: changing uid/guid" + usermod mayan -u ${MAYAN_USER_UID:-${DEFAULT_USER_UID}} + groupmod mayan -g ${MAYAN_USER_GID:-${DEFAULT_USER_GID}} } os_package_installs || true pip_installs || true +chown mayan:mayan /var/lib/mayan -R case "$1" in -mayan) # Check if this is a new install, otherwise try to upgrade the existing - # installation on subsequent starts - if [ ! -f $INSTALL_FLAG ]; then - initialize - else - upgrade - fi - start - ;; - -run-tests) # Check if this is a new install, otherwise try to upgrade the existing - # installation on subsequent starts - if [ ! -f $INSTALL_FLAG ]; then - initialize - else - upgrade - fi - run-tests.sh - ;; - -*) su mayan -c "$@"; - ;; +run_initialsetup) + initialsetup + ;; + +run_performupgrade) + performupgrade + ;; + +run_all) + make_ready + run_all + ;; + +run_celery) + run_celery.sh "${@:2}" + ;; + +run_frontend) + run_frontend.sh + ;; + +run_tests) + make_ready + run_tests.sh + ;; + +run_worker) + run_worker.sh "${@:2}" + ;; + +*) + su mayan -c "$@" + ;; esac From 1e1b4dedf4c5cc43eb045c6e903924e87dcd2140 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:22:30 -0400 Subject: [PATCH 075/402] Update Docker make file - Include PIP proxies. - Add docker compose targets. Signed-off-by: Roberto Rosario --- docker/Dockerfile | 2 +- docker/Makefile | 19 +++++++++++++-- docker/rootfs/usr/local/bin/run_celery.sh | 5 ++++ docker/rootfs/usr/local/bin/run_frontend.sh | 7 ++++++ docker/rootfs/usr/local/bin/run_worker.sh | 8 +++++++ mayan/apps/platform/classes.py | 23 +++++++++++++++++++ .../platform/supervisord_docker.tmpl | 16 +------------ .../templates/platform/worker_queues.tmpl | 1 + mayan/apps/task_manager/classes.py | 4 ++++ 9 files changed, 67 insertions(+), 18 deletions(-) create mode 100755 docker/rootfs/usr/local/bin/run_celery.sh create mode 100755 docker/rootfs/usr/local/bin/run_frontend.sh create mode 100755 docker/rootfs/usr/local/bin/run_worker.sh create mode 100644 mayan/apps/platform/templates/platform/worker_queues.tmpl diff --git a/docker/Dockerfile b/docker/Dockerfile index c9fa36c6092..58372d50a3b 100755 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -144,7 +144,7 @@ VOLUME ["/var/lib/mayan"] ENTRYPOINT ["entrypoint.sh"] EXPOSE 8000 -CMD ["mayan"] +CMD ["run_all"] RUN ${PROJECT_INSTALL_DIR}/bin/mayan-edms.py platformtemplate supervisord_docker > /etc/supervisor/conf.d/mayan.conf \ && apt-get clean autoclean \ diff --git a/docker/Makefile b/docker/Makefile index bd2743759b3..cbb25827b9c 100755 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,4 +1,9 @@ -APT_PROXY ?= `/sbin/ip route|awk '/docker0/ { print $$9 }'`:3142 +HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'` + +APT_PROXY ?= $(HOST_IP):3142 +PIP_INDEX_URL ?= http://$(HOST_IP):3141/root/pypi/+simple/ +PIP_TRUSTED_HOST ?= $(HOST_IP) + IMAGE_VERSION ?= `cat docker/rootfs/version` CONSOLE_COLUMNS ?= `echo $$(tput cols)` CONSOLE_LINES ?= `echo $$(tput lines)` @@ -7,7 +12,7 @@ docker-build: ## Build a new image locally. docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile . docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY. - docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) . + docker build -t mayanedms/mayanedms:$(IMAGE_VERSION) -f docker/Dockerfile --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) . docker-shell: ## Launch a bash instance inside a running container. Pass the container name via DOCKER_CONTAINER. docker exec -e TERM=$(TERM) -e "COLUMNS=$(CONSOLE_COLUMNS)" -e "LINES=$(CONSOLE_LINES)" -it $(DOCKER_CONTAINER) /bin/bash @@ -23,3 +28,13 @@ docker-test-cleanup: ## Delete the test container and the test volume. docker-test-all: ## Build and executed the test suite in a test container. docker-test-all: docker-build-with-proxy docker run --rm run-tests + +docker-compose-build: + docker-compose -f docker/docker-compose.yml -p mayan-edms build + +docker-compose-build-with-proxy: + docker-compose -f docker/docker-compose.yml -p mayan-edms build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) + +docker-compose-up: + docker-compose -f docker/docker-compose.yml -p mayan-edms up + diff --git a/docker/rootfs/usr/local/bin/run_celery.sh b/docker/rootfs/usr/local/bin/run_celery.sh new file mode 100755 index 00000000000..a4aba0c8a26 --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_celery.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Use -A and not --app. Both are the same but behave differently +# -A can be located before the command while --app cannot. +su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan $@" diff --git a/docker/rootfs/usr/local/bin/run_frontend.sh b/docker/rootfs/usr/local/bin/run_frontend.sh new file mode 100755 index 00000000000..f3c3967ffbf --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_frontend.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +MAYAN_GUNICORN_MAX_REQUESTS=${MAYAN_GUNICORN_MAX_REQUESTS:-500} +MAYAN_GUNICORN_MAX_REQUESTS_JITTERS=${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS:-50} +MAYAN_GUNICORN_WORKER_CLASS=${MAYAN_GUNICORN_WORKER_CLASS:-sync} + +su mayan -c "${MAYAN_PYTHON_BIN_DIR}gunicorn -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests ${MAYAN_GUNICORN_MAX_REQUESTS} --max-requests-jitter ${MAYAN_GUNICORN_MAX_REQUESTS_JITTERS} --worker-class ${MAYAN_GUNICORN_WORKER_CLASS} --bind 0.0.0.0:8000 --timeout ${MAYAN_GUNICORN_TIMEOUT}" diff --git a/docker/rootfs/usr/local/bin/run_worker.sh b/docker/rootfs/usr/local/bin/run_worker.sh new file mode 100755 index 00000000000..dd5db6068d1 --- /dev/null +++ b/docker/rootfs/usr/local/bin/run_worker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +QUEUE_LIST=`MAYAN_WORKER_NAME=$1 su mayan -c "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"` + +# Use -A and not --app. Both are the same but behave differently +# -A can be located before the command while --app cannot. +# Pass ${@:2} to allow overriding the defaults arguments +su mayan -c "${MAYAN_PYTHON_BIN_DIR}celery -A mayan worker -Ofair -l ERROR -Q $QUEUE_LIST ${@:2}" diff --git a/mayan/apps/platform/classes.py b/mayan/apps/platform/classes.py index 8db3ebdb370..8fc3a92cb63 100644 --- a/mayan/apps/platform/classes.py +++ b/mayan/apps/platform/classes.py @@ -181,5 +181,28 @@ def get_context(self): return {'workers': Worker.all()} +class PlatformTemplateWorkerQueues(PlatformTemplate): + label = _('Template showing the queues of a worker.') + name = 'worker_queues' + + variables = ( + Variable( + name='WORKER_NAME', default=None, + environment_name='MAYAN_WORKER_NAME' + ), + ) + + def get_context(self): + worker_name = self.get_variables_context().get('WORKER_NAME') + queues = Worker.get(name=worker_name).queues + + return { + 'queues': queues, 'queue_names': sorted( + map(lambda x: x.name, queues) + ) + } + + PlatformTemplate.register(klass=PlatformTemplateSupervisord) PlatformTemplate.register(klass=PlatformTemplateSupervisordDocker) +PlatformTemplate.register(klass=PlatformTemplateWorkerQueues) diff --git a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl index 3448c3b6185..7ca62781913 100644 --- a/mayan/apps/platform/templates/platform/supervisord_docker.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord_docker.tmpl @@ -1,11 +1,7 @@ -[supervisord] -environment= - DJANGO_SETTINGS_MODULE="%(ENV_MAYAN_SETTINGS_MODULE)s" - [program:mayan-gunicorn] autorestart = false autostart = true -command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class gevent --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT} +command = /bin/bash -c "${MAYAN_GUNICORN_BIN} -w ${MAYAN_GUNICORN_WORKERS} mayan.wsgi --max-requests 500 --max-requests-jitter 50 --worker-class sync --bind 0.0.0.0:8000 --env DJANGO_SETTINGS_MODULE=${MAYAN_SETTINGS_MODULE}" --timeout ${MAYAN_GUNICORN_TIMEOUT} redirect_stderr = true stderr_logfile = /dev/fd/2 stderr_logfile_maxbytes = 0 @@ -13,16 +9,6 @@ stdout_logfile = /dev/fd/1 stdout_logfile_maxbytes = 0 user = mayan -[program:redis] -autorestart = false -autostart = true -command = /bin/bash -c "if [ ${MAYAN_CELERY_BROKER_URL} == ${MAYAN_DEFAULT_CELERY_BROKER_URL} ] && [ ${MAYAN_CELERY_RESULT_BACKEND} == ${MAYAN_DEFAULT_CELERY_RESULT_BACKEND} ];then /usr/bin/redis-server /etc/redis/;fi" -stderr_logfile = /dev/fd/2 -stderr_logfile_maxbytes = 0 -stdout_logfile = /dev/fd/1 -stdout_logfile_maxbytes = 0 -user = root - {% for worker in workers %} [program:mayan-worker-{{ worker.name }}] autorestart = false diff --git a/mayan/apps/platform/templates/platform/worker_queues.tmpl b/mayan/apps/platform/templates/platform/worker_queues.tmpl new file mode 100644 index 00000000000..871cafccb88 --- /dev/null +++ b/mayan/apps/platform/templates/platform/worker_queues.tmpl @@ -0,0 +1 @@ +{{ queue_names|join:"," }} diff --git a/mayan/apps/task_manager/classes.py b/mayan/apps/task_manager/classes.py index b075acfb914..2fd398b125a 100644 --- a/mayan/apps/task_manager/classes.py +++ b/mayan/apps/task_manager/classes.py @@ -188,6 +188,10 @@ class Worker(object): def all(cls): return cls._registry.values() + @classmethod + def get(cls, name): + return cls._registry[name] + def __init__(self, name, label=None, nice_level=0): self.name = name self.label = label From 3563297d4817fefdc451cb998621269f00188d27 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:34:58 -0400 Subject: [PATCH 076/402] Update default Docker compose file - Launch a Redis container. - Include optional services examples. Signed-off-by: Roberto Rosario --- docker/docker-compose.yml | 174 +++++++++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 51 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 978b4171486..79fa4d7f82e 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,58 +1,130 @@ -version: '2.1' - -volumes: - broker: - driver: local - app: - driver: local - db: - driver: local - results: - driver: local +version: '3.7' + +networks: + mayan-bridge: + driver: bridge services: - broker: - container_name: mayan-edms-broker - image: healthcheck/rabbitmq - environment: - RABBITMQ_DEFAULT_USER: mayan - RABBITMQ_DEFAULT_PASS: mayan - RABBITMQ_DEFAULT_VHOST: mayan - volumes: - - broker:/var/lib/rabbitmq - results: - container_name: mayan-edms-results - image: healthcheck/redis + app: + build: + context: .. + dockerfile: ./docker/Dockerfile + depends_on: + - postgresql + - redis + # Enable to use RabbitMQ + #- rabbitmq + environment: &mayan_env + # Enable to use RabbitMQ + # MAYAN_CELERY_BROKER_URL: amqp://mayan:mayanrabbitpass@broker:5672/mayan + # Disable Redis Broker to use RabbitMQ as Broker + MAYAN_CELERY_BROKER_URL: redis://redis:6379/1 + MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0 + MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'postgresql'}}" + image: mayanedms/mayanedms:3.2.6 + networks: + - mayan-bridge + ports: + - "80:8000" + restart: unless-stopped volumes: - - results:/data - db: - container_name: mayan-edms-db - image: healthcheck/postgres + - /docker-volumes/mayan-edms/media:/var/lib/mayan + + postgresql: environment: POSTGRES_DB: mayan - POSTGRES_PASSWORD: mayan-password + POSTGRES_PASSWORD: mayandbpass POSTGRES_USER: mayan + image: postgres:9.6 + networks: + - mayan-bridge + restart: unless-stopped volumes: - - db:/var/lib/postgresql/data - mayan-edms: - container_name: mayan-edms-app - image: mayanedms/mayanedms:latest - depends_on: - broker: - condition: service_healthy - db: - condition: service_healthy - results: - condition: service_healthy - environment: - MAYAN_BROKER_URL: amqp://mayan:mayan@broker:5672/mayan - MAYAN_CELERY_RESULT_BACKEND: redis://results:6379/0 - MAYAN_DATABASE_ENGINE: django.db.backends.postgresql - MAYAN_DATABASE_HOST: db - MAYAN_DATABASE_NAME: mayan - MAYAN_DATABASE_PASSWORD: mayan-password - MAYAN_DATABASE_USER: mayan - ports: - - "80:8000" - volumes: - - app:/var/lib/mayan + - /docker-volumes/mayan-edms/postgres:/var/lib/postgresql/data + + redis: + command: + - redis-server + - --databases + - "2" + - --maxmemory-policy + - allkeys-lru + - --save + - "" + image: redis:5.0 + networks: + - mayan-bridge + restart: unless-stopped + + # Optional services + + # celery_flower: + # command: + # - run_celery + # - flower + # depends_on: + # - postgresql + # - redis + # # Enable to use RabbitMQ + # # - rabbitmq + # environment: + # <<: *mayan_env + # image: mayanedms/mayanedms:3.2.6 + # networks: + # - mayan-bridge + # ports: + # - "5555:5555" + # restart: unless-stopped + + # Enable to use RabbitMQ + # rabbitmq: + # container_name: mayan-edms-rabbitmq + # image: healthcheck/rabbitmq + # environment: + # RABBITMQ_DEFAULT_USER: mayan + # RABBITMQ_DEFAULT_PASS: mayanrabbitpass + # RABBITMQ_DEFAULT_VHOST: mayan + # networks: + # - mayan-bridge + # restart: unless-stopped + # volumes: + # - /docker-volumes/mayan-edms/rabbitmq:/var/lib/rabbitmq + + # Enable to run stand alone workers + # worker_fast: + # command: + # - run_worker + # - fast + # depends_on: + # - postgresql + # - redis + # # Enable to use RabbitMQ + # # - rabbitmq + # environment: + # <<: *mayan_env + # image: mayanedms/mayanedms:3.2.6 + # networks: + # - mayan-bridge + # restart: unless-stopped + # volumes: + # - /docker-volumes/mayan-edms/media:/var/lib/mayan + + # Enable to run stand frontend gunicorn + # frontend: + # command: + # - run_frontend + # depends_on: + # - postgresql + # - redis + # # Enable to use RabbitMQ + # # - rabbitmq + # environment: + # <<: *mayan_env + # image: mayanedms/mayanedms:3.2.6 + # networks: + # - mayan-bridge + # ports: + # - "81:8000" + # restart: unless-stopped + # volumes: + # - /docker-volumes/mayan-edms/media:/var/lib/mayan From adeea6247f8a4cb51f82fda1a629d0f95fd83972 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:38:48 -0400 Subject: [PATCH 077/402] Update Docker stack file Signed-off-by: Roberto Rosario --- docker/stack.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docker/stack.yml b/docker/stack.yml index c8a7b243bd7..95bff439094 100644 --- a/docker/stack.yml +++ b/docker/stack.yml @@ -9,24 +9,32 @@ volumes: services: db: - image: postgres environment: POSTGRES_DB: mayan - POSTGRES_PASSWORD: mayan-password + POSTGRES_PASSWORD: mayandbpass POSTGRES_USER: mayan + image: postgres volumes: - db:/var/lib/postgresql/data app: + environment: + MAYAN_CELERY_BROKER_URL: redis://redis:6379/1 + MAYAN_CELERY_RESULT_BACKEND: redis://redis:6379/0 + MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'mayan','PASSWORD':'mayandbpass','USER':'mayan','HOST':'db'}}" image: mayanedms/mayanedms:latest ports: - 80:8000 - environment: - MAYAN_DATABASE_ENGINE: django.db.backends.postgresql - MAYAN_DATABASE_HOST: db - MAYAN_DATABASE_NAME: mayan - MAYAN_DATABASE_PASSWORD: mayan-password - MAYAN_DATABASE_USER: mayan - MAYAN_DATABASE_CONN_MAX_AGE: 0 volumes: - app:/var/lib/mayan + + redis: + command: + - redis-server + - --databases + - "2" + - --maxmemory-policy + - allkeys-lru + - --save + - "" + image: redis:5.0 From 4558894faf31227b0ccf2e6e1b9492442b5b34d5 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:39:42 -0400 Subject: [PATCH 078/402] Include devpi-server as a development dependency Signed-off-by: Roberto Rosario --- mayan/apps/common/dependencies.py | 4 ++++ requirements/development.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/mayan/apps/common/dependencies.py b/mayan/apps/common/dependencies.py index 26aa3fd351c..aeeae7a7b6e 100644 --- a/mayan/apps/common/dependencies.py +++ b/mayan/apps/common/dependencies.py @@ -409,6 +409,10 @@ module=__name__, environment=environment_development, name='Werkzeug', version_string='==0.15.4' ) +PythonDependency( + module=__name__, environment=environment_development, name='devpi-server', + version_string='==5.0.0' +) PythonDependency( environment=environment_development, module=__name__, name='django-debug-toolbar', version_string='==1.11' diff --git a/requirements/development.txt b/requirements/development.txt index 8689376c36f..36332a99b32 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,5 @@ Werkzeug==0.15.4 +devpi-server==5.0.0 django-debug-toolbar==1.11 django-extensions==2.1.9 django-rosetta==0.9.3 From 0029d3074ffe0ee5494fea648580039d988b386c Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:40:10 -0400 Subject: [PATCH 079/402] Modify PYTHONPATH in-place Avoid including a hard coded path. Signed-off-by: Roberto Rosario --- mayan/apps/platform/templates/platform/supervisord.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/platform/templates/platform/supervisord.tmpl b/mayan/apps/platform/templates/platform/supervisord.tmpl index 513a03581a5..8f34d4406e7 100644 --- a/mayan/apps/platform/templates/platform/supervisord.tmpl +++ b/mayan/apps/platform/templates/platform/supervisord.tmpl @@ -1,6 +1,6 @@ [supervisord] environment= - PYTHONPATH={{ INSTALLATION_PATH }}/lib/python3.7/site-packages:{{ MEDIA_ROOT }}/mayan_settings, + PYTHONPATH="%(ENV_PYTHONPATH)s:{{ MEDIA_ROOT }}/mayan_settings", DJANGO_SETTINGS_MODULE=mayan.settings.production, MAYAN_MEDIA_ROOT="{{ MEDIA_ROOT }}", MAYAN_ALLOWED_HOSTS="{{ ALLOWED_HOSTS }}", From 070355033e8f2b99674ebedc8b26ec0471733efa Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Tue, 23 Jul 2019 21:41:44 -0400 Subject: [PATCH 080/402] Update changelog Signed-off-by: Roberto Rosario --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index aded82fb4fc..414217709df 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,7 +39,8 @@ - Backport file cache manager app. - Convert document image cache to use file cache manager app. Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. - +- Rename MAYAN_GUID to MAYAN_GID +- Switch Gunicorn worker to sync. 3.2.6 (2019-07-10) ================== From afc6b545200a7b764b575901892d80bacb362692 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 01:56:09 -0400 Subject: [PATCH 081/402] Update release notes and changelog Signed-off-by: Roberto Rosario --- HISTORY.rst | 29 +++++++++++++++++++++++------ docs/releases/3.3.rst | 30 +++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 414217709df..5016d38d2de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,16 +39,33 @@ - Backport file cache manager app. - Convert document image cache to use file cache manager app. Add setting DOCUMENTS_CACHE_MAXIMUM_SIZE defaults to 500 MB. +- Replace djcelery and replace it with django-celery-beat. +- Update Celery to version 4.3.0 + Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling) + for much of the research and code updates. +- Support wildcard MIME type associations for the file metadata drivers. - Rename MAYAN_GUID to MAYAN_GID -- Switch Gunicorn worker to sync. +- Update Gunicorn to use sync workers. +- Include devpi-server as a development dependency. +- Update default Docker stack file. +- Remove Redis from the Docker image. +- Add Celery flower to the Docker image. +- Allow PIP proxying to the Docker image during build. +- Default Celery worker concurrency to 0 (auto). +- Set DJANGO_SETTINGS_MODULE environment variable to make it + available to sub processes. +- Add entrypoint commands to run single workers, single gunicorn + or single celery commands like "flower". +- Add platform template to return queues for a worker. + 3.2.6 (2019-07-10) ================== -* Remove the smart settings app * import. -* Encode settings YAML before hashing. -* Fix document icon used in the workflow runtime links. -* Add trashed date time label. -* Fix thumbnail generation issue. GitLab issue #637. +- Remove the smart settings app * import. +- Encode settings YAML before hashing. +- Fix document icon used in the workflow runtime links. +- Add trashed date time label. +- Fix thumbnail generation issue. GitLab issue #637. Thanks to Giacomo Cariello (@giacomocariello) for the report and the merge request fixing the issue. diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index b71a273c841..bcb2c463067 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -55,6 +55,26 @@ Changes - Update Celery to version 4.3.0. Settings changed: MAYAN_BROKER_URL to MAYAN_CELERY_BROKER_URL, MAYAN_CELERY_ALWAYS_EAGER to MAYAN_CELERY_TASK_ALWAYS_EAGER. +- Replace djcelery and replace it with django-celery-beat. +- Update Celery to version 4.3.0 with 55e9b2263cbdb9b449361412fd18d8ee0a442dd3 + from versions/next, code from GitLab issue #594 and GitLab merge request !55. + Thanks to Jakob Haufe (@sur5r) and Jesaja Everling (@jeverling) + for much of the research and code updates. +- Support wildcard MIME type associations for the file metadata drivers. +- Rename MAYAN_GUID to MAYAN_GID +- Update Gunicorn to use sync workers. +- Include devpi-server as a development dependency. +- Update default Docker stack file. +- Remove Redis from the Docker image. +- Add Celery flower to the Docker image. +- Allow PIP proxying to the Docker image during build. +- Default Celery worker concurrency to 0 (auto). +- Set DJANGO_SETTINGS_MODULE environment variable to make it + available to sub processes. +- Add entrypoint commands to run single workers, single gunicorn + or single celery commands like "flower". +- Add platform template to return queues for a worker. + Removals -------- @@ -62,17 +82,17 @@ Removals - Database conversion. Reason for removal. The database conversions support provided by this feature (SQLite to PostgreSQL) was being confused with database migrations and upgrades. - + Database upgrades are the responsibility of the app and the framework. Database conversions however are not the responsibility of the app (Mayan), they are the responsibility of the framework. - + Database conversion is outside the scope of what Mayan does but we added the code, management command, instructions and testing setup to provide this to our users until the framework (Django) decided to add this themselves (like they did with migrations). - Continued confusion about the purpose of the feature and confusion about + Continued confusion about the purpose of the feature and confusion about how errors with this feature were a reflexion of the code quality of Mayannecessitated the removal of the database conversion feature. @@ -167,6 +187,10 @@ Bugs fixed or issues closed --------------------------- - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified +- :gitlab-issue:`594` 3.2b1: Unable to install/run under Python 3.5/3.6/3.7 - :gitlab-issue:`634` Failing docker entrypoint when using secret config +- :gitlab-issue:`635` Build a docker image for Python3 +- :gitlab-issue:`644` Update sane-utils package in docker image. + .. _PyPI: https://pypi.python.org/pypi/mayan-edms/ From 53928b2ab6ecd45b76761cc0ce94cdcb5ee92ffc Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 01:57:20 -0400 Subject: [PATCH 082/402] Run EXIFTOOL always regardless of MIME type Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + mayan/apps/file_metadata/drivers/exiftool.py | 40 +++++--------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5016d38d2de..585c2040bf0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -57,6 +57,8 @@ - Add entrypoint commands to run single workers, single gunicorn or single celery commands like "flower". - Add platform template to return queues for a worker. +- Update the EXIFTOOL driver to run for all documents + regardless of MIME type. 3.2.6 (2019-07-10) diff --git a/mayan/apps/file_metadata/drivers/exiftool.py b/mayan/apps/file_metadata/drivers/exiftool.py index cff210335e5..28eb3f065ca 100644 --- a/mayan/apps/file_metadata/drivers/exiftool.py +++ b/mayan/apps/file_metadata/drivers/exiftool.py @@ -40,8 +40,15 @@ def _process(self, document_version): try: document_version.save_to_file(file_object=temporary_fileobject) temporary_fileobject.seek(0) - result = self.command_exiftool(temporary_fileobject.name) - return json.loads(s=result.stdout)[0] + try: + result = self.command_exiftool(temporary_fileobject.name) + except sh.ErrorReturnCode_1 as exception: + result = json.loads(s=exception.stdout)[0] + if result.get('Error', '') == 'Unknown file type': + # Not a fatal error + return result + else: + return json.loads(s=result.stdout)[0] finally: temporary_fileobject.close() else: @@ -56,31 +63,4 @@ def read_settings(self): ).get('exiftool_path', DEFAULT_EXIF_PATH) -EXIFToolDriver.register( - mimetypes=( - 'application/msword', - 'application/pdf', - 'application/vnd.oasis.opendocument.text', - 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - 'application/x-bittorrent', - 'application/x-gzip', - 'application/x-rar-compressed', - 'application/x-shockwave-flash', - 'application/zip', - 'application/zip', - 'audio/x-pn-realaudio-plugin', - 'audio/x-wav', - 'image/jpeg', - 'image/png', - 'image/svg+xml', - 'image/tiff', - 'image/x-portable-pixmap', - 'text/html', - 'text/rtf', - 'text/x-sh', - 'video/mp4', - 'video/webm', - 'video/x-flv', - 'video/x-matroska' - ) -) +EXIFToolDriver.register(mimetypes=('*',)) From e652c7208cf9427524215cc911288e07878f387e Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 02:17:37 -0400 Subject: [PATCH 083/402] Move Celery dependencies to task_manager app Signed-off-by: Roberto Rosario --- mayan/apps/common/dependencies.py | 119 ---------------------- mayan/apps/task_manager/dependencies.py | 125 ++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 119 deletions(-) create mode 100644 mayan/apps/task_manager/dependencies.py diff --git a/mayan/apps/common/dependencies.py b/mayan/apps/common/dependencies.py index aeeae7a7b6e..d3492b93f21 100644 --- a/mayan/apps/common/dependencies.py +++ b/mayan/apps/common/dependencies.py @@ -61,128 +61,9 @@ SOFTWARE. ''', module=__name__, name='PyYAML', version_string='==5.1.1' ) -PythonDependency( - copyright_text=''' - Copyright (c) 2015 Ask Solem & contributors. All rights reserved. - Copyright (c) 2012-2014 GoPivotal, Inc. All rights reserved. - Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All rights reserved. - - Celery is licensed under The BSD License (3 Clause, also known as - the new BSD license). The license is an OSI approved Open Source - license and is GPL-compatible(1). - - The license text can also be found here: - http://www.opensource.org/licenses/BSD-3-Clause - - License - ======= - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Ask Solem, nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS - BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - Documentation License - ===================== - - The documentation portion of Celery (the rendered contents of the - "docs" directory of a software distribution or checkout) is supplied - under the Creative Commons Attribution-Noncommercial-Share Alike 3.0 - United States License as described by - http://creativecommons.org/licenses/by-nc-sa/3.0/us/ - - Footnotes - ========= - (1) A GPL-compatible license makes it possible to - combine Celery with other software that is released - under the GPL, it does not mean that we're distributing - Celery under the GPL license. The BSD license, unlike the GPL, - let you distribute a modified version without making your - changes open source. - ''', module=__name__, name='celery', version_string='==4.3.0' -) -PythonDependency( - copyright_text=''' - Copyright (c) 2015-2016 Ask Solem. All Rights Reserved. - Copyright (c) 2012-2014 GoPivotal, Inc. All Rights Reserved. - Copyright (c) 2009-2012 Ask Solem. All Rights Reserved. - - django-celery-beat is licensed under The BSD License (3 Clause, also known as - the new BSD license). The license is an OSI approved Open Source - license and is GPL-compatible(1). - - The license text can also be found here: - http://www.opensource.org/licenses/BSD-3-Clause - - License - ======= - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Ask Solem nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS - BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - Documentation License - ===================== - - The documentation portion of django-celery-beat (the rendered contents of the - "docs" directory of a software distribution or checkout) is supplied - under the "Creative Commons Attribution-ShareAlike 4.0 - International" (CC BY-SA 4.0) License as described by - http://creativecommons.org/licenses/by-sa/4.0/ - - Footnotes - ========= - (1) A GPL-compatible license makes it possible to - combine django-celery-beat with other software that is released - under the GPL, it does not mean that we're distributing - django-celery-beat under the GPL license. The BSD license, unlike the GPL, - let you distribute a modified version without making your - changes open source. - ''', module=__name__, name='django-celery-beat', version_string='==1.5.0' -) PythonDependency( module=__name__, name='django-downloadview', version_string='==1.9' ) -PythonDependency( - module=__name__, name='django-environ', version_string='==0.4.5' -) PythonDependency( module=__name__, name='django-formtools', version_string='==2.1' ) diff --git a/mayan/apps/task_manager/dependencies.py b/mayan/apps/task_manager/dependencies.py new file mode 100644 index 00000000000..84f10960dfd --- /dev/null +++ b/mayan/apps/task_manager/dependencies.py @@ -0,0 +1,125 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.dependencies.classes import ( + environment_build, environment_development, environment_testing, + PythonDependency +) + +PythonDependency( + copyright_text=''' + Copyright (c) 2015 Ask Solem & contributors. All rights reserved. + Copyright (c) 2012-2014 GoPivotal, Inc. All rights reserved. + Copyright (c) 2009, 2010, 2011, 2012 Ask Solem, and individual contributors. All rights reserved. + + Celery is licensed under The BSD License (3 Clause, also known as + the new BSD license). The license is an OSI approved Open Source + license and is GPL-compatible(1). + + The license text can also be found here: + http://www.opensource.org/licenses/BSD-3-Clause + + License + ======= + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Ask Solem, nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS + BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Documentation License + ===================== + + The documentation portion of Celery (the rendered contents of the + "docs" directory of a software distribution or checkout) is supplied + under the Creative Commons Attribution-Noncommercial-Share Alike 3.0 + United States License as described by + http://creativecommons.org/licenses/by-nc-sa/3.0/us/ + + Footnotes + ========= + (1) A GPL-compatible license makes it possible to + combine Celery with other software that is released + under the GPL, it does not mean that we're distributing + Celery under the GPL license. The BSD license, unlike the GPL, + let you distribute a modified version without making your + changes open source. + ''', module=__name__, name='celery', version_string='==4.3.0' +) +PythonDependency( + copyright_text=''' + Copyright (c) 2015-2016 Ask Solem. All Rights Reserved. + Copyright (c) 2012-2014 GoPivotal, Inc. All Rights Reserved. + Copyright (c) 2009-2012 Ask Solem. All Rights Reserved. + + django-celery-beat is licensed under The BSD License (3 Clause, also known as + the new BSD license). The license is an OSI approved Open Source + license and is GPL-compatible(1). + + The license text can also be found here: + http://www.opensource.org/licenses/BSD-3-Clause + + License + ======= + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Ask Solem nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL Ask Solem OR CONTRIBUTORS + BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Documentation License + ===================== + + The documentation portion of django-celery-beat (the rendered contents of the + "docs" directory of a software distribution or checkout) is supplied + under the "Creative Commons Attribution-ShareAlike 4.0 + International" (CC BY-SA 4.0) License as described by + http://creativecommons.org/licenses/by-sa/4.0/ + + Footnotes + ========= + (1) A GPL-compatible license makes it possible to + combine django-celery-beat with other software that is released + under the GPL, it does not mean that we're distributing + django-celery-beat under the GPL license. The BSD license, unlike the GPL, + let you distribute a modified version without making your + changes open source. + ''', module=__name__, name='django-celery-beat', version_string='==1.5.0' +) From 0c3b6e53880b9ed5ddb9669c42baa9214c18308f Mon Sep 17 00:00:00 2001 From: Jiri B Date: Sun, 14 Jul 2019 10:15:35 +0000 Subject: [PATCH 084/402] I was shocked my PDFs were deleted from source directory thus this needs to be clarified. --- docs/chapters/sources.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/chapters/sources.rst b/docs/chapters/sources.rst index 0b55bc06409..1d054bb3df4 100644 --- a/docs/chapters/sources.rst +++ b/docs/chapters/sources.rst @@ -15,7 +15,8 @@ The current document sources supported are: - IMAP email - Same as the ``POP3`` email source but for email accounts using the ``IMAP`` protocol. - Watch folder - A filesystem folder that is scanned periodically for files. - Any file in the watch folder is automatically uploaded. + Any file in the watch folder is automatically uploaded. When the upload for a + file is completed, the file is removed from source folder. - Staging folder - Folder where networked attached scanned can save image files. The files in these staging folders are scanned and a preview is generated to help the process of upload. Staging folders and Watch folders From fac77a2f73defc7f40566b267feaec9f03c96b35 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 02:25:49 -0400 Subject: [PATCH 085/402] Workaround for the OCR task-inside-task issue Thanks to Jakob Haufe (@sur5r) for the solution. https://gitlab.com/sur5r/mayan-edms/commit/2423254aa4fd85c4420b59f846c05b1a89f5ee99 Signed-off-by: Roberto Rosario --- mayan/apps/ocr/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/ocr/managers.py b/mayan/apps/ocr/managers.py index f6a0b689789..7d1e9124724 100644 --- a/mayan/apps/ocr/managers.py +++ b/mayan/apps/ocr/managers.py @@ -35,7 +35,7 @@ def process_document_page(self, document_page): ) ) - cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT) + cache_filename = task.get(timeout=DOCUMENT_IMAGE_TASK_TIMEOUT, disable_sync_subtasks=False) with document_page.cache_partition.get_file(filename=cache_filename).open() as file_object: document_page_content, created = DocumentPageOCRContent.objects.get_or_create( From 6f907d156a2d70363aa080872b238d021a07b38d Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 02:49:37 -0400 Subject: [PATCH 086/402] Remove task inspection from task manager app Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 +- docs/releases/3.3.rst | 3 ++ mayan/apps/task_manager/apps.py | 19 +------- mayan/apps/task_manager/classes.py | 26 ----------- mayan/apps/task_manager/links.py | 16 ------- mayan/apps/task_manager/tests/test_views.py | 48 ------------------- mayan/apps/task_manager/urls.py | 20 +------- mayan/apps/task_manager/views.py | 52 --------------------- 8 files changed, 6 insertions(+), 180 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 585c2040bf0..9f60710187a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -59,7 +59,7 @@ - Add platform template to return queues for a worker. - Update the EXIFTOOL driver to run for all documents regardless of MIME type. - +- Remove task inspection from task manager app. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index bcb2c463067..09d17e0429e 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -74,6 +74,7 @@ Changes - Add entrypoint commands to run single workers, single gunicorn or single celery commands like "flower". - Add platform template to return queues for a worker. +- Remove task inspection from task manager app. Removals @@ -186,7 +187,9 @@ Backward incompatible changes Bugs fixed or issues closed --------------------------- +- :gitlab-issue:`526` RuntimeWarning: Never call result.get() within a task! - :gitlab-issue:`532` Workflow preview isn't updated right after transitions are modified +- :gitlab-issue:`540` hint-outdated/update documentation - :gitlab-issue:`594` 3.2b1: Unable to install/run under Python 3.5/3.6/3.7 - :gitlab-issue:`634` Failing docker entrypoint when using secret config - :gitlab-issue:`635` Build a docker image for Python3 diff --git a/mayan/apps/task_manager/apps.py b/mayan/apps/task_manager/apps.py index 781632b9270..ce70a3597eb 100644 --- a/mayan/apps/task_manager/apps.py +++ b/mayan/apps/task_manager/apps.py @@ -8,18 +8,13 @@ from mayan.apps.navigation.classes import SourceColumn from .classes import CeleryQueue, Task -from .links import ( - link_queue_list, link_queue_active_task_list, - link_queue_scheduled_task_list, link_queue_reserved_task_list, - link_task_manager -) +from .links import link_task_manager from .settings import * # NOQA class TaskManagerApp(MayanAppConfig): app_namespace = 'task_manager' app_url = 'task_manager' - has_tests = True name = 'mayan.apps.task_manager' verbose_name = _('Task manager') @@ -65,16 +60,4 @@ def ready(self): func=lambda context: context['object'].kwargs['worker_pid'] ) - menu_object.bind_links( - links=( - link_queue_active_task_list, link_queue_scheduled_task_list, - link_queue_reserved_task_list, - ), sources=(CeleryQueue,) - ) - - menu_secondary.bind_links( - links=(link_queue_list,), - sources=(CeleryQueue, Task, 'task_manager:queue_list') - ) - menu_tools.bind_links(links=(link_task_manager,)) diff --git a/mayan/apps/task_manager/classes.py b/mayan/apps/task_manager/classes.py index 2fd398b125a..d37df2c0490 100644 --- a/mayan/apps/task_manager/classes.py +++ b/mayan/apps/task_manager/classes.py @@ -6,9 +6,6 @@ from kombu import Exchange, Queue -from celery.five import monotonic -from celery.task.control import inspect - from django.apps import apps from django.utils.encoding import force_text, python_2_unicode_compatible from django.utils.module_loading import import_string @@ -62,18 +59,10 @@ def __init__(self, task_type, kwargs): def __str__(self): return force_text(self.task_type) - def get_time_started(self): - time_start = self.kwargs.get('time_start') - if time_start: - return now() - timedelta(seconds=monotonic() - self.kwargs['time_start']) - else: - return None - @python_2_unicode_compatible class CeleryQueue(object): _registry = {} - _inspect_instance = inspect() @staticmethod def initialize(): @@ -132,21 +121,6 @@ def add_task_type(self, *args, **kwargs): self.task_types.append(task_type) return task_type - def get_active_tasks(self): - return self._process_task_dictionary( - task_dictionary=self.__class__._inspect_instance.active() - ) - - def get_reserved_tasks(self): - return self._process_task_dictionary( - task_dictionary=self.__class__._inspect_instance.reserved() - ) - - def get_scheduled_tasks(self): - return self._process_task_dictionary( - task_dictionary=self.__class__._inspect_instance.scheduled() - ) - def _update_celery(self): kwargs = { 'name': self.name, 'exchange': Exchange(self.name), diff --git a/mayan/apps/task_manager/links.py b/mayan/apps/task_manager/links.py index 767008ab591..c7e4a8d465d 100644 --- a/mayan/apps/task_manager/links.py +++ b/mayan/apps/task_manager/links.py @@ -11,19 +11,3 @@ icon_class=icon_task_manager, permissions=(permission_task_view,), text=_('Task manager'), view='task_manager:queue_list' ) -link_queue_list = Link( - icon_class=icon_queue_list, permissions=(permission_task_view,), - text=_('Background task queues'), view='task_manager:queue_list' -) -link_queue_active_task_list = Link( - args='resolved_object.name', permissions=(permission_task_view,), - text=_('Active tasks'), view='task_manager:queue_active_task_list' -) -link_queue_reserved_task_list = Link( - args='resolved_object.name', permissions=(permission_task_view,), - text=_('Reserved tasks'), view='task_manager:queue_reserved_task_list' -) -link_queue_scheduled_task_list = Link( - args='resolved_object.name', permissions=(permission_task_view,), - text=_('Scheduled tasks'), view='task_manager:queue_scheduled_task_list' -) diff --git a/mayan/apps/task_manager/tests/test_views.py b/mayan/apps/task_manager/tests/test_views.py index c672c48dd6a..4df67fa61c4 100644 --- a/mayan/apps/task_manager/tests/test_views.py +++ b/mayan/apps/task_manager/tests/test_views.py @@ -22,29 +22,11 @@ def setUp(self): super(TaskManagerViewTestCase, self).setUp() self._create_test_queue() - def _request_active_task_list(self): - return self.get( - viewname='task_manager:queue_active_task_list', - kwargs={'queue_name': self.test_queue.name}, follow=True - ) - def _request_queue_list(self): return self.get( viewname='task_manager:queue_list', follow=True ) - def _request_reserved_task_list(self): - return self.get( - viewname='task_manager:queue_reserved_task_list', - kwargs={'queue_name': self.test_queue.name}, follow=True - ) - - def _request_scheduled_task_list(self): - return self.get( - viewname='task_manager:queue_scheduled_task_list', - kwargs={'queue_name': self.test_queue.name}, follow=True - ) - def test_queue_list_view_no_permissions(self): response = self._request_queue_list() @@ -57,33 +39,3 @@ def test_queue_list_view_with_permissions(self): self.assertContains( response, text=self.test_queue.name, status_code=200 ) - - def test_active_task_list_view_no_permissions(self): - response = self._request_active_task_list() - self.assertEqual(response.status_code, 403) - - def test_active_task_list_view_with_permissions(self): - self.grant_permission(permission=permission_task_view) - - response = self._request_active_task_list() - self.assertEqual(response.status_code, 200) - - def test_reserved_task_list_view_no_permissions(self): - response = self._request_reserved_task_list() - self.assertEqual(response.status_code, 403) - - def test_reserved_task_list_view_with_permissions(self): - self.grant_permission(permission=permission_task_view) - - response = self._request_reserved_task_list() - self.assertEqual(response.status_code, 200) - - def test_scheduled_task_list_view_no_permissions(self): - response = self._request_scheduled_task_list() - self.assertEqual(response.status_code, 403) - - def test_scheduled_task_list_view_with_permissions(self): - self.grant_permission(permission=permission_task_view) - - response = self._request_scheduled_task_list() - self.assertEqual(response.status_code, 200) diff --git a/mayan/apps/task_manager/urls.py b/mayan/apps/task_manager/urls.py index de48bf12ff0..622a46fea42 100644 --- a/mayan/apps/task_manager/urls.py +++ b/mayan/apps/task_manager/urls.py @@ -2,29 +2,11 @@ from django.conf.urls import url -from .views import ( - QueueListView, QueueActiveTaskListView, QueueScheduledTaskListView, - QueueReservedTaskListView -) - +from .views import QueueListView urlpatterns = [ url( regex=r'^queues/$', view=QueueListView.as_view(), name='queue_list' ), - url( - regex=r'^queues/(?P[-\w]+)/tasks/active/$', - view=QueueActiveTaskListView.as_view(), name='queue_active_task_list' - ), - url( - regex=r'^queues/(?P[-\w]+)/tasks/scheduled/$', - view=QueueScheduledTaskListView.as_view(), - name='queue_scheduled_task_list' - ), - url( - regex=r'^queues/(?P[-\w]+)/tasks/reserved/$', - view=QueueReservedTaskListView.as_view(), - name='queue_reserved_task_list' - ), ] diff --git a/mayan/apps/task_manager/views.py b/mayan/apps/task_manager/views.py index ab229a0c87b..0cf63278eff 100644 --- a/mayan/apps/task_manager/views.py +++ b/mayan/apps/task_manager/views.py @@ -18,55 +18,3 @@ class QueueListView(SingleObjectListView): def get_source_queryset(self): return CeleryQueue.all() - - -class QueueActiveTaskListView(SingleObjectListView): - view_permission = permission_task_view - - def get_extra_context(self): - return { - 'hide_object': True, - 'object': self.get_object(), - 'title': _('Active tasks in queue: %s') % self.get_object() - } - - def get_object(self): - return CeleryQueue.get(queue_name=self.kwargs['queue_name']) - - def get_source_queryset(self): - try: - return self.get_task_list() - except Exception as exception: - messages.error( - message=_( - 'Unable to retrieve task list; %s' - ) % exception, request=self.request - ) - return () - - def get_task_list(self): - return self.get_object().get_active_tasks() - - -class QueueScheduledTaskListView(QueueActiveTaskListView): - def get_extra_context(self): - return { - 'hide_object': True, - 'object': self.get_object(), - 'title': _('Scheduled tasks in queue: %s') % self.get_object() - } - - def get_task_list(self): - return self.get_object().get_scheduled_tasks() - - -class QueueReservedTaskListView(QueueActiveTaskListView): - def get_extra_context(self): - return { - 'hide_object': True, - 'object': self.get_object(), - 'title': _('Reserved tasks in queue: %s') % self.get_object() - } - - def get_task_list(self): - return self.get_object().get_reserved_tasks() From 3d7e6b6fbe30902e49f9023b703f306309cbc2b2 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 02:50:55 -0400 Subject: [PATCH 087/402] Update GUID to GID in documentation Signed-off-by: Roberto Rosario --- docs/chapters/docker.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/chapters/docker.rst b/docs/chapters/docker.rst index fb5b377255f..d203fd5ac34 100644 --- a/docs/chapters/docker.rst +++ b/docs/chapters/docker.rst @@ -171,9 +171,9 @@ number of CPUs detected). Optional. Changes the UID of the ``mayan`` user internal to the Docker container. Defaults to 1000. -``MAYAN_USER_GUID`` +``MAYAN_USER_GID`` -Optional. Changes the GUID of the ``mayan`` user internal to the Docker +Optional. Changes the GID of the ``mayan`` user internal to the Docker container. Defaults to 1000. From 2e12a6af41d4154373c19898ad5f327aea8a1faf Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 02:58:29 -0400 Subject: [PATCH 088/402] Fix test case method resolution Signed-off-by: Roberto Rosario --- mayan/apps/user_management/tests/test_api.py | 4 ++-- .../apps/user_management/tests/test_events.py | 18 +++++++++++++----- mayan/apps/user_management/tests/test_views.py | 10 ++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/mayan/apps/user_management/tests/test_api.py b/mayan/apps/user_management/tests/test_api.py index 48cd5bc976f..8d2a009afd4 100644 --- a/mayan/apps/user_management/tests/test_api.py +++ b/mayan/apps/user_management/tests/test_api.py @@ -107,7 +107,7 @@ def test_group_edit_via_put_with_access(self): self.assertNotEqual(self.test_group.name, group_name) -class UserAPITestCase(UserAPITestMixin, UserTestMixin, BaseAPITestCase): +class UserAPITestCase(UserAPITestMixin, BaseAPITestCase): def test_user_create_api_view_no_permission(self): user_count = get_user_model().objects.count() @@ -194,7 +194,7 @@ def test_user_edit_put_api_view_with_access(self): self.assertNotEqual(self.test_user.username, user_username) -class UserGroupAPITestCase(GroupTestMixin, UserAPITestMixin, UserTestMixin, BaseAPITestCase): +class UserGroupAPITestCase(GroupTestMixin, UserAPITestMixin, BaseAPITestCase): def test_user_create_with_group_api_view_no_permission(self): self._create_test_group() diff --git a/mayan/apps/user_management/tests/test_events.py b/mayan/apps/user_management/tests/test_events.py index 4f0cab8d5ff..9e390dd3e08 100644 --- a/mayan/apps/user_management/tests/test_events.py +++ b/mayan/apps/user_management/tests/test_events.py @@ -21,7 +21,9 @@ ) -class GroupEventsViewTestCase(GroupTestMixin, GroupViewTestMixin, UserTestMixin, GenericViewTestCase): +class GroupEventsViewTestCase( + GroupTestMixin, GroupViewTestMixin, GenericViewTestCase +): def test_group_create_event(self): self.grant_permission( permission=permission_group_create @@ -53,7 +55,9 @@ def test_group_edit_event(self): self.assertEqual(action.verb, event_group_edited.id) -class GroupEventsAPITestCase(GroupAPITestMixin, GroupTestMixin, GroupViewTestMixin, BaseAPITestCase): +class GroupEventsAPITestCase( + GroupAPITestMixin, GroupTestMixin, GroupViewTestMixin, BaseAPITestCase +): def test_group_create_event_from_api_view(self): self.grant_permission( permission=permission_group_create @@ -91,7 +95,7 @@ def test_group_edit_event_from_api_view(self): self.assertEqual(action.verb, event_group_edited.id) -class UserEventsTestCase(UserTestMixin, GenericViewTestCase): +class UserEventsTestCase(GenericViewTestCase): auto_login_user = False create_test_case_user = False @@ -130,7 +134,9 @@ def test_user_logged_out_event_from_view(self): self.assertEqual(action.verb, event_user_logged_out.id) -class UserEventsViewTestCase(UserAPITestMixin, UserTestMixin, UserViewTestMixin, GenericViewTestCase): +class UserEventsViewTestCase( + UserAPITestMixin, UserViewTestMixin, GenericViewTestCase +): def test_user_create_event_from_view(self): self.grant_permission( permission=permission_user_create @@ -164,7 +170,9 @@ def test_user_edit_event_from_view(self): self.assertEqual(action.verb, event_user_edited.id) -class UserEventsAPITestCase(UserAPITestMixin, UserTestMixin, UserViewTestMixin, BaseAPITestCase): +class UserEventsAPITestCase( + UserAPITestMixin, UserViewTestMixin, BaseAPITestCase +): def test_user_create_event_from_api_view(self): self.grant_permission( permission=permission_user_create diff --git a/mayan/apps/user_management/tests/test_views.py b/mayan/apps/user_management/tests/test_views.py index 3733aadefe3..4e5b4c97289 100644 --- a/mayan/apps/user_management/tests/test_views.py +++ b/mayan/apps/user_management/tests/test_views.py @@ -19,7 +19,7 @@ ) -class GroupViewsTestCase(GroupTestMixin, GroupViewTestMixin, UserTestMixin, GenericViewTestCase): +class GroupViewsTestCase(GroupTestMixin, GroupViewTestMixin, GenericViewTestCase): def test_group_create_view_no_permission(self): group_count = Group.objects.count() @@ -158,7 +158,7 @@ def test_group_members_view_with_full_access(self): ) -class SuperUserViewTestCase(UserTestMixin, UserViewTestMixin, GenericViewTestCase): +class SuperUserViewTestCase(UserViewTestMixin, GenericViewTestCase): def setUp(self): super(SuperUserViewTestCase, self).setUp() self._create_test_superuser() @@ -199,7 +199,7 @@ def test_superuser_normal_user_detail_view_with_access(self): self.assertEqual(response.status_code, 404) -class UserViewTestCase(UserTestMixin, UserViewTestMixin, GenericViewTestCase): +class UserViewTestCase(UserViewTestMixin, GenericViewTestCase): def test_user_create_view_no_permission(self): user_count = get_user_model().objects.count() @@ -263,7 +263,9 @@ def test_user_multiple_delete_view_with_access(self): self.assertEqual(get_user_model().objects.count(), user_count - 1) -class UserGroupViewTestCase(GroupTestMixin, UserTestMixin, UserViewTestMixin, GenericViewTestCase): +class UserGroupViewTestCase( + GroupTestMixin, UserViewTestMixin, GenericViewTestCase +): def test_user_groups_view_no_permission(self): self._create_test_user() self._create_test_group() From 4c73239dde618f50c8b41007c02a7f3e5f8421ab Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 03:20:57 -0400 Subject: [PATCH 089/402] Fix http.URL class final URL generation Signed-off-by: Roberto Rosario --- mayan/apps/common/http.py | 6 ++---- mayan/apps/metadata/tests/test_wizard_steps.py | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mayan/apps/common/http.py b/mayan/apps/common/http.py index f353dfb99a3..c3610f9fb6d 100644 --- a/mayan/apps/common/http.py +++ b/mayan/apps/common/http.py @@ -20,9 +20,7 @@ def args(self): def to_string(self): if self._args.keys(): - query = force_bytes( - '?{}'.format(self._args.urlencode()) - ) + query = '?{}'.format(self._args.urlencode()) else: query = '' @@ -31,6 +29,6 @@ def to_string(self): else: path = '' - result = force_bytes('{}{}'.format(path, query)) + result = '{}{}'.format(path, query) return result diff --git a/mayan/apps/metadata/tests/test_wizard_steps.py b/mayan/apps/metadata/tests/test_wizard_steps.py index e6b7644c3e7..2d986710791 100644 --- a/mayan/apps/metadata/tests/test_wizard_steps.py +++ b/mayan/apps/metadata/tests/test_wizard_steps.py @@ -70,6 +70,7 @@ def test_upload_interactive_with_ampersand_metadata(self): self.grant_access( permission=permission_document_create, obj=self.test_document_type ) + # Upload the test document with open(TEST_SMALL_DOCUMENT_PATH, mode='rb') as file_descriptor: response = self.post( @@ -78,6 +79,7 @@ def test_upload_interactive_with_ampersand_metadata(self): 'document_type_id': self.test_document_type.pk, } ) + self.assertEqual(response.status_code, 302) self.assertEqual(Document.objects.count(), 1) From 84b329f661b639c0c851c7fa7f62342cef2fdb02 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 15:29:14 -0400 Subject: [PATCH 090/402] Fix more test case method resolution Signed-off-by: Roberto Rosario --- mayan/apps/authentication/tests/test_views.py | 3 +-- mayan/apps/common/tests/test_views.py | 3 +-- mayan/apps/permissions/tests/test_models.py | 6 ++++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mayan/apps/authentication/tests/test_views.py b/mayan/apps/authentication/tests/test_views.py index 8e61c9849ec..db2b2b7c6e0 100644 --- a/mayan/apps/authentication/tests/test_views.py +++ b/mayan/apps/authentication/tests/test_views.py @@ -14,7 +14,6 @@ from mayan.apps.common.tests import GenericViewTestCase from mayan.apps.smart_settings.classes import Namespace from mayan.apps.user_management.permissions import permission_user_edit -from mayan.apps.user_management.tests.mixins import UserTestMixin from mayan.apps.user_management.tests.literals import TEST_USER_PASSWORD_EDITED from ..settings import setting_maximum_session_length @@ -262,7 +261,7 @@ def test_username_login_redirect(self): self.assertEqual(response.redirect_chain, [(TEST_REDIRECT_URL, 302)]) -class UserViewTestCase(UserTestMixin, UserPasswordViewTestMixin, GenericViewTestCase): +class UserViewTestCase(UserPasswordViewTestMixin, GenericViewTestCase): def test_user_set_password_view_no_access(self): self._create_test_user() diff --git a/mayan/apps/common/tests/test_views.py b/mayan/apps/common/tests/test_views.py index bc13f1c3dca..9cf3b4bce08 100644 --- a/mayan/apps/common/tests/test_views.py +++ b/mayan/apps/common/tests/test_views.py @@ -4,7 +4,6 @@ from django.contrib.contenttypes.models import ContentType from mayan.apps.acls.classes import ModelPermission -from mayan.apps.user_management.tests.mixins import UserTestMixin from ..models import ErrorLogEntry from ..permissions_runtime import permission_error_log_view @@ -13,7 +12,7 @@ from .literals import TEST_ERROR_LOG_ENTRY_RESULT -class CommonViewTestCase(UserTestMixin, GenericViewTestCase): +class CommonViewTestCase(GenericViewTestCase): def _request_about_view(self): return self.get(viewname='common:about_view') diff --git a/mayan/apps/permissions/tests/test_models.py b/mayan/apps/permissions/tests/test_models.py index 74adff07888..3448cdbb150 100644 --- a/mayan/apps/permissions/tests/test_models.py +++ b/mayan/apps/permissions/tests/test_models.py @@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied from mayan.apps.common.tests import BaseTestCase -from mayan.apps.user_management.tests.mixins import GroupTestMixin, UserTestMixin +from mayan.apps.user_management.tests.mixins import GroupTestMixin from ..classes import Permission, PermissionNamespace from ..models import StoredPermission @@ -16,7 +16,9 @@ from .mixins import PermissionTestMixin, RoleTestMixin -class PermissionTestCase(GroupTestMixin, PermissionTestMixin, RoleTestMixin, UserTestMixin, BaseTestCase): +class PermissionTestCase( + GroupTestMixin, PermissionTestMixin, RoleTestMixin, BaseTestCase +): def setUp(self): super(PermissionTestCase, self).setUp() self._create_test_user() From e4bc007bba960ebb7aa5b30157ab432cf1173462 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Wed, 24 Jul 2019 16:06:45 -0400 Subject: [PATCH 091/402] Unify lists header markup Convert list headers into a separate template Signed-off-by: Roberto Rosario --- .../generic_list_items_subtemplate.html | 30 +------ .../appearance/generic_list_subtemplate.html | 29 +------ .../templates/appearance/list_header.html | 28 +++++++ .../templates/appearance/list_toolbar.html | 78 ++++++++++--------- 4 files changed, 72 insertions(+), 93 deletions(-) create mode 100644 mayan/apps/appearance/templates/appearance/list_header.html diff --git a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html index 63ddc71018d..7a326d3e2c5 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_items_subtemplate.html @@ -11,35 +11,9 @@ {% include 'appearance/no_results.html' %}
    {% else %} -

    - {% if page_obj %} - {% if page_obj.paginator.num_pages != 1 %} - {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %} - {% else %} - {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %} - {% endif %} - {% else %} - {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %} - {% endif %} -

    -
    - + {% include "appearance/list_header.html" %} + {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
    - - {% if object_list %} - {% if not hide_multi_item_actions %} - {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %} - {% endif %} - {% endif %} - -
    - {% include 'appearance/list_toolbar.html' %} -
    - - {% if links_multi_menus_results %} -
    - {% endif %} -
    {% for object in object_list %}
    diff --git a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html index e3e03e4d5a7..f139c8a7920 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html @@ -11,34 +11,9 @@ {% include 'appearance/no_results.html' %}
    {% else %} -

    - {% if page_obj %} - {% if page_obj.paginator.num_pages != 1 %} - {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %} - {% else %} - {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %} - {% endif %} - {% else %} - {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %} - {% endif %} -

    -
    - + {% include "appearance/list_header.html" %} + {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %}
    - {% if object_list %} - {% if not hide_multi_item_actions %} - {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %} - {% endif %} - {% endif %} - -
    - {% include 'appearance/list_toolbar.html' %} -
    - - {% if links_multi_menus_results %} -
    - {% endif %} -
    - +
    diff --git a/mayan/apps/appearance/templates/appearance/list_header.html b/mayan/apps/appearance/templates/appearance/list_header.html new file mode 100644 index 00000000000..ce310e8419b --- /dev/null +++ b/mayan/apps/appearance/templates/appearance/list_header.html @@ -0,0 +1,28 @@ +{% load i18n %} +{% load static %} + +{% load common_tags %} +{% load navigation_tags %} + +{% if object_list %} +

    + {% if page_obj %} + {% if page_obj.paginator.num_pages != 1 %} + {% blocktrans with page_obj.start_index as start and page_obj.end_index as end and page_obj.paginator.object_list|length as total and page_obj.number as page_number and page_obj.paginator.num_pages as total_pages %}Total ({{ start }} - {{ end }} out of {{ total }}) (Page {{ page_number }} of {{ total_pages }}){% endblocktrans %} + {% else %} + {% blocktrans with page_obj.paginator.object_list|length as total %}Total: {{ total }}{% endblocktrans %} + {% endif %} + {% else %} + {% blocktrans with object_list|length as total %}Total: {{ total }}{% endblocktrans %} + {% endif %} +

    +
    + + {% if not hide_multi_item_actions %} + {% navigation_resolve_menu name='multi item' sort_results=True source=object_list.0 as links_multi_menus_results %} + {% endif %} +{% endif %} + +
    + {% include 'appearance/list_toolbar.html' %} +
    diff --git a/mayan/apps/appearance/templates/appearance/list_toolbar.html b/mayan/apps/appearance/templates/appearance/list_toolbar.html index e94eedb3f79..23197e94e16 100644 --- a/mayan/apps/appearance/templates/appearance/list_toolbar.html +++ b/mayan/apps/appearance/templates/appearance/list_toolbar.html @@ -3,47 +3,49 @@ {% load common_tags %} {% load navigation_tags %} -
    - {% endfor %}
    - {% include 'pagination/pagination.html' %} {% endif %} diff --git a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html index f139c8a7920..c564f56a8be 100644 --- a/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html +++ b/mayan/apps/appearance/templates/appearance/generic_list_subtemplate.html @@ -142,7 +142,6 @@
    - {% include 'pagination/pagination.html' %}
    {% endif %}

    diff --git a/mayan/apps/appearance/templates/appearance/list_toolbar.html b/mayan/apps/appearance/templates/appearance/list_toolbar.html index 23197e94e16..d73de69556c 100644 --- a/mayan/apps/appearance/templates/appearance/list_toolbar.html +++ b/mayan/apps/appearance/templates/appearance/list_toolbar.html @@ -3,7 +3,9 @@ {% load common_tags %} {% load navigation_tags %} -
    +{% if is_paginated or links_multi_menus_results %} +
    +{% endif %}
    +{% if is_paginated %} +
    + +
    +{% endif %} + {% if links_multi_menus_results %}

    {% trans 'Select items to activate bulk actions. Use Shift + click to select many.' %}

    @@ -48,4 +86,7 @@
    {% endif %} -
    + +{% if is_paginated or links_multi_menus_results %} +
    +{% endif %} From 40a306996ccd0450a630f660046123d6f894b53b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 25 Jul 2019 00:48:47 -0400 Subject: [PATCH 094/402] Update transformation tests Signed-off-by: Roberto Rosario --- .../converter/tests/test_transformations.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mayan/apps/converter/tests/test_transformations.py b/mayan/apps/converter/tests/test_transformations.py index c596abfa275..4267c7fd78f 100644 --- a/mayan/apps/converter/tests/test_transformations.py +++ b/mayan/apps/converter/tests/test_transformations.py @@ -121,7 +121,7 @@ def test_crop_transformation_optional_arguments(self): arguments={'top': '10'} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) def test_crop_transformation_invalid_arguments(self): self._silence_logger(name='mayan.apps.converter.managers') @@ -132,8 +132,7 @@ def test_crop_transformation_invalid_arguments(self): obj=document_page, transformation=TransformationCrop, arguments={'top': 'x', 'left': '-'} ) - - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) def test_crop_transformation_non_valid_range_arguments(self): self._silence_logger(name='mayan.apps.converter.managers') @@ -145,7 +144,7 @@ def test_crop_transformation_non_valid_range_arguments(self): arguments={'top': '-1000', 'bottom': '100000000'} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) def test_crop_transformation_overlapping_ranges_arguments(self): self._silence_logger(name='mayan.apps.converter.managers') @@ -162,7 +161,7 @@ def test_crop_transformation_overlapping_ranges_arguments(self): arguments={'left': '1000', 'right': '10000'} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) def test_lineart_transformations(self): document_page = self.test_document.pages.first() @@ -172,7 +171,7 @@ def test_lineart_transformations(self): arguments={} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) def test_rotate_transformations(self): document_page = self.test_document.pages.first() @@ -182,18 +181,18 @@ def test_rotate_transformations(self): arguments={} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) Transformation.objects.add_to_object( obj=document_page, transformation=TransformationRotate180, arguments={} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) Transformation.objects.add_to_object( obj=document_page, transformation=TransformationRotate270, arguments={} ) - self.assertTrue(document_page.generate_image().startswith('page')) + self.assertTrue(document_page.generate_image()) From 9315776926e25b5eaaa9861f3018782a62ecef6b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 25 Jul 2019 00:52:21 -0400 Subject: [PATCH 095/402] Add missing migrations Signed-off-by: Roberto Rosario --- .../migrations/0008_checkedoutdocument.py | 26 +++++++++++++++++++ .../migrations/0013_auto_20190725_0452.py | 20 ++++++++++++++ .../migrations/0050_auto_20190725_0451.py | 20 ++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 mayan/apps/checkouts/migrations/0008_checkedoutdocument.py create mode 100644 mayan/apps/common/migrations/0013_auto_20190725_0452.py create mode 100644 mayan/apps/documents/migrations/0050_auto_20190725_0451.py diff --git a/mayan/apps/checkouts/migrations/0008_checkedoutdocument.py b/mayan/apps/checkouts/migrations/0008_checkedoutdocument.py new file mode 100644 index 00000000000..903a01d1a67 --- /dev/null +++ b/mayan/apps/checkouts/migrations/0008_checkedoutdocument.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-25 04:52 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0050_auto_20190725_0451'), + ('checkouts', '0007_auto_20180310_1715'), + ] + + operations = [ + migrations.CreateModel( + name='CheckedOutDocument', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + }, + bases=('documents.document',), + ), + ] diff --git a/mayan/apps/common/migrations/0013_auto_20190725_0452.py b/mayan/apps/common/migrations/0013_auto_20190725_0452.py new file mode 100644 index 00000000000..1a8d0b92cd3 --- /dev/null +++ b/mayan/apps/common/migrations/0013_auto_20190725_0452.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-25 04:52 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0012_auto_20190711_0548'), + ] + + operations = [ + migrations.AlterField( + model_name='userlocaleprofile', + name='timezone', + field=models.CharField(choices=[('Africa/Abidjan', 'Africa/Abidjan'), ('Africa/Accra', 'Africa/Accra'), ('Africa/Addis_Ababa', 'Africa/Addis_Ababa'), ('Africa/Algiers', 'Africa/Algiers'), ('Africa/Asmara', 'Africa/Asmara'), ('Africa/Bamako', 'Africa/Bamako'), ('Africa/Bangui', 'Africa/Bangui'), ('Africa/Banjul', 'Africa/Banjul'), ('Africa/Bissau', 'Africa/Bissau'), ('Africa/Blantyre', 'Africa/Blantyre'), ('Africa/Brazzaville', 'Africa/Brazzaville'), ('Africa/Bujumbura', 'Africa/Bujumbura'), ('Africa/Cairo', 'Africa/Cairo'), ('Africa/Casablanca', 'Africa/Casablanca'), ('Africa/Ceuta', 'Africa/Ceuta'), ('Africa/Conakry', 'Africa/Conakry'), ('Africa/Dakar', 'Africa/Dakar'), ('Africa/Dar_es_Salaam', 'Africa/Dar_es_Salaam'), ('Africa/Djibouti', 'Africa/Djibouti'), ('Africa/Douala', 'Africa/Douala'), ('Africa/El_Aaiun', 'Africa/El_Aaiun'), ('Africa/Freetown', 'Africa/Freetown'), ('Africa/Gaborone', 'Africa/Gaborone'), ('Africa/Harare', 'Africa/Harare'), ('Africa/Johannesburg', 'Africa/Johannesburg'), ('Africa/Juba', 'Africa/Juba'), ('Africa/Kampala', 'Africa/Kampala'), ('Africa/Khartoum', 'Africa/Khartoum'), ('Africa/Kigali', 'Africa/Kigali'), ('Africa/Kinshasa', 'Africa/Kinshasa'), ('Africa/Lagos', 'Africa/Lagos'), ('Africa/Libreville', 'Africa/Libreville'), ('Africa/Lome', 'Africa/Lome'), ('Africa/Luanda', 'Africa/Luanda'), ('Africa/Lubumbashi', 'Africa/Lubumbashi'), ('Africa/Lusaka', 'Africa/Lusaka'), ('Africa/Malabo', 'Africa/Malabo'), ('Africa/Maputo', 'Africa/Maputo'), ('Africa/Maseru', 'Africa/Maseru'), ('Africa/Mbabane', 'Africa/Mbabane'), ('Africa/Mogadishu', 'Africa/Mogadishu'), ('Africa/Monrovia', 'Africa/Monrovia'), ('Africa/Nairobi', 'Africa/Nairobi'), ('Africa/Ndjamena', 'Africa/Ndjamena'), ('Africa/Niamey', 'Africa/Niamey'), ('Africa/Nouakchott', 'Africa/Nouakchott'), ('Africa/Ouagadougou', 'Africa/Ouagadougou'), ('Africa/Porto-Novo', 'Africa/Porto-Novo'), ('Africa/Sao_Tome', 'Africa/Sao_Tome'), ('Africa/Tripoli', 'Africa/Tripoli'), ('Africa/Tunis', 'Africa/Tunis'), ('Africa/Windhoek', 'Africa/Windhoek'), ('America/Adak', 'America/Adak'), ('America/Anchorage', 'America/Anchorage'), ('America/Anguilla', 'America/Anguilla'), ('America/Antigua', 'America/Antigua'), ('America/Araguaina', 'America/Araguaina'), ('America/Argentina/Buenos_Aires', 'America/Argentina/Buenos_Aires'), ('America/Argentina/Catamarca', 'America/Argentina/Catamarca'), ('America/Argentina/Cordoba', 'America/Argentina/Cordoba'), ('America/Argentina/Jujuy', 'America/Argentina/Jujuy'), ('America/Argentina/La_Rioja', 'America/Argentina/La_Rioja'), ('America/Argentina/Mendoza', 'America/Argentina/Mendoza'), ('America/Argentina/Rio_Gallegos', 'America/Argentina/Rio_Gallegos'), ('America/Argentina/Salta', 'America/Argentina/Salta'), ('America/Argentina/San_Juan', 'America/Argentina/San_Juan'), ('America/Argentina/San_Luis', 'America/Argentina/San_Luis'), ('America/Argentina/Tucuman', 'America/Argentina/Tucuman'), ('America/Argentina/Ushuaia', 'America/Argentina/Ushuaia'), ('America/Aruba', 'America/Aruba'), ('America/Asuncion', 'America/Asuncion'), ('America/Atikokan', 'America/Atikokan'), ('America/Bahia', 'America/Bahia'), ('America/Bahia_Banderas', 'America/Bahia_Banderas'), ('America/Barbados', 'America/Barbados'), ('America/Belem', 'America/Belem'), ('America/Belize', 'America/Belize'), ('America/Blanc-Sablon', 'America/Blanc-Sablon'), ('America/Boa_Vista', 'America/Boa_Vista'), ('America/Bogota', 'America/Bogota'), ('America/Boise', 'America/Boise'), ('America/Cambridge_Bay', 'America/Cambridge_Bay'), ('America/Campo_Grande', 'America/Campo_Grande'), ('America/Cancun', 'America/Cancun'), ('America/Caracas', 'America/Caracas'), ('America/Cayenne', 'America/Cayenne'), ('America/Cayman', 'America/Cayman'), ('America/Chicago', 'America/Chicago'), ('America/Chihuahua', 'America/Chihuahua'), ('America/Costa_Rica', 'America/Costa_Rica'), ('America/Creston', 'America/Creston'), ('America/Cuiaba', 'America/Cuiaba'), ('America/Curacao', 'America/Curacao'), ('America/Danmarkshavn', 'America/Danmarkshavn'), ('America/Dawson', 'America/Dawson'), ('America/Dawson_Creek', 'America/Dawson_Creek'), ('America/Denver', 'America/Denver'), ('America/Detroit', 'America/Detroit'), ('America/Dominica', 'America/Dominica'), ('America/Edmonton', 'America/Edmonton'), ('America/Eirunepe', 'America/Eirunepe'), ('America/El_Salvador', 'America/El_Salvador'), ('America/Fort_Nelson', 'America/Fort_Nelson'), ('America/Fortaleza', 'America/Fortaleza'), ('America/Glace_Bay', 'America/Glace_Bay'), ('America/Godthab', 'America/Godthab'), ('America/Goose_Bay', 'America/Goose_Bay'), ('America/Grand_Turk', 'America/Grand_Turk'), ('America/Grenada', 'America/Grenada'), ('America/Guadeloupe', 'America/Guadeloupe'), ('America/Guatemala', 'America/Guatemala'), ('America/Guayaquil', 'America/Guayaquil'), ('America/Guyana', 'America/Guyana'), ('America/Halifax', 'America/Halifax'), ('America/Havana', 'America/Havana'), ('America/Hermosillo', 'America/Hermosillo'), ('America/Indiana/Indianapolis', 'America/Indiana/Indianapolis'), ('America/Indiana/Knox', 'America/Indiana/Knox'), ('America/Indiana/Marengo', 'America/Indiana/Marengo'), ('America/Indiana/Petersburg', 'America/Indiana/Petersburg'), ('America/Indiana/Tell_City', 'America/Indiana/Tell_City'), ('America/Indiana/Vevay', 'America/Indiana/Vevay'), ('America/Indiana/Vincennes', 'America/Indiana/Vincennes'), ('America/Indiana/Winamac', 'America/Indiana/Winamac'), ('America/Inuvik', 'America/Inuvik'), ('America/Iqaluit', 'America/Iqaluit'), ('America/Jamaica', 'America/Jamaica'), ('America/Juneau', 'America/Juneau'), ('America/Kentucky/Louisville', 'America/Kentucky/Louisville'), ('America/Kentucky/Monticello', 'America/Kentucky/Monticello'), ('America/Kralendijk', 'America/Kralendijk'), ('America/La_Paz', 'America/La_Paz'), ('America/Lima', 'America/Lima'), ('America/Los_Angeles', 'America/Los_Angeles'), ('America/Lower_Princes', 'America/Lower_Princes'), ('America/Maceio', 'America/Maceio'), ('America/Managua', 'America/Managua'), ('America/Manaus', 'America/Manaus'), ('America/Marigot', 'America/Marigot'), ('America/Martinique', 'America/Martinique'), ('America/Matamoros', 'America/Matamoros'), ('America/Mazatlan', 'America/Mazatlan'), ('America/Menominee', 'America/Menominee'), ('America/Merida', 'America/Merida'), ('America/Metlakatla', 'America/Metlakatla'), ('America/Mexico_City', 'America/Mexico_City'), ('America/Miquelon', 'America/Miquelon'), ('America/Moncton', 'America/Moncton'), ('America/Monterrey', 'America/Monterrey'), ('America/Montevideo', 'America/Montevideo'), ('America/Montserrat', 'America/Montserrat'), ('America/Nassau', 'America/Nassau'), ('America/New_York', 'America/New_York'), ('America/Nipigon', 'America/Nipigon'), ('America/Nome', 'America/Nome'), ('America/Noronha', 'America/Noronha'), ('America/North_Dakota/Beulah', 'America/North_Dakota/Beulah'), ('America/North_Dakota/Center', 'America/North_Dakota/Center'), ('America/North_Dakota/New_Salem', 'America/North_Dakota/New_Salem'), ('America/Ojinaga', 'America/Ojinaga'), ('America/Panama', 'America/Panama'), ('America/Pangnirtung', 'America/Pangnirtung'), ('America/Paramaribo', 'America/Paramaribo'), ('America/Phoenix', 'America/Phoenix'), ('America/Port-au-Prince', 'America/Port-au-Prince'), ('America/Port_of_Spain', 'America/Port_of_Spain'), ('America/Porto_Velho', 'America/Porto_Velho'), ('America/Puerto_Rico', 'America/Puerto_Rico'), ('America/Punta_Arenas', 'America/Punta_Arenas'), ('America/Rainy_River', 'America/Rainy_River'), ('America/Rankin_Inlet', 'America/Rankin_Inlet'), ('America/Recife', 'America/Recife'), ('America/Regina', 'America/Regina'), ('America/Resolute', 'America/Resolute'), ('America/Rio_Branco', 'America/Rio_Branco'), ('America/Santarem', 'America/Santarem'), ('America/Santiago', 'America/Santiago'), ('America/Santo_Domingo', 'America/Santo_Domingo'), ('America/Sao_Paulo', 'America/Sao_Paulo'), ('America/Scoresbysund', 'America/Scoresbysund'), ('America/Sitka', 'America/Sitka'), ('America/St_Barthelemy', 'America/St_Barthelemy'), ('America/St_Johns', 'America/St_Johns'), ('America/St_Kitts', 'America/St_Kitts'), ('America/St_Lucia', 'America/St_Lucia'), ('America/St_Thomas', 'America/St_Thomas'), ('America/St_Vincent', 'America/St_Vincent'), ('America/Swift_Current', 'America/Swift_Current'), ('America/Tegucigalpa', 'America/Tegucigalpa'), ('America/Thule', 'America/Thule'), ('America/Thunder_Bay', 'America/Thunder_Bay'), ('America/Tijuana', 'America/Tijuana'), ('America/Toronto', 'America/Toronto'), ('America/Tortola', 'America/Tortola'), ('America/Vancouver', 'America/Vancouver'), ('America/Whitehorse', 'America/Whitehorse'), ('America/Winnipeg', 'America/Winnipeg'), ('America/Yakutat', 'America/Yakutat'), ('America/Yellowknife', 'America/Yellowknife'), ('Antarctica/Casey', 'Antarctica/Casey'), ('Antarctica/Davis', 'Antarctica/Davis'), ('Antarctica/DumontDUrville', 'Antarctica/DumontDUrville'), ('Antarctica/Macquarie', 'Antarctica/Macquarie'), ('Antarctica/Mawson', 'Antarctica/Mawson'), ('Antarctica/McMurdo', 'Antarctica/McMurdo'), ('Antarctica/Palmer', 'Antarctica/Palmer'), ('Antarctica/Rothera', 'Antarctica/Rothera'), ('Antarctica/Syowa', 'Antarctica/Syowa'), ('Antarctica/Troll', 'Antarctica/Troll'), ('Antarctica/Vostok', 'Antarctica/Vostok'), ('Arctic/Longyearbyen', 'Arctic/Longyearbyen'), ('Asia/Aden', 'Asia/Aden'), ('Asia/Almaty', 'Asia/Almaty'), ('Asia/Amman', 'Asia/Amman'), ('Asia/Anadyr', 'Asia/Anadyr'), ('Asia/Aqtau', 'Asia/Aqtau'), ('Asia/Aqtobe', 'Asia/Aqtobe'), ('Asia/Ashgabat', 'Asia/Ashgabat'), ('Asia/Atyrau', 'Asia/Atyrau'), ('Asia/Baghdad', 'Asia/Baghdad'), ('Asia/Bahrain', 'Asia/Bahrain'), ('Asia/Baku', 'Asia/Baku'), ('Asia/Bangkok', 'Asia/Bangkok'), ('Asia/Barnaul', 'Asia/Barnaul'), ('Asia/Beirut', 'Asia/Beirut'), ('Asia/Bishkek', 'Asia/Bishkek'), ('Asia/Brunei', 'Asia/Brunei'), ('Asia/Chita', 'Asia/Chita'), ('Asia/Choibalsan', 'Asia/Choibalsan'), ('Asia/Colombo', 'Asia/Colombo'), ('Asia/Damascus', 'Asia/Damascus'), ('Asia/Dhaka', 'Asia/Dhaka'), ('Asia/Dili', 'Asia/Dili'), ('Asia/Dubai', 'Asia/Dubai'), ('Asia/Dushanbe', 'Asia/Dushanbe'), ('Asia/Famagusta', 'Asia/Famagusta'), ('Asia/Gaza', 'Asia/Gaza'), ('Asia/Hebron', 'Asia/Hebron'), ('Asia/Ho_Chi_Minh', 'Asia/Ho_Chi_Minh'), ('Asia/Hong_Kong', 'Asia/Hong_Kong'), ('Asia/Hovd', 'Asia/Hovd'), ('Asia/Irkutsk', 'Asia/Irkutsk'), ('Asia/Jakarta', 'Asia/Jakarta'), ('Asia/Jayapura', 'Asia/Jayapura'), ('Asia/Jerusalem', 'Asia/Jerusalem'), ('Asia/Kabul', 'Asia/Kabul'), ('Asia/Kamchatka', 'Asia/Kamchatka'), ('Asia/Karachi', 'Asia/Karachi'), ('Asia/Kathmandu', 'Asia/Kathmandu'), ('Asia/Khandyga', 'Asia/Khandyga'), ('Asia/Kolkata', 'Asia/Kolkata'), ('Asia/Krasnoyarsk', 'Asia/Krasnoyarsk'), ('Asia/Kuala_Lumpur', 'Asia/Kuala_Lumpur'), ('Asia/Kuching', 'Asia/Kuching'), ('Asia/Kuwait', 'Asia/Kuwait'), ('Asia/Macau', 'Asia/Macau'), ('Asia/Magadan', 'Asia/Magadan'), ('Asia/Makassar', 'Asia/Makassar'), ('Asia/Manila', 'Asia/Manila'), ('Asia/Muscat', 'Asia/Muscat'), ('Asia/Nicosia', 'Asia/Nicosia'), ('Asia/Novokuznetsk', 'Asia/Novokuznetsk'), ('Asia/Novosibirsk', 'Asia/Novosibirsk'), ('Asia/Omsk', 'Asia/Omsk'), ('Asia/Oral', 'Asia/Oral'), ('Asia/Phnom_Penh', 'Asia/Phnom_Penh'), ('Asia/Pontianak', 'Asia/Pontianak'), ('Asia/Pyongyang', 'Asia/Pyongyang'), ('Asia/Qatar', 'Asia/Qatar'), ('Asia/Qostanay', 'Asia/Qostanay'), ('Asia/Qyzylorda', 'Asia/Qyzylorda'), ('Asia/Riyadh', 'Asia/Riyadh'), ('Asia/Sakhalin', 'Asia/Sakhalin'), ('Asia/Samarkand', 'Asia/Samarkand'), ('Asia/Seoul', 'Asia/Seoul'), ('Asia/Shanghai', 'Asia/Shanghai'), ('Asia/Singapore', 'Asia/Singapore'), ('Asia/Srednekolymsk', 'Asia/Srednekolymsk'), ('Asia/Taipei', 'Asia/Taipei'), ('Asia/Tashkent', 'Asia/Tashkent'), ('Asia/Tbilisi', 'Asia/Tbilisi'), ('Asia/Tehran', 'Asia/Tehran'), ('Asia/Thimphu', 'Asia/Thimphu'), ('Asia/Tokyo', 'Asia/Tokyo'), ('Asia/Tomsk', 'Asia/Tomsk'), ('Asia/Ulaanbaatar', 'Asia/Ulaanbaatar'), ('Asia/Urumqi', 'Asia/Urumqi'), ('Asia/Ust-Nera', 'Asia/Ust-Nera'), ('Asia/Vientiane', 'Asia/Vientiane'), ('Asia/Vladivostok', 'Asia/Vladivostok'), ('Asia/Yakutsk', 'Asia/Yakutsk'), ('Asia/Yangon', 'Asia/Yangon'), ('Asia/Yekaterinburg', 'Asia/Yekaterinburg'), ('Asia/Yerevan', 'Asia/Yerevan'), ('Atlantic/Azores', 'Atlantic/Azores'), ('Atlantic/Bermuda', 'Atlantic/Bermuda'), ('Atlantic/Canary', 'Atlantic/Canary'), ('Atlantic/Cape_Verde', 'Atlantic/Cape_Verde'), ('Atlantic/Faroe', 'Atlantic/Faroe'), ('Atlantic/Madeira', 'Atlantic/Madeira'), ('Atlantic/Reykjavik', 'Atlantic/Reykjavik'), ('Atlantic/South_Georgia', 'Atlantic/South_Georgia'), ('Atlantic/St_Helena', 'Atlantic/St_Helena'), ('Atlantic/Stanley', 'Atlantic/Stanley'), ('Australia/Adelaide', 'Australia/Adelaide'), ('Australia/Brisbane', 'Australia/Brisbane'), ('Australia/Broken_Hill', 'Australia/Broken_Hill'), ('Australia/Currie', 'Australia/Currie'), ('Australia/Darwin', 'Australia/Darwin'), ('Australia/Eucla', 'Australia/Eucla'), ('Australia/Hobart', 'Australia/Hobart'), ('Australia/Lindeman', 'Australia/Lindeman'), ('Australia/Lord_Howe', 'Australia/Lord_Howe'), ('Australia/Melbourne', 'Australia/Melbourne'), ('Australia/Perth', 'Australia/Perth'), ('Australia/Sydney', 'Australia/Sydney'), ('Canada/Atlantic', 'Canada/Atlantic'), ('Canada/Central', 'Canada/Central'), ('Canada/Eastern', 'Canada/Eastern'), ('Canada/Mountain', 'Canada/Mountain'), ('Canada/Newfoundland', 'Canada/Newfoundland'), ('Canada/Pacific', 'Canada/Pacific'), ('Europe/Amsterdam', 'Europe/Amsterdam'), ('Europe/Andorra', 'Europe/Andorra'), ('Europe/Astrakhan', 'Europe/Astrakhan'), ('Europe/Athens', 'Europe/Athens'), ('Europe/Belgrade', 'Europe/Belgrade'), ('Europe/Berlin', 'Europe/Berlin'), ('Europe/Bratislava', 'Europe/Bratislava'), ('Europe/Brussels', 'Europe/Brussels'), ('Europe/Bucharest', 'Europe/Bucharest'), ('Europe/Budapest', 'Europe/Budapest'), ('Europe/Busingen', 'Europe/Busingen'), ('Europe/Chisinau', 'Europe/Chisinau'), ('Europe/Copenhagen', 'Europe/Copenhagen'), ('Europe/Dublin', 'Europe/Dublin'), ('Europe/Gibraltar', 'Europe/Gibraltar'), ('Europe/Guernsey', 'Europe/Guernsey'), ('Europe/Helsinki', 'Europe/Helsinki'), ('Europe/Isle_of_Man', 'Europe/Isle_of_Man'), ('Europe/Istanbul', 'Europe/Istanbul'), ('Europe/Jersey', 'Europe/Jersey'), ('Europe/Kaliningrad', 'Europe/Kaliningrad'), ('Europe/Kiev', 'Europe/Kiev'), ('Europe/Kirov', 'Europe/Kirov'), ('Europe/Lisbon', 'Europe/Lisbon'), ('Europe/Ljubljana', 'Europe/Ljubljana'), ('Europe/London', 'Europe/London'), ('Europe/Luxembourg', 'Europe/Luxembourg'), ('Europe/Madrid', 'Europe/Madrid'), ('Europe/Malta', 'Europe/Malta'), ('Europe/Mariehamn', 'Europe/Mariehamn'), ('Europe/Minsk', 'Europe/Minsk'), ('Europe/Monaco', 'Europe/Monaco'), ('Europe/Moscow', 'Europe/Moscow'), ('Europe/Oslo', 'Europe/Oslo'), ('Europe/Paris', 'Europe/Paris'), ('Europe/Podgorica', 'Europe/Podgorica'), ('Europe/Prague', 'Europe/Prague'), ('Europe/Riga', 'Europe/Riga'), ('Europe/Rome', 'Europe/Rome'), ('Europe/Samara', 'Europe/Samara'), ('Europe/San_Marino', 'Europe/San_Marino'), ('Europe/Sarajevo', 'Europe/Sarajevo'), ('Europe/Saratov', 'Europe/Saratov'), ('Europe/Simferopol', 'Europe/Simferopol'), ('Europe/Skopje', 'Europe/Skopje'), ('Europe/Sofia', 'Europe/Sofia'), ('Europe/Stockholm', 'Europe/Stockholm'), ('Europe/Tallinn', 'Europe/Tallinn'), ('Europe/Tirane', 'Europe/Tirane'), ('Europe/Ulyanovsk', 'Europe/Ulyanovsk'), ('Europe/Uzhgorod', 'Europe/Uzhgorod'), ('Europe/Vaduz', 'Europe/Vaduz'), ('Europe/Vatican', 'Europe/Vatican'), ('Europe/Vienna', 'Europe/Vienna'), ('Europe/Vilnius', 'Europe/Vilnius'), ('Europe/Volgograd', 'Europe/Volgograd'), ('Europe/Warsaw', 'Europe/Warsaw'), ('Europe/Zagreb', 'Europe/Zagreb'), ('Europe/Zaporozhye', 'Europe/Zaporozhye'), ('Europe/Zurich', 'Europe/Zurich'), ('GMT', 'GMT'), ('Indian/Antananarivo', 'Indian/Antananarivo'), ('Indian/Chagos', 'Indian/Chagos'), ('Indian/Christmas', 'Indian/Christmas'), ('Indian/Cocos', 'Indian/Cocos'), ('Indian/Comoro', 'Indian/Comoro'), ('Indian/Kerguelen', 'Indian/Kerguelen'), ('Indian/Mahe', 'Indian/Mahe'), ('Indian/Maldives', 'Indian/Maldives'), ('Indian/Mauritius', 'Indian/Mauritius'), ('Indian/Mayotte', 'Indian/Mayotte'), ('Indian/Reunion', 'Indian/Reunion'), ('Pacific/Apia', 'Pacific/Apia'), ('Pacific/Auckland', 'Pacific/Auckland'), ('Pacific/Bougainville', 'Pacific/Bougainville'), ('Pacific/Chatham', 'Pacific/Chatham'), ('Pacific/Chuuk', 'Pacific/Chuuk'), ('Pacific/Easter', 'Pacific/Easter'), ('Pacific/Efate', 'Pacific/Efate'), ('Pacific/Enderbury', 'Pacific/Enderbury'), ('Pacific/Fakaofo', 'Pacific/Fakaofo'), ('Pacific/Fiji', 'Pacific/Fiji'), ('Pacific/Funafuti', 'Pacific/Funafuti'), ('Pacific/Galapagos', 'Pacific/Galapagos'), ('Pacific/Gambier', 'Pacific/Gambier'), ('Pacific/Guadalcanal', 'Pacific/Guadalcanal'), ('Pacific/Guam', 'Pacific/Guam'), ('Pacific/Honolulu', 'Pacific/Honolulu'), ('Pacific/Kiritimati', 'Pacific/Kiritimati'), ('Pacific/Kosrae', 'Pacific/Kosrae'), ('Pacific/Kwajalein', 'Pacific/Kwajalein'), ('Pacific/Majuro', 'Pacific/Majuro'), ('Pacific/Marquesas', 'Pacific/Marquesas'), ('Pacific/Midway', 'Pacific/Midway'), ('Pacific/Nauru', 'Pacific/Nauru'), ('Pacific/Niue', 'Pacific/Niue'), ('Pacific/Norfolk', 'Pacific/Norfolk'), ('Pacific/Noumea', 'Pacific/Noumea'), ('Pacific/Pago_Pago', 'Pacific/Pago_Pago'), ('Pacific/Palau', 'Pacific/Palau'), ('Pacific/Pitcairn', 'Pacific/Pitcairn'), ('Pacific/Pohnpei', 'Pacific/Pohnpei'), ('Pacific/Port_Moresby', 'Pacific/Port_Moresby'), ('Pacific/Rarotonga', 'Pacific/Rarotonga'), ('Pacific/Saipan', 'Pacific/Saipan'), ('Pacific/Tahiti', 'Pacific/Tahiti'), ('Pacific/Tarawa', 'Pacific/Tarawa'), ('Pacific/Tongatapu', 'Pacific/Tongatapu'), ('Pacific/Wake', 'Pacific/Wake'), ('Pacific/Wallis', 'Pacific/Wallis'), ('US/Alaska', 'US/Alaska'), ('US/Arizona', 'US/Arizona'), ('US/Central', 'US/Central'), ('US/Eastern', 'US/Eastern'), ('US/Hawaii', 'US/Hawaii'), ('US/Mountain', 'US/Mountain'), ('US/Pacific', 'US/Pacific'), ('UTC', 'UTC')], max_length=48, verbose_name='Timezone'), + ), + ] diff --git a/mayan/apps/documents/migrations/0050_auto_20190725_0451.py b/mayan/apps/documents/migrations/0050_auto_20190725_0451.py new file mode 100644 index 00000000000..cfb54c92c2a --- /dev/null +++ b/mayan/apps/documents/migrations/0050_auto_20190725_0451.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-07-25 04:51 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '0049_auto_20190715_0454'), + ] + + operations = [ + migrations.AlterField( + model_name='document', + name='language', + field=models.CharField(blank=True, default='eng', help_text='The dominant language in the document.', max_length=8, verbose_name='Language'), + ), + ] From 88bc29e4d78f7467c7de2800ded9c5b21f9bbb00 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 25 Jul 2019 02:22:57 -0400 Subject: [PATCH 096/402] Update the file caching app - Add view to list available caches. - Add links to view and purge caches. - Add permissions. - Add events. - Add purge task. - Remove document image clear link and view. This is now handled by the file caching app. Signed-off-by: Roberto Rosario --- HISTORY.rst | 2 + docs/releases/3.3.rst | 2 + mayan/apps/documents/apps.py | 4 +- mayan/apps/documents/icons.py | 2 - mayan/apps/documents/links.py | 12 +--- mayan/apps/documents/queues.py | 4 -- mayan/apps/documents/tasks.py | 11 ---- mayan/apps/documents/urls.py | 6 +- mayan/apps/documents/views/misc_views.py | 17 +----- mayan/apps/file_caching/apps.py | 71 ++++++++++++++++++++++++ mayan/apps/file_caching/events.py | 19 +++++++ mayan/apps/file_caching/icons.py | 11 ++++ mayan/apps/file_caching/links.py | 22 ++++++++ mayan/apps/file_caching/models.py | 60 ++++++++++++++++++-- mayan/apps/file_caching/permissions.py | 14 +++++ mayan/apps/file_caching/queues.py | 10 ++++ mayan/apps/file_caching/tasks.py | 25 +++++++++ mayan/apps/file_caching/urls.py | 20 +++++++ mayan/apps/file_caching/views.py | 53 ++++++++++++++++++ 19 files changed, 309 insertions(+), 56 deletions(-) create mode 100644 mayan/apps/file_caching/events.py create mode 100644 mayan/apps/file_caching/icons.py create mode 100644 mayan/apps/file_caching/links.py create mode 100644 mayan/apps/file_caching/permissions.py create mode 100644 mayan/apps/file_caching/queues.py create mode 100644 mayan/apps/file_caching/tasks.py create mode 100644 mayan/apps/file_caching/urls.py create mode 100644 mayan/apps/file_caching/views.py diff --git a/HISTORY.rst b/HISTORY.rst index fa09d0aa141..942e3fa898d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -61,6 +61,8 @@ regardless of MIME type. - Remove task inspection from task manager app. - Move pagination navigation inside the toolbar. +- Remove document image clear link and view. + This is now handled by the file caching app. 3.2.6 (2019-07-10) ================== diff --git a/docs/releases/3.3.rst b/docs/releases/3.3.rst index 1daaceae322..3ff4c6d77e1 100644 --- a/docs/releases/3.3.rst +++ b/docs/releases/3.3.rst @@ -76,6 +76,8 @@ Changes - Add platform template to return queues for a worker. - Remove task inspection from task manager app. - Move pagination navigation inside the toolbar. +- Remove document image clear link and view. + This is now handled by the file caching app. Removals diff --git a/mayan/apps/documents/apps.py b/mayan/apps/documents/apps.py index 25a943294b0..aec4a06e214 100644 --- a/mayan/apps/documents/apps.py +++ b/mayan/apps/documents/apps.py @@ -47,7 +47,7 @@ handler_remove_empty_duplicates_lists, handler_scan_duplicates_for ) from .links import ( - link_clear_image_cache, link_document_clear_transformations, + link_document_clear_transformations, link_document_clone_transformations, link_document_delete, link_document_document_type_edit, link_document_download, link_document_duplicates_list, link_document_edit, @@ -377,7 +377,7 @@ def ready(self): menu_setup.bind_links(links=(link_document_type_setup,)) menu_tools.bind_links( - links=(link_clear_image_cache, link_duplicated_document_scan) + links=(link_duplicated_document_scan,) ) # Document type links diff --git a/mayan/apps/documents/icons.py b/mayan/apps/documents/icons.py index 8b9b4067d9c..fe745fe5384 100644 --- a/mayan/apps/documents/icons.py +++ b/mayan/apps/documents/icons.py @@ -12,8 +12,6 @@ icon_menu_documents = Icon(driver_name='fontawesome', symbol='book') -icon_clear_image_cache = Icon(driver_name='fontawesome', symbol='file-image') - icon_dashboard_document_types = icon_document_type icon_dashboard_documents_in_trash = Icon( driver_name='fontawesome', symbol='trash-alt' diff --git a/mayan/apps/documents/links.py b/mayan/apps/documents/links.py index 98920e1dc1f..17ed67313e6 100644 --- a/mayan/apps/documents/links.py +++ b/mayan/apps/documents/links.py @@ -8,7 +8,7 @@ from mayan.apps.navigation.classes import Link from .icons import ( - icon_clear_image_cache, icon_document_list_recent_access, + icon_document_list_recent_access, icon_recent_added_document_list, icon_document_page_navigation_first, icon_document_page_navigation_last, icon_document_page_navigation_next, icon_document_page_navigation_previous, icon_document_page_return, @@ -264,16 +264,6 @@ def is_min_zoom(context): text=_('Trash can'), view='documents:document_list_deleted' ) -# Tools -link_clear_image_cache = Link( - icon_class=icon_clear_image_cache, - description=_( - 'Clear the graphics representations used to speed up the documents\' ' - 'display and interactive transformations results.' - ), permissions=(permission_document_tools,), - text=_('Clear document image cache'), - view='documents:document_clear_image_cache' -) link_trash_can_empty = Link( permissions=(permission_empty_trash,), text=_('Empty trash'), view='documents:trash_can_empty' diff --git a/mayan/apps/documents/queues.py b/mayan/apps/documents/queues.py index 57f4c4b32a1..5ac0fa5f011 100644 --- a/mayan/apps/documents/queues.py +++ b/mayan/apps/documents/queues.py @@ -61,10 +61,6 @@ schedule=timedelta(seconds=DELETE_STALE_STUBS_INTERVAL), ) -queue_tools.add_task_type( - dotted_path='mayan.apps.documents.tasks.task_clear_image_cache', - label=_('Clear image cache') -) queue_tools.add_task_type( dotted_path='mayan.apps.documents.tasks.task_scan_duplicates_all', label=_('Duplicated document scan') diff --git a/mayan/apps/documents/tasks.py b/mayan/apps/documents/tasks.py index 857eb38e661..98e08d1e250 100644 --- a/mayan/apps/documents/tasks.py +++ b/mayan/apps/documents/tasks.py @@ -41,17 +41,6 @@ def task_check_trash_periods(): DocumentType.objects.check_trash_periods() -@app.task(ignore_result=True) -def task_clear_image_cache(): - Document = apps.get_model( - app_label='documents', model_name='Document' - ) - - logger.info('Starting document cache invalidation') - Document.objects.invalidate_cache() - logger.info('Finished document cache invalidation') - - @app.task(ignore_result=True) def task_delete_document(trashed_document_id): DeletedDocument = apps.get_model( diff --git a/mayan/apps/documents/urls.py b/mayan/apps/documents/urls.py index 6d954eebb0e..e36181662e3 100644 --- a/mayan/apps/documents/urls.py +++ b/mayan/apps/documents/urls.py @@ -13,7 +13,7 @@ APIRecentDocumentListView ) from .views import ( - ClearImageCacheView, DocumentDocumentTypeEditView, DocumentDownloadFormView, + DocumentDocumentTypeEditView, DocumentDownloadFormView, DocumentDownloadView, DocumentDuplicatesListView, DocumentEditView, DocumentListView, DocumentPageListView, DocumentPageNavigationFirst, DocumentPageNavigationLast, DocumentPageNavigationNext, @@ -272,10 +272,6 @@ view=DocumentTransformationsClearView.as_view(), name='document_multiple_clear_transformations' ), - url( - regex=r'^cache/clear/$', view=ClearImageCacheView.as_view(), - name='document_clear_image_cache' - ), url( regex=r'^page/(?P\d+)/$', view=DocumentPageView.as_view(), name='document_page_view' diff --git a/mayan/apps/documents/views/misc_views.py b/mayan/apps/documents/views/misc_views.py index 0f2bf4fab8d..5500f53951b 100644 --- a/mayan/apps/documents/views/misc_views.py +++ b/mayan/apps/documents/views/misc_views.py @@ -8,25 +8,12 @@ from mayan.apps.common.generics import ConfirmView from ..permissions import permission_document_tools -from ..tasks import task_clear_image_cache, task_scan_duplicates_all +from ..tasks import task_scan_duplicates_all -__all__ = ('ClearImageCacheView', 'ScanDuplicatedDocuments') +__all__ = ('ScanDuplicatedDocuments',) logger = logging.getLogger(__name__) -class ClearImageCacheView(ConfirmView): - extra_context = { - 'title': _('Clear the document image cache?') - } - view_permission = permission_document_tools - - def view_action(self): - task_clear_image_cache.apply_async() - messages.success( - self.request, _('Document cache clearing queued successfully.') - ) - - class ScanDuplicatedDocuments(ConfirmView): extra_context = { 'title': _('Scan for duplicated documents?') diff --git a/mayan/apps/file_caching/apps.py b/mayan/apps/file_caching/apps.py index bb333be6e03..9d505c304b5 100644 --- a/mayan/apps/file_caching/apps.py +++ b/mayan/apps/file_caching/apps.py @@ -1,8 +1,79 @@ from __future__ import unicode_literals +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.acls.classes import ModelPermission +from mayan.apps.acls.links import link_acl_list +from mayan.apps.acls.permissions import permission_acl_edit, permission_acl_view from mayan.apps.common.apps import MayanAppConfig +from mayan.apps.common.menus import ( + menu_list_facet, menu_multi_item, menu_object, menu_secondary, menu_tools +) +from mayan.apps.events.classes import ModelEventType +from mayan.apps.events.links import ( + link_events_for_object, link_object_event_types_user_subcriptions_list +) +from mayan.apps.navigation.classes import SourceColumn + +from .events import event_cache_edited, event_cache_purged +from .links import ( + link_caches_list, link_cache_multiple_purge, link_cache_purge +) +from .permissions import permission_cache_purge, permission_cache_view class FileCachingConfig(MayanAppConfig): + app_namespace = 'file_caching' + app_url = 'file_caching' has_tests = False name = 'mayan.apps.file_caching' + verbose_name = _('File caching') + + def ready(self): + super(FileCachingConfig, self).ready() + from actstream import registry + + Cache = self.get_model(model_name='Cache') + + ModelEventType.register( + event_types=(event_cache_edited, event_cache_purged,), + model=Cache + ) + + ModelPermission.register( + model=Cache, permissions=( + permission_acl_edit, permission_acl_view, + permission_cache_purge, permission_cache_view + ) + ) + + SourceColumn(attribute='name', source=Cache) + SourceColumn(attribute='label', source=Cache) + SourceColumn(attribute='storage_instance_path', source=Cache) + SourceColumn(attribute='get_maximum_size_display', source=Cache) + SourceColumn(attribute='get_total_size_display', source=Cache) + + menu_list_facet.bind_links( + links=( + link_acl_list, link_events_for_object, + link_object_event_types_user_subcriptions_list, + ), sources=(Cache,) + ) + + menu_object.bind_links( + links=(link_cache_purge,), + sources=(Cache,) + ) + menu_multi_item.bind_links( + links=(link_cache_multiple_purge,), + sources=(Cache,) + ) + menu_secondary.bind_links( + links=(link_caches_list,), sources=( + Cache, + ) + ) + + menu_tools.bind_links(links=(link_caches_list,)) + + registry.register(Cache) diff --git a/mayan/apps/file_caching/events.py b/mayan/apps/file_caching/events.py new file mode 100644 index 00000000000..529710d64b2 --- /dev/null +++ b/mayan/apps/file_caching/events.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.events.classes import EventTypeNamespace + +namespace = EventTypeNamespace( + label=_('File caching'), name='file_caching' +) + +event_cache_created = namespace.add_event_type( + label=_('Cache created'), name='cache_created' +) +event_cache_edited = namespace.add_event_type( + label=_('Cache edited'), name='cache_edited' +) +event_cache_purged = namespace.add_event_type( + label=_('Cache purge'), name='cache_purged' +) diff --git a/mayan/apps/file_caching/icons.py b/mayan/apps/file_caching/icons.py new file mode 100644 index 00000000000..94b42874565 --- /dev/null +++ b/mayan/apps/file_caching/icons.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, unicode_literals + +from mayan.apps.appearance.classes import Icon + +icon_file_caching = Icon( + driver_name='fontawesome', symbol='warehouse' +) +icon_cache_purge = Icon( + driver_name='fontawesome-dual', primary_symbol='warehouse', + secondary_symbol='check' +) diff --git a/mayan/apps/file_caching/links.py b/mayan/apps/file_caching/links.py new file mode 100644 index 00000000000..b0bc57cae40 --- /dev/null +++ b/mayan/apps/file_caching/links.py @@ -0,0 +1,22 @@ +from __future__ import unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.navigation.classes import Link + +from .icons import icon_cache_purge, icon_file_caching +from .permissions import permission_cache_purge, permission_cache_view + +link_caches_list = Link( + icon_class=icon_file_caching, permissions=(permission_cache_view,), + text=_('File caches'), view='file_caching:cache_list' +) +link_cache_purge = Link( + icon_class=icon_cache_purge, kwargs={'cache_id': 'resolved_object.id'}, + permissions=(permission_cache_purge,), text=_('Purge cache'), + view='file_caching:cache_purge' +) +link_cache_multiple_purge = Link( + icon_class=icon_cache_purge, text=_('Purge cache'), + view='file_caching:cache_multiple_purge' +) diff --git a/mayan/apps/file_caching/models.py b/mayan/apps/file_caching/models.py index 6b8e6bb3ccc..8ea22ee4aae 100644 --- a/mayan/apps/file_caching/models.py +++ b/mayan/apps/file_caching/models.py @@ -6,6 +6,7 @@ from django.core.files.base import ContentFile from django.db import models, transaction from django.db.models import Sum +from django.template.defaultfilters import filesizeformat from django.utils.encoding import python_2_unicode_compatible from django.utils.functional import cached_property from django.utils.module_loading import import_string @@ -14,18 +15,31 @@ from mayan.apps.lock_manager.exceptions import LockError from mayan.apps.lock_manager.runtime import locking_backend +from .events import ( + event_cache_created, event_cache_edited, event_cache_purged +) + logger = logging.getLogger(__name__) @python_2_unicode_compatible class Cache(models.Model): name = models.CharField( - max_length=128, unique=True, verbose_name=_('Name') + help_text=_('Internal name of the cache.'), max_length=128, + unique=True, verbose_name=_('Name') + ) + label = models.CharField( + help_text=_('A short text describing the cache.'), max_length=128, + verbose_name=_('Label') + ) + maximum_size = models.PositiveIntegerField( + help_text=_('Maximum size of the cache in bytes.'), + verbose_name=_('Maximum size') ) - label = models.CharField(max_length=128, verbose_name=_('Label')) - maximum_size = models.PositiveIntegerField(verbose_name=_('Maximum size')) storage_instance_path = models.CharField( - max_length=255, unique=True, verbose_name=_('Storage instance path') + help_text=_( + 'Dotted path to the actual storage class used for the cache.' + ), max_length=255, unique=True, verbose_name=_('Storage instance path') ) class Meta: @@ -38,21 +52,55 @@ def __str__(self): def get_files(self): return CachePartitionFile.objects.filter(partition__cache__id=self.pk) + def get_maximum_size_display(self): + return filesizeformat(bytes_=self.maximum_size) + + get_maximum_size_display.short_description = _('Maximum size') + def get_total_size(self): + """ + Return the actual usage of the cache. + """ return self.get_files().aggregate( file_size__sum=Sum('file_size') )['file_size__sum'] or 0 + def get_total_size_display(self): + return filesizeformat(bytes_=self.get_total_size()) + + get_total_size_display.short_description = _('Total size') + def prune(self): + """ + Deletes files until the total size of the cache is below the allowed + maximum size of the cache. + """ while self.get_total_size() > self.maximum_size: self.get_files().earliest().delete() - def purge(self): + def purge(self, _user=None): + """ + Deletes the entire cache. + """ for partition in self.partitions.all(): partition.purge() + event_cache_purged.commit(actor=_user, target=self) + def save(self, *args, **kwargs): - result = super(Cache, self).save(*args, **kwargs) + _user = kwargs.pop('_user', None) + with transaction.atomic(): + is_new = not self.pk + result = super(Cache, self).save(*args, **kwargs) + if is_new: + event_cache_created.commit( + actor=_user, target=self + ) + else: + event_cache_edited.commit( + actor=_user, target=self + ) + self.prune() return result diff --git a/mayan/apps/file_caching/permissions.py b/mayan/apps/file_caching/permissions.py new file mode 100644 index 00000000000..4c2609ee109 --- /dev/null +++ b/mayan/apps/file_caching/permissions.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.permissions import PermissionNamespace + +namespace = PermissionNamespace(label=_('File caching'), name='file_caching') + +permission_cache_purge = namespace.add_permission( + label=_('Purge a file cache'), name='file_caching_cache_purge' +) +permission_cache_view = namespace.add_permission( + label=_('View a file cache'), name='file_caching_cache_view' +) diff --git a/mayan/apps/file_caching/queues.py b/mayan/apps/file_caching/queues.py new file mode 100644 index 00000000000..77d8af40cb4 --- /dev/null +++ b/mayan/apps/file_caching/queues.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ + +from mayan.apps.common.queues import queue_tools + +queue_tools.add_task_type( + dotted_path='mayan.apps.file_caching.tasks.task_cache_purge', + label=_('Purge a file cache') +) diff --git a/mayan/apps/file_caching/tasks.py b/mayan/apps/file_caching/tasks.py new file mode 100644 index 00000000000..06696a1c9db --- /dev/null +++ b/mayan/apps/file_caching/tasks.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + +import logging + +from django.apps import apps +from django.contrib.auth import get_user_model + +from mayan.celery import app + +logger = logging.getLogger(__name__) + + +@app.task(ignore_result=True) +def task_cache_purge(cache_id, user_id=None): + Cache = apps.get_model( + app_label='file_caching', model_name='Cache' + ) + User = get_user_model() + + cache = Cache.objects.get(pk=cache_id) + user = User.objects.get(pk=user_id) + + logger.info('Starting cache id %s purge', cache) + cache.purge(_user=user) + logger.info('Finished cache id %s purge', cache) diff --git a/mayan/apps/file_caching/urls.py b/mayan/apps/file_caching/urls.py new file mode 100644 index 00000000000..4ce37409f6f --- /dev/null +++ b/mayan/apps/file_caching/urls.py @@ -0,0 +1,20 @@ +from __future__ import unicode_literals + +from django.conf.urls import url + +from .views import CacheListView, CachePurgeView + +urlpatterns = [ + url( + regex=r'^caches/$', + name='cache_list', view=CacheListView.as_view() + ), + url( + regex=r'^caches/(?P\d+)/purge/$', + name='cache_purge', view=CachePurgeView.as_view() + ), + url( + regex=r'^caches/multiple/purge/$', name='cache_multiple_purge', + view=CachePurgeView.as_view() + ), +] diff --git a/mayan/apps/file_caching/views.py b/mayan/apps/file_caching/views.py new file mode 100644 index 00000000000..bf0d86ef7ef --- /dev/null +++ b/mayan/apps/file_caching/views.py @@ -0,0 +1,53 @@ +from __future__ import absolute_import, unicode_literals + +from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ungettext + +from mayan.apps.common.generics import ( + MultipleObjectConfirmActionView, SingleObjectListView +) + +from .models import Cache +from .permissions import permission_cache_purge, permission_cache_view + +from .tasks import task_cache_purge + + +class CacheListView(SingleObjectListView): + model = Cache + permission = permission_cache_view + + def get_extra_context(self): + return { + 'hide_object': True, + 'title': _('File caches list') + } + + +class CachePurgeView(MultipleObjectConfirmActionView): + model = Cache + object_permission = permission_cache_purge + pk_url_kwarg = 'cache_id' + success_message_singular = '%(count)d cache submitted for purging.' + success_message_plural = '%(count)d caches submitted for purging.' + + def get_extra_context(self): + queryset = self.object_list + + result = { + 'title': ungettext( + singular='Submit the selected cache for purging?', + plural='Submit the selected caches for purging?', + number=queryset.count() + ) + } + + if queryset.count() == 1: + result['object'] = queryset.first() + + return result + + def object_action(self, form, instance): + task_cache_purge.apply_async( + kwargs={'cache_id': instance.pk, 'user_id': self.request.user.pk} + ) From 6635bb42352ea6dc67a9a39f987f1c44949acfc6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Thu, 25 Jul 2019 20:36:47 -0400 Subject: [PATCH 097/402] Tweak CSS to unify widths in plain template Signed-off-by: Roberto Rosario --- mayan/apps/appearance/static/appearance/css/base.css | 5 +++++ mayan/apps/appearance/templates/appearance/base_plain.html | 2 +- .../apps/authentication/templates/authentication/login.html | 2 +- mayan/apps/autoadmin/templates/autoadmin/credentials.html | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mayan/apps/appearance/static/appearance/css/base.css b/mayan/apps/appearance/static/appearance/css/base.css index bed433804ed..008fb218fb0 100644 --- a/mayan/apps/appearance/static/appearance/css/base.css +++ b/mayan/apps/appearance/static/appearance/css/base.css @@ -552,3 +552,8 @@ a i { padding-right: 15px; padding-top: 8px; } + +#body-plain { + padding-top: 0px; + margin-top: 10px; +} diff --git a/mayan/apps/appearance/templates/appearance/base_plain.html b/mayan/apps/appearance/templates/appearance/base_plain.html index 0496426dad4..5475f564f4a 100644 --- a/mayan/apps/appearance/templates/appearance/base_plain.html +++ b/mayan/apps/appearance/templates/appearance/base_plain.html @@ -33,7 +33,7 @@ } - + {% block content_plain %}{% endblock %} diff --git a/mayan/apps/authentication/templates/authentication/login.html b/mayan/apps/authentication/templates/authentication/login.html index dcbe3ed42dc..9398db0f375 100644 --- a/mayan/apps/authentication/templates/authentication/login.html +++ b/mayan/apps/authentication/templates/authentication/login.html @@ -17,7 +17,7 @@ {% motd %}
    -
    +

     

    diff --git a/mayan/apps/autoadmin/templates/autoadmin/credentials.html b/mayan/apps/autoadmin/templates/autoadmin/credentials.html index 7e89a5be15c..73490c86922 100755 --- a/mayan/apps/autoadmin/templates/autoadmin/credentials.html +++ b/mayan/apps/autoadmin/templates/autoadmin/credentials.html @@ -4,7 +4,7 @@ {% if autoadmin_properties.account %}
    -
    +

    From 669dfeb30a570999f1e37d768f1dbf73fb2c29e7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 01:21:01 -0400 Subject: [PATCH 098/402] Use common app serialization util Signed-off-by: Roberto Rosario --- mayan/apps/document_states/models.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mayan/apps/document_states/models.py b/mayan/apps/document_states/models.py index d3d41038146..d7fab264a71 100644 --- a/mayan/apps/document_states/models.py +++ b/mayan/apps/document_states/models.py @@ -6,11 +6,6 @@ from furl import furl from graphviz import Digraph -import yaml -try: - from yaml import CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeLoader from django.conf import settings from django.core import serializers @@ -24,6 +19,7 @@ from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.models import AccessControlList +from mayan.apps.common.serialization import yaml_load from mayan.apps.common.validators import YAMLValidator, validate_internal_name from mayan.apps.documents.models import Document, DocumentType from mayan.apps.documents.permissions import permission_document_view @@ -465,7 +461,7 @@ def __str__(self): return self.label def get_widget_kwargs(self): - return yaml.load(stream=self.widget_kwargs, Loader=SafeLoader) + return yaml_load(stream=self.widget_kwargs) @python_2_unicode_compatible From ff6674cc4a9de4bdc1aa2957015a057e4d255d1f Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 01:24:55 -0400 Subject: [PATCH 099/402] Fix workflow preview under Python 3 Signed-off-by: Roberto Rosario --- mayan/apps/document_states/models.py | 8 ++++++-- mayan/apps/document_states/tests/test_models.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 mayan/apps/document_states/tests/test_models.py diff --git a/mayan/apps/document_states/models.py b/mayan/apps/document_states/models.py index d7fab264a71..a8538c4a1b5 100644 --- a/mayan/apps/document_states/models.py +++ b/mayan/apps/document_states/models.py @@ -14,7 +14,9 @@ from django.db import IntegrityError, models, transaction from django.db.models import F, Max, Q from django.urls import reverse -from django.utils.encoding import force_text, python_2_unicode_compatible +from django.utils.encoding import ( + force_bytes, force_text, python_2_unicode_compatible +) from django.utils.module_loading import import_string from django.utils.translation import ugettext_lazy as _ @@ -110,7 +112,9 @@ def get_hash(self): ) return hashlib.sha256( - serializers.serialize('json', objects_lists) + force_bytes( + serializers.serialize('json', objects_lists) + ) ).hexdigest() def get_initial_state(self): diff --git a/mayan/apps/document_states/tests/test_models.py b/mayan/apps/document_states/tests/test_models.py new file mode 100644 index 00000000000..ebe512f2448 --- /dev/null +++ b/mayan/apps/document_states/tests/test_models.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from mayan.apps.common.tests import BaseTestCase + +from .mixins import WorkflowTestMixin + + +class WorkflowModelTestCase(WorkflowTestMixin, BaseTestCase): + def test_workflow_template_preview(self): + self._create_test_workflow() + self.assertTrue(self.test_workflow.get_api_image_url()) From c2e99e6efbb2403dbf26779761a1cda5adcbcde6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 01:33:14 -0400 Subject: [PATCH 100/402] Purge cache partition before deleting them Signed-off-by: Roberto Rosario --- mayan/apps/file_caching/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mayan/apps/file_caching/models.py b/mayan/apps/file_caching/models.py index 8ea22ee4aae..64f8513563c 100644 --- a/mayan/apps/file_caching/models.py +++ b/mayan/apps/file_caching/models.py @@ -167,6 +167,10 @@ def create_file(self, filename): logger.debug('unable to obtain lock: %s' % lock_id) raise + def delete(self, *args, **kwargs): + self.purge() + return super(CachePartition, self).delete(*args, **kwargs) + def get_file(self, filename): try: return self.files.get(filename=filename) From f920dffc0193d400b447e0ff65cd0bdff4d542b7 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 01:33:41 -0400 Subject: [PATCH 101/402] Remove document model cache invalidation The cache invalidation is now handled by the file caching app. Signed-off-by: Roberto Rosario --- mayan/apps/documents/managers.py | 4 ---- mayan/apps/documents/models/document_models.py | 4 ---- mayan/apps/documents/models/document_page_models.py | 5 +---- mayan/apps/documents/models/document_version_models.py | 6 +----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/mayan/apps/documents/managers.py b/mayan/apps/documents/managers.py index b047b9d5e94..54e512ca712 100644 --- a/mayan/apps/documents/managers.py +++ b/mayan/apps/documents/managers.py @@ -25,10 +25,6 @@ def get_queryset(self): self.model, using=self._db ).filter(in_trash=False).filter(is_stub=False) - def invalidate_cache(self): - for document in self.model.objects.all(): - document.invalidate_cache() - class DocumentPageCachedImage(models.Manager): def get_by_natural_key(self, filename, document_page_natural_key): diff --git a/mayan/apps/documents/models/document_models.py b/mayan/apps/documents/models/document_models.py index 9f205226c88..0934a95f022 100644 --- a/mayan/apps/documents/models/document_models.py +++ b/mayan/apps/documents/models/document_models.py @@ -136,10 +136,6 @@ def get_api_image_url(self, *args, **kwargs): if latest_version: return latest_version.get_api_image_url(*args, **kwargs) - def invalidate_cache(self): - for document_version in self.versions.all(): - document_version.invalidate_cache() - @property def is_in_trash(self): return self.in_trash diff --git a/mayan/apps/documents/models/document_page_models.py b/mayan/apps/documents/models/document_page_models.py index 3461d1f1527..b4fa0ea1df4 100644 --- a/mayan/apps/documents/models/document_page_models.py +++ b/mayan/apps/documents/models/document_page_models.py @@ -64,7 +64,7 @@ def cache_partition(self): return partition def delete(self, *args, **kwargs): - self.invalidate_cache() + self.cache_partition.delete() super(DocumentPage, self).delete(*args, **kwargs) def detect_orientation(self): @@ -234,9 +234,6 @@ def get_image(self, transformations=None): ) raise - def invalidate_cache(self): - self.cache_partition.purge() - @property def is_in_trash(self): return self.document.is_in_trash diff --git a/mayan/apps/documents/models/document_version_models.py b/mayan/apps/documents/models/document_version_models.py index 1428f9e6156..be47d6329b1 100644 --- a/mayan/apps/documents/models/document_version_models.py +++ b/mayan/apps/documents/models/document_version_models.py @@ -139,6 +139,7 @@ def delete(self, *args, **kwargs): page.delete() self.file.storage.delete(self.file.name) + self.cache_partition.delete() return super(DocumentVersion, self).delete(*args, **kwargs) @@ -225,11 +226,6 @@ def natural_key(self): return (self.checksum, self.document.natural_key()) natural_key.dependencies = ['documents.Document'] - def invalidate_cache(self): - self.cache_partition.purge() - for page in self.pages.all(): - page.invalidate_cache() - @property def is_in_trash(self): return self.document.is_in_trash From 93ba547350c9135a148622e59f274ba4e140fcd6 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 02:22:04 -0400 Subject: [PATCH 102/402] Convert workflow previews app to use file caching Signed-off-by: Roberto Rosario --- mayan/apps/document_states/api_views.py | 4 +-- mayan/apps/document_states/apps.py | 9 +++-- mayan/apps/document_states/handlers.py | 16 +++++++++ mayan/apps/document_states/literals.py | 4 +++ mayan/apps/document_states/models.py | 47 ++++++++++++++++++------- mayan/apps/document_states/settings.py | 12 +++++++ mayan/apps/document_states/utils.py | 12 +++++++ 7 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 mayan/apps/document_states/utils.py diff --git a/mayan/apps/document_states/api_views.py b/mayan/apps/document_states/api_views.py index 90df8863cb3..29ba5096676 100644 --- a/mayan/apps/document_states/api_views.py +++ b/mayan/apps/document_states/api_views.py @@ -27,7 +27,6 @@ ) from .settings import settings_workflow_image_cache_time -from .storages import storage_workflowimagecache from .tasks import task_generate_workflow_image @@ -204,7 +203,8 @@ def retrieve(self, request, *args, **kwargs): ) cache_filename = task.get(timeout=WORKFLOW_IMAGE_TASK_TIMEOUT) - with storage_workflowimagecache.open(cache_filename) as file_object: + cache_file = self.get_object().cache_partition.get_file(filename=cache_filename) + with cache_file.open() as file_object: response = HttpResponse(file_object.read(), content_type='image') if '_hash' in request.GET: patch_cache_control( diff --git a/mayan/apps/document_states/apps.py b/mayan/apps/document_states/apps.py index dcbcf3e574b..2d46c6bde21 100644 --- a/mayan/apps/document_states/apps.py +++ b/mayan/apps/document_states/apps.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from django.apps import apps -from django.db.models.signals import post_save +from django.db.models.signals import post_migrate, post_save from django.utils.translation import ugettext_lazy as _ from mayan.apps.acls.classes import ModelPermission @@ -25,7 +25,8 @@ from .events import event_workflow_created, event_workflow_edited from .dependencies import * # NOQA from .handlers import ( - handler_index_document, handler_launch_workflow, handler_trigger_transition + handler_create_workflow_image_cache, handler_index_document, + handler_launch_workflow, handler_trigger_transition ) from .html_widgets import WorkflowLogExtraDataWidget, widget_transition_events from .links import ( @@ -452,6 +453,10 @@ def ready(self): # Index updating + post_migrate.connect( + dispatch_uid='workflows_handler_create_workflow_image_cache', + receiver=handler_create_workflow_image_cache, + ) post_save.connect( dispatch_uid='workflows_handler_index_document_save', receiver=handler_index_document, diff --git a/mayan/apps/document_states/handlers.py b/mayan/apps/document_states/handlers.py index a0e3ee68990..8fc226b59b0 100644 --- a/mayan/apps/document_states/handlers.py +++ b/mayan/apps/document_states/handlers.py @@ -6,6 +6,22 @@ from mayan.apps.document_indexing.tasks import task_index_document from mayan.apps.events.classes import EventType +from .literals import ( + WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH, WORKFLOW_IMAGE_CACHE_NAME +) +from .settings import setting_workflow_image_cache_maximum_size + + +def handler_create_workflow_image_cache(sender, **kwargs): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + Cache.objects.update_or_create( + defaults={ + 'label': _('Workflow images'), + 'storage_instance_path': WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH, + 'maximum_size': setting_workflow_image_cache_maximum_size.value, + }, name=WORKFLOW_IMAGE_CACHE_NAME, + ) + def handler_index_document(sender, **kwargs): task_index_document.apply_async( diff --git a/mayan/apps/document_states/literals.py b/mayan/apps/document_states/literals.py index ad6906fd3bb..ee4d5a085ef 100644 --- a/mayan/apps/document_states/literals.py +++ b/mayan/apps/document_states/literals.py @@ -2,6 +2,8 @@ from django.utils.translation import ugettext_lazy as _ +DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE = 50 * 2 ** 20 # 50 Megabytes + FIELD_TYPE_CHOICE_CHAR = 1 FIELD_TYPE_CHOICE_INTEGER = 2 FIELD_TYPE_CHOICES = ( @@ -30,4 +32,6 @@ (WORKFLOW_ACTION_ON_ENTRY, _('On entry')), (WORKFLOW_ACTION_ON_EXIT, _('On exit')), ) +WORKFLOW_IMAGE_CACHE_NAME = 'workflow_images' +WORKFLOW_IMAGE_CACHE_STORAGE_INSTANCE_PATH = 'mayan.apps.document_states.storages.storage_workflowimagecache' WORKFLOW_IMAGE_TASK_TIMEOUT = 60 diff --git a/mayan/apps/document_states/models.py b/mayan/apps/document_states/models.py index a8538c4a1b5..9deb97d96aa 100644 --- a/mayan/apps/document_states/models.py +++ b/mayan/apps/document_states/models.py @@ -7,16 +7,17 @@ from furl import furl from graphviz import Digraph +from django.apps import apps from django.conf import settings from django.core import serializers from django.core.exceptions import PermissionDenied, ValidationError -from django.core.files.base import ContentFile from django.db import IntegrityError, models, transaction from django.db.models import F, Max, Q from django.urls import reverse from django.utils.encoding import ( force_bytes, force_text, python_2_unicode_compatible ) +from django.utils.functional import cached_property from django.utils.module_loading import import_string from django.utils.translation import ugettext_lazy as _ @@ -31,11 +32,11 @@ from .events import event_workflow_created, event_workflow_edited from .literals import ( FIELD_TYPE_CHOICES, WIDGET_CLASS_CHOICES, WORKFLOW_ACTION_WHEN_CHOICES, - WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT + WORKFLOW_ACTION_ON_ENTRY, WORKFLOW_ACTION_ON_EXIT, + WORKFLOW_IMAGE_CACHE_NAME ) from .managers import WorkflowManager from .permissions import permission_workflow_transition -from .storages import storage_workflowimagecache logger = logging.getLogger(__name__) @@ -72,19 +73,37 @@ class Meta: def __str__(self): return self.label + @cached_property + def cache(self): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + return Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME) + + @cached_property + def cache_partition(self): + partition, created = self.cache.partitions.get_or_create( + name='{}'.format(self.pk) + ) + return partition + + def delete(self, *args, **kwargs): + self.cache_partition.delete() + return super(Workflow, self).delete(*args, **kwargs) + def generate_image(self): - cache_filename = '{}-{}'.format(self.id, self.get_hash()) - image = self.render() - - # Since open "wb+" doesn't create files, check if the file - # exists, if not then create it - if not storage_workflowimagecache.exists(cache_filename): - storage_workflowimagecache.save( - name=cache_filename, content=ContentFile(content='') + cache_filename = '{}'.format(self.get_hash()) + + if self.cache_partition.get_file(filename=cache_filename): + logger.debug( + 'workflow cache file "%s" found', cache_filename + ) + else: + logger.debug( + 'workflow cache file "%s" not found', cache_filename ) - with storage_workflowimagecache.open(cache_filename, mode='wb+') as file_object: - file_object.write(image) + image = self.render() + with self.cache_partition.create_file(filename=cache_filename) as file_object: + file_object.write(image) return cache_filename @@ -107,6 +126,8 @@ def get_hash(self): Workflow.objects.filter(pk=self.pk) ) + list( WorkflowState.objects.filter(workflow__pk=self.pk) + ) + list( + WorkflowStateAction.objects.filter(state__workflow__pk=self.pk) ) + list( WorkflowTransition.objects.filter(workflow__pk=self.pk) ) diff --git a/mayan/apps/document_states/settings.py b/mayan/apps/document_states/settings.py index 26298fbdf92..657754856be 100644 --- a/mayan/apps/document_states/settings.py +++ b/mayan/apps/document_states/settings.py @@ -7,8 +7,20 @@ from mayan.apps.smart_settings.classes import Namespace +from .literals import DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE +from .utils import callback_update_workflow_image_cache_size + namespace = Namespace(label=_('Workflows'), name='document_states') +setting_workflow_image_cache_maximum_size = namespace.add_setting( + global_name='WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE', + default=DEFAULT_WORKFLOW_IMAGE_CACHE_MAXIMUM_SIZE, + help_text=_( + 'The threshold at which the WORKFLOW_IMAGE_CACHE_STORAGE_BACKEND will ' + 'start deleting the oldest workflow image cache files. Specify the ' + 'size in bytes.' + ), post_edit_function=callback_update_workflow_image_cache_size +) settings_workflow_image_cache_time = namespace.add_setting( global_name='WORKFLOWS_IMAGE_CACHE_TIME', default='31556926', help_text=_( diff --git a/mayan/apps/document_states/utils.py b/mayan/apps/document_states/utils.py new file mode 100644 index 00000000000..252ffd51658 --- /dev/null +++ b/mayan/apps/document_states/utils.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals + +from django.apps import apps + +from .literals import WORKFLOW_IMAGE_CACHE_NAME + + +def callback_update_workflow_image_cache_size(setting): + Cache = apps.get_model(app_label='file_caching', model_name='Cache') + cache = Cache.objects.get(name=WORKFLOW_IMAGE_CACHE_NAME) + cache.maximum_size = setting.value + cache.save() From 150c5d8cc29d8e2d888f17c67a0ed232d4d264b4 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 02:22:34 -0400 Subject: [PATCH 103/402] Make cache columns sortable Signed-off-by: Roberto Rosario --- mayan/apps/file_caching/apps.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mayan/apps/file_caching/apps.py b/mayan/apps/file_caching/apps.py index 9d505c304b5..a4eaa248b78 100644 --- a/mayan/apps/file_caching/apps.py +++ b/mayan/apps/file_caching/apps.py @@ -47,10 +47,15 @@ def ready(self): ) ) - SourceColumn(attribute='name', source=Cache) - SourceColumn(attribute='label', source=Cache) - SourceColumn(attribute='storage_instance_path', source=Cache) - SourceColumn(attribute='get_maximum_size_display', source=Cache) + SourceColumn(attribute='name', is_sortable=True, source=Cache) + SourceColumn(attribute='label', is_sortable=True, source=Cache) + SourceColumn( + attribute='storage_instance_path', is_sortable=True, source=Cache + ) + SourceColumn( + attribute='get_maximum_size_display', is_sortable=True, + sort_field='maximum_size', source=Cache + ) SourceColumn(attribute='get_total_size_display', source=Cache) menu_list_facet.bind_links( From 3a7025d9c4b040f8ed008080cee1c788bc9ef09a Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 02:22:50 -0400 Subject: [PATCH 104/402] Add exists method to cache file model Signed-off-by: Roberto Rosario --- mayan/apps/file_caching/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mayan/apps/file_caching/models.py b/mayan/apps/file_caching/models.py index 64f8513563c..24e573fc3c2 100644 --- a/mayan/apps/file_caching/models.py +++ b/mayan/apps/file_caching/models.py @@ -210,6 +210,9 @@ def delete(self, *args, **kwargs): self.partition.cache.storage.delete(name=self.full_filename) return super(CachePartitionFile, self).delete(*args, **kwargs) + def exists(self): + return self.partition.cache.storage.exists(name=self.full_filename) + @cached_property def full_filename(self): return CachePartition.get_combined_filename( From 88863fd6d0eee9047c5dca2c9d8268373dfcdd26 Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 02:23:09 -0400 Subject: [PATCH 105/402] Fix typo in Cache get_model Signed-off-by: Roberto Rosario --- mayan/apps/documents/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mayan/apps/documents/utils.py b/mayan/apps/documents/utils.py index 7a7e16d4554..e9ae97a24b6 100644 --- a/mayan/apps/documents/utils.py +++ b/mayan/apps/documents/utils.py @@ -9,7 +9,7 @@ def callback_update_cache_size(setting): - Cache = apps.get_model(app_label='common', model_name='Cache') + Cache = apps.get_model(app_label='file_caching', model_name='Cache') cache = Cache.objects.get(name=DOCUMENT_IMAGES_CACHE_NAME) cache.maximum_size = setting.value cache.save() From fd0d5728a1ed728fb7eaa6b166ef5e795fe0472b Mon Sep 17 00:00:00 2001 From: Roberto Rosario Date: Fri, 26 Jul 2019 02:34:01 -0400 Subject: [PATCH 106/402] Improve toolbar display logic Signed-off-by: Roberto Rosario --- .../templates/appearance/list_toolbar.html | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/mayan/apps/appearance/templates/appearance/list_toolbar.html b/mayan/apps/appearance/templates/appearance/list_toolbar.html index d73de69556c..72a4148390e 100644 --- a/mayan/apps/appearance/templates/appearance/list_toolbar.html +++ b/mayan/apps/appearance/templates/appearance/list_toolbar.html @@ -7,53 +7,51 @@
    {% endif %} -
    -