From a877b16c0363245b95ba7ea2d158c4ceb2893b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Horv=C3=A1th?= Date: Sat, 16 Sep 2023 09:01:20 +0200 Subject: [PATCH] Preview generator setup (#7) * Preview generator setup Enable the variable nextcloud_preview_enabled and you are good to go. Some important aspects of usage: - the preview generator has two stages [according to their readme](https://github.com/nextcloud/previewgenerator) - a generate-all phase, which has to be executed only a single time - a pre-generate phase, that should be run in a cronjob. That runs quite fast if the generate-all phase finishd. We do not want to run the generate-all phase multiple times, so its execution has to be followed somehow. This is done by creating a file on the host side and both the task that executes generate-all and both the cronjob checks its existance. Multiple vaiables are also defined and the corresponding default values are also set. These values are based on the [upstream readme](https://github.com/nextcloud/previewgenerator) and also on experience. Feel free to change anything. Once installed, the playbook needs to be called with the adjust-nextcloud-config tag. This tag sets up the variables and calls the generate-all script, that will also create the file---signalling its finished state---on the host. *** As this may take a long time, be sure to only call it when you have time to leave it running!!! *** The playbook calls generate-all asynchronously, but it will timeout after about 27h. On 60GBs, most if images, it took about 10 minutes to finish. If it takes more time, you may want to start it from the host by calling ```sh /usr/bin/env docker exec mash-nextcloud-server php /var/www/html/occ preview:generate-all ``` If the nextcloud_preview_enabled value is set back to false, the host side files are cleaned up and also the cron job is changed, not to call prevew generation again however, the database and generated previews are kept intact. * fix playbook format suggestions by Slavi (see conversation in !7) --- defaults/main.yml | 44 ++++++++++++++ tasks/adjust_config.yml | 57 +++++++++++++++++++ tasks/install.yml | 12 ++++ .../preview-generator-cron.sh.j2 | 9 +++ templates/systemd/cron.service.j2 | 3 + 5 files changed, 125 insertions(+) create mode 100644 templates/preview-generator/preview-generator-cron.sh.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 3f80c33..dbefa89 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -36,6 +36,50 @@ nextcloud_customized_container_src_path: "{{ nextcloud_base_path }}/customized-c nextcloud_container_image_customizations_php_imageick_installation_enabled: false nextcloud_container_image_customizations_php_imageick_installation_package: "libmagickcore-6.q16-6-extra" +# Preview generator setup +# +# Enable the variable nextcloud_preview_enabled and you are good to go. +# +# Some important aspects of usage: +# - the preview generator has two stages [according to their readme](https://github.com/nextcloud/previewgenerator) +# - a generate-all phase, which has to be executed only a single time +# - a pre-generate phase, that should be run in a cronjob. +# That runs quite fast if the generate-all phase finishd. +# We do not want to run the generate-all phase multiple times, so its execution has to be followed somehow. +# This is done by creating a file on the host side and both the task that executes generate-all +# and both the cronjob checks its existance. +# +# Multiple vaiables are also defined and the corresponding default values are also set. +# These values are based on the [upstream readme](https://github.com/nextcloud/previewgenerator) and also on experience. +# Feel free to change anything. +# +# Once installed, the playbook needs to be called with the adjust-nextcloud-config tag. +# This tag sets up the variables and calls the generate-all script, that will also create the file---signalling +# its finished state---on the host. +# *** As this may take a long time, be sure to only call it when you have time to leave it running!!! *** +# The playbook calls generate-all asynchronously, but it will timeout after about 27h. +# On 60GBs, most if images, it took about 10 minutes to finish. +# If it takes more time, you may want to start it from the host by calling +# ```sh +# /usr/bin/env docker exec mash-nextcloud-server php /var/www/html/occ preview:generate-all +# ``` +# +# If the nextcloud_preview_enabled value is set back to false, the host side files are cleaned up +# and also the cron job is changed, not to call prevew generation again however, the database and generated +# previews are kept intact. +nextcloud_preview_enabled: false +nextcloud_preview_folder_name: "preview-generator" +nextcloud_preview_docker_folder: "/{{ nextcloud_preview_folder_name }}" +nextcloud_preview_host_folder: "{{ nextcloud_customized_container_src_path }}/{{ nextcloud_preview_folder_name }}" +nextcloud_preview_first_run_finished_filename: "finished-first-run.keepit" +nextcloud_preview_squareSizes: "\"64 256 1024 2048\"" +nextcloud_preview_widthSizes: "\"64 256 1024 2048\"" +nextcloud_preview_heightSizes: "\"64 256 1024 2048\"" +nextcloud_preview_preview_max_x: 2048 +nextcloud_preview_preview_max_y: 2048 +nextcloud_preview_system_jpeg_quality: 60 +nextcloud_preview_app_jpeg_quality: "60" + # nextcloud_container_image_customizations_enabled controls whether a customized Nextcloud image will be built. # # We toggle this variable to `true` when certain features which require a custom build are enabled. diff --git a/tasks/adjust_config.yml b/tasks/adjust_config.yml index bfa1286..fdbd196 100644 --- a/tasks/adjust_config.yml +++ b/tasks/adjust_config.yml @@ -33,3 +33,60 @@ ansible.builtin.command: cmd: |- docker exec --user={{ nextcloud_uid }}:{{ nextcloud_gid }} {{ nextcloud_identifier }}-server php /var/www/html/occ db:add-missing-indices + +- when: nextcloud_preview_enabled | bool + block: + - name: Adjust Nextcloud configuration values related to preview generator + ansible.builtin.command: + cmd: |- + docker exec --user={{ nextcloud_uid }}:{{ nextcloud_gid }} {{ nextcloud_identifier }}-server php /var/www/html/occ {{ item.appType }} {{ item.appConfig }} {{ item.configName }} --value={{ item.value }} + with_items: + - appType: "config:app:set" + appConfig: "previewgenerator" + configName: "squareSizes" + value: "{{ nextcloud_preview_squareSizes }}" + - appType: "config:app:set" + appConfig: "previewgenerator" + configName: "widthSizes" + value: "{{ nextcloud_preview_widthSizes }}" + - appType: "config:app:set" + appConfig: "previewgenerator" + configName: "heightSizes" + value: "{{ nextcloud_preview_heightSizes }}" + - appType: "config:system:set" + appConfig: "" + configName: "preview_max_x" + value: "{{ nextcloud_preview_preview_max_x }}" + - appType: "config:system:set" + appConfig: "" + configName: "preview_max_y" + value: "{{ nextcloud_preview_preview_max_y }}" + - appType: "config:system:set" + appConfig: "" + configName: "jpeg_quality" + value: "{{ nextcloud_preview_system_jpeg_quality }}" + - appType: "config:app:set" + appConfig: "preview" + configName: "jpeg_quality" + value: "{{ nextcloud_preview_app_jpeg_quality }}" + + - name: "Check if {{ nextcloud_preview_first_run_finished_filename }} exists" + ansible.builtin.stat: + path: "{{ nextcloud_preview_host_folder }}/{{ nextcloud_preview_first_run_finished_filename }}" + register: nextcloud_preview_file_stats + + - when: not nextcloud_preview_file_stats.stat.exists + name: Start preview generate-all + ansible.builtin.command: + cmd: |- + docker exec --user={{ nextcloud_uid }}:{{ nextcloud_gid }} {{ nextcloud_identifier }}-server php /var/www/html/occ preview:generate-all + async: 100000 + poll: 10 + + - name: "Create {{ nextcloud_preview_first_run_finished_filename } to signal, that preview generate-all is finished" + ansible.builtin.file: + path: "{{ nextcloud_preview_host_folder }}/{{ nextcloud_preview_first_run_finished_filename }}" + state: "touch" + mode: 0644 + owner: "{{ nextcloud_uid }}" + group: "{{ nextcloud_gid }}" diff --git a/tasks/install.yml b/tasks/install.yml index 45b40bb..b98e001 100644 --- a/tasks/install.yml +++ b/tasks/install.yml @@ -23,6 +23,9 @@ - path: "{{ nextcloud_redis_session_ini_path }}" state: "{{ 'touch' if nextcloud_redis_is_configured else 'absent' }}" when: true + - path: "{{ nextcloud_preview_host_folder }}" + state: "{{ 'directory' if nextcloud_preview_enabled else 'absent' }}" + when: true when: item.when | bool - name: Ensure Nextcloud support files installed @@ -34,6 +37,15 @@ - env - labels +- when: "nextcloud_preview_enabled | bool" + name: Ensure Nextcloud preview generator files are present + ansible.builtin.template: + src: "{{ role_path }}/templates/preview-generator/{{ item }}.j2" + dest: "{{ nextcloud_preview_host_folder }}/{{ item }}" + mode: 0740 + with_items: + - preview-generator-cron.sh + - name: Ensure Nextcloud container image is pulled community.docker.docker_image: name: "{{ nextcloud_container_image }}" diff --git a/templates/preview-generator/preview-generator-cron.sh.j2 b/templates/preview-generator/preview-generator-cron.sh.j2 new file mode 100644 index 0000000..a9f3be6 --- /dev/null +++ b/templates/preview-generator/preview-generator-cron.sh.j2 @@ -0,0 +1,9 @@ +#! /usr/bin/env bash + +if [ -f {{ nextcloud_preview_host_folder }}/{{ nextcloud_preview_first_run_finished_filename }} ] +then + echo "Preview scan is finished, regeneration starts" + docker exec --user={{ nextcloud_uid }}:{{ nextcloud_gid }} {{ nextcloud_identifier }}-server php /var/www/html/occ preview:pre-generate +else + echo "Preview scan hasn't finished, regeneration skipped" +fi diff --git a/templates/systemd/cron.service.j2 b/templates/systemd/cron.service.j2 index 2e17bdb..b1b45a5 100644 --- a/templates/systemd/cron.service.j2 +++ b/templates/systemd/cron.service.j2 @@ -4,3 +4,6 @@ Description=Runs the cronjobs For Nextcloud ({{ nextcloud_identifier }}) [Service] Type=oneshot ExecStart={{ devture_systemd_docker_base_host_command_docker }} exec {{ nextcloud_identifier }}-server php /var/www/html/cron.php +{% if nextcloud_preview_enabled %} +ExecStart={{ nextcloud_preview_host_folder }}/preview-generator-cron.sh +{% endif %}