Initial work on optimization commands

The playbook can now optimize itself based on the enabled components in
for all hosts in the inventory (`just optimize`) or for a specific host
(`just optimize-for-host HOSTNAME`).

The optimized playbook will have:

- fewer requirements (fewer roles need to be installed by `just roles`)
- a shorter and quicker to evaluate `group_vars/mash_servers` file
- a `setup.yml` file which includes less roles

Running the playbook optimized is still work in progress.
There still probably exist various role dependencies in the group-vars file, etc.

The `optimize-reset` command aims to restore your playbook to a
non-optimized state, which should work as before (and not experience bugs).

The playbook takes care to notice of changes to the various files in
`templates/` (`setup.yml`, `requirements.yml`, `group_vars_mash_servers`)
and update your optimized or non-optimized copies that are derived from
these templates. To do this, it keeps `.srchash` files in the `run/` directory.
When it notices a change in the source file's hash (by comparing to the `.srchash` file),
it will update you to the new template.

Optimization state is stored in a file in `run/` as well (`optimization-vars-files.state`).
Should the playbook notice changes in the source `template/` files, it
should update you and re-optimize using the same parameters as before (read from the state file).

.editorconfig

@@ -27,7 +27,7 @@ indent_size = 4
 indent_style = space
 indent_size = 2
 
-[group_vars/mash_servers_all]
+[templates/group_vars_mash_servers]
 indent_style = space
 indent_size = 2
 

.gitignore

@@ -11,10 +11,11 @@
 .DS_Store
 
 /requirements.yml
-/requirements.yml.srchash
-
 /setup.yml
-/setup.yml.srchash
-
 /group_vars/mash_servers
-/group_vars/mash_servers.srchash
+
+/run/*
+!/run/.gitkeep
+
+/group_vars/*
+!/group_vars/.gitkeep

bin/optimize.py

@@ -0,0 +1,161 @@
+import argparse
+import regex
+import sys
+import yaml
+
+parser = argparse.ArgumentParser(description='Optimizes the playbook based on enabled components found in vars.yml files')
+parser.add_argument('--vars-paths', help='Path to vars.yml configuration files to process', required=True)
+parser.add_argument('--src-requirements-yml-path', help='Path to source requirements.yml file with all role definitions', required=True)
+parser.add_argument('--src-setup-yml-path', help='Path to source setup.yml file', required=True)
+parser.add_argument('--src-group-vars-yml-path', help='Path to source group vars file', required=True)
+parser.add_argument('--dst-requirements-yml-path', help='Path to destination requirements.yml file, where role definitions will be saved', required=True)
+parser.add_argument('--dst-setup-yml-path', help='Path to destination setup.yml file', required=True)
+parser.add_argument('--dst-group-vars-yml-path', help='Path to destination group vars file', required=True)
+
+args = parser.parse_args()
+
+def load_combined_variable_names_from_files(vars_yml_file_paths):
+    variable_names = set({})
+    for vars_path in vars_yml_file_paths:
+        with open(vars_path, 'r') as file:
+            yaml_data = yaml.safe_load(file)
+
+            variable_names = variable_names | set(yaml_data.keys())
+    return variable_names
+
+def load_yaml_file(path):
+    with open(path, 'r') as file:
+        return yaml.safe_load(file)
+
+def is_role_definition_in_use(role_definition, used_variable_names):
+    for variable_name in used_variable_names:
+        if 'activation_prefix' in role_definition:
+            if role_definition['activation_prefix'] == '':
+                # Special value indicating "always activate".
+                # We don't really need this dedicated if, but it's more obvious with it.
+                return True
+            if variable_name.startswith(role_definition['activation_prefix']):
+                return True
+    return False
+
+def write_yaml_to_file(definitions, path):
+    with open(path, 'w') as file:
+        yaml.dump(definitions, file)
+
+def read_file(path):
+    with open(path, 'r') as file:
+        return file.read()
+
+def write_to_file(contents, path):
+    with open(path, 'w') as file:
+        file.write(contents)
+
+# Matches the beginning of role-specific blocks.
+# Example: `# role-specific:playbook_help`
+regex_role_specific_block_start = regex.compile('^\s*#\s*role-specific\:\s*([^\s]+)$')
+
+# Matches the end of role-specific blocks.
+# Example: `# /role-specific:playbook_help`
+regex_role_specific_block_end = regex.compile('^\s*#\s*/role-specific\:\s*([^\s]+)$')
+
+def process_file_contents(file_name, enabled_role_names, known_role_names):
+    contents = read_file(file_name)
+
+    lines_preserved = []
+    role_specific_stack = []
+
+    for line_number, line in enumerate(contents.split("\n")):
+        # Stage 1: looking for a role-specific starting block
+        start_role_matches = regex_role_specific_block_start.match(line)
+        if start_role_matches is not None:
+            role_name = start_role_matches.group(1)
+            if role_name not in known_role_names:
+                raise Exception('Found start block for role {0} on line {1} in file {2}, but it is not a known role name found among: {3}'.format(
+                    role_name,
+                    line_number,
+                    file_name,
+                    known_role_names,
+                ))
+            role_specific_stack.append(role_name)
+            continue
+
+        # Stage 2: looking for role-specific closing blocks
+        end_role_matches = regex_role_specific_block_end.match(line)
+        if end_role_matches is not None:
+            role_name = end_role_matches.group(1)
+            if role_name not in known_role_names:
+                raise Exception('Found end block for role {0} on line {1} in file {2}, but it is not a known role name found among: {3}'.format(
+                    role_name,
+                    line_number,
+                    file_name,
+                    known_role_names,
+                ))
+
+            if len(role_specific_stack) == 0:
+                raise Exception('Found end block for role {0} on line {1} in file {2}, but there is no opening statement for it'.format(
+                    role_name,
+                    line_number,
+                    file_name,
+                ))
+
+            last_role_name = role_specific_stack[len(role_specific_stack) - 1]
+            if role_name != last_role_name:
+                raise Exception('Found end block for role {0} on line {1} in file {2}, but the last starting block was for role {3}'.format(
+                    role_name,
+                    line_number,
+                    file_name,
+                    last_role_name,
+                ))
+
+            role_specific_stack.pop()
+
+            continue
+
+        # Stage 3: regular line
+        all_roles_allowed = True
+        for role_name in role_specific_stack:
+            if role_name not in enabled_role_names:
+                all_roles_allowed = False
+                break
+
+        if all_roles_allowed:
+            lines_preserved.append(line)
+
+    if len(role_specific_stack) != 0:
+        raise Exception('Expected one or more closing block for role-specific tags in file {0}: {1}'.format(file_name, role_specific_stack))
+
+    lines_final = []
+    sequential_blank_lines_count = 0
+    for line in lines_preserved:
+        if line != "":
+            lines_final.append(line)
+            sequential_blank_lines_count = 0
+            continue
+
+        if sequential_blank_lines_count <= 1:
+            lines_final.append(line)
+            sequential_blank_lines_count += 1
+            continue
+
+    return "\n".join(lines_final)
+
+vars_paths = args.vars_paths.split(' ')
+used_variable_names = load_combined_variable_names_from_files(vars_paths)
+
+all_role_definitions = load_yaml_file(args.src_requirements_yml_path)
+
+enabled_role_definitions = []
+for role_definition in all_role_definitions:
+    if is_role_definition_in_use(role_definition, used_variable_names):
+        enabled_role_definitions.append(role_definition)
+
+write_yaml_to_file(enabled_role_definitions, args.dst_requirements_yml_path)
+
+known_role_names = tuple(map(lambda definition: definition['name'], all_role_definitions))
+enabled_role_names = tuple(map(lambda definition: definition['name'], enabled_role_definitions))
+
+setup_yml_processed = process_file_contents(args.src_setup_yml_path, enabled_role_names, known_role_names)
+write_to_file(setup_yml_processed, args.dst_setup_yml_path)
+
+group_vars_yml_processed = process_file_contents(args.src_group_vars_yml_path, enabled_role_names, known_role_names)
+write_to_file(group_vars_yml_processed, args.dst_group_vars_yml_path)

justfile

@@ -2,19 +2,75 @@
 default:
     @just --list --justfile {{ justfile() }}
 
+run_directory_path := justfile_directory() + "/run"
+templates_directory_path := justfile_directory() + "/templates"
+optimization_vars_files_file_path := run_directory_path + "/optimization-vars-files.state"
+
 # Pulls external Ansible roles
-roles: requirements-yml
+roles: _requirements-yml
     #!/usr/bin/env sh
     if [ -x "$(command -v agru)" ]; then
         agru -r {{ justfile_directory() }}/requirements.yml
     else
         rm -rf roles/galaxy
-        ansible-galaxy install -r requirements.yml -p roles/galaxy/ --force
+        ansible-galaxy install -r {{ justfile_directory() }}/requirements.yml -p roles/galaxy/ --force
     fi
 
+# Optimizes the playbook based on stored configuration (vars.yml paths)
+optimize-restore:
+    #!/usr/bin/env sh
+    if [ -f "$optimization_vars_files_file_path" ]; then
+        just --justfile {{ justfile() }} \
+        _optimize-for-var-paths \
+        $(cat $optimization_vars_files_file_path)
+    else
+        echo "Cannot restore optimization state from a file ($optimization_vars_files_file_path), because it doesn't exist"
+        exit 1
+    fi
+
+# Clears optimizations and resets the playbook to a non-optimized state
+optimize-reset: && _clean_template_derived_files
+    #!/usr/bin/env sh
+    rm -f {{ run_directory_path }}/*.srchash
+    rm -f {{ optimization_vars_files_file_path }}
+
+# Optimizes the playbook based on the enabled components for all hosts in the inventory
+optimize inventory_path='inventory': _reconfigure-for-all-hosts
+
+_reconfigure-for-all-hosts inventory_path='inventory':
+    #!/usr/bin/env sh
+    just --justfile {{ justfile() }} \
+    _optimize-for-var-paths \
+    $(find {{ inventory_path }}/host_vars/ -maxdepth 2 -name '*.yml' -exec readlink -f {} \;)
+
+# Optimizes the playbook based on the enabled components for a single host
+optimize-for-host hostname inventory_path='inventory':
+    #!/usr/bin/env sh
+    just --justfile {{ justfile() }} \
+    _optimize-for-var-paths \
+    $(find {{ inventory_path }}/host_vars/{{ hostname }} -maxdepth 1 -name '*.yml' -exec readlink -f {} \;)
+
+# Optimizes the playbook based on the enabled components found in the given vars.yml files
+_optimize-for-var-paths +PATHS:
+    #!/usr/bin/env sh
+    echo '{{ PATHS }}' > {{ optimization_vars_files_file_path }}
+
+    just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/requirements.yml {{ justfile_directory() }}/requirements.yml
+    just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/setup.yml {{ justfile_directory() }}/setup.yml
+    just --justfile {{ justfile() }} _save_hash_for_file {{ templates_directory_path }}/group_vars_mash_servers {{ justfile_directory() }}/group_vars/mash_servers
+
+    /usr/bin/env python {{ justfile_directory() }}/bin/optimize.py \
+    --vars-paths='{{ PATHS }}' \
+    --src-requirements-yml-path={{ templates_directory_path }}/requirements.yml \
+    --dst-requirements-yml-path={{ justfile_directory() }}/requirements.yml \
+    --src-setup-yml-path={{ templates_directory_path }}/setup.yml \
+    --dst-setup-yml-path={{ justfile_directory() }}/setup.yml \
+    --src-group-vars-yml-path={{ templates_directory_path }}/group_vars_mash_servers \
+    --dst-group-vars-yml-path={{ justfile_directory() }}/group_vars/mash_servers
+
 # Updates requirements.yml if there are any new tags available. Requires agru
 update: && opml
-    @agru -r {{ justfile_directory() }}/requirements.all.yml -u
+    @agru -r {{ templates_directory_path }}/requirements.yml -u
 
 # Runs ansible-lint against all roles in the playbook
 lint:
@@ -39,7 +95,7 @@ install-service service *extra_args:
 setup-all *extra_args: (run-tags "setup-all,start" extra_args)
 
 # Runs the playbook with the given list of arguments
-run +extra_args: requirements-yml setup-yml group-vars-mash-servers
+run +extra_args: _requirements-yml _setup-yml _group-vars-mash-servers
     ansible-playbook -i inventory/hosts setup.yml {{ extra_args }}
 
 # Runs the playbook with the given list of comma-separated tags and optional arguments
@@ -61,20 +117,21 @@ stop-group group *extra_args:
     @just --justfile {{ justfile() }} run-tags stop-group --extra-vars="group={{ group }}" {{ extra_args }}
 
 # Prepares the requirements.yml file
-requirements-yml:
+_requirements-yml:
-    @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/requirements.all.yml {{ justfile_directory() }}/requirements.yml
+    @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/requirements.yml {{ justfile_directory() }}/requirements.yml
 
 # Prepares the setup.yml file
-setup-yml:
+_setup-yml:
-    @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/setup.all.yml {{ justfile_directory() }}/setup.yml
+    @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/setup.yml {{ justfile_directory() }}/setup.yml
 
 # Prepares the group_vars/mash_servers file
-group-vars-mash-servers:
+_group-vars-mash-servers:
-    @just --justfile {{ justfile() }} _ensure_file_prepared {{ justfile_directory() }}/group_vars/mash_servers_all {{ justfile_directory() }}/group_vars/mash_servers
+    @just --justfile {{ justfile() }} _ensure_file_prepared {{ templates_directory_path }}/group_vars_mash_servers {{ justfile_directory() }}/group_vars/mash_servers
 
 _ensure_file_prepared src_path dst_path:
     #!/usr/bin/env sh
-    hash_path={{ dst_path }}.srchash
+    dst_file_name=$(basename "{{ dst_path }}")
+    hash_path={{ run_directory_path }}"/"$dst_file_name".srchash"
     src_hash=$(md5sum {{ src_path }} | cut -d ' ' -f 1)
 
     if [ ! -f "{{ dst_path }}" ] || [ ! -f "$hash_path" ]; then
@@ -86,5 +143,32 @@ _ensure_file_prepared src_path dst_path:
         if [ "$current_hash" != "$src_hash" ]; then
             cp {{ src_path }} {{ dst_path }}
             echo $src_hash > $hash_path
+
+            if [ -f "$optimization_vars_files_file_path" ]; then
+                just --justfile {{ justfile() }} \
+                _optimize-for-var-paths \
+                $(cat $optimization_vars_files_file_path)
             fi
         fi
+    fi
+
+_save_hash_for_file src_path dst_path:
+    #!/usr/bin/env sh
+    dst_file_name=$(basename "{{ dst_path }}")
+    hash_path={{ run_directory_path }}"/"$dst_file_name".srchash"
+    src_hash=$(md5sum {{ src_path }} | cut -d ' ' -f 1)
+    echo $src_hash > $hash_path
+
+_clean_template_derived_files:
+    #!/usr/bin/env sh
+    if [ -f "{{ justfile_directory() }}/requirements.yml" ]; then
+        rm {{ justfile_directory() }}/requirements.yml
+    fi
+
+    if [ -f "{{ justfile_directory() }}/setup.yml" ]; then
+        rm {{ justfile_directory() }}/setup.yml
+    fi
+
+    if [ -f "{{ justfile_directory() }}/group_vars/mash_servers" ]; then
+        rm {{ justfile_directory() }}/group_vars/mash_servers
+    fi