diff --git a/src/IntuneCD/backup_intune.py b/src/IntuneCD/backup_intune.py index 82e7f1e6..90b85ec9 100644 --- a/src/IntuneCD/backup_intune.py +++ b/src/IntuneCD/backup_intune.py @@ -39,6 +39,7 @@ def backup_intune( args, max_workers, platforms, + enrich_documentation=False, ): """ Imports all the backup functions dynamically and runs them in parallel. @@ -66,6 +67,21 @@ def backup_intune( "platforms": platforms, } + # Enrich data if the --enrich-documentation flag is set + if enrich_documentation: + from .intunecdlib.BaseGraphModule import BaseGraphModule + import os + import json + + graph = BaseGraphModule() + graph.token = token + settings = graph.make_graph_request("https://graph.microsoft.com/beta/deviceManagement/configurationSettings") + with open(os.path.join(path, "configurationSettings.json"), "w", encoding="utf-8") as f: + json.dump(settings, f, indent=2) + categories = graph.make_graph_request("https://graph.microsoft.com/beta/deviceManagement/configurationCategories") + with open(os.path.join(path, "configurationCategories.json"), "w", encoding="utf-8") as f: + json.dump(categories, f, indent=2) + # List of backup modules to dynamically import and execute backup_modules = [ ( diff --git a/src/IntuneCD/document_intune.py b/src/IntuneCD/document_intune.py index 4e9f39c5..e500794b 100644 --- a/src/IntuneCD/document_intune.py +++ b/src/IntuneCD/document_intune.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- from concurrent.futures import ThreadPoolExecutor, as_completed +import os +import json from .intunecdlib.documentation_functions import ( document_configs, document_management_intents, + document_settings_catalog, ) from .decorators import time_command @@ -18,6 +21,7 @@ def document_intune( decode, split_per_config, max_workers, + enrich_documentation=False, ): """ This function is used to document Intune configuration using threading. @@ -30,6 +34,7 @@ def document_intune( :param decode: Decode base64 values :param split_per_config: Whether to split each config into its own Markdown file :param max_workers: Maximum number of concurrent threads + :param enrich_documentation: Whether to enrich Settings Catalog documentation with additional details """ # Ensure the output directory exists @@ -77,22 +82,59 @@ def document_intune( # sort doc_tasks alphabetically doc_tasks = sorted(doc_tasks, key=lambda x: x[1]) + settings_lookup = None + categories_lookup = None + + if enrich_documentation: + settings_lookup = {} + categories_lookup = {} + + settings_path = os.path.join(configpath, "configurationSettings.json") + categories_path = os.path.join(configpath, "configurationCategories.json") + if os.path.exists(settings_path): + with open(settings_path, "r", encoding="utf-8") as f: + settings_json = json.load(f) + if os.path.exists(categories_path): + with open(categories_path, "r", encoding="utf-8") as f: + categories_json = json.load(f) + + # Build lookup dictionaries for enrichment + if settings_json: + for s in settings_json.get("value", []): + settings_lookup[s.get("id")] = s + if categories_json: + for c in categories_json.get("value", []): + categories_lookup[c.get("id")] = c + if split or split_per_config: with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - document_configs, - f"{configpath}/{task[0]}", - outpath, - task[1], - maxlength, - split, - cleanup, - decode, - split_per_config, - ): task[1] - for task in doc_tasks - } + futures = {} + for task in doc_tasks: + # Submit Settings Catalog with enrichment if enabled + if task[1] == "Settings Catalog" and enrich_documentation: + futures[executor.submit( + document_settings_catalog, + f"{configpath}/{task[0]}", + outpath, + task[1], + maxlength, + split, + split_per_config, + settings_lookup, + categories_lookup, + )] = task[1] + else: + futures[executor.submit( + document_configs, + f"{configpath}/{task[0]}", + outpath, + task[1], + maxlength, + split, + cleanup, + decode, + split_per_config, + )] = task[1] for future in as_completed(futures): task_name = futures[future] @@ -104,16 +146,29 @@ def document_intune( else: # Run sequentially if split options are disabled for task in doc_tasks: - document_configs( - f"{configpath}/{task[0]}", - outpath, - task[1], - maxlength, - split, - cleanup, - decode, - split_per_config, - ) + # Submit Settings Catalog with enrichment if enabled + if task[1] == "Settings Catalog" and enrich_documentation: + document_settings_catalog( + f"{configpath}/{task[0]}", + outpath, + task[1], + maxlength, + split, + split_per_config, + settings_lookup, + categories_lookup, + ) + else: + document_configs( + f"{configpath}/{task[0]}", + outpath, + task[1], + maxlength, + split, + cleanup, + decode, + split_per_config, + ) # **Run Management Intents Sequentially** document_management_intents( diff --git a/src/IntuneCD/intunecdlib/documentation_functions.py b/src/IntuneCD/intunecdlib/documentation_functions.py index f5fb6f1f..64a59108 100644 --- a/src/IntuneCD/intunecdlib/documentation_functions.py +++ b/src/IntuneCD/intunecdlib/documentation_functions.py @@ -12,9 +12,9 @@ import os import platform import re - import yaml from pytablewriter import MarkdownTableWriter +from collections import defaultdict def md_file(outpath): @@ -29,16 +29,16 @@ def md_file(outpath): open(outpath, "w", encoding="utf-8").close() -def write_table(data): +def write_table(data, headers=None): """ This function creates the markdown table. :param data: The data to be written to the table + :param headers: The headers for the table :return: The Markdown table writer """ - writer = MarkdownTableWriter( - headers=["setting", "value"], + headers=headers if headers else ["setting", "value"], value_matrix=data, ) @@ -47,16 +47,49 @@ def write_table(data): def escape_markdown(text): """ - This function escapes markdown characters. + Escapes markdown characters except inside http/https links. :param text: The text to be escaped :return: The escaped text """ + # Regex to match http/https links + link_pattern = re.compile(r'(https?://[^\s\)\]\}]+)') + parts = [] + last_end = 0 + for match in link_pattern.finditer(text): + # Escape markdown in text before the link + before = text[last_end:match.start()] + escaped = re.sub(r"([\_*\[\]()\{\}`>\#\+\-=|\.!])", r"\\\1", before) + parts.append(escaped) + # Add the link unescaped + parts.append(match.group(0)) + last_end = match.end() + # Escape markdown in the remaining text + after = text[last_end:] + escaped_after = re.sub(r"([\_*\[\]()\{\}`>\#\+\-=|\.!])", r"\\\1", after) + parts.append(escaped_after) + return ''.join(parts) + + +def sanitize_text(text): + """ + Sanitizes the input text by removing extra spaces, newlines, and non-printable/control characters. + :param text: The text to be sanitized + :return: The sanitized text + """ + text = re.sub(r'[ \t]+', ' ', text) + text = re.sub(r'[\r\n]+', '\n', text) + text = re.sub(r'[^\x20-\x7E\n]', '', text) + return text.strip() - # Escape markdown characters - parse = re.sub(r"([\_*\[\]()\{\}`>\#\+\-=|\.!])", r"\\\1", text) - return parse +def convert_newlines_to_br(text): + """ + Converts any newline characters in the text to
. + :param text: The input text + :return: Text with newlines replaced by
+ """ + return text.replace('\n', '
') def assignment_table(data): @@ -598,3 +631,238 @@ def get_md_files(configpath): md_files.sort(key=lambda f: os.path.splitext(os.path.basename(f))[0].lower()) return md_files + + +def extract_setting(setting_instance, settings_lookup): + """ + Extracts setting information from a setting instance using the provided settings lookup. + :param setting_instance: The setting instance dictionary + :param settings_lookup: The settings lookup dictionary + :return: A list of lists containing setting name, formatted value, and description + """ + + def escape_backslash_for_md(value): + """ + This function processes the input string to ensure that backslashes preceding Markdown special characters are properly escaped, preventing unintended formatting when rendered. It is recommended to pass the input as a raw string to avoid Python interpreting escape sequences. + :param value: The input string to be processed, pass value as raw string: Example: escape_backslash_for_md(rf"{value}") + :return: The processed string with backslashes properly escaped for Markdown + """ + escapable = r"_*\[\](){}#`>+-=|.!" + value = re.sub(rf'(?".join([f'[{url}]({url})' for i, url in enumerate(info_urls)]) + description = f"{description}
InfoUrls:
{links}" if description else links + description = f"
Click to expand...{description}
" if description else "" + + if "simpleSettingValue" in setting_instance: + value = setting_instance["simpleSettingValue"].get("value", "") + formatted_value = escape_backslash_for_md(rf"{value}") if value != "" else "Not configured" + return [[display_name, formatted_value, description]] + + elif "simpleSettingCollectionValue" in setting_instance: + collection = setting_instance["simpleSettingCollectionValue"] + if isinstance(collection, list) and collection: + values = [] + for item in collection: + value = item.get("value", "") + if value != "": + values.append(str(escape_backslash_for_md(rf"{value}"))) + formatted_value = ", ".join(values) if values else "Not configured" + return [[display_name, formatted_value, description]] + else: + return [[display_name, "Not configured", description]] + + elif "choiceSettingValue" in setting_instance: + choice_value_obj = setting_instance["choiceSettingValue"] + value = choice_value_obj.get("value", "") + children = choice_value_obj.get("children", []) + option_display_name = None + if value and "options" in definition: + for option in definition["options"]: + if option.get("value") == value or option.get("itemId") == value: + option_display_name = option.get("displayName") or option.get("name") + break + formatted_value = option_display_name if option_display_name else (value if value else "Not configured") + rows = [] + rows.append([display_name, formatted_value, description]) + for child in children: + rows.extend(extract_setting(child, settings_lookup)) + return rows + + elif "groupSettingCollectionValue" in setting_instance: + collection = setting_instance["groupSettingCollectionValue"] + rows = [] + if isinstance(collection, list): + for item in collection: + children = item.get("children", []) + for child in children: + rows.extend(extract_setting(child, settings_lookup)) + return rows if rows else [[display_name, "Collection value", description]] + + return [[display_name, "Not configured", description]] + + +def document_settings_catalog( + configpath, + outpath, + header, + max_length, + split, + split_per_config, + settings_lookup=None, + categories_lookup=None, +): + """ + Documents Settings Catalog configurations, enriched with configurationSettings and configurationCategories. This function is only started when backup and documentation are started with --enrich-documentation. + + :param configpath: Path to backup files + :param outpath: Base path for Markdown output + :param header: Configuration type header (e.g., "AppConfigurations") + :param max_length: Max length for displayed values + :param split: Split into one file per type + :param split_per_config: Split into one file per individual config + :param settings_lookup: Lookup dictionary for configurationSettings + :param categories_lookup: Lookup dictionary for configurationCategories + """ + if not os.path.exists(configpath): + return + + # Prepare output path for split mode + if split and not split_per_config: + outpath = os.path.join(configpath, f"{header}.md") + md_file(outpath) + + if split_per_config is False: + with open(outpath, "a", encoding="utf-8") as md: + md.write("## " + header + "\n") + + pattern = os.path.join(configpath, "**", "*.json") + files = sorted(glob.glob(pattern, recursive=True), key=str.casefold) + if not files: + return + + for filename in files: + if filename.endswith(".md") or os.path.isdir(filename): + continue + + try: + with open(filename, encoding="utf-8") as f: + repo_data = json.load(f) + + # Assignments Table + assignments_table = assignment_table(repo_data) + repo_data.pop("assignments", None) + + # Basics Table + basics_table = [ + ["Name", repo_data.get("name", "")], + ["Profile type", "Settings catalog"], + ["Platform supported", repo_data.get("platforms", "")], + ["Technologies", repo_data.get("technologies", "")], + ["Scope tags", ", ".join(repo_data.get("roleScopeTagIds", []))], + ] + basics_md_table = write_table(basics_table) + + # Configuration Table + config_table_list = [] + + for setting in repo_data.get("settings", []): + rows = extract_setting(setting.get("settingInstance", {}), settings_lookup) + for row in rows: + setting_name = row[0] + value = row[1] + description = row[2] + setting_definition_id = setting.get("settingInstance", {}).get("settingDefinitionId", "") + definition = settings_lookup.get(setting_definition_id, {}) + category_id = definition.get("categoryId", "") + category_name = categories_lookup.get(category_id, {}).get("displayName", "") + root_category_id = categories_lookup.get(category_id, {}).get("rootCategoryId", "") + root_category_name = categories_lookup.get(root_category_id, {}).get("displayName", "") + + if max_length and isinstance(value, str) and len(value) > max_length: + value = "Value too long to display" + config_table_list.append({ + "setting_name": setting_name, + "value": value, + "description": description, + "category_name": category_name, + "root_category_name": root_category_name + }) + + # Sort by category_name, then root_category_name + config_table_list_sorted = sorted( + config_table_list, + key=lambda x: (x["root_category_name"], x["category_name"]) + ) + + # Group items by root_category_name and category_name + grouped = defaultdict(lambda: defaultdict(list)) + for item in config_table_list_sorted: + grouped[item["root_category_name"]][item["category_name"]].append(item) + + + # Output file logic + config_name = repo_data.get("name", os.path.splitext(os.path.basename(filename))[0]) + safe_config_name = re.sub(r'[<>:"/\\|?*]', "_", config_name) + if split_per_config: + if not os.path.exists(f"{configpath}/docs"): + os.makedirs(f"{configpath}/docs") + config_outpath = os.path.join(f"{configpath}/docs", f"{safe_config_name}.md") + md_file(config_outpath) + target_md = config_outpath + top_header = f"# {config_name}" + split_per_config_index_md(configpath, header) + elif split: + target_md = outpath + top_header = f"### {config_name}" + else: + target_md = outpath + top_header = f"### {config_name}" + + # Write markdown + with open(target_md, "a", encoding="utf-8") as md: + md.write(top_header + "\n") + if assignments_table: + md.write("#### Assignments\n") + md.write(str(assignments_table) + "\n") + md.write("#### Basics\n") + md.write(str(basics_md_table) + "\n") + md.write("#### Configuration\n") + + # Write grouped tables + table_data = [] + for root_cat, categories in grouped.items(): + + for cat, items in categories.items(): + if cat == root_cat: + table_data.append([f"**{root_cat}**", "", ""]) + else: + table_data.append([f"**{root_cat}** > **{cat}**", "", ""]) + for i in items: + table_data.append([i["setting_name"], i["value"], i["description"]]) + table_md = write_table(table_data, headers=["Setting", "Value", "Description"]) + md.write(str(table_md) + "\n") + + except Exception as e: + print(f"[DEBUG] Error processing {filename}: {type(e).__name__}: {e}") diff --git a/src/IntuneCD/run_backup.py b/src/IntuneCD/run_backup.py index 707420f7..eecc9203 100644 --- a/src/IntuneCD/run_backup.py +++ b/src/IntuneCD/run_backup.py @@ -206,6 +206,11 @@ def get_parser(include_help=True): help="When set, the script will not move files to archive. Might require manual cleanup.", action="store_true", ) + parser.add_argument( + "--enrich-documentation", + help="If set, fetches and stores Intune configurationSettings and configurationCategories for SettingsCatalog documentation enrichment. Requires the documentation process to be run with --enrich-documentation as well.", + action="store_true", + ) return parser @@ -266,7 +271,7 @@ def selected_mode(argument): azure_token = obtain_azure_token(os.environ.get("TENANT_ID"), args.path) def run_backup( - path, output, exclude, token, prefix, append_id, max_workers, platforms + path, output, exclude, token, prefix, append_id, max_workers, platforms, enrich_documentation ): results = [] @@ -288,6 +293,7 @@ def run_backup( args, max_workers, platforms, + enrich_documentation, ) from .intunecdlib.assignment_report import AssignmentReport @@ -353,6 +359,7 @@ def run_backup( args.append_id, args.max_workers, platforms, + args.enrich_documentation, ) sys.stdout = old_stdout feed_bytes = feedstdout.getvalue().encode("utf-8") @@ -373,6 +380,7 @@ def run_backup( args.append_id, args.max_workers, platforms, + args.enrich_documentation, ) else: diff --git a/src/IntuneCD/run_documentation.py b/src/IntuneCD/run_documentation.py index 584e6612..fbc360e8 100644 --- a/src/IntuneCD/run_documentation.py +++ b/src/IntuneCD/run_documentation.py @@ -86,6 +86,11 @@ def get_parser(include_help=True): type=int, default=10, ) + parser.add_argument( + "--enrich-documentation", + help="If set, enriches documentation with configurationSettings and configurationCategories if available. Requires the backup process to have been run with --enrich-documentation as well.", + action="store_true", + ) return parser @@ -105,6 +110,7 @@ def run_documentation( decode, split_per_config, max_workers, + enrich_documentation, ): now = datetime.now() current_date = now.strftime("%d/%m/%Y %H:%M:%S") @@ -123,6 +129,7 @@ def run_documentation( decode, split_per_config, max_workers, + enrich_documentation, ) write_type_header(split, outpath, "Entra") @@ -196,6 +203,7 @@ def run_documentation( args.decode, args.split_per_config, args.max_workers, + args.enrich_documentation, ) diff --git a/tests/test_documentation_functions.py b/tests/test_documentation_functions.py index 7301f6fd..f907109f 100644 --- a/tests/test_documentation_functions.py +++ b/tests/test_documentation_functions.py @@ -87,11 +87,11 @@ def test_remove_characters(self): def test_escape_markdown(self): """The escaped string should be returned.""" - self.string = "\\`*_{}[]()#+-.!Hello World" + self.string = "\\`*_{}[]()#+-.!Hello World Check this link: https://example.com/test_path?param=1&other=2" self.assertEqual( escape_markdown(self.string), - "\\\\`\\*\\_\\{\\}\\[\\]\\(\\)\\#\\+\\-\\.\\!Hello World", + "\\\\`\\*\\_\\{\\}\\[\\]\\(\\)\\#\\+\\-\\.\\!Hello World Check this link: https://example.com/test_path?param=1&other=2", ) def test_clean_list_list(self):