# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license """ Automates building and post-processing of MkDocs documentation, especially for multilingual projects. This script streamlines generating localized documentation and updating HTML links for correct formatting. Key Features: - Automated building of MkDocs documentation: Compiles main documentation and localized versions from separate MkDocs configuration files. - Post-processing of generated HTML files: Updates HTML files to remove '.md' from internal links, ensuring correct navigation in web-based documentation. Usage: - Run from the root directory of your MkDocs project. - Ensure MkDocs is installed and configuration files (main and localized) are present. - The script builds documentation using MkDocs, then scans HTML files in 'site' to update links. - Ideal for projects with Markdown documentation served as a static website. Note: - Requires Python and MkDocs to be installed and configured. """ from __future__ import annotations import os import re import shutil import subprocess from pathlib import Path from bs4 import BeautifulSoup from ultralytics.utils import LINUX, LOGGER, MACOS from ultralytics.utils.tqdm import TQDM os.environ["JUPYTER_PLATFORM_DIRS"] = "1" # fix DeprecationWarning: Jupyter is migrating to use standard platformdirs DOCS = Path(__file__).parent.resolve() SITE = DOCS.parent / "site" LINK_PATTERN = re.compile(r"(https?://[^\s()<>]*[^\s()<>.,:;!?\'\"])") def prepare_docs_markdown(clone_repos: bool = True): """Build docs using mkdocs.""" LOGGER.info("Removing existing build artifacts") shutil.rmtree(SITE, ignore_errors=True) shutil.rmtree(DOCS / "repos", ignore_errors=True) if clone_repos: # Get hub-sdk repo repo = "https://github.com/ultralytics/hub-sdk" local_dir = DOCS / "repos" / Path(repo).name subprocess.run( ["git", "clone", repo, str(local_dir), "--depth", "1", "--single-branch", "--branch", "main"], check=True ) shutil.rmtree(DOCS / "en/hub/sdk", ignore_errors=True) # delete if exists shutil.copytree(local_dir / "docs", DOCS / "en/hub/sdk") # for docs shutil.rmtree(DOCS.parent / "hub_sdk", ignore_errors=True) # delete if exists shutil.copytree(local_dir / "hub_sdk", DOCS.parent / "hub_sdk") # for mkdocstrings LOGGER.info(f"Cloned/Updated {repo} in {local_dir}") # Get docs repo repo = "https://github.com/ultralytics/docs" local_dir = DOCS / "repos" / Path(repo).name subprocess.run( ["git", "clone", repo, str(local_dir), "--depth", "1", "--single-branch", "--branch", "main"], check=True ) shutil.rmtree(DOCS / "en/compare", ignore_errors=True) # delete if exists shutil.copytree(local_dir / "docs/en/compare", DOCS / "en/compare") # for docs LOGGER.info(f"Cloned/Updated {repo} in {local_dir}") # Add frontmatter for file in TQDM((DOCS / "en").rglob("*.md"), desc="Adding frontmatter"): update_markdown_files(file) def update_page_title(file_path: Path, new_title: str): """Update the title of an HTML file.""" with open(file_path, encoding="utf-8") as file: content = file.read() # Replace the existing title with the new title updated_content = re.sub(r".*?", f"{new_title}", content) # Write the updated content back to the file with open(file_path, "w", encoding="utf-8") as file: file.write(updated_content) def update_html_head(script: str = ""): """Update the HTML head section of each file.""" html_files = Path(SITE).rglob("*.html") for html_file in TQDM(html_files, desc="Processing HTML files"): with html_file.open("r", encoding="utf-8") as file: html_content = file.read() if script in html_content: # script already in HTML file return head_end_index = html_content.lower().rfind("") if head_end_index != -1: # Add the specified JavaScript to the HTML file just before the end of the head tag. new_html_content = html_content[:head_end_index] + script + html_content[head_end_index:] with html_file.open("w", encoding="utf-8") as file: file.write(new_html_content) def update_subdir_edit_links(subdir: str = "", docs_url: str = ""): """Update the HTML head section of each file.""" if str(subdir[0]) == "/": subdir = str(subdir[0])[1:] html_files = (SITE / subdir).rglob("*.html") for html_file in TQDM(html_files, desc="Processing subdir files", mininterval=1.0): with html_file.open("r", encoding="utf-8") as file: soup = BeautifulSoup(file, "html.parser") # Find the anchor tag and update its href attribute a_tag = soup.find("a", {"class": "md-content__button md-icon"}) if a_tag and a_tag["title"] == "Edit this page": a_tag["href"] = f"{docs_url}{a_tag['href'].rpartition(subdir)[-1]}" # Write the updated HTML back to the file with open(html_file, "w", encoding="utf-8") as file: file.write(str(soup)) def update_markdown_files(md_filepath: Path): """Create or update a Markdown file, ensuring frontmatter is present.""" if md_filepath.exists(): content = md_filepath.read_text().strip() # Replace apostrophes content = content.replace("‘", "'").replace("’", "'") # Add frontmatter if missing if not content.strip().startswith("---\n") and "macros" not in md_filepath.parts: # skip macros directory header = "---\ncomments: true\ndescription: TODO ADD DESCRIPTION\nkeywords: TODO ADD KEYWORDS\n---\n\n" content = header + content # Ensure MkDocs admonitions "=== " lines are preceded and followed by empty newlines lines = content.split("\n") new_lines = [] for i, line in enumerate(lines): stripped_line = line.strip() if stripped_line.startswith("=== "): if i > 0 and new_lines[-1] != "": new_lines.append("") new_lines.append(line) if i < len(lines) - 1 and lines[i + 1].strip() != "": new_lines.append("") else: new_lines.append(line) content = "\n".join(new_lines) # Add EOF newline if missing if not content.endswith("\n"): content += "\n" # Save page md_filepath.write_text(content) return def update_docs_html(): """Update titles, edit links, head sections, and convert plaintext links in HTML documentation.""" # Update 404 titles update_page_title(SITE / "404.html", new_title="Ultralytics Docs - Not Found") # Update edit button links for subdir, docs_url in ( ("hub/sdk/", "https://github.com/ultralytics/hub-sdk/tree/main/docs/"), # do not use leading slash ("compare/", "https://github.com/ultralytics/docs/tree/main/docs/en/compare/"), ): update_subdir_edit_links(subdir=subdir, docs_url=docs_url) # Convert plaintext links to HTML hyperlinks files_modified = 0 for html_file in TQDM(SITE.rglob("*.html"), desc="Updating bs4 soup", mininterval=1.0): with open(html_file, encoding="utf-8") as file: content = file.read() updated_content = update_docs_soup(content, html_file=html_file) if updated_content != content: with open(html_file, "w", encoding="utf-8") as file: file.write(updated_content) files_modified += 1 LOGGER.info(f"Modified bs4 soup in {files_modified} files.") # Update HTML file head section script = "" if any(script): update_html_head(script) # Delete the /macros directory from the built site macros_dir = SITE / "macros" if macros_dir.exists(): LOGGER.info(f"Removing /macros directory from site: {macros_dir}") shutil.rmtree(macros_dir) def update_docs_soup(content: str, html_file: Path | None = None, max_title_length: int = 70) -> str: """Convert plaintext links to HTML hyperlinks, truncate long meta titles, and remove code line hrefs.""" soup = BeautifulSoup(content, "html.parser") modified = False # Truncate long meta title if needed title_tag = soup.find("title") if title_tag and len(title_tag.text) > max_title_length and "-" in title_tag.text: title_tag.string = title_tag.text.rsplit("-", 1)[0].strip() modified = True # Find the main content area main_content = soup.find("main") or soup.find("div", class_="md-content") if not main_content: return str(soup) if modified else content # Convert plaintext links to HTML hyperlinks for paragraph in main_content.select("p, li"): for text_node in paragraph.find_all(string=True, recursive=False): if text_node.parent.name not in {"a", "code"}: new_text = LINK_PATTERN.sub(r'\1', str(text_node)) if " str: """Remove comments and empty lines from a string of code, preserving newlines and URLs. Args: content (str): Code content to process. file_type (str): Type of file ('html', 'css', or 'js'). Returns: (str): Cleaned content with comments and empty lines removed. Notes: Typical reductions for Ultralytics Docs are: - Total HTML reduction: 2.83% (1301.56 KB saved) - Total CSS reduction: 1.75% (2.61 KB saved) - Total JS reduction: 13.51% (99.31 KB saved) """ if file_type == "html": # Remove HTML comments content = re.sub(r"", "", content) # Only remove empty lines for HTML, preserve indentation content = re.sub(r"^\s*$\n", "", content, flags=re.MULTILINE) elif file_type == "css": # Remove CSS comments content = re.sub(r"/\*[\s\S]*?\*/", "", content) # Remove whitespace around specific characters content = re.sub(r"\s*([{}:;,])\s*", r"\1", content) # Remove empty lines content = re.sub(r"^\s*\n", "", content, flags=re.MULTILINE) # Collapse multiple spaces to single space content = re.sub(r"\s{2,}", " ", content) # Remove all newlines content = re.sub(r"\n", "", content) elif file_type == "js": # Handle JS single-line comments (preserving http:// and https://) lines = content.split("\n") processed_lines = [] for line in lines: # Only remove comments if they're not part of a URL if "//" in line and "http://" not in line and "https://" not in line: processed_lines.append(line.partition("//")[0]) else: processed_lines.append(line) content = "\n".join(processed_lines) # Remove JS multi-line comments and clean whitespace content = re.sub(r"/\*[\s\S]*?\*/", "", content) # Remove empty lines content = re.sub(r"^\s*\n", "", content, flags=re.MULTILINE) # Collapse multiple spaces to single space content = re.sub(r"\s{2,}", " ", content) # Safe space removal around punctuation and operators (NEVER include colons - breaks JS) content = re.sub(r"\s*([,;{}])\s*", r"\1", content) content = re.sub(r"(\w)\s*\(|\)\s*{|\s*([+\-*/=])\s*", lambda m: m.group(0).replace(" ", ""), content) return content def minify_files(html: bool = True, css: bool = True, js: bool = True): """Minify HTML, CSS, and JS files and print total reduction stats.""" minify, compress, jsmin = None, None, None try: if html: from minify_html import minify if css: from csscompressor import compress if js: import jsmin except ImportError as e: LOGGER.info(f"Missing required package: {e}") return stats = {} for ext, minifier in { "html": (lambda x: minify(x, keep_closing_tags=True, minify_css=True, minify_js=True)) if html else None, "css": compress if css else None, "js": jsmin.jsmin if js else None, }.items(): stats[ext] = {"original": 0, "minified": 0} directory = "" # "stylesheets" if ext == css else "javascript" if ext == "js" else "" for f in TQDM((SITE / directory).rglob(f"*.{ext}"), desc=f"Minifying {ext.upper()}", mininterval=1.0): content = f.read_text(encoding="utf-8") minified = minifier(content) if minifier else remove_comments_and_empty_lines(content, ext) stats[ext]["original"] += len(content) stats[ext]["minified"] += len(minified) f.write_text(minified, encoding="utf-8") for ext, data in stats.items(): if data["original"]: r = data["original"] - data["minified"] # reduction LOGGER.info(f"Total {ext.upper()} reduction: {(r / data['original']) * 100:.2f}% ({r / 1024:.2f} KB saved)") def main(): """Build docs, update titles and edit links, minify HTML, and print local server command.""" prepare_docs_markdown() # Build the main documentation LOGGER.info(f"Building docs from {DOCS}") subprocess.run(["mkdocs", "build", "-f", str(DOCS.parent / "mkdocs.yml"), "--strict"], check=True) remove_macros() LOGGER.info(f"Site built at {SITE}") # Update docs HTML pages update_docs_html() # Minify files minify_files(html=False, css=False, js=False) # Cleanup shutil.rmtree(DOCS.parent / "hub_sdk", ignore_errors=True) shutil.rmtree(DOCS / "repos", ignore_errors=True) # Print results and auto-serve on macOS size = sum(f.stat().st_size for f in SITE.rglob("*") if f.is_file()) >> 20 LOGGER.info(f"Docs built correctly ✅ ({size:.1f} MB)") if (MACOS or LINUX) and not os.getenv("GITHUB_ACTIONS"): import webbrowser webbrowser.open("http://localhost:8000") try: subprocess.run(["python", "-m", "http.server", "--directory", str(SITE), "8000"], check=True) except KeyboardInterrupt: LOGGER.info("\nServer stopped.") else: LOGGER.info('Serve site at http://localhost:8000 with "python -m http.server --directory site"') if __name__ == "__main__": main()