From 4023aecbe052950e4c8ea0b4fb78f7c48404ad26 Mon Sep 17 00:00:00 2001 From: Jonathan Cubides Date: Mon, 25 Nov 2024 01:16:42 -0500 Subject: [PATCH] Bump up to v0.4.3 (#12) - Needed refactors due to caching conflicts and data races (still some work to be done) --- .github/workflows/ci.yml | 4 +- mkdocs_juvix/common/preprocesors/links.py | 39 +- mkdocs_juvix/env.py | 12 +- mkdocs_juvix/main.py | 613 ++++++++++++---------- mkdocs_juvix/utils.py | 9 - poetry.lock | 10 +- pyproject.toml | 6 +- src/cli.py | 30 +- src/fixtures/ci.yml | 4 +- 9 files changed, 387 insertions(+), 340 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14d07b4..d8a05a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,11 +68,13 @@ jobs: run: poetry run pytest - name: Create MkDocs Project run: | - juvix-mkdocs new -f -n --no-run-server --no-open --project-name my-juvix-project --anoma-setup + juvix-mkdocs new -f -n --no-run-server --no-open --project-name + my-juvix-project --anoma-setup - name: Build MkDocs Project run: juvix-mkdocs build -p my-juvix-project env: SITE_URL: https://anoma.github.io/juvix-mkdocs + SITE_NAME: Juvix MkDocs - if: success() uses: JamesIves/github-pages-deploy-action@v4.6.4 with: diff --git a/mkdocs_juvix/common/preprocesors/links.py b/mkdocs_juvix/common/preprocesors/links.py index 805e38b..f348da4 100644 --- a/mkdocs_juvix/common/preprocesors/links.py +++ b/mkdocs_juvix/common/preprocesors/links.py @@ -204,31 +204,34 @@ def _run(self, content: str) -> str: log.error(f"Error occurred while processing ignore patterns: {str(e)}") return content - intervals_where_not_to_look = None - if intervals: - starts, ends, ids = map(np.array, zip(*intervals)) - intervals_where_not_to_look = NCLS(starts, ends, ids) + # intervals_where_not_to_look = None + # if intervals: + # starts, ends, ids = map(np.array, zip(*intervals)) + # intervals_where_not_to_look = NCLS(starts, ends, ids) # Find all wikilinks str_wikilinks = list(WIKILINK_PATTERN.finditer(content)) - + log.debug(f"Found {len(str_wikilinks)} wikilinks") replacements = [] for m in str_wikilinks: start, end = m.start(), m.end() - if intervals_where_not_to_look and not list( - intervals_where_not_to_look.find_overlap(start, end) - ): - link: Optional[WikiLink] = process_wikilink( - self.config, content, m, self.absolute_path - ) - if link is not None: - replacements.append( - ( - start, - end, - link.markdown(), - ) + + # TODO: review this + # if intervals_where_not_to_look and not list( + # intervals_where_not_to_look.find_overlap(start, end) + # ): + link: Optional[WikiLink] = process_wikilink( + self.config, content, m, self.absolute_path + ) + log.debug(f"Processing wikilink: {link}") + if link is not None: + replacements.append( + ( + start, + end, + link.markdown(), ) + ) for start, end, new_text in reversed(replacements): content = content[:start] + new_text + content[end:] return content diff --git a/mkdocs_juvix/env.py b/mkdocs_juvix/env.py index 6d0f046..de43dbf 100644 --- a/mkdocs_juvix/env.py +++ b/mkdocs_juvix/env.py @@ -19,12 +19,7 @@ import mkdocs_juvix.utils as utils from mkdocs_juvix.juvix_version import MIN_JUVIX_VERSION -from mkdocs_juvix.utils import ( - compute_sha_over_folder, - fix_site_url, - hash_content_of, - is_juvix_markdown_file, -) +from mkdocs_juvix.utils import is_juvix_markdown_file log = get_plugin_logger(f"{Fore.BLUE}[juvix_mkdocs] (env) {Style.RESET_ALL}") @@ -43,7 +38,7 @@ class ENV: DIFF_AVAILABLE: bool DIFF_DIR: Path DIFF_OPTIONS: List[str] - SITE_URL: str + SITE_URL: str = getenv("SITE_URL", "/") SITE_DIR: Optional[str] JUVIX_VERSION: str = "" USE_DOT: bool @@ -52,6 +47,7 @@ class ENV: IMAGES_ENABLED: bool CLEAN_DEPS: bool = bool(getenv("CLEAN_DEPS", False)) UPDATE_DEPS: bool = bool(getenv("UPDATE_DEPS", False)) + TIMELIMIT: int = int(getenv("TIMELIMIT", 10)) REMOVE_CACHE: bool = bool(getenv("REMOVE_CACHE", False)) @@ -119,10 +115,8 @@ def __init__(self, config: Optional[MkDocsConfig] = None): exit(1) self.ROOT_PATH = Path(config_file).parent - self.SITE_URL = config.get("site_url", "") else: self.ROOT_PATH = Path(".").resolve() - self.SITE_URL = "" self.ROOT_ABSPATH = self.ROOT_PATH.absolute() self.CACHE_ABSPATH = self.ROOT_ABSPATH / self.CACHE_DIRNAME diff --git a/mkdocs_juvix/main.py b/mkdocs_juvix/main.py index 810cd57..671a31a 100644 --- a/mkdocs_juvix/main.py +++ b/mkdocs_juvix/main.py @@ -1,5 +1,5 @@ +import asyncio import json -import re import shutil import subprocess import textwrap @@ -10,6 +10,7 @@ from urllib.parse import urljoin import pathspec +import questionary import yaml # type:ignore from bs4 import BeautifulSoup # type:ignore from colorama import Back, Fore, Style # type: ignore @@ -19,7 +20,8 @@ from mkdocs.structure.files import Files from mkdocs.structure.pages import Page from semver import Version -from tqdm import tqdm # type: ignore +from tqdm import tqdm as sync_tqdm # type: ignore +from tqdm.asyncio import tqdm as async_tqdm # type: ignore from watchdog.events import FileSystemEvent from mkdocs_juvix.common.preprocesors.links import WLPreprocessor @@ -82,11 +84,6 @@ def time_spent(message: Optional[Any] = None, print_result: bool = False): T = TypeVar("T") -REGEX_MARKDOWN_WITH_METADATA = re.compile( - r"(?:^(?P---\n(?P.*)\n---\n\s*)(?P.*)(?:\n\s*$|\Z))", - re.DOTALL, -) - def template_error_message( filepath: Optional[Path], command: List[str], error_message: str @@ -110,74 +107,64 @@ class EnhancedMarkdownFile: def __init__(self, filepath: Path, env: ENV, config: MkDocsConfig): self.env: ENV = env self.config: MkDocsConfig = config - try: - # File paths and locations - self.absolute_filepath: Path = filepath.absolute() - self.original_in_cache_filepath: Path = ( - self.env.CACHE_ORIGINALS_ABSPATH - / self.absolute_filepath.relative_to(self.env.DOCS_ABSPATH) - ) - # copy the original file to the cache folder for faster lookup and control - self.copy_original_file_to_cache() - - self.src_uri: str = filepath.as_posix() - self.relative_filepath: Path = self.absolute_filepath.relative_to( - env.DOCS_ABSPATH - ) - self.url: str = urljoin( - env.SITE_URL, - self.relative_filepath.as_posix() - .replace(".juvix", "") - .replace(".md", ".html"), - ) + self.absolute_filepath: Path = filepath.absolute() + self.original_in_cache_filepath: Path = ( + self.env.CACHE_ORIGINALS_ABSPATH + / self.absolute_filepath.relative_to(self.env.DOCS_ABSPATH) + ) - # Module information - self.module_name: Optional[str] = env.unqualified_module_name(filepath) - self.qualified_module_name: Optional[str] = env.qualified_module_name( - filepath - ) + self.src_uri: str = filepath.as_posix() + self.relative_filepath: Path = self.absolute_filepath.relative_to( + env.DOCS_ABSPATH + ) + self.url: str = urljoin( + env.SITE_URL, + self.relative_filepath.as_posix() + .replace(".juvix", "") + .replace(".md", ".html"), + ) - # Markdown related, some filled later in the process - self._markdown_output: Optional[str] = None - self._metadata: Optional[dict] = None - self.cache_filepath: Path = env.compute_processed_filepath(filepath) - # the hash cache file is used to check if the file has changed - self.hash_cache_filepath: Path = env.compute_filepath_for_cached_hash_for( + # Module information + self.module_name: Optional[str] = env.unqualified_module_name(filepath) + self.qualified_module_name: Optional[str] = env.qualified_module_name(filepath) + + # Markdown related, some filled later in the process + self._markdown_output: Optional[str] = None + self._metadata: Optional[dict] = None + self.cache_filepath: Path = env.compute_processed_filepath(filepath) + # the hash cache file is used to check if the file has changed + self.hash_cache_filepath: Path = env.compute_filepath_for_cached_hash_for( + self.absolute_filepath + ) + self._root_juvix_project_path: Optional[Path] = None + + # Processing flags for each task (updating during the process) + self._processed_juvix_markdown: bool = False + self._processed_juvix_isabelle: bool = False + self._processed_juvix_html: bool = False + self._processed_images: bool = False + self._processed_wikilinks: bool = False + self._processed_snippets: bool = False + self._processed_errors: bool = False + + # Isabelle related + self._needs_isabelle: bool = False # Fill later in the process + self._include_isabelle_at_bottom: bool = False # Fill later in the process + self.cached_isabelle_filepath: Optional[Path] = ( + self.env.compute_filepath_for_juvix_isabelle_output_in_cache( self.absolute_filepath ) - self._root_juvix_project_path: Optional[Path] = None - - # Processing flags for each task (updating during the process) - self._processed_juvix_markdown: bool = False - self._processed_juvix_isabelle: bool = False - self._processed_juvix_html: bool = False - self._processed_images: bool = False - self._processed_wikilinks: bool = False - self._processed_snippets: bool = False - self._processed_errors: bool = False - - # Isabelle related - self._needs_isabelle: bool = False # Fill later in the process - self._include_isabelle_at_bottom: bool = False # Fill later in the process - self.cached_isabelle_filepath: Optional[Path] = ( - self.env.compute_filepath_for_juvix_isabelle_output_in_cache( - self.absolute_filepath - ) - ) - - # Error handling - self._cached_error_messages: Dict[str, Optional[str]] = { - "juvix_markdown": None, - "juvix_isabelle": None, - "juvix_html": None, - "images": None, - "wikilinks": None, - "snippets": None, - } + ) - except Exception as e: - log.error(f"Error initializing JuvixMarkdownFile: {e}") - raise + # Error handling + self._cached_error_messages: Dict[str, Optional[str]] = { + "juvix_markdown": None, + "juvix_isabelle": None, + "juvix_html": None, + "images": None, + "wikilinks": None, + "snippets": None, + } def __str__(self) -> str: return f"{Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}" @@ -193,11 +180,15 @@ def to_dict(self) -> Dict[str, Any]: else None, "url": self.url, "content_hash": self.hash, - # Processing flags - "needs_isabelle": self._needs_isabelle, - "include_isabelle_at_bottom": self._include_isabelle_at_bottom, # Error handling "error_messages": self._cached_error_messages, + "processed": { + "juvix_markdown": self._processed_juvix_markdown, + "isabelle": self._processed_juvix_isabelle, + "images": self._processed_images, + "snippets": self._processed_snippets, + "wikilinks": self._processed_wikilinks, + }, } # ------------------------------------------------------------------ @@ -253,8 +244,8 @@ def hash(self) -> Optional[str]: def reset_processed_flags(self): self._processed_juvix_markdown = False + self._processed_juvix_isabelle = False self._processed_images = False - self._processed_isabelle = False self._processed_snippets = False self._processed_wikilinks = False self._processed_errors = False @@ -326,14 +317,21 @@ def run_pipeline(self, save_markdown: bool = True, force: bool = False) -> None: """ if self.changed_since_last_run(): self.generate_original_markdown(save_markdown=save_markdown) - if is_juvix_markdown_file(self.absolute_filepath): - self.generate_juvix_markdown(save_markdown=save_markdown, force=force) + if ( + is_juvix_markdown_file(self.absolute_filepath) + and self.env.juvix_enabled + ): + self.generate_juvix_markdown_output( + save_markdown=save_markdown, force=force + ) self.generate_isabelle_theories( save_markdown=save_markdown, force=force ) self.generate_images(save_markdown=save_markdown, force=force) - self.generate_wikilinks(save_markdown=save_markdown, force=force) - self.generate_snippets(save_markdown=save_markdown, force=force) + self.replaces_wikilinks_by_markdown_links( + save_markdown=save_markdown, force=force + ) + self.render_snippets(save_markdown=save_markdown, force=force) @time_spent(message="> saving markdown output") def save_markdown_output(self, md_output: str) -> Optional[Path]: @@ -524,21 +522,17 @@ def _run_juvix_root_project_path(self) -> Optional[Path]: self._build_juvix_root_project_path_command(), capture_output=True, text=True, + timeout=self.env.TIMELIMIT, ) return Path(result.stdout.strip()) except Exception as e: self.save_error_message(str(e), "juvix_root") - log.error( - template_error_message( - self.relative_filepath, - self._build_juvix_root_project_path_command(), - str(e), - ) - ) return None @property def root_juvix_project_path(self) -> Optional[Path]: + if not self.env.juvix_enabled: + return None if self._root_juvix_project_path is None: self._root_juvix_project_path = self._run_juvix_root_project_path() return self._root_juvix_project_path @@ -569,7 +563,10 @@ def _run_command_juvix_markdown(self, force: bool = False) -> Optional[str]: The error message is saved to be displayed in the markdown output of the file. """ - if not is_juvix_markdown_file(self.absolute_filepath): + if ( + not is_juvix_markdown_file(self.absolute_filepath) + or not self.env.juvix_enabled + ): return None self._processed_juvix_markdown = ( @@ -587,6 +584,7 @@ def _run_command_juvix_markdown(self, force: bool = False) -> Optional[str]: check=True, capture_output=True, text=True, + timeout=self.env.TIMELIMIT, ) if result.returncode != 0: raise Exception(result.stderr) @@ -633,29 +631,36 @@ def _process_juvix_html(self, update_assets: bool = False) -> None: since the last time `juvix html` was run on it. Otherwise, it reads the cached HTML output from the cache file. """ - if not is_juvix_markdown_file(self.absolute_filepath): + if ( + not is_juvix_markdown_file(self.absolute_filepath) + or not self.env.juvix_enabled + ): + return None + + if self._processed_juvix_html and not self.changed_since_last_run(): + log.debug( + f"> Skipping HTML generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" + ) return None self.env.CACHE_HTML_PATH.mkdir(parents=True, exist_ok=True) self.clear_error_messages("juvix_html") try: - clear_line() - log.info( - f"> running juvix html on {Fore.MAGENTA}{self.relative_filepath}{Style.RESET_ALL}" - ) output = subprocess.run( self._build_juvix_html_command(), cwd=self.env.DOCS_ABSPATH, check=True, capture_output=True, text=True, + timeout=self.env.TIMELIMIT, ) if output.returncode != 0: self.save_error_message(output.stderr, "juvix_html") return None + self._processed_juvix_html = True # ------------------------------------------------------------ # Rename the HTML files of Juvix Markdown files, .html -> .judoc.html # ------------------------------------------------------------ @@ -735,7 +740,10 @@ def _run_juvix_isabelle(self) -> Optional[str]: saved to be displayed in the markdown output of the file. The output is not Markdown, it is Isabelle/HOL code. """ - if not is_juvix_markdown_file(self.absolute_filepath): + if ( + not is_juvix_markdown_file(self.absolute_filepath) + or not self.env.juvix_enabled + ): return None self.clear_error_messages("juvix_isabelle") @@ -749,6 +757,7 @@ def _run_juvix_isabelle(self) -> Optional[str]: # check=True, capture_output=True, text=True, + timeout=self.env.TIMELIMIT, ) if result.returncode != 0: self.save_error_message(result.stderr, "juvix_isabelle") @@ -851,12 +860,12 @@ def _fix_unclosed_snippet_annotations(isabelle_output: str) -> str: # Juvix Markdown files # ------------------------------------------------------------------------ - def _skip_generation(self, process_tags: List[str] = []) -> bool: + def _skip_generation(self, processed_tags: List[str] = []) -> bool: """ Skip the generation of the markdown output if the file has not changed since the last time the pipeline was run on it. """ - for tag in process_tags: + for tag in processed_tags: if tag == "juvix_markdown" and not self._processed_juvix_markdown: return False if tag == "isabelle" and not self._processed_juvix_isabelle: @@ -869,26 +878,32 @@ def _skip_generation(self, process_tags: List[str] = []) -> bool: return False if tag == "errors" and not self._processed_errors: return False - return not self.changed_since_last_run() + return False def skip_and_use_cache_for_process( - self, process_tag: str, force: bool = False + self, processed_tag: str, force: bool = False ) -> Optional[str]: if force: - setattr(self, process_tag, False) + setattr(self, processed_tag, False) try: - if self._skip_generation(process_tags=[process_tag]): - log.debug(f"Reading cached markdown from {self.cache_filepath}") + if not self.changed_since_last_run(): self.load_and_print_saved_error_messages() + return None + if self._skip_generation(processed_tags=[processed_tag]): + log.debug(f"Reading cached markdown from {self.cache_filepath}") return self.cache_filepath.read_text() except Exception as e: log.error( f"Failed to read cached markdown from" f"{Fore.GREEN}{self.cache_filepath}{Style.RESET_ALL}:\n{e}" ) - return None return None + # ------------------------------------------------------------------------ + # Generate and save markdown output, the task run over either markdown or + # Juvix Markdown files + # ------------------------------------------------------------------------ + def generate_original_markdown(self, save_markdown: bool = True) -> None: """ Save the original markdown output for the file for later use. @@ -900,27 +915,28 @@ def generate_original_markdown(self, save_markdown: bool = True) -> None: except Exception as e: log.error(f"Failed to save markdown output, we however continue: {e}") - def generate_juvix_markdown( + def generate_juvix_markdown_output( self, save_markdown: bool = True, force: bool = False - ) -> Optional[str]: + ) -> None: """ Generate the markdown output for the file. """ if not is_juvix_markdown_file(self.absolute_filepath): log.debug( - f"> Skipping markdown generation for {Fore.GREEN}{self}{Style.RESET_ALL} " + f"> Skipping markdown generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} " f"because it is not a Juvix Markdown file" ) return None - if result := self.skip_and_use_cache_for_process( - force=force, - process_tag="juvix_markdown", + if ( + self._processed_juvix_markdown + and not force + and not self.changed_since_last_run() ): log.debug( - f"> Returning cached markdown output for {Fore.GREEN}{self}{Style.RESET_ALL}" + f"> Skipping markdown generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" ) - return result + return None markdown_output: str = self.cache_filepath.read_text() metadata = parse_front_matter(markdown_output) or {} @@ -936,23 +952,26 @@ def generate_juvix_markdown( self.save_markdown_output(_output) else: self._processed_juvix_markdown = False - return _output @time_spent(message="> generating isabelle theories") def generate_isabelle_theories( self, save_markdown: bool = True, force: bool = False - ) -> Optional[str]: + ) -> None: """ Process the Isabelle translation, saving the output to the cache folder. """ if not is_juvix_markdown_file(self.absolute_filepath): return None - if result := self.skip_and_use_cache_for_process( - force=force, - process_tag="isabelle", + if ( + self._processed_juvix_isabelle + and not force + and not self.changed_since_last_run() ): - return result + log.debug( + f"> Skipping isabelle generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" + ) + return None markdown_output: str = self.cache_filepath.read_text() metadata = parse_front_matter(markdown_output) or {} @@ -963,10 +982,8 @@ def generate_isabelle_theories( self._needs_isabelle, ) _output = None - if ( - (self._needs_isabelle or self._needs_isabelle_at_bottom) - and not self._processed_juvix_isabelle - or force + if (self._needs_isabelle or self._needs_isabelle_at_bottom) and ( + not self._processed_juvix_isabelle or force ): _output = self.process_isabelle_translation( content=markdown_output, @@ -977,16 +994,23 @@ def generate_isabelle_theories( self.save_markdown_output(_output) else: self._processed_juvix_isabelle = False - return _output @time_spent(message="> extracting snippets") - def generate_snippets( - self, save_markdown: bool = True, force: bool = False - ) -> Optional[str]: + def render_snippets(self, save_markdown: bool = True, force: bool = False) -> None: """ - Modify the markdown output by adding the snippets. This requires the - preprocess of Juvix and Isabelle to be ocurred before. + Modify the markdown output by adding the snippets found in the file. + This requires the preprocess of Juvix and Isabelle to be ocurred before. """ + if self._processed_snippets and not force and not self.changed_since_last_run(): + log.debug( + f"> Skipping snippets generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" + ) + return None + + log.debug( + f"Processing snippets for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL}" + ) + _markdown_output: str = self.cache_filepath.read_text() metadata = parse_front_matter(_markdown_output) or {} preprocess = metadata.get("preprocess", {}) @@ -997,7 +1021,8 @@ def generate_snippets( if _output and save_markdown: self._processed_snippets = True self.save_markdown_output(_output) - return _output + else: + self._processed_snippets = False @time_spent(message="> processing snippets") def run_snippet_preprocessor( @@ -1025,33 +1050,46 @@ def run_snippet_preprocessor( return content or "Something went wrong processing snippets" @time_spent(message="> generating wikilinks") - def generate_wikilinks( + def replaces_wikilinks_by_markdown_links( self, save_markdown: bool = True, force: bool = False - ) -> Optional[str]: + ) -> None: """ - Modify the markdown output by adding the wikilinks. This requires the - preprocess of Juvix and Isabelle to be ocurred before. + Modify the markdown output by replacing the wikilinks by markdown links. + This requires the preprocess of Juvix and Isabelle to be ocurred before. """ - _output = None - self._processed_wikilinks = False if force else self._processed_wikilinks - _markdown_output = self.markdown_output - if _markdown_output is None: - log.error( - f"Failed to read markdown output from {self.relative_filepath} when processing for wikilinks" + + if ( + self._processed_wikilinks + and not force + and not self.changed_since_last_run() + ): + log.debug( + f"> Skipping wikilinks generation for {Fore.GREEN}{self}{Style.RESET_ALL} using cached output" ) return None + + log.debug(f"Processing wikilinks for {Fore.GREEN}{self}{Style.RESET_ALL}") + _output = None + _markdown_output = self.cache_filepath.read_text() + log.debug(f"Read markdown content from {self.cache_filepath}") metadata = parse_front_matter(_markdown_output) or {} + log.debug(f"Parsed front matter: {metadata}") preprocess = metadata.get("preprocess", {}) needs_wikilinks = preprocess.get("wikilinks", True) - if needs_wikilinks and not self._processed_wikilinks: + log.debug(f"Needs wikilinks: {needs_wikilinks}") + if needs_wikilinks and (not self._processed_wikilinks or force): _output = self.run_wikilinks_preprocessor( content=_markdown_output, modify_markdown_output=True, ) + log.debug(f"Ran wikilinks preprocessor, output: {_output}") if _output and save_markdown: - self.save_markdown_output(_output) self._processed_wikilinks = True - return _output + self.save_markdown_output(_output) + log.debug(f"Saved markdown output for {self.relative_filepath}") + else: + self._processed_wikilinks = False + log.debug(f"Did not save markdown output for {self.relative_filepath}") @time_spent(message="> processing wikilinks") def run_wikilinks_preprocessor( @@ -1069,7 +1107,9 @@ def run_wikilinks_preprocessor( if modify_markdown_output: content = wl_preprocessor._run(content) - return content + "\n" + TOKEN_LIST_WIKILINKS + "\n" + if TOKEN_LIST_WIKILINKS not in content: + content = content + "\n" + TOKEN_LIST_WIKILINKS + "\n" + return content def generate_images( self, save_markdown: bool = True, force: bool = False @@ -1078,6 +1118,15 @@ def generate_images( Modify the markdown output by adding the images. This requires the preprocess of Juvix and Isabelle to be ocurred before. """ + # if result := self.skip_and_use_cache_for_process( + # force=force, + # processed_tag="images", + # ): + # log.debug( + # f"> Skipping images generation for {Fore.GREEN}{self.relative_filepath}{Style.RESET_ALL} using cached output" + # ) + # return result + _output = None _markdown_output = self.cache_filepath.read_text() metadata = parse_front_matter(_markdown_output) or {} @@ -1096,28 +1145,6 @@ def generate_images( self._processed_images = False return _output - @time_spent(message="> generating errors") - def write_errors_in_markdown( - self, save_markdown: bool = True, force: bool = False - ) -> Optional[str]: - """ - Modify the markdown output by adding the errors. This requires the - preprocess of Juvix and Isabelle to be ocurred before. - """ - _output = None - _markdown_output = self.cache_filepath.read_text() - metadata = parse_front_matter(_markdown_output) or {} - preprocess = metadata.get("preprocess", {}) - needs_errors = preprocess.get("errors", True) - if needs_errors and (not self._processed_errors or force): - _output = self.add_errors_to_markdown(content=_markdown_output) - if save_markdown: - self.save_markdown_output(_output) - self._processed_errors = True - else: - self._processed_errors = False - return _output - class EnhancedMarkdownCollection: """ @@ -1126,14 +1153,15 @@ class EnhancedMarkdownCollection: env: ENV config: MkDocsConfig + force_wikilinks_generation: bool = False - @time_spent(message="> initializing enhanced markdown collection") + # @time_spent(message="> initializing enhanced markdown collection") def __init__(self, config, env: ENV, docs: Optional[Path] = None): self.env: ENV = env self.config = config - try: - self.docs_path: Path = docs or env.DOCS_ABSPATH + self.docs_path: Path = docs or env.DOCS_ABSPATH + try: # The list of markdown files to be processed, filled later self.files: Optional[List[EnhancedMarkdownFile]] = None @@ -1158,7 +1186,7 @@ def __init__(self, config, env: ENV, docs: Optional[Path] = None): raise @time_spent(message="> storing original files in cache for faster lookup") - def cache_orginals(self) -> List[EnhancedMarkdownFile]: + def scanning_originals(self) -> List[EnhancedMarkdownFile]: """ Cache the original Juvix Markdown files in the cache folder for faster lookup. @@ -1177,25 +1205,24 @@ def cache_orginals(self) -> List[EnhancedMarkdownFile]: file for file in md_files if not set(file.parts) & set(SKIP_DIRS) ] - with tqdm( - total=len(files_to_process), desc="> creating cache database" + with sync_tqdm( + total=len(files_to_process), desc="Scanning files for faster lookup" ) as pbar: for file in files_to_process: + pbar.set_postfix_str( + f"{Fore.MAGENTA}{file.relative_to(self.docs_path)}{Style.RESET_ALL}" + ) enhanced_file = EnhancedMarkdownFile(file, self.env, self.config) self.files.append(enhanced_file) enhanced_file.original_in_cache_filepath.parent.mkdir( parents=True, exist_ok=True ) - current_file = enhanced_file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - log.debug( f"Copying original content from {file} to {enhanced_file.original_in_cache_filepath} for safe content extraction" ) shutil.copy(file, enhanced_file.original_in_cache_filepath) pbar.update(1) + return self.files except Exception as e: @@ -1271,11 +1298,7 @@ def save_juvix_modules_json(self) -> Optional[Path]: try: json_path = self.env.CACHE_ABSPATH / "juvix_modules.json" json_content = json.dumps( - [ - file.to_dict() - for file in self.files - if is_juvix_markdown_file(file.absolute_filepath) - ], + [file.to_dict() for file in self.files if file.absolute_filepath], indent=2, ) json_path.write_text(json_content) @@ -1302,12 +1325,11 @@ def run_pipeline_on_collection( log.debug("> no files to process") return - clear_screen() log.info( f"> running pipeline on {Fore.GREEN}{len(self.files)}{Style.RESET_ALL} files" ) - files_to_process = [ + files_to_process: List[EnhancedMarkdownFile] = [ file for file in self.files if file.changed_since_last_run() or file.has_error_message() @@ -1315,93 +1337,105 @@ def run_pipeline_on_collection( if len(files_to_process) == 0: log.debug(f"{Fore.YELLOW}no files to process{Style.RESET_ALL}") - else: - log.info( - f"> {Fore.GREEN}{len(files_to_process)}{Style.RESET_ALL} file{'s' if len(files_to_process) > 1 else ''} need to be processed due to changes or errors on previous processing" + return + + log.info( + f"> {Fore.GREEN}{len(files_to_process)}{Style.RESET_ALL} file" + f"{'s' if len(files_to_process) > 1 else ''} need to be processed " + "because it's the first time, their cached version is not up to date, " + "or the file has errors on previous processing" + ) + + @time_spent(message="collecting original markdowns for caching") + async def process_original_markdowns(): + await async_tqdm.gather( + *[ + asyncio.to_thread(file.generate_original_markdown) + for file in files_to_process + ], + desc="> collecting original markdowns for caching", ) - with tqdm( - total=len(files_to_process), - desc="> collecting original markdown for safe lookup", + asyncio.run(process_original_markdowns()) + clear_line() + + juvix_files = [ + file + for file in files_to_process + if is_juvix_markdown_file(file.absolute_filepath) + ] + + if generate_juvix_markdown and self.env.juvix_enabled: + with sync_tqdm( + total=len(juvix_files), desc="> running Juvix markdown" ) as pbar: - for file in files_to_process: + for file in juvix_files: current_file = file.relative_filepath pbar.set_postfix_str( f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" ) - file.generate_original_markdown() + file.generate_juvix_markdown_output() pbar.update(1) - clear_line() - juvix_files = [ - file - for file in files_to_process - if is_juvix_markdown_file(file.absolute_filepath) - ] - if generate_juvix_markdown: - with tqdm( - total=len(juvix_files), desc="> processing Juvix markdown" - ) as pbar: - for file in juvix_files: - current_file = file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - file.generate_juvix_markdown() - pbar.update(1) + if generate_juvix_isabelle and self.env.juvix_enabled: + with sync_tqdm( + total=len(juvix_files), desc="> generating Isabelle theories" + ) as pbar: + for file in juvix_files: + current_file = file.relative_filepath + pbar.set_postfix_str( + f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" + ) + file.generate_isabelle_theories() + pbar.update(1) clear_line() - if generate_juvix_isabelle: - with tqdm( - total=len(juvix_files), desc="> processing Isabelle theories" - ) as pbar: - for file in juvix_files: - current_file = file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - file.generate_isabelle_theories() - pbar.update(1) - clear_line() - if generate_images: - with tqdm( - total=len(files_to_process), desc="> processing images" - ) as pbar: - for file in files_to_process: - file.generate_images() - current_file = file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - pbar.update(1) + # clear_line() + # if generate_images: + # with tqdm(total=len(files_to_process), desc="> processing images") as pbar: + # for file in files_to_process: + # file.generate_images() + # current_file = file.relative_filepath + # pbar.set_postfix_str( + # f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" + # ) + # pbar.update(1) + + if generate_wikilinks: + + @time_spent(message="> processing wikilinks") + async def process_wikilinks(): + # if mkdocs + flist = ( + files_to_process + if not self.force_wikilinks_generation + else self.files + ) + await async_tqdm.gather( + *[ + asyncio.to_thread(file.replaces_wikilinks_by_markdown_links) + for file in flist + ], + total=len(flist), + desc="> processing wikilinks", + ) + asyncio.run(process_wikilinks()) clear_line() - if generate_wikilinks: - with tqdm( - total=len(files_to_process), desc="> processing wikilinks" - ) as pbar: - for file in files_to_process: - current_file = file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - file.generate_wikilinks() - pbar.update(1) - clear_line() - # snippets are generated on all the files if generate_snippets: - with tqdm(total=len(self.files), desc="> extracting snippets") as pbar: - for file in self.files: - current_file = file.relative_filepath - pbar.set_postfix_str( - f"{Fore.MAGENTA}{current_file}{Style.RESET_ALL}" - ) - file.generate_snippets() - pbar.update(1) - clear_line() + async def process_snippets(): + await async_tqdm.gather( + *[asyncio.to_thread(file.render_snippets) for file in self.files], + total=len(self.files), + desc="> processing snippets", + ) + + asyncio.run(process_snippets()) + clear_line() + self.update_cached_hash() self.save_juvix_modules_json() @@ -1419,27 +1453,44 @@ def generate_html(self, force: bool = False) -> None: all the Juvix Markdown files. Otherwise, we generate the HTML output for every Juvix Markdown file individually (not recommended). """ + if not self.env.JUVIX_ENABLED: + log.info( + f"{Fore.YELLOW}Juvix is not enabled, skipping HTML generation{Style.RESET_ALL}" + ) + return needs_to_generate_html = self.is_html_cache_empty() or self.has_changes() - if self.files is None and not needs_to_generate_html and not force: + if not needs_to_generate_html and not force: log.info("No files or changes detected, skipping HTML generation") return log.info("> adding auxiliary HTML files...") if self.everything_html_file and (needs_to_generate_html or force): self.everything_html_file._process_juvix_html(update_assets=True) + + if not self.is_html_cache_empty(): return + # we'll run the html generation over the entire collection so we can + # have at least one html file to be used as the "everything" file + self.remove_html_cache() self.env.CACHE_HTML_PATH.mkdir(parents=True, exist_ok=True) @time_spent(message="> generating HTML for files") def run_html_generation(files: List[EnhancedMarkdownFile]) -> None: - for file in files: - if is_juvix_markdown_file(file.absolute_filepath) and ( - needs_to_generate_html or force - ): + juvix_files = [ + file for file in files if is_juvix_markdown_file(file.absolute_filepath) + ] + with sync_tqdm( + total=len(juvix_files), desc="> generating HTML for files" + ) as pbar: + for file in juvix_files: + pbar.set_postfix_str( + f"{Fore.MAGENTA}{file.relative_filepath}{Style.RESET_ALL}" + ) file._process_juvix_html(update_assets=True) + pbar.update(1) run_html_generation(self.files) @@ -1465,6 +1516,7 @@ def clean_juvix_dependencies(self) -> None: clean_command, cwd=self.env.DOCS_ABSPATH, capture_output=True, + timeout=self.env.TIMELIMIT, ) if res.returncode != 0: log.error( @@ -1485,6 +1537,7 @@ def update_juvix_dependencies(self) -> bool: update_command, cwd=self.env.DOCS_ABSPATH, capture_output=True, + timeout=self.env.TIMELIMIT, ) if res.returncode != 0: log.error( @@ -1502,6 +1555,12 @@ class JuvixPlugin(BasePlugin): enhanced_collection: EnhancedMarkdownCollection wikilinks_plugin: WikilinksPlugin first_run: bool = True + response: Optional[str] = None + use_juvix_question = questionary.select( + "Do you want to process Juvix Markdown files (this will take longer)?", + choices=["yes", "no", "always", "never"], + default="no", + ) def on_startup(self, *, command: str, dirty: bool) -> None: clear_screen() @@ -1510,12 +1569,16 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig: self.env = ENV(config) config = fix_site_url(config) - # Initialize the wikilinks plugin which requires some preprocessing - self.wikilinks_plugin = WikilinksPlugin(config, self.env) - config = self.wikilinks_plugin.on_config(config) - self.env.SITE_DIR = config.get("site_dir", getenv("SITE_DIR", None)) - self.env.SITE_URL = config.get("site_url", getenv("SITE_URL", "")) + + # ask the user if they want to process Juvix Markdown files, options, + # yes no, always, never + if self.response in ["yes", "no"] or self.response is None: + self.response = self.use_juvix_question.ask() + if self.response == "never": + self.env.JUVIX_ENABLED = False + elif self.response == "always": + self.env.JUVIX_ENABLED = True if self.env.JUVIX_ENABLED and not self.env.JUVIX_AVAILABLE: log.error( @@ -1526,12 +1589,14 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig: "\n- JUVIX_PATH" ) - if self.env.juvix_enabled and self.first_run: + if self.first_run: + self.wikilinks_plugin = WikilinksPlugin(config, self.env) + config = self.wikilinks_plugin.on_config(config) self.enhanced_collection = EnhancedMarkdownCollection( env=self.env, config=config, ) - self.enhanced_collection.cache_orginals() + self.enhanced_collection.scanning_originals() self.add_footer_css_file_to_extra_css() if self.env.CLEAN_DEPS: @@ -1556,9 +1621,8 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]: List of the files to be included in the final build. These are copied to the site directory. """ - self.wikilinks_plugin.on_files(files, config) - return Files( + Files( [ file for file in files @@ -1567,6 +1631,11 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Optional[Files]: & set([".juvix-build", ".git"]) ] ) + # if mkdocs.yml is in the files, we need to process the wikilinks + mkdocs_config_filepath = self.env.ROOT_ABSPATH / "mkdocs.yml" + if mkdocs_config_filepath in [file.abs_src_path for file in files]: + self.enhanced_collection.force_wikilinks_generation = True + return files def on_nav(self, nav, config: MkDocsConfig, files: Files): return nav @@ -1585,11 +1654,12 @@ def on_page_read_source(self, page: Page, config: MkDocsConfig) -> Optional[str] self.enhanced_collection.get_enhanced_file_entry(abs_src_path) ) if file: - return file.write_errors_in_markdown() + output = file.cache_filepath.read_text() + return file.add_errors_to_markdown(output) else: log.error( f"{Fore.RED}File not found in collection: " - f"{Fore.YELLOW}{abs_src_path}{Style.RESET_ALL}, Try rerun " + f"{Fore.GREEN}{abs_src_path}{Style.RESET_ALL}, Try rerun " f"the build process." ) except Exception as e: @@ -1631,11 +1701,11 @@ def on_post_page(self, output: str, page: Page, config: MkDocsConfig) -> str: def on_post_build(self, config: MkDocsConfig) -> None: log.debug("> post build task: generating HTML for files") - self.enhanced_collection.generate_html() - self.enhanced_collection.update_cached_hash() - self.enhanced_collection.save_juvix_modules_json() - self.move_html_cache_to_site_dir() - # self.wikilinks_plugin.on_post_build(config) + if self.env.JUVIX_ENABLED: + self.enhanced_collection.generate_html() + self.move_html_cache_to_site_dir() + self.enhanced_collection.update_cached_hash() + self.enhanced_collection.save_juvix_modules_json() files_to_check = ( self.enhanced_collection.files if self.enhanced_collection.files else [] @@ -1655,7 +1725,6 @@ def move_html_cache_to_site_dir(self) -> None: log.error("No site directory specified. Skipping HTML cache move.") return - clear_line() log.info( f"> moving HTML cache to site directory: {Fore.GREEN}{self.env.SITE_DIR}{Style.RESET_ALL}" ) @@ -1711,7 +1780,7 @@ def wrapper(event: FileSystemEvent) -> None: return # clear the console - print("\033[H\033[J", end="", flush=True) + clear_screen() fpath = Path(fpathstr) if fpath.is_relative_to(self.env.DOCS_ABSPATH): log.info( diff --git a/mkdocs_juvix/utils.py b/mkdocs_juvix/utils.py index b277701..ba10ec2 100644 --- a/mkdocs_juvix/utils.py +++ b/mkdocs_juvix/utils.py @@ -92,17 +92,8 @@ def find_file_in_subdirs( def fix_site_url(config: MkDocsConfig) -> MkDocsConfig: site_url = os.getenv("SITE_URL") - if site_url: config["site_url"] = site_url - else: - mike_docs_version = os.getenv("MIKE_DOCS_VERSION") - if mike_docs_version: - log.debug( - f"Using MIKE_DOCS_VERSION environment variable: {mike_docs_version}" - ) - config["docs_version"] = mike_docs_version - # Ensure site_url ends with a slash if not config.get("site_url", None): config["site_url"] = "" diff --git a/poetry.lock b/poetry.lock index 128b5b0..7730f2e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1903,20 +1903,20 @@ files = [ [[package]] name = "tqdm" -version = "4.67.0" +version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, - {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] discord = ["requests"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] @@ -2087,4 +2087,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.14" -content-hash = "27752a7366fb2b0a47ae4937a8d52162ff7a58fa3e9566885c70e155c2d06156" +content-hash = "d1c396152b638e112a6ce0bdb492f29d39827e0b4578e921511d487e498873d5" diff --git a/pyproject.toml b/pyproject.toml index e24ec13..b383c18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mkdocs-juvix-plugin" -version = "0.4.2" +version = "0.4.3" description = "MkDocs documentation with support for Juvix Markdown files" authors = ["Jonathan Prieto-Cubides, and GitHub contributors"] license = "MIT" @@ -35,7 +35,7 @@ graphviz = "^0.20.3" python-levenshtein = "^0.26.0" intervaltree = "^3.1.0" ncls = "^0.0.68" -tqdm = "^4.66.5" +tqdm = "^4.67.1" trio = "^0.27.0" [tool.poetry.group.dev.dependencies] @@ -48,6 +48,8 @@ pytest = "^8.3.3" requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + + [tool.poetry.group.test.dependencies] pytest = "*" diff --git a/src/cli.py b/src/cli.py index ea95d22..3befce2 100644 --- a/src/cli.py +++ b/src/cli.py @@ -784,7 +784,6 @@ def install_poetry_package(package_name, skip_flag=False, development_flag=False help="Path to the mkdocs configuration file", show_default=True, ) -@click.option("--debug", is_flag=True, help="Set the environment variable DEBUG to 1") @click.option( "--remove-cache", "-r", is_flag=True, help="Remove the cache before serving" ) @@ -796,7 +795,6 @@ def serve( no_open: bool, quiet: bool, config_file: Path, - debug: bool, verbose: bool, remove_cache: bool, ): @@ -812,17 +810,16 @@ def serve( fg="red", ) return - previous_debug: str | None = os.environ.get("DEBUG") - if debug: - os.environ["DEBUG"] = "1" + if remove_cache: try: - shutil.rmtree(project_path / ".cache-juvix-mkdocs") + if (project_path / ".cache-juvix-mkdocs").exists(): + shutil.rmtree(project_path / ".cache-juvix-mkdocs") + else: + click.secho("Cache folder not found.", fg="yellow") except Exception: click.secho("Failed to remove .cache-juvix-mkdocs folder.", fg="red") - if previous_debug: - os.environ["DEBUG"] = previous_debug - return + mkdocs_serve_cmd = ["poetry", "run", "mkdocs", "serve", "--clean"] if not no_open: mkdocs_serve_cmd.append("--open") @@ -833,19 +830,15 @@ def serve( if config_file: mkdocs_serve_cmd.append(f"--config-file={config_file}") try: + click.secho(f"> Running command: {' '.join(mkdocs_serve_cmd)}", fg="yellow") subprocess.run(mkdocs_serve_cmd, cwd=project_path, check=True) except subprocess.CalledProcessError as e: click.secho("Failed to start the server.", fg="red") click.secho(f"Error: {e}", fg="red") - if previous_debug: - os.environ["DEBUG"] = previous_debug except FileNotFoundError: click.secho("Failed to start the server.", fg="red") click.secho("Make sure Poetry is installed and in your system PATH.", fg="red") - if previous_debug: - os.environ["DEBUG"] = previous_debug - @cli.command() @click.option( @@ -863,7 +856,6 @@ def serve( help="Path to the mkdocs configuration file", show_default=True, ) -@click.option("--debug", is_flag=True, help="Set the environment variable DEBUG to 1") @click.option( "--remove-cache", "-r", is_flag=True, help="Remove the cache before building" ) @@ -874,7 +866,6 @@ def serve( def build( project_path: Path, config_file: Path, - debug: bool, remove_cache: bool, quiet: bool, verbose: bool, @@ -889,9 +880,6 @@ def build( fg="red", ) return - previous_debug: str | None = os.environ.get("DEBUG") - if debug: - os.environ["DEBUG"] = "1" mkdocs_build_cmd = ["poetry", "run", "mkdocs", "build"] if config_file: mkdocs_build_cmd.append(f"--config-file={config_file}") @@ -904,16 +892,12 @@ def build( shutil.rmtree(project_path / ".cache-juvix-mkdocs") except Exception: click.secho("Failed to remove .cache-juvix-mkdocs folder.", fg="red") - if previous_debug: - os.environ["DEBUG"] = previous_debug return try: subprocess.run(mkdocs_build_cmd, cwd=project_path, check=True) except subprocess.CalledProcessError as e: click.secho("Failed to build the project.", fg="red") click.secho(f"Error: {e}", fg="red") - if previous_debug: - os.environ["DEBUG"] = previous_debug if __name__ == "__main__": diff --git a/src/fixtures/ci.yml b/src/fixtures/ci.yml index f3128ba..48e33c9 100644 --- a/src/fixtures/ci.yml +++ b/src/fixtures/ci.yml @@ -67,7 +67,9 @@ jobs: - name: Build MkDocs Project run: poetry run mkdocs build --clean --config-file mkdocs.yml env: - SITE_URL: https://${{{{ github.repository_owner }}}}.github.io/${{{{ github.event.repository.name }}}} + SITE_URL: https://${{{{ github.repository_owner }}}}.github.io/${{{{ + github.event.repository.name }}}} + - if: success() uses: JamesIves/github-pages-deploy-action@v4.6.4 with: