implement: Make pandoc and the filters optional (t9)

Author: peterbjohnson

docs/source/filters.py

@@ -1,6 +1,5 @@
 import importlib
 import os
-import pkgutil
 import shutil
 import subprocess
 from pathlib import Path
@@ -19,13 +18,7 @@ def generate_filters_docs():
     autosummary_directory.mkdir(exist_ok=True, parents=True)
     static_pdf_directory.mkdir(exist_ok=True)
 
-    filters = (
-        i.name
-        for i in pkgutil.iter_modules(in2lambda.filters.__path__)
-        if i.name != "markdown"
-    )
-
-    for filter_name in filters:
+    for filter_name in in2lambda.filters.builtin_filters():
         filter_module = importlib.import_module(
             f"in2lambda.filters.{filter_name}.filter"
         )

in2lambda/api/set.py

@@ -136,6 +136,42 @@ def to_json(self, output_dir: str) -> None:
 
         json_convert.main(self, output_dir)
 
+    @classmethod
+    def from_json(cls, path: str) -> "Set":
+        """Loads a Lambda Feedback export, as a folder or a zip, into a Set.
+
+        Only what the Set holds is read: the name, description, visibilities, and each
+        question's title, main text, parts, worked solutions and images. A zip is
+        extracted to a temporary directory that is not removed afterwards, because the
+        loaded images point into it.
+
+        Args:
+            path: The exported set's folder or zip.
+
+        Returns:
+            The loaded set.
+
+        Raises:
+            ValueError: If the export does not hold exactly one ``set_*.json``.
+
+        Examples:
+            >>> import tempfile
+            >>> s = Set()
+            >>> s.add_question("Question 1")
+            >>> s.add_question("Question 2")
+            >>> with tempfile.TemporaryDirectory() as temp_dir:
+            ...     s.to_json(temp_dir)
+            ...     from_folder = Set.from_json(f"{temp_dir}/set")
+            ...     from_zip = Set.from_json(f"{temp_dir}/set.zip")
+            >>> [question.title for question in from_folder.questions]
+            ['Question 1', 'Question 2']
+            >>> [question.title for question in from_zip.questions]
+            ['Question 1', 'Question 2']
+        """
+        from in2lambda.json_convert import json_convert
+
+        return json_convert.load(path)
+
     def set_name(self, name: str) -> None:
         """Sets the name of the set.
 

in2lambda/filters/__init__.py

@@ -1 +1,19 @@
 """Subject specific panflute filters for parsing LaTeX documents."""
+
+import pkgutil
+
+
+def builtin_filters() -> list[str]:
+    """Lists the filters shipped with in2lambda.
+
+    Each filter is a subpackage of this one; ``markdown`` is a helper module they share.
+
+    Returns:
+        The filter names, as accepted by :func:`in2lambda.main.runner`.
+
+    Examples:
+        >>> from in2lambda.filters import builtin_filters
+        >>> "PartsSepSol" in builtin_filters()
+        True
+    """
+    return [i.name for i in pkgutil.iter_modules(__path__) if i.name != "markdown"]

in2lambda/json_convert/json_convert.py

@@ -1,15 +1,19 @@
-"""Converts questions from a Python set object into Lambda Feedback JSON."""
+"""Converts questions between a Python set object and Lambda Feedback JSON."""
 
 import json
 import os
 import re
 import shutil
+import tempfile
 import zipfile
 from copy import deepcopy
 from pathlib import Path
 from typing import Any
 
+from in2lambda.api.part import Part
+from in2lambda.api.question import Question
 from in2lambda.api.set import Set
+from in2lambda.api.visibility_status import VisibilityController, VisibilityStatus
 
 MINIMAL_QUESTION_TEMPLATE = "minimal_template_question.json"
 MINIMAL_SET_TEMPLATE = "minimal_template_set.json"
@@ -96,12 +100,14 @@ def converter(
                     ListQuestions[i].parts[j].worked_solution
                 )
 
-        # Output file
+        # Lambda Feedback names the file after the title with only spaces made
+        # underscores. Path separators go too, so a title cannot leave the set folder,
+        # and so do the characters Windows forbids in file names.
         filename = (
             "question_"
             + str(i).zfill(3)
             + "_"
-            + re.sub(r"[^\w\-_.]", "_", output["title"].strip())
+            + re.sub(r'[\s/\\<>:"|?*]', "_", output["title"].strip())
         )
 
         # write questions into directory
@@ -145,3 +151,84 @@ def main(set_questions: Set, output_dir: str) -> None:
         except OSError as e:
             print("Error: %s : %s" % (output_dir, e.strerror))
     converter(question_template, set_template, set_questions, output_dir)
+
+
+def load(path: str) -> Set:
+    """Reads a Lambda Feedback export into a Set, keeping only what the model holds.
+
+    A zip is extracted to a new temporary directory, which is left for the operating
+    system to clear: the loaded images point into it and must still exist when the
+    set is written out.
+
+    Args:
+        path: An exported set, as a folder or a zip, with or without a top-level folder.
+
+    Returns:
+        The set, with each question's images as absolute paths into ``media/``.
+
+    Raises:
+        ValueError: If the export does not hold exactly one ``set_*.json``.
+    """
+    root = Path(path)
+    if root.suffix == ".zip":
+        extracted = tempfile.mkdtemp(prefix="in2lambda-")
+        with zipfile.ZipFile(root) as zf:
+            zf.extractall(extracted)
+        root = Path(extracted)
+
+    set_files = list(root.rglob("set_*.json"))
+    if len(set_files) != 1:
+        raise ValueError(f"Expected one set_*.json in {path}, found {len(set_files)}")
+    (set_file,) = set_files
+    export_dir = set_file.parent
+
+    set_json = json.loads(set_file.read_text())
+    question_set = Set(
+        _name=set_json["name"],
+        _description=set_json["description"],
+        _finalAnswerVisibility=VisibilityController(
+            VisibilityStatus(set_json["finalAnswerVisibility"])
+        ),
+        _workedSolutionVisibility=VisibilityController(
+            VisibilityStatus(set_json["workedSolutionVisibility"])
+        ),
+        _structuredTutorialVisibility=VisibilityController(
+            VisibilityStatus(set_json["structuredTutorialVisibility"])
+        ),
+    )
+
+    question_files = sorted(
+        export_dir.glob("question_*.json"),
+        key=lambda file: json.loads(file.read_text())["orderNumber"],
+    )
+    media = sorted((export_dir / "media").glob("*"))
+    for question_file in question_files:
+        question_json = json.loads(question_file.read_text())
+        parts = [
+            Part(
+                text=part["content"],
+                worked_solution=(
+                    part["workedSolution"]["content"]
+                    if "workedSolution" in part
+                    else ""
+                ),
+            )
+            for part in question_json["parts"]
+        ]
+        question_set.questions.append(
+            Question(
+                title=question_json["title"],
+                main_text=question_json["masterContent"],
+                parts=parts,
+                images=[
+                    str(image)
+                    for image in media
+                    if image.name.startswith(f"{question_file.stem}_")
+                ],
+                # Every loaded part already has its text and solution, so further
+                # add_part_text/add_solution calls must add parts after them rather
+                # than overwrite the first.
+                _last_part={"solution": len(parts), "text": len(parts)},
+            )
+        )
+    return question_set

in2lambda/json_convert/minimal_template_question.json

@@ -14,7 +14,6 @@
       "answerContent": "",
       "responseAreas": [],
       "workedSolution": {
-        "title": "",
         "content": "Part worked solution here",
         "children": []
       }

in2lambda/main.py

@@ -8,7 +8,6 @@
 
 import importlib
 import importlib.util
-import pkgutil
 import shutil
 import subprocess
 from typing import Optional
@@ -191,14 +190,7 @@ def runner(
 # Python files in the subjects directory
 @click.argument(
     "chosen_filter",
-    type=click.Choice(
-        [
-            i.name
-            for i in pkgutil.iter_modules(in2lambda.filters.__path__)
-            if i.name != "markdown"
-        ],
-        case_sensitive=False,
-    ),
+    type=click.Choice(in2lambda.filters.builtin_filters(), case_sensitive=False),
 )
 @click.option(
     "--out",

in2lambda/validation/__init__.py

@@ -0,0 +1,37 @@
+"""Pre-flight checks for the ``#``/``##`` and ``$``/``$$`` markdown delimiters.
+
+The markdown that in2lambda converts - however it was produced - is a shared
+contract with Lambda Feedback. These checks catch structural mistakes in that
+markdown, currently unbalanced or misplaced math delimiters, before it is
+converted.
+"""
+
+from in2lambda.validation.delimiters import MathDelimiterError, math_delimiter_checker
+
+__all__ = ["MathDelimiterError", "math_delimiter_checker", "check_markdown"]
+
+
+def check_markdown(md_content: str) -> list[MathDelimiterError]:
+    """Run every markdown check and return the problems found.
+
+    Args:
+        md_content: The markdown text to validate.
+
+    Returns:
+        A list of :class:`MathDelimiterError` members, one per problem found.
+        An empty list means the markdown passed every check.
+
+    Examples:
+        >>> from in2lambda.validation import check_markdown
+        >>> check_markdown("Inline $x = y$ is fine.")
+        []
+        >>> check_markdown("Unbalanced $x = y")
+        [<MathDelimiterError.MISSING_CLOSING_SINGLE_DOLLAR: 'unclosed inline $ ... $'>]
+    """
+    problems: list[MathDelimiterError] = []
+
+    result = math_delimiter_checker(md_content)
+    if result is not MathDelimiterError.PASSED:
+        problems.append(result)
+
+    return problems

in2lambda/validation/delimiters.py

@@ -0,0 +1,119 @@
+"""Checks that ``$ ... $`` and ``$$ ... $$`` math delimiters are balanced and placed correctly.
+
+KaTeX (and Lambda Feedback) expect inline math wrapped in single dollar signs on
+one line, and display math wrapped in ``$$`` that each sit alone on their own
+line. This module scans markdown character by character and reports the first
+delimiter mistake it finds.
+"""
+
+from enum import Enum
+
+
+class MathDelimiterError(Enum):
+    """Outcome of :func:`math_delimiter_checker`.
+
+    ``PASSED`` means no problem was found; every other member describes a
+    specific delimiter mistake. The value is a short human-readable message
+    suitable for showing on the command line.
+    """
+
+    PASSED = "ok"
+    MISSING_NEWLINE_BEFORE_OPENING_DISPLAY = "opening $$ must start its own line"
+    MISSING_NEWLINE_AFTER_OPENING_DISPLAY = "opening $$ must be followed by a newline"
+    DOUBLE_DOLLAR_INSTEAD_OF_CLOSING_SINGLE = "inline $ ... $ closed with $$"
+    MISSING_CLOSING_DOUBLE_INSTEAD_OF_SINGLE = (
+        "display $$ ... $$ closed with a single $"
+    )
+    MISSING_NEWLINE_BEFORE_CLOSING_DISPLAY = "closing $$ must start its own line"
+    MISSING_NEWLINE_AFTER_CLOSING_DISPLAY = "closing $$ must be followed by a newline"
+    INVALID_NEWLINE_INSIDE_INLINE = "newline inside an inline $ ... $ expression"
+    MISSING_CLOSING_SINGLE_DOLLAR = "unclosed inline $ ... $"
+    MISSING_CLOSING_DOUBLE_DOLLAR = "unclosed display $$ ... $$"
+
+
+def math_delimiter_checker(md_content: str) -> MathDelimiterError:
+    r"""Scan markdown for the first math-delimiter mistake.
+
+    ``\$`` is treated as a literal dollar sign, not a delimiter.
+
+    Args:
+        md_content: The markdown text to check.
+
+    Returns:
+        ``MathDelimiterError.PASSED`` if the delimiters are well formed,
+        otherwise the member describing the first problem found.
+
+    Examples:
+        >>> from in2lambda.validation.delimiters import math_delimiter_checker
+        >>> math_delimiter_checker("An inline $x = y$ expression.")
+        <MathDelimiterError.PASSED: 'ok'>
+        >>> math_delimiter_checker("Display:\n$$\nx = y\n$$")
+        <MathDelimiterError.PASSED: 'ok'>
+        >>> math_delimiter_checker("This costs \\$5, no math here.")
+        <MathDelimiterError.PASSED: 'ok'>
+        >>> math_delimiter_checker("Broken $x = y")
+        <MathDelimiterError.MISSING_CLOSING_SINGLE_DOLLAR: 'unclosed inline $ ... $'>
+    """
+    # False once we are inside a math expression and awaiting its closing delimiter.
+    expect_open_delimiter = True
+    # While inside an expression, whether it opened with a single "$" (inline) or "$$" (display).
+    expect_single_dollar = True
+
+    idx = 0
+    while idx < len(md_content):
+        prev_character = md_content[idx - 1] if idx > 0 else None
+        character = md_content[idx]
+        next_character = md_content[idx + 1] if idx + 1 < len(md_content) else None
+
+        if character == "$" and prev_character != "\\":
+            if expect_open_delimiter:
+                expect_open_delimiter = False
+
+                if next_character == "$":
+                    next_next_character = (
+                        md_content[idx + 2] if idx + 2 < len(md_content) else None
+                    )
+                    # "$$" must sit alone on its own line.
+                    if prev_character != "\n" and prev_character is not None:
+                        return MathDelimiterError.MISSING_NEWLINE_BEFORE_OPENING_DISPLAY
+                    if next_next_character != "\n":
+                        return MathDelimiterError.MISSING_NEWLINE_AFTER_OPENING_DISPLAY
+
+                    expect_single_dollar = False
+                    idx += 1  # Skip the second "$"; the loop increments idx again.
+                else:
+                    expect_single_dollar = True
+            else:
+                expect_open_delimiter = True
+
+                if expect_single_dollar and next_character == "$":
+                    return MathDelimiterError.DOUBLE_DOLLAR_INSTEAD_OF_CLOSING_SINGLE
+
+                elif not expect_single_dollar:
+                    if next_character != "$":
+                        return (
+                            MathDelimiterError.MISSING_CLOSING_DOUBLE_INSTEAD_OF_SINGLE
+                        )
+
+                    next_next_character = (
+                        md_content[idx + 2] if idx + 2 < len(md_content) else None
+                    )
+                    if prev_character != "\n" and prev_character is not None:
+                        return MathDelimiterError.MISSING_NEWLINE_BEFORE_CLOSING_DISPLAY
+                    if next_next_character != "\n" and next_next_character is not None:
+                        return MathDelimiterError.MISSING_NEWLINE_AFTER_CLOSING_DISPLAY
+
+                    idx += 1  # Skip the second "$"; the loop increments idx again.
+
+        # A newline may not appear inside an inline "$ ... $" expression.
+        elif character == "\n" and not expect_open_delimiter and expect_single_dollar:
+            return MathDelimiterError.INVALID_NEWLINE_INSIDE_INLINE
+
+        idx += 1
+
+    if expect_open_delimiter:
+        return MathDelimiterError.PASSED
+    elif expect_single_dollar:
+        return MathDelimiterError.MISSING_CLOSING_SINGLE_DOLLAR
+    else:
+        return MathDelimiterError.MISSING_CLOSING_DOUBLE_DOLLAR