Cell diffs show context, outputs, directives, and word-level changes

nbdev/_modidx.py

@@ -82,10 +82,17 @@
             'nbdev.diff': { 'nbdev.diff._cell_changes': ('api/diff.html#_cell_changes', 'nbdev/diff.py'),
                             'nbdev.diff._diff_order': ('api/diff.html#_diff_order', 'nbdev/diff.py'),
                             'nbdev.diff._dline': ('api/diff.html#_dline', 'nbdev/diff.py'),
-                            'nbdev.diff._file_srcs': ('api/diff.html#_file_srcs', 'nbdev/diff.py'),
+                            'nbdev.diff._dtag': ('api/diff.html#_dtag', 'nbdev/diff.py'),
+                            'nbdev.diff._file_cells': ('api/diff.html#_file_cells', 'nbdev/diff.py'),
                             'nbdev.diff._nb_srcdict': ('api/diff.html#_nb_srcdict', 'nbdev/diff.py'),
+                            'nbdev.diff._out_lines': ('api/diff.html#_out_lines', 'nbdev/diff.py'),
+                            'nbdev.diff._refine': ('api/diff.html#_refine', 'nbdev/diff.py'),
                             'nbdev.diff._src': ('api/diff.html#_src', 'nbdev/diff.py'),
                             'nbdev.diff._srcdict': ('api/diff.html#_srcdict', 'nbdev/diff.py'),
+                            'nbdev.diff._tokens': ('api/diff.html#_tokens', 'nbdev/diff.py'),
+                            'nbdev.diff._trunc': ('api/diff.html#_trunc', 'nbdev/diff.py'),
+                            'nbdev.diff._wline': ('api/diff.html#_wline', 'nbdev/diff.py'),
+                            'nbdev.diff._word_diff': ('api/diff.html#_word_diff', 'nbdev/diff.py'),
                             'nbdev.diff.cell_diffs': ('api/diff.html#cell_diffs', 'nbdev/diff.py'),
                             'nbdev.diff.changed_cells': ('api/diff.html#changed_cells', 'nbdev/diff.py'),
                             'nbdev.diff.nb_diff': ('api/diff.html#nb_diff', 'nbdev/diff.py'),

nbdev/diff.py

@@ -5,19 +5,20 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/19_diff.ipynb.
 
 # %% auto #0
-__all__ = ['MAXLEN', 'read_nb_from_git', 'nbs_pair', 'changed_cells', 'source_diff', 'cell_diffs', 'render_diff', 'nb_diff',
-           'nbdev_diff_driver', 'nbdev_diff']
+__all__ = ['MAXLEN', 'OUT_MAXLEN', 'MIN_RATIO', 'read_nb_from_git', 'nbs_pair', 'changed_cells', 'source_diff', 'cell_diffs',
+           'render_diff', 'nb_diff', 'nbdev_diff_driver', 'nbdev_diff']
 
 # %% ../nbs/api/19_diff.ipynb #10c8ad0b
 import json
 from fastcore.utils import *
 from fastcore.meta import delegates
 from fastcore.script import *
-from difflib import unified_diff
+from difflib import unified_diff, SequenceMatcher
 from subprocess import CalledProcessError
 from typing import Annotated
 from fastgit import Git
 from fastcore.nbio import *
+from fastcore.nbio import _directive
 from .doclinks import nbglob
 
 # %% ../nbs/api/19_diff.ipynb #a8981115
@@ -37,14 +38,24 @@ def read_nb_from_git(
     return dict2nb(json.loads(raw))
 
 # %% ../nbs/api/19_diff.ipynb #3ac25702
-def _src(c): return c.get('source','')
+def _src(c):
+    "Source of cell `c` without its leading directive lines"
+    dirs,code = c._partition()
+    return ''.join([l for l in dirs if not _directive(l, c.lang_)] + code)
 
 def _srcdict(nb, f=noop):
     "Dict of cell id->`f(cell)`, with positional fallback ids for pre-4.5 notebooks"
     return {c.get('id', f'c{c.idx_}'): f(c) for c in nb.cells}
 
 def _nb_srcdict(g:Git, nb_path, ref=None, f=noop): return _srcdict(read_nb_from_git(g, nb_path, ref), f)
 
+# %% ../nbs/api/19_diff.ipynb #17749428
+def _dtag(c):
+    "Cell `c`'s merged directives as a ` [k k=v]` header tag, in `dir_tag`'s form; `''` if none"
+    tag = ' '.join(k if not v else f'{k}={v}' for k,v in c.directives.items())
+    return f' [{tag}]' if tag else ''
+
+
 # %% ../nbs/api/19_diff.ipynb #27ff9a7b
 def nbs_pair(
     nb_path, # Path to the notebook
@@ -82,7 +93,6 @@ def cell_content(c):
     if dels: res |= {cid: fn(cid, old[cid], '') for cid in old if cid not in new}
     return res
 
-# %% ../nbs/api/19_diff.ipynb #6a723bc8
 @delegates(_cell_changes)
 def changed_cells(nb_path, **kwargs):
     "Return set of cell IDs for changed/added/deleted cells between two refs"
@@ -106,14 +116,78 @@ def f(cid,o,n): return source_diff(o,n)
 
 # %% ../nbs/api/19_diff.ipynb #d670e58e
 MAXLEN = 180 # Most characters shown per displayed line
-_dcolors = {'-':'\x1b[31m', '+':'\x1b[32m', '@':'\x1b[36m', '#':'\x1b[1m', '# ':'\x1b[1;7m'}
+OUT_MAXLEN = 512 # Most characters shown of a cell's stored output
+MIN_RATIO = 0.5 # Least similar (by shared words) a changed line pair can be and still merge into one word-diff line
+_dcolors = {'## …':'\x1b[34m', '# ':'\x1b[1;7m', '#':'\x1b[1m', '-':'\x1b[31m', '+':'\x1b[32m', '@':'\x1b[36m'}
+
+def _trunc(s, maxlen):
+    "`s` cut to `maxlen`, ending `…[n]` with `n` the number of characters missing"
+    if not maxlen or len(s)<=maxlen: return s
+    return truncstr(s, maxlen, suf=lambda n: f'…[{humanize(n)}]')
 
 def _dline(l, maxlen=MAXLEN, color=False):
-    "Truncate `l` to `maxlen` with an ellipsis, optionally colored by its leading diff marker"
-    if maxlen and len(l)>maxlen: l = l[:maxlen]+'…'
-    c = (_dcolors.get(l[:2]) or _dcolors.get(l[:1])) if color else None
+    "Truncate `l` with `_trunc`, optionally colored by its leading diff marker; word-diff spans render via `_wline`"
+    if not isinstance(l, str): return _wline(l, maxlen, color)
+    l = _trunc(l, maxlen)
+    c = first(v for k,v in _dcolors.items() if l.startswith(k)) if color else None
     return f'{c}{l}\x1b[0m' if c else l
 
+# %% ../nbs/api/19_diff.ipynb #7a1d7b28
+_wcolors = {'-':'\x1b[31;9m', '+':'\x1b[32m'}
+_wmarks = {'-':('[-','-]'), '+':('{+','+}')}
+
+def _tokens(s): return re.findall(r'\w+|\s+|[^\w\s]', s)
+
+def _word_diff(a, b, maxlen=MAXLEN):
+    "`(kind, text)` spans merging lines `a` and `b` (' ' shared, '-' only in `a`, '+' only in `b`); None when too different, or longer than `maxlen`"
+    if maxlen and max(len(a),len(b))>maxlen: return None
+    if SequenceMatcher(None, re.findall(r'\w+', a), re.findall(r'\w+', b), autojunk=False).ratio()<MIN_RATIO: return None
+    ta,tb = _tokens(a),_tokens(b)
+    res = []
+    for tag,i1,i2,j1,j2 in SequenceMatcher(None, ta, tb, autojunk=False).get_opcodes():
+        if tag=='equal': res.append((' ', ''.join(ta[i1:i2])))
+        else:
+            if i2>i1: res.append(('-', ''.join(ta[i1:i2])))
+            if j2>j1: res.append(('+', ''.join(tb[j1:j2])))
+    return res
+
+def _wline(spans, maxlen=MAXLEN, color=False):
+    "One `!` line from word-diff `spans`: deletions red and struck out, additions green, or `[-…-]`/`{+…+}` markers without `color`"
+    if not color: return _dline('!'+''.join(t if k==' ' else _wmarks[k][0]+t+_wmarks[k][1] for k,t in spans), maxlen)
+    full = '!'+''.join(t for _,t in spans)
+    cut = _trunc(full, maxlen)
+    suf = cut[cut.rfind('…['):] if cut!=full else ''
+    res,room = '!',len(cut)-len(suf)-1
+    for k,t in spans:
+        t = t[:room]
+        room -= len(t)
+        res += t if k==' ' else f'{_wcolors[k]}{t}\x1b[0m'
+        if not room: break
+    return res+suf
+
+# %% ../nbs/api/19_diff.ipynb #2bf82340
+def _refine(lines, maxlen=MAXLEN):
+    "Diff `lines` with each positionally paired `-`/`+` line merged into word-diff spans where `_word_diff` allows"
+    res,i = [],0
+    while i<len(lines):
+        if not lines[i].startswith('-'):
+            res.append(lines[i])
+            i += 1
+            continue
+        j = i
+        while j<len(lines) and lines[j].startswith('-'): j += 1
+        k = j
+        while k<len(lines) and lines[k].startswith('+'): k += 1
+        olds,news = lines[i:j],lines[j:k]
+        bo,bn = [],[]
+        for o,n in zip(olds,news):
+            w = _word_diff(o[1:], n[1:], maxlen)
+            if w: res,bo,bn = res+bo+bn+[w],[],[]
+            else: bo,bn = bo+[o],bn+[n]
+        res += bo+olds[len(news):]+bn+news[len(olds):]
+        i = k
+    return res
+
 # %% ../nbs/api/19_diff.ipynb #342fc231
 def _diff_order(old, new):
     "All cell ids in notebook order: `new`'s order, with deleted ids after their predecessor in `old`"
@@ -125,19 +199,36 @@ def _diff_order(old, new):
     return res
 
 # %% ../nbs/api/19_diff.ipynb #eb3fc128
+def _out_lines(c):
+    "Stored outputs of cell `c` rendered as text, capped at `OUT_MAXLEN` chars, each line prefixed `| `"
+    r = _trunc(render_text(c.get('outputs', [])), OUT_MAXLEN)
+    return [f'| {l}' for l in r.splitlines()]
+
 def render_diff(
-    old, # `{id: source}` for the old version of the notebook
-    new, # `{id: source}` for the new version
+    old, # `{id: cell}` for the old version of the notebook
+    new, # `{id: cell}` for the new version
     maxlen:int=MAXLEN, # Truncate diff lines to this width (falsy: no limit)
-    color:bool=False # Add ANSI colors?
-): # One section per changed cell, in notebook order
-    "Render cell-level changes between two notebooks as truncated unified diffs"
-    res = []
-    for cid in _diff_order(old, new):
-        o,n = old.get(cid,''), new.get(cid,'')
-        if cid in old and cid in new and o==n: continue
-        kind = 'modified' if cid in old and cid in new else 'added' if cid in new else 'deleted'
-        lines = [f'## {kind} {cid}:'] + source_diff(o,n).splitlines()[2:]
+    color:bool=False, # Add ANSI colors?
+    context:int=1, # Unchanged cells to show either side of each change
+    show_out:bool=True # Show the stored outputs of changed and added cells?
+): # One section per shown cell, in notebook order
+    "Render cell-level changes between two notebooks as truncated unified diffs, with context cells and outputs"
+    order = _diff_order(old, new)
+    def _key(cid, d): return (_src(d[cid]), d[cid].directives) if cid in d else None
+    def _changed(cid): return _key(cid, old)!=_key(cid, new)
+    show = {j for i,cid in enumerate(order) if _changed(cid) for j in range(i-context, i+context+1)}
+    res,prev = [],-1
+    for i,cid in enumerate(order):
+        if i not in show: continue
+        if 0<=prev<i-1: res.append(_dline(f'## … {i-prev-1} cells', maxlen, color))
+        prev = i
+        c = new.get(cid) or old[cid]
+        if not _changed(cid): lines = [f'## {cid}{_dtag(c)}:'] + [' '+l for l in _src(c).splitlines()]
+        else:
+            o,n = (_src(old[cid]) if cid in old else ''), (_src(new[cid]) if cid in new else '')
+            kind = 'modified' if cid in old and cid in new else 'added' if cid in new else 'deleted'
+            lines = [f'## {kind} {cid}{_dtag(c)}:'] + _refine(source_diff(o,n).splitlines()[2:], maxlen)
+            if show_out and cid in new: lines += _out_lines(new[cid])
         res.append('\n'.join(_dline(l, maxlen, color) for l in lines))
     return '\n\n'.join(res)
 
@@ -147,18 +238,20 @@ def nb_diff(
     ref_a='HEAD', # First git ref
     ref_b=None, # Second git ref; None for working dir
     maxlen:int=MAXLEN, # Truncate diff lines to this width (falsy: no limit)
-    color:bool=False # Add ANSI colors?
+    color:bool=False, # Add ANSI colors?
+    context:int=1, # Unchanged cells to show either side of each change
+    show_out:bool=True # Show the stored outputs of changed and added cells?
 ): # Rendered diff of changed cells
     "Rendered cell diff for `nb_path` between two refs"
-    a,b = nbs_pair(nb_path, ref_a, ref_b, f=_src)
-    return render_diff(a, b, maxlen=maxlen, color=color)
+    a,b = nbs_pair(nb_path, ref_a, ref_b)
+    return render_diff(a, b, maxlen=maxlen, color=color, context=context, show_out=show_out)
 
 # %% ../nbs/api/19_diff.ipynb #ab9f0b26
-def _file_srcs(path):
-    "`{id: source}` from an ipynb file; `{}` when missing or empty (e.g. `/dev/null`)"
+def _file_cells(path):
+    "`{id: cell}` from an ipynb file; `{}` when missing or empty (e.g. `/dev/null`)"
     p = Path(path)
     if not p.exists() or not p.stat().st_size: return {}
-    return _srcdict(read_nb(p), _src)
+    return _srcdict(read_nb(p))
 
 _pos = Annotated[str, {'opt':False, 'nargs':'?'}]  # optional positional CLI arg, as git passes them
 
@@ -173,12 +266,14 @@ def nbdev_diff_driver(
     new_mode:_pos=None, # Post-change file mode
     rename_to:_pos=None, # New repo path, when git detected a rename
     similarity:_pos=None, # Similarity score, when git detected a rename
-    maxlen:int=MAXLEN # Truncate diff lines to this width (0 for no limit)
+    maxlen:int=MAXLEN, # Truncate diff lines to this width (0 for no limit)
+    context:int=1, # Unchanged cells to show either side of each change
+    show_out:bool=True # Show the stored outputs of changed and added cells?
 ):
     "Git external diff driver for notebooks; installed by `nbdev-install-hooks`"
     if not new_file: return print(f'unmerged: {path}')
     color = sys.stdout.isatty() or os.environ.get('GIT_PAGER_IN_USE')=='true'
-    res = render_diff(_file_srcs(old_file), _file_srcs(new_file), maxlen=maxlen, color=color)
+    res = render_diff(_file_cells(old_file), _file_cells(new_file), maxlen=maxlen, color=color, context=context, show_out=show_out)
     if not res: return
     print(_dline(f'# {path}', 0, color))
     print(res, end='\n\n')
@@ -190,10 +285,12 @@ def nbdev_diff(
     ref_a:str='HEAD', # First git ref
     ref_b:str=None, # Second git ref (default: working directory)
     maxlen:int=MAXLEN, # Truncate diff lines to this width (0 for no limit)
-    color:bool_arg=None # Add ANSI colors? (default: only if stdout is a tty)
+    color:bool_arg=None, # Add ANSI colors? (default: only if stdout is a tty)
+    context:int=1, # Unchanged cells to show either side of each change
+    show_out:bool=True # Show the stored outputs of changed and added cells?
 ):
     "Cell-level diffs for changed notebooks between two git refs"
     if color is None: color = sys.stdout.isatty()
     for p in sorted(nbglob(path)):
-        res = nb_diff(p, ref_a, ref_b, maxlen=maxlen, color=color)
+        res = nb_diff(p, ref_a, ref_b, maxlen=maxlen, color=color, context=context, show_out=show_out)
         if res: print(_dline(f'# {p}', 0, color), res, sep='\n', end='\n\n')