From 33efbe3028a41ba04550b013935b92b229628241 Mon Sep 17 00:00:00 2001 From: Roland Walker Date: Sat, 12 Sep 2026 09:18:26 -0400 Subject: [PATCH] make approximate-matching thresholds tuneable Expose the previously hardcoded rapidfuzz matching thresholds in ~/.myclirc. Motivation: this complements making candidate match order configurable; some users may prefer rapidfuzz over ordered character matching. Limitation: the hardcoded cap of 20 approximate matches is left in place. --- changelog.md | 1 + mycli/client.py | 5 ++++ mycli/client_query.py | 3 +++ mycli/myclirc | 13 ++++++++++ mycli/sqlcompleter.py | 12 +++++++--- test/myclirc | 13 ++++++++++ test/pytests/test_client.py | 37 ++++++++++++++++++++++++++++ test/pytests/test_client_query.py | 12 ++++++++++ test/pytests/test_sqlcompleter.py | 40 +++++++++++++++++++++++++++++++ 9 files changed, 133 insertions(+), 3 deletions(-) diff --git a/changelog.md b/changelog.md index 250b4321..0f2be987 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,7 @@ Features -------- * Preserve the query as metadata when saving to Parquet with `.>`. * Make completion candidate match order configurable. +* Make approximate-matching thresholds configurable. 2.23.0 (2026/09/09) diff --git a/mycli/client.py b/mycli/client.py index 6ba6abf4..8ea067a0 100644 --- a/mycli/client.py +++ b/mycli/client.py @@ -213,6 +213,11 @@ def __init__( indexed_column_suffix=indexed_column_suffix, config_property_names=get_config_property_names(self.config), completion_match_order=c['main'].as_list('completion_match_order') if 'completion_match_order' in c['main'] else (), + rapidfuzz_min_length=c['main'].as_int('rapidfuzz_min_length') if c['main'].get('rapidfuzz_min_length') else 4, + rapidfuzz_score_cutoff=c['main'].as_float('rapidfuzz_score_cutoff') if c['main'].get('rapidfuzz_score_cutoff') else 75.0, + rapidfuzz_length_coverage=c['main'].as_float('rapidfuzz_length_coverage') + if c['main'].get('rapidfuzz_length_coverage') + else 0.67, ) for error in self.completer.completion_config_errors: self.echo(error, err=True, fg='red') diff --git a/mycli/client_query.py b/mycli/client_query.py index e4a455fe..0a12281a 100644 --- a/mycli/client_query.py +++ b/mycli/client_query.py @@ -61,6 +61,9 @@ def refresh_completions(self, reset: bool = False) -> list[SQLResult]: "config_property_names": self.completer.config_property_names, 'frecency_provider': self.completer.frecency_provider, 'completion_match_order': self.completer.completion_match_order, + 'rapidfuzz_min_length': self.completer.rapidfuzz_min_length, + 'rapidfuzz_length_coverage': self.completer.rapidfuzz_length_coverage, + 'rapidfuzz_score_cutoff': self.completer.rapidfuzz_score_cutoff, }, ) diff --git a/mycli/myclirc b/mycli/myclirc index 947c32a5..211c5862 100644 --- a/mycli/myclirc +++ b/mycli/myclirc @@ -20,6 +20,19 @@ smart_completion = True # * rapidfuzz - true approximate matching, _ie_ autcorrect completion_match_order = perfect, regex, under_words, slash_words, camel_case, rapidfuzz +# Minimum input characters before using rapidfuzz approximate matching. +# Empty uses the default of 4. Set to 0 or less to remove the minimum. +rapidfuzz_min_length = 4 + +# Minimum rapidfuzz candidate length as a fraction of the input length. +# Empty uses 0.67. Lower is more liberal. Set to 0 or less to suppress +# the length filter. +rapidfuzz_length_coverage = 0.67 + +# Minimum rapidfuzz similarity score (0-100). Lower is more liberal. +# Empty uses the default of 75. +rapidfuzz_score_cutoff = 75 + # Text appended to indexed column names in the completion menu. This text is # not inserted into the query. Leave empty to disable the marker. Quote values # containing spaces, commas, or comment characters. diff --git a/mycli/sqlcompleter.py b/mycli/sqlcompleter.py index 89409cfa..d9f72eab 100644 --- a/mycli/sqlcompleter.py +++ b/mycli/sqlcompleter.py @@ -956,12 +956,18 @@ def __init__( config_property_names: Collection[str] = (), frecency_provider: Callable[[], Mapping[str, float]] | None = None, completion_match_order: Collection[str] = (), + rapidfuzz_min_length: int = 4, + rapidfuzz_length_coverage: float = 0.67, + rapidfuzz_score_cutoff: float = 75.0, ) -> None: super(self.__class__, self).__init__() self.smart_completion = smart_completion self.indexed_column_suffix = indexed_column_suffix self.config_property_names = tuple(sorted(config_property_names)) self.frecency_provider = frecency_provider + self.rapidfuzz_min_length = max(0, rapidfuzz_min_length) + self.rapidfuzz_length_coverage = max(0.0, rapidfuzz_length_coverage) + self.rapidfuzz_score_cutoff = max(0.0, min(100.0, rapidfuzz_score_cutoff)) self.completion_config_errors: list[str] = [] default_order = tuple(category.name.lower() for category in Fuzziness) order = tuple(name.strip().lower() for name in completion_match_order if name.strip()) @@ -1355,7 +1361,7 @@ def find_fuzzy_matches( if fuzziness is not None: completions.append((item, fuzziness)) - if len(text) >= 4: + if len(text) >= self.rapidfuzz_min_length: rapidfuzz_matches = rapidfuzz.process.extract( text, collection, @@ -1364,11 +1370,11 @@ def find_fuzzy_matches( # because underscores are valuable info processor=rapidfuzz.utils.default_process, limit=20, - score_cutoff=75, + score_cutoff=self.rapidfuzz_score_cutoff, ) existing = {c[0]: index for index, c in enumerate(completions)} for item, _score, _type in rapidfuzz_matches: - if len(item) < len(text) / 1.5: + if len(item) < len(text) * self.rapidfuzz_length_coverage: continue if item in existing: index = existing[item] diff --git a/test/myclirc b/test/myclirc index 37202862..a9b4dff1 100644 --- a/test/myclirc +++ b/test/myclirc @@ -20,6 +20,19 @@ smart_completion = True # * rapidfuzz - true approximate matching, _ie_ autcorrect completion_match_order = perfect, regex, under_words, slash_words, camel_case, rapidfuzz +# Minimum input characters before using rapidfuzz approximate matching. +# Empty uses the default of 4. Set to 0 or less to remove the minimum. +rapidfuzz_min_length = 4 + +# Minimum rapidfuzz candidate length as a fraction of the input length. +# Empty uses 0.67. Lower is more liberal. Set to 0 or less to suppress +# the length filter. +rapidfuzz_length_coverage = 0.67 + +# Minimum rapidfuzz similarity score (0-100). Lower is more liberal. +# Empty uses the default of 75. +rapidfuzz_score_cutoff = 75 + # Text appended to indexed column names in the completion menu. This text is # not inserted into the query. Leave empty to disable the marker. Quote values # containing spaces, commas, or comment characters. diff --git a/test/pytests/test_client.py b/test/pytests/test_client.py index 2f2283fb..c2955987 100644 --- a/test/pytests/test_client.py +++ b/test/pytests/test_client.py @@ -36,6 +36,43 @@ def test_init_configures_completion_ranking(monkeypatch: pytest.MonkeyPatch, tmp assert cli.completer.completion_match_order == ('camel_case', 'under_words', 'perfect', 'regex', 'slash_words', 'rapidfuzz') +@pytest.mark.parametrize(('value', 'expected'), [(None, 4), ('', 4), ('2', 2), ('0', 0), ('-1', 0)]) +def test_init_configures_rapidfuzz_min_length(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, value: str | None, expected: int) -> None: + patch_constructor_side_effects(monkeypatch) + setting = f'rapidfuzz_min_length = {value}\n' if value is not None else '' + myclirc = write_myclirc(tmp_path, f'[main]\n{setting}') + + cli = MyCli(myclirc=myclirc) + + assert cli.completer.rapidfuzz_min_length == expected + + +@pytest.mark.parametrize(('value', 'expected'), [(None, 0.67), ('', 0.67), ('0.5', 0.5), ('0', 0.0), ('-1', 0.0)]) +def test_init_configures_rapidfuzz_length_coverage( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, value: str | None, expected: float +) -> None: + patch_constructor_side_effects(monkeypatch) + setting = f'rapidfuzz_length_coverage = {value}\n' if value is not None else '' + myclirc = write_myclirc(tmp_path, f'[main]\n{setting}') + + cli = MyCli(myclirc=myclirc) + + assert cli.completer.rapidfuzz_length_coverage == expected + + +@pytest.mark.parametrize(('value', 'expected'), [(None, 75.0), ('', 75.0), ('82.5', 82.5), ('0', 0.0), ('-1', 0.0), ('101', 100.0)]) +def test_init_configures_rapidfuzz_score_cutoff( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, value: str | None, expected: float +) -> None: + patch_constructor_side_effects(monkeypatch) + setting = f'rapidfuzz_score_cutoff = {value}\n' if value is not None else '' + myclirc = write_myclirc(tmp_path, f'[main]\n{setting}') + + cli = MyCli(myclirc=myclirc) + + assert cli.completer.rapidfuzz_score_cutoff == expected + + @pytest.mark.parametrize('value', ['', 'rapidfuzz']) def test_init_reads_empty_or_single_match_order(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, value: str) -> None: patch_constructor_side_effects(monkeypatch) diff --git a/test/pytests/test_client_query.py b/test/pytests/test_client_query.py index 4afccbeb..f442bf22 100644 --- a/test/pytests/test_client_query.py +++ b/test/pytests/test_client_query.py @@ -29,6 +29,9 @@ def make_refresh_cli() -> tuple[Any, dict[str, Any]]: config_property_names=('main.show_warnings',), frecency_provider=lambda: {'select': 1.0}, completion_match_order=('under_words', 'regex'), + rapidfuzz_min_length=2, + rapidfuzz_length_coverage=0.5, + rapidfuzz_score_cutoff=82.5, keyword_casing='upper', indexed_column_suffix=' [indexed]', set_dbname=lambda dbname: state['set_dbname_calls'].append(dbname), @@ -76,6 +79,9 @@ def test_refresh_completions_passes_options_to_refresher() -> None: 'config_property_names': ('main.show_warnings',), 'frecency_provider': cli.completer.frecency_provider, 'completion_match_order': ('under_words', 'regex'), + 'rapidfuzz_min_length': 2, + 'rapidfuzz_length_coverage': 0.5, + 'rapidfuzz_score_cutoff': 82.5, }, ) ] @@ -107,6 +113,9 @@ def test_refresh_completions_updates_dbname_when_reset() -> None: config_property_names=(), frecency_provider=None, completion_match_order=(), + rapidfuzz_min_length=4, + rapidfuzz_length_coverage=0.67, + rapidfuzz_score_cutoff=75.0, keyword_casing='lower', indexed_column_suffix='*', set_dbname=lambda dbname: set_dbname_calls.append(dbname), @@ -129,6 +138,9 @@ def test_refresh_completions_uses_lock_when_reset() -> None: config_property_names=(), frecency_provider=None, completion_match_order=(), + rapidfuzz_min_length=4, + rapidfuzz_length_coverage=0.67, + rapidfuzz_score_cutoff=75.0, keyword_casing='lower', indexed_column_suffix='*', set_dbname=lambda dbname: None, diff --git a/test/pytests/test_sqlcompleter.py b/test/pytests/test_sqlcompleter.py index 1d387ccd..54c96494 100644 --- a/test/pytests/test_sqlcompleter.py +++ b/test/pytests/test_sqlcompleter.py @@ -147,6 +147,46 @@ def fail_extract(*args, **kwargs): assert matches == [] +@pytest.mark.parametrize( + ('minimum', 'text', 'should_run'), + [(2, 's', False), (2, 'se', True), (6, 'selec', False), (6, 'select', True), (0, '', True)], +) +def test_find_fuzzy_matches_uses_configured_minimum(monkeypatch: pytest.MonkeyPatch, minimum: int, text: str, should_run: bool) -> None: + calls: list[str] = [] + + def extract(query: str, *args: object, **kwargs: object) -> list[tuple[str, int, int]]: + calls.append(query) + return [('SELECT', 100, 0)] + + monkeypatch.setattr(mycli.sqlcompleter.rapidfuzz.process, 'extract', extract) + completer = SQLCompleter(rapidfuzz_min_length=minimum, completion_match_order=('rapidfuzz',)) + + matches = completer.find_fuzzy_matches(text, text, ['SELECT']) + + assert calls == ([text] if should_run else []) + assert (('SELECT', Fuzziness.RAPIDFUZZ) in matches) == should_run + + +@pytest.mark.parametrize(('coverage', 'accepted'), [(0.0, True), (0.5, True), (0.75, True), (0.76, False), (1.0, False)]) +def test_find_fuzzy_matches_filters_candidate_length(monkeypatch: pytest.MonkeyPatch, coverage: float, accepted: bool) -> None: + monkeypatch.setattr(SQLCompleter, 'find_fuzzy_match', lambda *args: None) + monkeypatch.setattr(mycli.sqlcompleter.rapidfuzz.process, 'extract', lambda *args, **kwargs: [('abc', 90, 0)]) + completer = SQLCompleter(rapidfuzz_length_coverage=coverage) + + matches = completer.find_fuzzy_matches('abcd', 'abcd', ['abc']) + + assert matches == ([('abc', Fuzziness.RAPIDFUZZ)] if accepted else []) + + +@pytest.mark.parametrize(('cutoff', 'accepted'), [(0.0, True), (75.0, True), (75.1, False), (100.0, False)]) +def test_find_fuzzy_matches_applies_score_cutoff(cutoff: float, accepted: bool) -> None: + completer = SQLCompleter(rapidfuzz_score_cutoff=cutoff) + + matches = completer.find_fuzzy_matches('abcd', 'abcd', ['abce']) + + assert matches == ([('abce', Fuzziness.RAPIDFUZZ)] if accepted else []) + + def test_find_fuzzy_matches_appends_rapidfuzz_results_and_skips_duplicates(monkeypatch) -> None: monkeypatch.setattr( SQLCompleter,