gh-151464: exclude '<>' token from tokenize output

Grammar/python.gram

@@ -807,13 +807,21 @@ noteq_bitwise_or[CmpopExprPair*]:
     | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) }
 lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) }
 lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) }
+    | invalid_noteq
 gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) }
 gt_bitwise_or[CmpopExprPair*]: '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) }
 notin_bitwise_or[CmpopExprPair*]: 'not' 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotIn, a) }
 in_bitwise_or[CmpopExprPair*]: 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, In, a) }
 isnot_bitwise_or[CmpopExprPair*]: 'is' 'not' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, IsNot, a) }
 is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Is, a) }
 
+invalid_noteq:
+    | a='<' b='>' {
+      _PyPegen_tokens_are_adjacent(a, b)
+      ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax.  Maybe you meant '!=' instead of '<>'?")
+      : NULL
+    }
+
 # Bitwise operators
 # -----------------
 

Include/internal/pycore_token.h

@@ -101,7 +101,7 @@ extern "C" {
 // Export these 4 symbols for 'test_peg_generator'
 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
 PyAPI_FUNC(int) _PyToken_OneChar(int);
-PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
+PyAPI_FUNC(int) _PyToken_TwoChars(int, int, int);
 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
 
 #ifdef __cplusplus

Lib/test/test_flufl.py

@@ -58,7 +58,30 @@ def test_barry_as_bdfl_relative_import(self):
         self.assertEqual(cm.exception.lineno, 1)
         self.assertEqual(cm.exception.offset, len(code) - 4)
 
+    def test_guido_as_bdfl_ineq_tokens(self):
+        code = """
+from io import BytesIO
+import tokenize
+from test.test_tokenize import stringify_tokens_from_source
 
+s = "{0}"
+f = BytesIO(s.encode('utf-8'))
+globals()['result'] = stringify_tokens_from_source(tokenize.tokenize(f.readline), s)
+"""
+        ns = {}
+        exec(code.format('1 != 2'), ns)
+        self.assertEqual(ns['result'],
+                         ["    ENCODING   'utf-8'       (0, 0) (0, 0)",
+                          "    NUMBER     '1'           (1, 0) (1, 1)",
+                          "    OP         '!='          (1, 2) (1, 4)",
+                          "    NUMBER     '2'           (1, 5) (1, 6)"])
+        exec(code.format('1 <> 2'), ns)
+        self.assertEqual(ns['result'],
+                         ["    ENCODING   'utf-8'       (0, 0) (0, 0)",
+                          "    NUMBER     '1'           (1, 0) (1, 1)",
+                          "    OP         '<'           (1, 2) (1, 3)",
+                          "    OP         '>'           (1, 3) (1, 4)",
+                          "    NUMBER     '2'           (1, 5) (1, 6)"])
 
 
 if __name__ == '__main__':

Lib/test/test_syntax.py

@@ -3608,6 +3608,31 @@ def test_ifexp_body_stmt_else_stmt(self):
         ]:
             self._check_error(f"x = {lhs_stmt} if 1 else {rhs_stmt}", msg)
 
+    def test_diamond_operator(self):
+        self._check_error(
+            "1<>2",
+            r"Maybe you meant '!=' instead of '<>'\?",
+            lineno=1,
+            end_lineno=1,
+            offset=2,
+            end_offset=4,
+        )
+
+    def test_diamond_operator_barry_as_flufl(self):
+        # Under barry_as_FLUFL, '<>' is the valid "not equal" operator
+        compile(
+            "from __future__ import barry_as_FLUFL\n1<>2",
+            "<test>", "exec",
+        )
+        self._check_error(
+            "from __future__ import barry_as_FLUFL\na != b",
+            "with Barry as BDFL, use '<>' instead of '!='",
+            lineno=2,
+            end_lineno=2,
+            offset=3,
+            end_offset=5,
+        )
+
     def test_double_ampersand(self):
         self._check_error(
             "a && b",

Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst

@@ -0,0 +1 @@
+Exclude invalid token ``<>`` from :mod:`tokenize` output.

Parser/action_helpers.c

@@ -2076,6 +2076,7 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name,
             alias_ty alias = asdl_seq_GET(names, i);
             if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) {
                 p->flags |= PyPARSE_BARRY_AS_BDFL;
+                p->tok->BARRY_AS_BDFL = 1;
             }
         }
     }

Parser/lexer/lexer.c

@@ -562,7 +562,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     /* Check for two-character token */
     {
         int c2 = tok_nextc(tok);
-        int current_token = _PyToken_TwoChars(c, c2);
+        int current_token = _PyToken_TwoChars(c, c2, tok->BARRY_AS_BDFL);
         if (current_token != OP) {
             int c3 = tok_nextc(tok);
             int current_token3 = _PyToken_ThreeChars(c, c2, c3);

Parser/lexer/state.c

@@ -63,6 +63,7 @@ _PyTokenizer_tok_new(void)
 #ifdef Py_DEBUG
     tok->debug = _Py_GetConfig()->parser_debug;
 #endif
+    tok->BARRY_AS_BDFL = 0;
     return tok;
 }
 

Parser/lexer/state.h

@@ -137,6 +137,7 @@ struct tok_state {
 #ifdef Py_DEBUG
     int debug;
 #endif
+    int BARRY_AS_BDFL;
 };
 
 int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,