diff --git a/Grammar/python.gram b/Grammar/python.gram index 2713ba9466a0b1..27ba3523bfdd44 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -807,6 +807,7 @@ noteq_bitwise_or[CmpopExprPair*]: | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } + | invalid_noteq gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } gt_bitwise_or[CmpopExprPair*]: '>' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Gt, a) } notin_bitwise_or[CmpopExprPair*]: 'not' 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotIn, a) } @@ -814,6 +815,13 @@ in_bitwise_or[CmpopExprPair*]: 'in' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, I isnot_bitwise_or[CmpopExprPair*]: 'is' 'not' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, IsNot, a) } is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Is, a) } +invalid_noteq: + | a='<' b='>' { + _PyPegen_tokens_are_adjacent(a, b) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '!=' instead of '<>'?") + : NULL + } + # Bitwise operators # ----------------- diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 5de1f719a2f1a2..8dcd4226745dd0 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -101,7 +101,7 @@ extern "C" { // Export these 4 symbols for 'test_peg_generator' PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) _PyToken_OneChar(int); -PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int, int); PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); #ifdef __cplusplus diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index d77e481c81d598..74e645556d9bbc 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -58,7 +58,30 @@ def test_barry_as_bdfl_relative_import(self): self.assertEqual(cm.exception.lineno, 1) self.assertEqual(cm.exception.offset, len(code) - 4) + def test_guido_as_bdfl_ineq_tokens(self): + code = """ +from io import BytesIO +import tokenize +from test.test_tokenize import stringify_tokens_from_source +s = "{0}" +f = BytesIO(s.encode('utf-8')) +globals()['result'] = stringify_tokens_from_source(tokenize.tokenize(f.readline), s) +""" + ns = {} + exec(code.format('1 != 2'), ns) + self.assertEqual(ns['result'], + [" ENCODING 'utf-8' (0, 0) (0, 0)", + " NUMBER '1' (1, 0) (1, 1)", + " OP '!=' (1, 2) (1, 4)", + " NUMBER '2' (1, 5) (1, 6)"]) + exec(code.format('1 <> 2'), ns) + self.assertEqual(ns['result'], + [" ENCODING 'utf-8' (0, 0) (0, 0)", + " NUMBER '1' (1, 0) (1, 1)", + " OP '<' (1, 2) (1, 3)", + " OP '>' (1, 3) (1, 4)", + " NUMBER '2' (1, 5) (1, 6)"]) if __name__ == '__main__': diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index a63042055210c5..11a948caf03877 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3608,6 +3608,31 @@ def test_ifexp_body_stmt_else_stmt(self): ]: self._check_error(f"x = {lhs_stmt} if 1 else {rhs_stmt}", msg) + def test_diamond_operator(self): + self._check_error( + "1<>2", + r"Maybe you meant '!=' instead of '<>'\?", + lineno=1, + end_lineno=1, + offset=2, + end_offset=4, + ) + + def test_diamond_operator_barry_as_flufl(self): + # Under barry_as_FLUFL, '<>' is the valid "not equal" operator + compile( + "from __future__ import barry_as_FLUFL\n1<>2", + "", "exec", + ) + self._check_error( + "from __future__ import barry_as_FLUFL\na != b", + "with Barry as BDFL, use '<>' instead of '!='", + lineno=2, + end_lineno=2, + offset=3, + end_offset=5, + ) + def test_double_ampersand(self): self._check_error( "a && b", diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst new file mode 100644 index 00000000000000..a667304505e585 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-29-08-43-14.gh-issue-151464.o_mtmz.rst @@ -0,0 +1 @@ +Exclude invalid token ``<>`` from :mod:`tokenize` output. diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 809f3c0a7b270e..7e79a36e661138 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -2076,6 +2076,7 @@ _PyPegen_checked_from_import(Parser *p, asdl_seq *dots, expr_ty module_name, alias_ty alias = asdl_seq_GET(names, i); if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { p->flags |= PyPARSE_BARRY_AS_BDFL; + p->tok->BARRY_AS_BDFL = 1; } } } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 41f03a408cfc75..17e531ee7f2c3d 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -562,7 +562,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str /* Check for two-character token */ { int c2 = tok_nextc(tok); - int current_token = _PyToken_TwoChars(c, c2); + int current_token = _PyToken_TwoChars(c, c2, tok->BARRY_AS_BDFL); if (current_token != OP) { int c3 = tok_nextc(tok); int current_token3 = _PyToken_ThreeChars(c, c2, c3); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 5cf9b4d768c3eb..72daaa2a4646e9 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -63,6 +63,7 @@ _PyTokenizer_tok_new(void) #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; #endif + tok->BARRY_AS_BDFL = 0; return tok; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9cd196a114c7cb..c367a4a0b7d883 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -137,6 +137,7 @@ struct tok_state { #ifdef Py_DEBUG int debug; #endif + int BARRY_AS_BDFL; }; int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, diff --git a/Parser/parser.c b/Parser/parser.c index 800db5b490fea9..eb1889256a39a2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -221,330 +221,331 @@ static char *soft_keywords[] = { #define in_bitwise_or_type 1132 #define isnot_bitwise_or_type 1133 #define is_bitwise_or_type 1134 -#define bitwise_or_type 1135 // Left-recursive -#define bitwise_xor_type 1136 // Left-recursive -#define bitwise_and_type 1137 // Left-recursive -#define shift_expr_type 1138 // Left-recursive -#define sum_type 1139 // Left-recursive -#define term_type 1140 // Left-recursive -#define factor_type 1141 -#define power_type 1142 -#define await_primary_type 1143 -#define primary_type 1144 // Left-recursive -#define slices_type 1145 -#define slice_type 1146 -#define atom_type 1147 -#define group_type 1148 -#define lambdef_type 1149 -#define lambda_params_type 1150 -#define lambda_parameters_type 1151 -#define lambda_slash_no_default_type 1152 -#define lambda_slash_with_default_type 1153 -#define lambda_star_etc_type 1154 -#define lambda_kwds_type 1155 -#define lambda_param_no_default_type 1156 -#define lambda_param_with_default_type 1157 -#define lambda_param_maybe_default_type 1158 -#define lambda_param_type 1159 -#define fstring_middle_type 1160 -#define fstring_replacement_field_type 1161 -#define fstring_conversion_type 1162 -#define fstring_full_format_spec_type 1163 -#define fstring_format_spec_type 1164 -#define fstring_type 1165 -#define tstring_format_spec_replacement_field_type 1166 -#define tstring_format_spec_type 1167 -#define tstring_full_format_spec_type 1168 -#define tstring_replacement_field_type 1169 -#define tstring_middle_type 1170 -#define tstring_type 1171 -#define string_type 1172 -#define strings_type 1173 -#define list_type 1174 -#define tuple_type 1175 -#define set_type 1176 -#define dict_type 1177 -#define double_starred_kvpairs_type 1178 -#define double_starred_kvpair_type 1179 -#define kvpair_type 1180 -#define for_if_clauses_type 1181 -#define for_if_clause_type 1182 -#define listcomp_type 1183 -#define setcomp_type 1184 -#define genexp_type 1185 -#define dictcomp_type 1186 -#define arguments_type 1187 -#define args_type 1188 -#define kwargs_type 1189 -#define starred_expression_type 1190 -#define kwarg_or_starred_type 1191 -#define kwarg_or_double_starred_type 1192 -#define star_targets_type 1193 -#define star_targets_list_seq_type 1194 -#define star_targets_tuple_seq_type 1195 -#define star_target_type 1196 -#define target_with_star_atom_type 1197 -#define star_atom_type 1198 -#define single_target_type 1199 -#define single_subscript_attribute_target_type 1200 -#define t_primary_type 1201 // Left-recursive -#define t_lookahead_type 1202 -#define del_targets_type 1203 -#define del_target_type 1204 -#define del_t_atom_type 1205 -#define type_expressions_type 1206 -#define func_type_comment_type 1207 -#define invalid_arguments_type 1208 -#define invalid_kwarg_type 1209 -#define expression_without_invalid_type 1210 -#define invalid_legacy_expression_type 1211 -#define invalid_type_param_type 1212 -#define invalid_expression_type 1213 -#define invalid_if_expression_type 1214 -#define invalid_named_expression_type 1215 -#define invalid_assignment_type 1216 -#define invalid_ann_assign_target_type 1217 -#define invalid_raise_stmt_type 1218 -#define invalid_del_stmt_type 1219 -#define invalid_assert_stmt_type 1220 -#define invalid_block_type 1221 -#define invalid_comprehension_type 1222 -#define invalid_parameters_type 1223 -#define invalid_default_type 1224 -#define invalid_star_etc_type 1225 -#define invalid_kwds_type 1226 -#define invalid_parameters_helper_type 1227 -#define invalid_lambda_parameters_type 1228 -#define invalid_lambda_parameters_helper_type 1229 -#define invalid_lambda_star_etc_type 1230 -#define invalid_lambda_kwds_type 1231 -#define invalid_double_type_comments_type 1232 -#define invalid_with_item_type 1233 -#define invalid_for_if_clause_type 1234 -#define invalid_for_target_type 1235 -#define invalid_group_type 1236 -#define invalid_import_type 1237 -#define invalid_dotted_as_name_type 1238 -#define invalid_import_from_as_name_type 1239 -#define invalid_import_from_type 1240 -#define invalid_import_from_targets_type 1241 -#define invalid_with_stmt_type 1242 -#define invalid_with_stmt_indent_type 1243 -#define invalid_try_stmt_type 1244 -#define invalid_except_stmt_type 1245 -#define invalid_except_star_stmt_type 1246 -#define invalid_finally_stmt_type 1247 -#define invalid_except_stmt_indent_type 1248 -#define invalid_except_star_stmt_indent_type 1249 -#define invalid_match_stmt_type 1250 -#define invalid_case_block_type 1251 -#define invalid_as_pattern_type 1252 -#define invalid_class_pattern_type 1253 -#define invalid_mapping_pattern_type 1254 -#define invalid_class_argument_pattern_type 1255 -#define invalid_if_stmt_type 1256 -#define invalid_elif_stmt_type 1257 -#define invalid_else_stmt_type 1258 -#define invalid_while_stmt_type 1259 -#define invalid_for_stmt_type 1260 -#define invalid_def_raw_type 1261 -#define invalid_class_def_raw_type 1262 -#define invalid_double_starred_kvpairs_type 1263 -#define invalid_kvpair_unpacking_type 1264 -#define invalid_kvpair_type 1265 -#define invalid_starred_expression_unpacking_type 1266 -#define invalid_starred_expression_unpacking_sequence_type 1267 -#define invalid_starred_expression_type 1268 -#define invalid_fstring_replacement_field_type 1269 -#define invalid_fstring_conversion_character_type 1270 -#define invalid_tstring_replacement_field_type 1271 -#define invalid_tstring_conversion_character_type 1272 -#define invalid_string_tstring_concat_type 1273 -#define invalid_arithmetic_type 1274 // Left-recursive -#define invalid_factor_type 1275 -#define invalid_type_params_type 1276 -#define invalid_bitwise_and_type 1277 // Left-recursive -#define invalid_bitwise_or_type 1278 // Left-recursive -#define _loop0_1_type 1279 -#define _loop1_2_type 1280 -#define _loop0_3_type 1281 -#define _gather_4_type 1282 -#define _tmp_5_type 1283 -#define _tmp_6_type 1284 -#define _tmp_7_type 1285 -#define _tmp_8_type 1286 -#define _tmp_9_type 1287 -#define _tmp_10_type 1288 -#define _tmp_11_type 1289 -#define _loop1_12_type 1290 -#define _loop0_13_type 1291 -#define _gather_14_type 1292 -#define _tmp_15_type 1293 -#define _tmp_16_type 1294 -#define _loop0_17_type 1295 -#define _loop1_18_type 1296 -#define _loop0_19_type 1297 -#define _gather_20_type 1298 -#define _tmp_21_type 1299 -#define _loop0_22_type 1300 -#define _gather_23_type 1301 -#define _loop1_24_type 1302 -#define _tmp_25_type 1303 -#define _tmp_26_type 1304 -#define _loop0_27_type 1305 -#define _loop0_28_type 1306 -#define _loop1_29_type 1307 -#define _loop1_30_type 1308 -#define _loop0_31_type 1309 -#define _loop1_32_type 1310 -#define _loop0_33_type 1311 -#define _gather_34_type 1312 -#define _tmp_35_type 1313 -#define _loop1_36_type 1314 -#define _loop1_37_type 1315 -#define _loop1_38_type 1316 -#define _loop0_39_type 1317 -#define _gather_40_type 1318 -#define _tmp_41_type 1319 -#define _tmp_42_type 1320 -#define _tmp_43_type 1321 -#define _loop0_44_type 1322 -#define _gather_45_type 1323 -#define _loop0_46_type 1324 -#define _gather_47_type 1325 -#define _tmp_48_type 1326 -#define _loop0_49_type 1327 -#define _gather_50_type 1328 -#define _loop0_51_type 1329 -#define _gather_52_type 1330 -#define _loop0_53_type 1331 -#define _gather_54_type 1332 -#define _loop1_55_type 1333 -#define _loop1_56_type 1334 -#define _loop0_57_type 1335 -#define _gather_58_type 1336 -#define _loop0_59_type 1337 -#define _gather_60_type 1338 -#define _loop1_61_type 1339 -#define _loop1_62_type 1340 -#define _loop1_63_type 1341 -#define _tmp_64_type 1342 -#define _loop0_65_type 1343 -#define _gather_66_type 1344 -#define _tmp_67_type 1345 -#define _tmp_68_type 1346 -#define _tmp_69_type 1347 -#define _tmp_70_type 1348 -#define _tmp_71_type 1349 -#define _loop0_72_type 1350 -#define _loop0_73_type 1351 -#define _loop1_74_type 1352 -#define _loop1_75_type 1353 -#define _loop0_76_type 1354 -#define _loop1_77_type 1355 -#define _loop0_78_type 1356 -#define _loop0_79_type 1357 -#define _loop0_80_type 1358 -#define _loop0_81_type 1359 -#define _loop1_82_type 1360 -#define _loop1_83_type 1361 -#define _tmp_84_type 1362 -#define _loop0_85_type 1363 -#define _gather_86_type 1364 -#define _loop1_87_type 1365 -#define _loop0_88_type 1366 -#define _tmp_89_type 1367 -#define _loop0_90_type 1368 -#define _gather_91_type 1369 -#define _tmp_92_type 1370 -#define _loop0_93_type 1371 -#define _gather_94_type 1372 -#define _loop0_95_type 1373 -#define _gather_96_type 1374 -#define _loop0_97_type 1375 -#define _loop0_98_type 1376 -#define _gather_99_type 1377 -#define _loop1_100_type 1378 -#define _tmp_101_type 1379 -#define _loop0_102_type 1380 -#define _gather_103_type 1381 -#define _loop0_104_type 1382 -#define _gather_105_type 1383 -#define _tmp_106_type 1384 -#define _tmp_107_type 1385 -#define _loop0_108_type 1386 -#define _gather_109_type 1387 -#define _tmp_110_type 1388 -#define _tmp_111_type 1389 -#define _tmp_112_type 1390 -#define _tmp_113_type 1391 -#define _tmp_114_type 1392 -#define _loop1_115_type 1393 -#define _tmp_116_type 1394 -#define _tmp_117_type 1395 -#define _tmp_118_type 1396 -#define _tmp_119_type 1397 -#define _tmp_120_type 1398 -#define _loop0_121_type 1399 -#define _loop0_122_type 1400 -#define _tmp_123_type 1401 -#define _tmp_124_type 1402 -#define _tmp_125_type 1403 -#define _tmp_126_type 1404 -#define _tmp_127_type 1405 -#define _tmp_128_type 1406 -#define _tmp_129_type 1407 -#define _tmp_130_type 1408 -#define _loop0_131_type 1409 -#define _gather_132_type 1410 -#define _tmp_133_type 1411 -#define _tmp_134_type 1412 -#define _tmp_135_type 1413 -#define _tmp_136_type 1414 -#define _loop0_137_type 1415 -#define _gather_138_type 1416 -#define _tmp_139_type 1417 -#define _loop0_140_type 1418 -#define _gather_141_type 1419 -#define _loop0_142_type 1420 -#define _gather_143_type 1421 -#define _tmp_144_type 1422 -#define _loop0_145_type 1423 -#define _tmp_146_type 1424 -#define _tmp_147_type 1425 -#define _tmp_148_type 1426 -#define _tmp_149_type 1427 -#define _tmp_150_type 1428 -#define _tmp_151_type 1429 -#define _tmp_152_type 1430 -#define _tmp_153_type 1431 -#define _tmp_154_type 1432 -#define _tmp_155_type 1433 -#define _tmp_156_type 1434 -#define _tmp_157_type 1435 -#define _tmp_158_type 1436 -#define _tmp_159_type 1437 -#define _tmp_160_type 1438 -#define _tmp_161_type 1439 -#define _tmp_162_type 1440 -#define _tmp_163_type 1441 -#define _tmp_164_type 1442 -#define _tmp_165_type 1443 -#define _tmp_166_type 1444 -#define _tmp_167_type 1445 -#define _tmp_168_type 1446 -#define _tmp_169_type 1447 -#define _tmp_170_type 1448 -#define _tmp_171_type 1449 -#define _tmp_172_type 1450 -#define _tmp_173_type 1451 -#define _loop0_174_type 1452 -#define _tmp_175_type 1453 -#define _tmp_176_type 1454 -#define _tmp_177_type 1455 -#define _tmp_178_type 1456 -#define _tmp_179_type 1457 -#define _tmp_180_type 1458 +#define invalid_noteq_type 1135 +#define bitwise_or_type 1136 // Left-recursive +#define bitwise_xor_type 1137 // Left-recursive +#define bitwise_and_type 1138 // Left-recursive +#define shift_expr_type 1139 // Left-recursive +#define sum_type 1140 // Left-recursive +#define term_type 1141 // Left-recursive +#define factor_type 1142 +#define power_type 1143 +#define await_primary_type 1144 +#define primary_type 1145 // Left-recursive +#define slices_type 1146 +#define slice_type 1147 +#define atom_type 1148 +#define group_type 1149 +#define lambdef_type 1150 +#define lambda_params_type 1151 +#define lambda_parameters_type 1152 +#define lambda_slash_no_default_type 1153 +#define lambda_slash_with_default_type 1154 +#define lambda_star_etc_type 1155 +#define lambda_kwds_type 1156 +#define lambda_param_no_default_type 1157 +#define lambda_param_with_default_type 1158 +#define lambda_param_maybe_default_type 1159 +#define lambda_param_type 1160 +#define fstring_middle_type 1161 +#define fstring_replacement_field_type 1162 +#define fstring_conversion_type 1163 +#define fstring_full_format_spec_type 1164 +#define fstring_format_spec_type 1165 +#define fstring_type 1166 +#define tstring_format_spec_replacement_field_type 1167 +#define tstring_format_spec_type 1168 +#define tstring_full_format_spec_type 1169 +#define tstring_replacement_field_type 1170 +#define tstring_middle_type 1171 +#define tstring_type 1172 +#define string_type 1173 +#define strings_type 1174 +#define list_type 1175 +#define tuple_type 1176 +#define set_type 1177 +#define dict_type 1178 +#define double_starred_kvpairs_type 1179 +#define double_starred_kvpair_type 1180 +#define kvpair_type 1181 +#define for_if_clauses_type 1182 +#define for_if_clause_type 1183 +#define listcomp_type 1184 +#define setcomp_type 1185 +#define genexp_type 1186 +#define dictcomp_type 1187 +#define arguments_type 1188 +#define args_type 1189 +#define kwargs_type 1190 +#define starred_expression_type 1191 +#define kwarg_or_starred_type 1192 +#define kwarg_or_double_starred_type 1193 +#define star_targets_type 1194 +#define star_targets_list_seq_type 1195 +#define star_targets_tuple_seq_type 1196 +#define star_target_type 1197 +#define target_with_star_atom_type 1198 +#define star_atom_type 1199 +#define single_target_type 1200 +#define single_subscript_attribute_target_type 1201 +#define t_primary_type 1202 // Left-recursive +#define t_lookahead_type 1203 +#define del_targets_type 1204 +#define del_target_type 1205 +#define del_t_atom_type 1206 +#define type_expressions_type 1207 +#define func_type_comment_type 1208 +#define invalid_arguments_type 1209 +#define invalid_kwarg_type 1210 +#define expression_without_invalid_type 1211 +#define invalid_legacy_expression_type 1212 +#define invalid_type_param_type 1213 +#define invalid_expression_type 1214 +#define invalid_if_expression_type 1215 +#define invalid_named_expression_type 1216 +#define invalid_assignment_type 1217 +#define invalid_ann_assign_target_type 1218 +#define invalid_raise_stmt_type 1219 +#define invalid_del_stmt_type 1220 +#define invalid_assert_stmt_type 1221 +#define invalid_block_type 1222 +#define invalid_comprehension_type 1223 +#define invalid_parameters_type 1224 +#define invalid_default_type 1225 +#define invalid_star_etc_type 1226 +#define invalid_kwds_type 1227 +#define invalid_parameters_helper_type 1228 +#define invalid_lambda_parameters_type 1229 +#define invalid_lambda_parameters_helper_type 1230 +#define invalid_lambda_star_etc_type 1231 +#define invalid_lambda_kwds_type 1232 +#define invalid_double_type_comments_type 1233 +#define invalid_with_item_type 1234 +#define invalid_for_if_clause_type 1235 +#define invalid_for_target_type 1236 +#define invalid_group_type 1237 +#define invalid_import_type 1238 +#define invalid_dotted_as_name_type 1239 +#define invalid_import_from_as_name_type 1240 +#define invalid_import_from_type 1241 +#define invalid_import_from_targets_type 1242 +#define invalid_with_stmt_type 1243 +#define invalid_with_stmt_indent_type 1244 +#define invalid_try_stmt_type 1245 +#define invalid_except_stmt_type 1246 +#define invalid_except_star_stmt_type 1247 +#define invalid_finally_stmt_type 1248 +#define invalid_except_stmt_indent_type 1249 +#define invalid_except_star_stmt_indent_type 1250 +#define invalid_match_stmt_type 1251 +#define invalid_case_block_type 1252 +#define invalid_as_pattern_type 1253 +#define invalid_class_pattern_type 1254 +#define invalid_mapping_pattern_type 1255 +#define invalid_class_argument_pattern_type 1256 +#define invalid_if_stmt_type 1257 +#define invalid_elif_stmt_type 1258 +#define invalid_else_stmt_type 1259 +#define invalid_while_stmt_type 1260 +#define invalid_for_stmt_type 1261 +#define invalid_def_raw_type 1262 +#define invalid_class_def_raw_type 1263 +#define invalid_double_starred_kvpairs_type 1264 +#define invalid_kvpair_unpacking_type 1265 +#define invalid_kvpair_type 1266 +#define invalid_starred_expression_unpacking_type 1267 +#define invalid_starred_expression_unpacking_sequence_type 1268 +#define invalid_starred_expression_type 1269 +#define invalid_fstring_replacement_field_type 1270 +#define invalid_fstring_conversion_character_type 1271 +#define invalid_tstring_replacement_field_type 1272 +#define invalid_tstring_conversion_character_type 1273 +#define invalid_string_tstring_concat_type 1274 +#define invalid_arithmetic_type 1275 // Left-recursive +#define invalid_factor_type 1276 +#define invalid_type_params_type 1277 +#define invalid_bitwise_and_type 1278 // Left-recursive +#define invalid_bitwise_or_type 1279 // Left-recursive +#define _loop0_1_type 1280 +#define _loop1_2_type 1281 +#define _loop0_3_type 1282 +#define _gather_4_type 1283 +#define _tmp_5_type 1284 +#define _tmp_6_type 1285 +#define _tmp_7_type 1286 +#define _tmp_8_type 1287 +#define _tmp_9_type 1288 +#define _tmp_10_type 1289 +#define _tmp_11_type 1290 +#define _loop1_12_type 1291 +#define _loop0_13_type 1292 +#define _gather_14_type 1293 +#define _tmp_15_type 1294 +#define _tmp_16_type 1295 +#define _loop0_17_type 1296 +#define _loop1_18_type 1297 +#define _loop0_19_type 1298 +#define _gather_20_type 1299 +#define _tmp_21_type 1300 +#define _loop0_22_type 1301 +#define _gather_23_type 1302 +#define _loop1_24_type 1303 +#define _tmp_25_type 1304 +#define _tmp_26_type 1305 +#define _loop0_27_type 1306 +#define _loop0_28_type 1307 +#define _loop1_29_type 1308 +#define _loop1_30_type 1309 +#define _loop0_31_type 1310 +#define _loop1_32_type 1311 +#define _loop0_33_type 1312 +#define _gather_34_type 1313 +#define _tmp_35_type 1314 +#define _loop1_36_type 1315 +#define _loop1_37_type 1316 +#define _loop1_38_type 1317 +#define _loop0_39_type 1318 +#define _gather_40_type 1319 +#define _tmp_41_type 1320 +#define _tmp_42_type 1321 +#define _tmp_43_type 1322 +#define _loop0_44_type 1323 +#define _gather_45_type 1324 +#define _loop0_46_type 1325 +#define _gather_47_type 1326 +#define _tmp_48_type 1327 +#define _loop0_49_type 1328 +#define _gather_50_type 1329 +#define _loop0_51_type 1330 +#define _gather_52_type 1331 +#define _loop0_53_type 1332 +#define _gather_54_type 1333 +#define _loop1_55_type 1334 +#define _loop1_56_type 1335 +#define _loop0_57_type 1336 +#define _gather_58_type 1337 +#define _loop0_59_type 1338 +#define _gather_60_type 1339 +#define _loop1_61_type 1340 +#define _loop1_62_type 1341 +#define _loop1_63_type 1342 +#define _tmp_64_type 1343 +#define _loop0_65_type 1344 +#define _gather_66_type 1345 +#define _tmp_67_type 1346 +#define _tmp_68_type 1347 +#define _tmp_69_type 1348 +#define _tmp_70_type 1349 +#define _tmp_71_type 1350 +#define _loop0_72_type 1351 +#define _loop0_73_type 1352 +#define _loop1_74_type 1353 +#define _loop1_75_type 1354 +#define _loop0_76_type 1355 +#define _loop1_77_type 1356 +#define _loop0_78_type 1357 +#define _loop0_79_type 1358 +#define _loop0_80_type 1359 +#define _loop0_81_type 1360 +#define _loop1_82_type 1361 +#define _loop1_83_type 1362 +#define _tmp_84_type 1363 +#define _loop0_85_type 1364 +#define _gather_86_type 1365 +#define _loop1_87_type 1366 +#define _loop0_88_type 1367 +#define _tmp_89_type 1368 +#define _loop0_90_type 1369 +#define _gather_91_type 1370 +#define _tmp_92_type 1371 +#define _loop0_93_type 1372 +#define _gather_94_type 1373 +#define _loop0_95_type 1374 +#define _gather_96_type 1375 +#define _loop0_97_type 1376 +#define _loop0_98_type 1377 +#define _gather_99_type 1378 +#define _loop1_100_type 1379 +#define _tmp_101_type 1380 +#define _loop0_102_type 1381 +#define _gather_103_type 1382 +#define _loop0_104_type 1383 +#define _gather_105_type 1384 +#define _tmp_106_type 1385 +#define _tmp_107_type 1386 +#define _loop0_108_type 1387 +#define _gather_109_type 1388 +#define _tmp_110_type 1389 +#define _tmp_111_type 1390 +#define _tmp_112_type 1391 +#define _tmp_113_type 1392 +#define _tmp_114_type 1393 +#define _loop1_115_type 1394 +#define _tmp_116_type 1395 +#define _tmp_117_type 1396 +#define _tmp_118_type 1397 +#define _tmp_119_type 1398 +#define _tmp_120_type 1399 +#define _loop0_121_type 1400 +#define _loop0_122_type 1401 +#define _tmp_123_type 1402 +#define _tmp_124_type 1403 +#define _tmp_125_type 1404 +#define _tmp_126_type 1405 +#define _tmp_127_type 1406 +#define _tmp_128_type 1407 +#define _tmp_129_type 1408 +#define _tmp_130_type 1409 +#define _loop0_131_type 1410 +#define _gather_132_type 1411 +#define _tmp_133_type 1412 +#define _tmp_134_type 1413 +#define _tmp_135_type 1414 +#define _tmp_136_type 1415 +#define _loop0_137_type 1416 +#define _gather_138_type 1417 +#define _tmp_139_type 1418 +#define _loop0_140_type 1419 +#define _gather_141_type 1420 +#define _loop0_142_type 1421 +#define _gather_143_type 1422 +#define _tmp_144_type 1423 +#define _loop0_145_type 1424 +#define _tmp_146_type 1425 +#define _tmp_147_type 1426 +#define _tmp_148_type 1427 +#define _tmp_149_type 1428 +#define _tmp_150_type 1429 +#define _tmp_151_type 1430 +#define _tmp_152_type 1431 +#define _tmp_153_type 1432 +#define _tmp_154_type 1433 +#define _tmp_155_type 1434 +#define _tmp_156_type 1435 +#define _tmp_157_type 1436 +#define _tmp_158_type 1437 +#define _tmp_159_type 1438 +#define _tmp_160_type 1439 +#define _tmp_161_type 1440 +#define _tmp_162_type 1441 +#define _tmp_163_type 1442 +#define _tmp_164_type 1443 +#define _tmp_165_type 1444 +#define _tmp_166_type 1445 +#define _tmp_167_type 1446 +#define _tmp_168_type 1447 +#define _tmp_169_type 1448 +#define _tmp_170_type 1449 +#define _tmp_171_type 1450 +#define _tmp_172_type 1451 +#define _tmp_173_type 1452 +#define _loop0_174_type 1453 +#define _tmp_175_type 1454 +#define _tmp_176_type 1455 +#define _tmp_177_type 1456 +#define _tmp_178_type 1457 +#define _tmp_179_type 1458 +#define _tmp_180_type 1459 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -681,6 +682,7 @@ static CmpopExprPair* notin_bitwise_or_rule(Parser *p); static CmpopExprPair* in_bitwise_or_rule(Parser *p); static CmpopExprPair* isnot_bitwise_or_rule(Parser *p); static CmpopExprPair* is_bitwise_or_rule(Parser *p); +static void *invalid_noteq_rule(Parser *p); static expr_ty bitwise_or_rule(Parser *p); static expr_ty bitwise_xor_rule(Parser *p); static expr_ty bitwise_and_rule(Parser *p); @@ -13173,7 +13175,7 @@ lte_bitwise_or_rule(Parser *p) return _res; } -// lt_bitwise_or: '<' bitwise_or +// lt_bitwise_or: '<' bitwise_or | invalid_noteq static CmpopExprPair* lt_bitwise_or_rule(Parser *p) { @@ -13213,6 +13215,25 @@ lt_bitwise_or_rule(Parser *p) D(fprintf(stderr, "%*c%s lt_bitwise_or[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'<' bitwise_or")); } + if (p->call_invalid_rules) { // invalid_noteq + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> lt_bitwise_or[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_noteq")); + void *invalid_noteq_var; + if ( + (invalid_noteq_var = invalid_noteq_rule(p)) // invalid_noteq + ) + { + D(fprintf(stderr, "%*c+ lt_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_noteq")); + _res = invalid_noteq_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s lt_bitwise_or[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_noteq")); + } _res = NULL; done: p->level--; @@ -13501,6 +13522,52 @@ is_bitwise_or_rule(Parser *p) return _res; } +// invalid_noteq: '<' '>' +static void * +invalid_noteq_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '<' '>' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_noteq[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'<' '>'")); + Token * a; + Token * b; + if ( + (a = _PyPegen_expect_token(p, 20)) // token='<' + && + (b = _PyPegen_expect_token(p, 21)) // token='>' + ) + { + D(fprintf(stderr, "%*c+ invalid_noteq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<' '>'")); + _res = _PyPegen_tokens_are_adjacent ( a , b ) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Maybe you meant '!=' instead of '<>'?" ) : NULL; + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_noteq[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'<' '>'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // Left-recursive // bitwise_or: bitwise_or '|' bitwise_xor | invalid_bitwise_or | bitwise_xor static expr_ty bitwise_or_raw(Parser *); diff --git a/Parser/pegen.c b/Parser/pegen.c index fcec810037e98d..c4f62dc56ba324 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1054,7 +1054,9 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena PyCompilerFlags *flags, int *errcode, PyObject **interactive_src, PyArena *arena) { - struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2); + int parser_flags = compute_parser_flags(flags); + struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2, + parser_flags & PyPARSE_BARRY_AS_BDFL); if (tok == NULL) { if (PyErr_Occurred()) { _PyTokenizer_raise_init_error(filename_ob); @@ -1080,7 +1082,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena goto error; } - int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); if (p == NULL) { @@ -1109,12 +1110,13 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; + int parser_flags = compute_parser_flags(flags); struct tok_state *tok; if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) { - tok = _PyTokenizer_FromUTF8(str, exec_input, 0); + tok = _PyTokenizer_FromUTF8(str, exec_input, 0, parser_flags & PyPARSE_BARRY_AS_BDFL); } else { - tok = _PyTokenizer_FromString(str, exec_input, 0); + tok = _PyTokenizer_FromString(str, exec_input, 0, parser_flags & PyPARSE_BARRY_AS_BDFL); } if (tok == NULL) { if (PyErr_Occurred()) { @@ -1133,7 +1135,6 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen // We need to clear up from here on mod_ty result = NULL; - int parser_flags = compute_parser_flags(flags); int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ? flags->cf_feature_version : PY_MINOR_VERSION; Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version, diff --git a/Parser/token.c b/Parser/token.c index a6e7ce35dfad85..0a729e5d8a368b 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -113,7 +113,7 @@ _PyToken_OneChar(int c1) } int -_PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL) { switch (c1) { case '!': @@ -163,7 +163,7 @@ _PyToken_TwoChars(int c1, int c2) switch (c2) { case '<': return LEFTSHIFT; case '=': return LESSEQUAL; - case '>': return NOTEQUAL; + case '>': return BARRY_AS_BDFL ? NOTEQUAL : OP; } break; case '=': diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index a11702557a07af..e588f135aed540 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -371,7 +371,8 @@ tok_underflow_file(struct tok_state *tok) /* Set up tokenizer for file */ struct tok_state * _PyTokenizer_FromFile(FILE *fp, const char* enc, - const char *ps1, const char *ps2) + const char *ps1, const char *ps2, + int BARRY_AS_BDFL) { struct tok_state *tok = _PyTokenizer_tok_new(); if (tok == NULL) @@ -401,6 +402,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, } tok->decoding_state = STATE_NORMAL; } + tok->BARRY_AS_BDFL = BARRY_AS_BDFL; return tok; } @@ -457,7 +459,7 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) if (fp == NULL) { return NULL; } - tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL); + tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL, 0); if (tok == NULL) { fclose(fp); return NULL; diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 7f07cca37ee019..3d5a03423340fb 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -128,7 +128,7 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr /* Set up tokenizer for string */ struct tok_state * -_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) +_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf, int BARRY_AS_BDFL) { struct tok_state *tok = _PyTokenizer_tok_new(); char *decoded; @@ -144,5 +144,6 @@ _PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) tok->buf = tok->cur = tok->inp = decoded; tok->end = decoded; tok->underflow = &tok_underflow_string; + tok->BARRY_AS_BDFL = BARRY_AS_BDFL; return tok; } diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 8fbeb2d6ae6df1..7117c6bc0c9754 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -3,11 +3,11 @@ #include "Python.h" -struct tok_state *_PyTokenizer_FromString(const char *, int, int); -struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); +struct tok_state *_PyTokenizer_FromString(const char *, int, int, int); +struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int, int); struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int); struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, - const char *, const char *); + const char *, const char *, int); #define tok_dump _Py_tok_dump diff --git a/Parser/tokenizer/utf8_tokenizer.c b/Parser/tokenizer/utf8_tokenizer.c index 1a925f445400fa..f31a7fe606a9df 100644 --- a/Parser/tokenizer/utf8_tokenizer.c +++ b/Parser/tokenizer/utf8_tokenizer.c @@ -28,7 +28,8 @@ tok_underflow_string(struct tok_state *tok) { /* Set up tokenizer for UTF-8 string */ struct tok_state * -_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) +_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf, + int BARRY_AS_BDFL) { struct tok_state *tok = _PyTokenizer_tok_new(); char *translated; @@ -51,5 +52,6 @@ _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) tok->buf = tok->cur = tok->inp = translated; tok->end = translated; tok->underflow = &tok_underflow_string; + tok->BARRY_AS_BDFL = BARRY_AS_BDFL; return tok; } diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 9ee5ec86e75d47..c7603e212d2529 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -94,7 +94,7 @@ def update_file(file, content): // Export these 4 symbols for 'test_peg_generator' PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) _PyToken_OneChar(int); -PyAPI_FUNC(int) _PyToken_TwoChars(int, int); +PyAPI_FUNC(int) _PyToken_TwoChars(int, int, int); PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); #ifdef __cplusplus @@ -142,7 +142,7 @@ def make_h(infile, outfile='Include/internal/pycore_token.h'): } int -_PyToken_TwoChars(int c1, int c2) +_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL) { %s\ return OP; @@ -171,7 +171,10 @@ def generate_chars_to_token(mapping, n=1): write(indent) write(' break;\n') else: - write("case '%s': return %s;\n" % (c, value)) + if c == '>' and value == 'NOTEQUAL': + write("case '%s': return BARRY_AS_BDFL ? %s : OP;\n" % (c, value)) + else: + write("case '%s': return %s;\n" % (c, value)) write(indent) write('}\n') return ''.join(result)