diff --git a/LICENSE b/LICENSE index 90d19f757b2..2b332548d80 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013-2025 Damien P. George +Copyright (c) 2013-2026 Damien P. George Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/library/index.rst b/docs/library/index.rst index 4e763e8a27e..e23e1915e03 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -34,7 +34,9 @@ These libraries are not currently enabled in any CircuitPython build, but may be platform.rst re.rst select.rst + string.templatelib.rst sys.rst + weakref.rst Omitted ``string`` functions ---------------------------- diff --git a/docs/library/re.rst b/docs/library/re.rst index b8aeefd90cf..47f623c5600 100644 --- a/docs/library/re.rst +++ b/docs/library/re.rst @@ -54,6 +54,10 @@ Supported operators and special sequences are: Grouping. Each group is capturing (a substring it captures can be accessed with `match.group()` method). +``(?:...)`` + Non-capturing grouping. Each group is matched using the same rules as + regular grouping, but will not be part of the match object. + ``\d`` Matches digit. Equivalent to ``[0-9]``. @@ -87,7 +91,6 @@ Supported operators and special sequences are: * counted repetitions (``{m,n}``) * named groups (``(?P...)``) -* non-capturing groups (``(?:...)``) * more advanced assertions (``\b``, ``\B``) * special character escapes like ``\r``, ``\n`` - use Python's own escaping instead diff --git a/docs/library/select.rst b/docs/library/select.rst index 08bbe08df2c..dcf1aa781b6 100644 --- a/docs/library/select.rst +++ b/docs/library/select.rst @@ -76,6 +76,9 @@ Methods In case of timeout, an empty list is returned. + Calling ``poll.poll`` is guaranteed to call pending callback functions + before entering the polling loop. + .. admonition:: Difference to CPython :class: attention @@ -93,6 +96,9 @@ Methods won't be processed until new mask is set with `poll.modify()`. This behaviour is useful for asynchronous I/O schedulers. + Calling ``poll.ipoll`` is guaranteed to call pending callback functions + before entering the polling loop. + .. admonition:: Difference to CPython :class: attention diff --git a/docs/library/string.templatelib.rst b/docs/library/string.templatelib.rst new file mode 100644 index 00000000000..901aa045ca1 --- /dev/null +++ b/docs/library/string.templatelib.rst @@ -0,0 +1,230 @@ +:mod:`string.templatelib` -- Template String Support +==================================================== + +.. module:: string.templatelib + :synopsis: PEP 750 template string support + +This module provides support for template strings (t-strings) as defined in +`PEP 750 `_. Template strings are created +using the ``t`` prefix and provide access to both the literal string parts and +interpolated values before they are combined. + +**Availability:** template strings require ``MICROPY_PY_TSTRINGS`` to be enabled +at compile time. They are enabled by default at the full feature level, which +includes the alif, mimxrt and samd (SAMD51 only) ports, the unix coverage variant +and the webassembly pyscript variant. + +Classes +------- + +.. class:: Template(*args) + + Represents a template string. Template objects are typically created by + t-string syntax (``t"..."``) but can also be constructed directly using + the constructor. + + .. attribute:: strings + + A tuple of string literals that appear between interpolations. + + .. attribute:: interpolations + + A tuple of :class:`Interpolation` objects representing the interpolated + expressions. + + .. attribute:: values + + A read-only property that returns a tuple containing the ``value`` + attribute from each interpolation in the template. + + .. method:: __iter__() + + Iterate over the template contents, yielding string parts and + :class:`Interpolation` objects in the order they appear. Empty strings + are omitted. + + .. method:: __add__(other) + + Concatenate two templates. Returns a new :class:`Template` combining + the strings and interpolations from both templates. + + :raises TypeError: if *other* is not a :class:`Template` + + Template concatenation with ``str`` is prohibited to avoid ambiguity + about whether the string should be treated as a literal or interpolation:: + + t1 = t"Hello " + t2 = t"World" + result = t1 + t2 # Valid + + # TypeError: cannot concatenate str to Template + result = t1 + "World" + +.. class:: Interpolation(value, expression='', conversion=None, format_spec='') + + Represents an interpolated expression within a template string. All + arguments can be passed as keyword arguments. + + .. attribute:: value + + The evaluated value of the interpolated expression. + + .. attribute:: expression + + The string representation of the expression as it appeared in the + template string. + + .. attribute:: conversion + + The conversion specifier (``'s'`` or ``'r'``) if present, otherwise ``None``. + Note that MicroPython does not support the ``'a'`` conversion. + + .. attribute:: format_spec + + The format specification string if present, otherwise an empty string. + +Template String Syntax +---------------------- + +Template strings use the same syntax as f-strings but with a ``t`` prefix:: + + name = "World" + template = t"Hello {name}!" + + # Access template components + print(template.strings) # ('Hello ', '!') + print(template.values) # ('World',) + print(template.interpolations[0].expression) # 'name' + +Conversion Specifiers +~~~~~~~~~~~~~~~~~~~~~ + +Template strings store conversion specifiers as metadata. Unlike f-strings, +the conversion is not applied automatically:: + + value = "test" + t = t"{value!r}" + # t.interpolations[0].value == "test" (not repr(value)) + # t.interpolations[0].conversion == "r" + +Processing code must explicitly apply conversions when needed. + +Format Specifications +~~~~~~~~~~~~~~~~~~~~~ + +Format specifications are stored as metadata in the ``Interpolation`` object. +Unlike f-strings, formatting is not applied automatically:: + + pi = 3.14159 + t = t"{pi:.2f}" + # t.interpolations[0].value == 3.14159 (not formatted) + # t.interpolations[0].format_spec == ".2f" + +Per PEP 750, processing code is not required to use format specifications, but +when present they should be respected and match f-string behavior where possible. + +Debug Format +~~~~~~~~~~~~ + +The debug format ``{expr=}`` is supported:: + + x = 42 + t = t"{x=}" + # t.strings == ("x=", "") + # t.interpolations[0].expression == "x" + # t.interpolations[0].conversion == "r" + +.. admonition:: Important + :class: attention + + As per PEP 750, unlike f-strings, template strings do not automatically + apply conversions or format specifications. This is by design to allow + processing code to control how these are handled. Processing code must + explicitly handle these attributes. + + MicroPython does not provide the ``format()`` built-in function. Use + string formatting methods like ``str.format()`` instead. + +Example Usage +------------- + +Basic processing without format support:: + + def simple_process(template): + """Simple template processing""" + parts = [] + for item in template: + if isinstance(item, str): + parts.append(item) + else: + parts.append(str(item.value)) + return "".join(parts) + +Processing template with format support:: + + from string.templatelib import Template, Interpolation + + def convert(value, conversion): + """Apply conversion specifier to value""" + if conversion == "r": + return repr(value) + elif conversion == "s": + return str(value) + return value + + def process_template(template): + """Process template with conversion and format support""" + result = [] + for part in template: + if isinstance(part, str): + result.append(part) + else: # Interpolation + value = convert(part.value, part.conversion) + if part.format_spec: + # Apply format specification using str.format + value = ("{:" + part.format_spec + "}").format(value) + else: + value = str(value) + result.append(value) + return "".join(result) + + pi = 3.14159 + name = "Alice" + t = t"{name!r}: {pi:.2f}" + print(process_template(t)) + # Output: "'Alice': 3.14" + + # Other format specifications work too + value = 42 + print(process_template(t"{value:>10}")) # " 42" + print(process_template(t"{value:04d}")) # "0042" + +HTML escaping example:: + + def html_escape(value): + """Escape HTML special characters""" + if not isinstance(value, str): + value = str(value) + return value.replace("&", "&").replace("<", "<").replace(">", ">") + + def safe_html(template): + """Convert template to HTML-safe string""" + result = [] + for part in template: + if isinstance(part, str): + result.append(part) + else: + result.append(html_escape(part.value)) + return "".join(result) + + user_input = "" + t = t"User said: {user_input}" + print(safe_html(t)) + # Output: "User said: <script>alert('xss')</script>" + +See Also +-------- + +* `PEP 750 `_ - Template Strings specification +* :ref:`python:formatstrings` - Format string syntax +* `Formatted string literals `_ - f-strings in Python diff --git a/docs/library/weakref.rst b/docs/library/weakref.rst new file mode 100644 index 00000000000..ffebc91277b --- /dev/null +++ b/docs/library/weakref.rst @@ -0,0 +1,78 @@ +:mod:`weakref` -- Python object lifetime management +=================================================== + +.. module:: weakref + :synopsis: Create weak references to Python objects + +|see_cpython_module| :mod:`python:weakref`. + +This module allows creation of weak references to Python objects. A weak reference +is a non-traceable reference to a heap-allocated Python object, so the garbage +collector can still reclaim the object even though the weak reference refers to it. + +Python callbacks can be registered to be called when an object is reclaimed by the +garbage collector. This provides a safe way to clean up when objects are no longer +needed. + +**Availability:** the weakref module requires ``MICROPY_PY_WEAKREF`` to be enabled +at compile time. It is enabled on the unix coverage variant and the webassembly +pyscript variant. + +ref objects +----------- + +A ref object is the simplest way to make a weak reference. + +.. class:: ref(object [, callback], /) + + Return a weak reference to the given *object*. + + If *callback* is given and is not ``None`` then, when *object* is reclaimed + by the garbage collector and if the weak reference object is still alive, the + *callback* will be called. The *callback* will be passed the weak reference + object as its single argument. + +.. method:: ref.__call__() + + Calling the weak reference object will return its referenced object if that + object is still alive. Otherwise ``None`` will be returned. + +finalize objects +---------------- + +A finalize object is an extended version of a ref object that is more convenient to +use, and allows more control over the callback. + +.. class:: finalize(object, callback, /, *args, **kwargs) + + Return a weak reference to the given *object*. In contrast to *weakref.ref* + objects, finalize objects are held onto internally and will not be collected until + *object* is collected. + + A finalize object starts off alive. It transitions to the dead state when the + finalize object is called, either explicitly or when *object* is collected. It also + transitions to dead if the `finalize.detach()` method is called. + + When *object* is reclaimed by the garbage collector (or the finalize object is + explicitly called by user code) and the finalize object is still in the alive state, + the *callback* will be called. The *callback* will be passed arguments as: + ``callback(*args, **kwargs)``. + +.. method:: finalize.__call__() + + If the finalize object is alive then it transitions to the dead state and returns + the value of ``callback(*args, **kwargs)``. Otherwise ``None`` will be returned. + +.. method:: finalize.alive + + Read-only boolean attribute that indicates if the finalizer is in the alive state. + +.. method:: finalize.peek() + + If the finalize object is alive then return ``(object, callback, args, kwargs)``. + Otherwise return ``None``. + +.. method:: finalize.detach() + + If the finalize object is alive then it transitions to the dead state and returns + ``(object, callback, args, kwargs)``. Otherwise ``None`` will be returned. diff --git a/extmod/modos.c b/extmod/modos.c index 69fdc3fac0a..c330162e102 100644 --- a/extmod/modos.c +++ b/extmod/modos.c @@ -145,6 +145,21 @@ static mp_obj_t mp_os_dupterm_notify(mp_obj_t obj_in) { static MP_DEFINE_CONST_FUN_OBJ_1(mp_os_dupterm_notify_obj, mp_os_dupterm_notify); #endif +#if MICROPY_PY_OS_URANDOM +// This wraps the port-specific mp_hal_get_random(), which is usually defined in mphalport.c. +static mp_obj_t mp_os_urandom(mp_obj_t num) { + mp_int_t n = mp_obj_get_int(num); + if (n < 0) { + mp_raise_ValueError(NULL); + } + vstr_t vstr; + vstr_init_len(&vstr, n); + mp_hal_get_random(n, (uint8_t *)vstr.buf); + return mp_obj_new_bytes_from_vstr(&vstr); +} +static MP_DEFINE_CONST_FUN_OBJ_1(mp_os_urandom_obj, mp_os_urandom); +#endif + static const mp_rom_map_elem_t os_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_os) }, diff --git a/extmod/modselect.c b/extmod/modselect.c index c4edee6a56b..72566e024f0 100644 --- a/extmod/modselect.c +++ b/extmod/modselect.c @@ -344,6 +344,8 @@ static mp_uint_t poll_set_poll_until_ready_or_timeout(poll_set_t *poll_set, size mp_uint_t start_ticks = mp_hal_ticks_ms(); bool has_timeout = timeout != (mp_uint_t)-1; + mp_handle_pending(true); + #if MICROPY_PY_SELECT_POSIX_OPTIMISATIONS for (;;) { diff --git a/extmod/vfs_lfsx_file.c b/extmod/vfs_lfsx_file.c index 56daa53e068..a0e572d80cb 100644 --- a/extmod/vfs_lfsx_file.c +++ b/extmod/vfs_lfsx_file.c @@ -152,11 +152,8 @@ static mp_uint_t MP_VFS_LFSx(file_write)(mp_obj_t self_in, const void *buf, mp_u static mp_uint_t MP_VFS_LFSx(file_ioctl)(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) { MP_OBJ_VFS_LFSx_FILE *self = MP_OBJ_TO_PTR(self_in); - if (request != MP_STREAM_CLOSE) { - MP_VFS_LFSx(check_open)(self); - } - if (request == MP_STREAM_SEEK) { + MP_VFS_LFSx(check_open)(self); struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)(uintptr_t)arg; int res = LFSx_API(file_seek)(&self->vfs->lfs, &self->file, s->offset, s->whence); if (res < 0) { @@ -171,6 +168,7 @@ static mp_uint_t MP_VFS_LFSx(file_ioctl)(mp_obj_t self_in, mp_uint_t request, ui s->offset = res; return 0; } else if (request == MP_STREAM_FLUSH) { + MP_VFS_LFSx(check_open)(self); int res = LFSx_API(file_sync)(&self->vfs->lfs, &self->file); if (res < 0) { *errcode = -res; diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c index 501550cb502..5fe330d7fcf 100644 --- a/extmod/vfs_posix_file.c +++ b/extmod/vfs_posix_file.c @@ -146,12 +146,9 @@ static mp_uint_t vfs_posix_file_write(mp_obj_t o_in, const void *buf, mp_uint_t static mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); - if (request != MP_STREAM_CLOSE) { - check_fd_is_open(o); - } - switch (request) { case MP_STREAM_FLUSH: { + check_fd_is_open(o); int ret; // fsync(stdin/stdout/stderr) may fail with EINVAL (or ENOTSUP on macos or EBADF // on windows), because the OS doesn't buffer these except for instance when they @@ -190,6 +187,7 @@ static mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ return 0; } case MP_STREAM_SEEK: { + check_fd_is_open(o); struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg; MP_THREAD_GIL_EXIT(); off_t off = lseek(o->fd, s->offset, s->whence); @@ -210,12 +208,14 @@ static mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_ o->fd = -1; return 0; case MP_STREAM_GET_FILENO: + check_fd_is_open(o); return o->fd; #if MICROPY_PY_SELECT && !MICROPY_PY_SELECT_POSIX_OPTIMISATIONS case MP_STREAM_POLL: { #ifdef _WIN32 mp_raise_NotImplementedError(MP_ERROR_TEXT("poll on file not available on win32")); #else + check_fd_is_open(o); mp_uint_t ret = 0; uint8_t pollevents = 0; if (arg & MP_STREAM_POLL_RD) { diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 6c36770295b..139ecafb35b 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -56,6 +56,7 @@ static mp_uint_t mp_reader_vfs_readbyte(void *data) { return MP_READER_EOF; } else { int errcode; + mp_event_handle_nowait(); reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); if (errcode != 0) { // TODO handle errors properly diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot index c55efbcf8c1..ae94af84ce2 100644 --- a/locale/circuitpython.pot +++ b/locale/circuitpython.pot @@ -1041,9 +1041,9 @@ msgstr "" #: ports/cxd56/common-hal/sdioio/SDCard.c #: ports/espressif/common-hal/sdioio/SDCard.c #: ports/raspberrypi/common-hal/sdioio/SDCard.c -#: ports/stm/common-hal/sdioio/SDCard.c shared-bindings/emmcio/EMMC.c -#: shared-bindings/floppyio/__init__.c shared-bindings/picogame/Canvas.c -#: shared-module/sdcardio/SDCard.c +#: ports/stm/common-hal/sdioio/SDCard.c shared-bindings/audiocore/WaveFile.c +#: shared-bindings/emmcio/EMMC.c shared-bindings/floppyio/__init__.c +#: shared-bindings/picogame/Canvas.c shared-module/sdcardio/SDCard.c #, c-format msgid "Buffer must be a multiple of %d bytes" msgstr "" @@ -1797,15 +1797,15 @@ msgid "Cannot get temperature" msgstr "" #: ports/nordic/common-hal/watchdog/WatchDogTimer.c -msgid "WatchDogTimer cannot be deinitialized once mode is set to RESET" +msgid "timeout duration exceeded the maximum supported value" msgstr "" #: ports/nordic/common-hal/watchdog/WatchDogTimer.c -msgid "timeout duration exceeded the maximum supported value" +msgid "%q cannot be changed once mode is set to %q" msgstr "" #: ports/nordic/common-hal/watchdog/WatchDogTimer.c -msgid "%q cannot be changed once mode is set to %q" +msgid "WatchDogTimer cannot be deinitialized once mode is set to RESET" msgstr "" #: ports/nordic/sd_mutex.c @@ -2910,6 +2910,10 @@ msgstr "" msgid "expecting a dict for keyword args" msgstr "" +#: py/modweakref.c +msgid "not a heap object" +msgstr "" + #: py/nativeglue.c msgid "set unsupported" msgstr "" @@ -3315,6 +3319,10 @@ msgstr "" msgid "string index out of range" msgstr "" +#: py/objtemplate.c +msgid "expected str or Interpolation" +msgstr "" + #: py/objtype.c msgid "Call super().__init__() before accessing native object." msgstr "" @@ -3437,6 +3445,10 @@ msgstr "" msgid "incompatible .mpy arch" msgstr "" +#: py/persistentcode.c +msgid "function must be bytecode" +msgstr "" + #: py/proto.c shared-bindings/digitalio/DigitalInOutProtocol.c msgid "'%q' object does not support '%q'" msgstr "" diff --git a/mpy-cross/main.c b/mpy-cross/main.c index add07c3d49b..8cdd124ef46 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -146,8 +146,8 @@ static int usage(char **argv) { "-march= : set architecture for native emitter;\n" " x86, x64, armv6, armv6m, armv7m, armv7em, armv7emsp,\n" " armv7emdp, xtensa, xtensawin, rv32imc, rv64imc, host, debug\n" - "-march-flags= : set architecture-specific flags (can be either a dec/hex/bin value or a string)\n" - " supported flags for rv32imc: zba\n" + "-march-flags= : set architecture-specific flags (can be either a dec/hex/bin value or a comma-separated flags string)\n" + " supported flags for rv32imc: zba, zcmp\n" "\n" "Implementation specific options:\n", argv[0] ); @@ -260,6 +260,34 @@ static bool parse_integer(const char *value, mp_uint_t *integer) { return valid; } +#if MICROPY_EMIT_NATIVE && MICROPY_EMIT_RV32 +static bool parse_rv32_flags_string(const char *source, mp_uint_t *flags) { + assert(source && "Flag arguments string is NULL."); + assert(flags && "Collected flags pointer is NULL."); + + const char *current = source; + const char *end = source + strlen(source); + mp_uint_t collected_flags = 0; + while (current < end) { + const char *separator = strchr(current, ','); + if (separator == NULL) { + separator = end; + } + ptrdiff_t length = separator - current; + if (length == (sizeof("zba") - 1) && memcmp(current, "zba", length) == 0) { + collected_flags |= RV32_EXT_ZBA; + } else if (length == (sizeof("zcmp") - 1) && memcmp(current, "zcmp", length) == 0) { + collected_flags |= RV32_EXT_ZCMP; + } else { + return false; + } + current = separator + 1; + } + *flags = collected_flags; + return collected_flags != 0; +} +#endif + MP_NOINLINE int main_(int argc, char **argv) { pre_process_options(argc, argv); @@ -414,14 +442,11 @@ MP_NOINLINE int main_(int argc, char **argv) { if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_RV32IMC) { mp_dynamic_compiler.backend_options = (void *)&rv32_options; mp_uint_t raw_flags = 0; - if (parse_integer(arch_flags, &raw_flags)) { + if (parse_integer(arch_flags, &raw_flags) || parse_rv32_flags_string(arch_flags, &raw_flags)) { if ((raw_flags & ~((mp_uint_t)RV32_EXT_ALL)) == 0) { rv32_options.allowed_extensions = raw_flags; processed = true; } - } else if (strncmp(arch_flags, "zba", sizeof("zba") - 1) == 0) { - rv32_options.allowed_extensions |= RV32_EXT_ZBA; - processed = true; } } #endif diff --git a/mpy-cross/mpconfigport.h b/mpy-cross/mpconfigport.h index 36bcbc11394..56c5963fa9c 100644 --- a/mpy-cross/mpconfigport.h +++ b/mpy-cross/mpconfigport.h @@ -32,6 +32,7 @@ #define MICROPY_ALLOC_PATH_MAX (PATH_MAX) #define MICROPY_PERSISTENT_CODE_LOAD (0) +#define MICROPY_PERSISTENT_CODE_LOAD_NATIVE (0) #define MICROPY_PERSISTENT_CODE_SAVE (1) #ifndef MICROPY_PERSISTENT_CODE_SAVE_FILE @@ -84,6 +85,7 @@ #define MICROPY_USE_INTERNAL_PRINTF (0) #define MICROPY_PY_FSTRINGS (1) +#define MICROPY_PY_TSTRINGS (1) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) #if !(defined(MICROPY_GCREGS_SETJMP) || defined(__x86_64__) || defined(__i386__) || defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) diff --git a/ports/espressif/common-hal/wifi/Network.c b/ports/espressif/common-hal/wifi/Network.c index 5e29cad3193..ac7d963dd08 100644 --- a/ports/espressif/common-hal/wifi/Network.c +++ b/ports/espressif/common-hal/wifi/Network.c @@ -6,6 +6,8 @@ #include +#include "py/objlist.h" + #include "shared-bindings/wifi/Network.h" #include "shared-bindings/wifi/AuthMode.h" diff --git a/ports/raspberrypi/common-hal/wifi/Network.c b/ports/raspberrypi/common-hal/wifi/Network.c index 1cd3effbd0f..dd3e4d0f2f1 100644 --- a/ports/raspberrypi/common-hal/wifi/Network.c +++ b/ports/raspberrypi/common-hal/wifi/Network.c @@ -6,6 +6,8 @@ #include +#include "py/objlist.h" + #include "shared-bindings/wifi/Network.h" #include "shared-bindings/wifi/AuthMode.h" diff --git a/ports/unix/alloc.c b/ports/unix/alloc.c index 9ab2ca04ebc..7e1b261261e 100644 --- a/ports/unix/alloc.c +++ b/ports/unix/alloc.c @@ -32,7 +32,7 @@ #include "py/mpstate.h" -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE #if defined(__OpenBSD__) || defined(__MACH__) #define MAP_ANONYMOUS MAP_ANON @@ -80,4 +80,4 @@ void mp_unix_free_exec(void *ptr, size_t size) { MP_REGISTER_ROOT_POINTER(void *mmap_region_head); -#endif // MICROPY_EMIT_NATIVE +#endif // MICROPY_ENABLE_NATIVE_CODE diff --git a/ports/unix/modos.c b/ports/unix/modos.c index af8a1cc4da8..28f232e42bc 100644 --- a/ports/unix/modos.c +++ b/ports/unix/modos.c @@ -107,15 +107,6 @@ static mp_obj_t mp_os_system(mp_obj_t cmd_in) { } static MP_DEFINE_CONST_FUN_OBJ_1(mp_os_system_obj, mp_os_system); -static mp_obj_t mp_os_urandom(mp_obj_t num) { - mp_int_t n = mp_obj_get_int(num); - vstr_t vstr; - vstr_init_len(&vstr, n); - mp_hal_get_random(n, vstr.buf); - return mp_obj_new_bytes_from_vstr(&vstr); -} -static MP_DEFINE_CONST_FUN_OBJ_1(mp_os_urandom_obj, mp_os_urandom); - static mp_obj_t mp_os_errno(size_t n_args, const mp_obj_t *args) { if (n_args == 0) { return MP_OBJ_NEW_SMALL_INT(errno); diff --git a/ports/unix/modtime.c b/ports/unix/modtime.c index 4f0550dbea7..97c9b292d06 100644 --- a/ports/unix/modtime.c +++ b/ports/unix/modtime.c @@ -95,6 +95,7 @@ static mp_obj_t mp_time_sleep(mp_obj_t arg) { tv.tv_sec = (suseconds_t)ipart; int res; while (1) { + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); MP_THREAD_GIL_EXIT(); res = sleep_select(0, NULL, NULL, NULL, &tv); MP_THREAD_GIL_ENTER(); @@ -104,7 +105,6 @@ static mp_obj_t mp_time_sleep(mp_obj_t arg) { if (res != -1 || errno != EINTR) { break; } - mp_handle_pending(true); // printf("select: EINTR: %ld:%ld\n", tv.tv_sec, tv.tv_usec); #else break; @@ -114,13 +114,13 @@ static mp_obj_t mp_time_sleep(mp_obj_t arg) { #else int seconds = mp_obj_get_int(arg); for (;;) { + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); MP_THREAD_GIL_EXIT(); seconds = sleep(seconds); MP_THREAD_GIL_ENTER(); if (seconds == 0) { break; } - mp_handle_pending(true); } #endif return mp_const_none; diff --git a/ports/unix/unix_mphal.c b/ports/unix/unix_mphal.c index 238d1cb5ee8..54fd4c94ad7 100644 --- a/ports/unix/unix_mphal.c +++ b/ports/unix/unix_mphal.c @@ -251,9 +251,13 @@ uint64_t mp_hal_time_ns(void) { #ifndef mp_hal_delay_ms void mp_hal_delay_ms(mp_uint_t ms) { - mp_uint_t start = mp_hal_ticks_ms(); - while (mp_hal_ticks_ms() - start < ms) { - mp_event_wait_ms(1); + if (ms) { + mp_uint_t start = mp_hal_ticks_ms(); + while (mp_hal_ticks_ms() - start < ms) { + mp_event_wait_ms(1); + } + } else { + mp_handle_pending(true); } } #endif diff --git a/ports/zephyr-cp/boards/nordic/nrf7002dk/circuitpython.toml b/ports/zephyr-cp/boards/nordic/nrf7002dk/circuitpython.toml index badaffb8e1b..0ed50c6ad2b 100644 --- a/ports/zephyr-cp/boards/nordic/nrf7002dk/circuitpython.toml +++ b/ports/zephyr-cp/boards/nordic/nrf7002dk/circuitpython.toml @@ -2,7 +2,7 @@ CIRCUITPY_BUILD_EXTENSIONS = ["hex"] USB_VID=0x239A USB_PID=0x8168 BLOBS=["nrf_wifi"] -DISABLED_MODULES=["aesio", "adafruit_bus_device", "zlib", "jpegio", "tilepalettemapper", "gifio"] +DISABLED_MODULES=["aesio", "adafruit_bus_device", "zlib", "jpegio", "tilepalettemapper", "gifio", "msgpack"] # ulab is on by default. This board has under 3 KB of flash headroom, and # ulab costs about 90 KB, so it opts out. @@ -10,3 +10,7 @@ CIRCUITPY_ULAB = false # The same 3 KB is why radio.ping() is off here too. CIRCUITPY_WIFI_PING = false + +# msgpack is off to fit the v1.28.0 MicroPython update, which needs 176 more +# bytes than the ja translation had left. Revert once #11335 gives slot0 the +# unused 32 KB storage partition. diff --git a/ports/zephyr-cp/common-hal/wifi/Network.c b/ports/zephyr-cp/common-hal/wifi/Network.c index 510f8b9f5c2..dad9293a149 100644 --- a/ports/zephyr-cp/common-hal/wifi/Network.c +++ b/ports/zephyr-cp/common-hal/wifi/Network.c @@ -6,6 +6,8 @@ #include +#include "py/objlist.h" + #include "shared-bindings/wifi/Network.h" #include "shared-bindings/wifi/AuthMode.h" diff --git a/py/asmarm.h b/py/asmarm.h index f3fd586a375..74e43de7f90 100644 --- a/py/asmarm.h +++ b/py/asmarm.h @@ -231,6 +231,8 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src); #define ASM_STORE16_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_arm_strh_reg_reg_reg((as), (reg_val), (reg_base), (reg_index)) #define ASM_STORE32_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_arm_str_reg_reg_reg((as), (reg_val), (reg_base), (reg_index)) +#define ASM_CLR_REG(as, reg_dest) asm_arm_eor_reg_reg_reg((as), (reg_dest), (reg_dest), (reg_dest)) + #endif // GENERIC_ASM_API #endif // MICROPY_INCLUDED_PY_ASMARM_H diff --git a/py/asmrv32.c b/py/asmrv32.c index 1d0cea6c026..e58e42012d9 100644 --- a/py/asmrv32.c +++ b/py/asmrv32.c @@ -53,6 +53,14 @@ ((((value) & ~((1U << ((bits) - 1)) - 1)) == 0) || \ (((value) & ~((1U << ((bits) - 1)) - 1)) == ~((1U << ((bits) - 1)) - 1))) +static bool asm_rv32_allow_zba_opcodes(void) { + return asm_rv32_allowed_extensions() & RV32_EXT_ZBA; +} + +static bool asm_rv32_allow_zcmp_opcodes(void) { + return asm_rv32_allowed_extensions() & RV32_EXT_ZCMP; +} + /////////////////////////////////////////////////////////////////////////////// void asm_rv32_emit_word_opcode(asm_rv32_t *state, mp_uint_t word) { @@ -214,6 +222,14 @@ static void adjust_stack(asm_rv32_t *state, mp_int_t stack_size) { return; } + // WARNING: If REG_TEMP0 is not set to a caller-saved register, then this + // bit has to be rewritten to avoid clobbering the temporary + // register when performing the stack adjustment. + + MP_STATIC_ASSERT(((REG_TEMP0 >= ASM_RV32_REG_T0) && (REG_TEMP0 <= ASM_RV32_REG_T2)) || \ + ((REG_TEMP0 >= ASM_RV32_REG_A0) && (REG_TEMP0 <= ASM_RV32_REG_A7)) || \ + ((REG_TEMP0 >= ASM_RV32_REG_T3) && (REG_TEMP0 <= ASM_RV32_REG_T6))); + // li temporary, stack_size // c.add sp, temporary load_full_immediate(state, REG_TEMP0, stack_size); @@ -245,6 +261,45 @@ static void emit_function_epilogue(asm_rv32_t *state, mp_uint_t registers) { state->saved_registers_mask = old_saved_registers_mask; } +static mp_uint_t compute_zcmp_sequence_length(mp_uint_t registers) { + // Can only handle RA and S0..S11 and must have at least one entry. + assert((registers != 0) && (registers & (~0x0FFC0302U)) == 0 && "Invalid Zcmp registers set."); + mp_uint_t length = 32 - mp_clz(((registers & 0x00000002) >> 1) | ((registers & 0x00000300) >> 7) | ((registers & 0x0FFC0000) >> 15)); + return length == 12 ? 13 : length; +} + +#define EMIT_ASSERT(state, condition, message) assert((((state)->base.pass != MP_ASM_PASS_EMIT) ? true : (condition)) && (message)) + +static void emit_compressed_function_prologue(asm_rv32_t *state, mp_uint_t registers_mask) { + mp_uint_t sequence_length = compute_zcmp_sequence_length(registers_mask); + mp_uint_t allocated_stack = (sequence_length + 3) & (mp_uint_t)-4; + EMIT_ASSERT(state, allocated_stack >= sequence_length, "Incorrect allocated stack calculation."); + mp_uint_t tail_slack = allocated_stack - sequence_length; + mp_uint_t locals_left = (state->locals_count < tail_slack) ? 0 : (state->locals_count - tail_slack); + mp_uint_t adjustment_chunks = MIN(3, locals_left / 4); + EMIT_ASSERT(state, (adjustment_chunks * 4) <= locals_left, "Incorrect adjustment chunks rounding."); + locals_left -= adjustment_chunks * 4; + EMIT_ASSERT(state, locals_left <= (MP_INT_MAX / sizeof(uint32_t)), "Too many locals."); + mp_int_t stack_size = (mp_int_t)(locals_left * sizeof(uint32_t)); + asm_rv32_opcode_cmpush(state, MIN(3 + sequence_length, 15), adjustment_chunks); + // CM.PUSH allocates a stack block and then puts the registers *at the end* + // of the block, so for example "CM.PUSH {RA, S0-S11}, -64" will put RA at + // SP + 60, not at SP + 0. + adjust_stack(state, -stack_size); + // The stack size is expressed in bytes and as a multiple of 4, hence the + // bottom two bits are not used. Since there can be up to three adjustment + // chunks, that number can be expressed in two bits, fitting nicely in the + // existing variable. + state->stack_size = ((mp_uint_t)stack_size) | adjustment_chunks; +} + +static void emit_compressed_function_epilogue(asm_rv32_t *state, mp_uint_t registers_mask) { + mp_uint_t sequence_length = compute_zcmp_sequence_length(registers_mask); + mp_uint_t stack_size = state->stack_size & (mp_uint_t)(~0x03U); + adjust_stack(state, stack_size); + asm_rv32_opcode_cmpopret(state, MIN(3 + sequence_length, 15), state->stack_size & 0x03); +} + static bool calculate_displacement_for_label(asm_rv32_t *state, mp_uint_t label, ptrdiff_t *displacement) { assert(displacement != NULL && "Displacement pointer is NULL"); @@ -256,16 +311,24 @@ static bool calculate_displacement_for_label(asm_rv32_t *state, mp_uint_t label, /////////////////////////////////////////////////////////////////////////////// void asm_rv32_entry(asm_rv32_t *state, mp_uint_t locals) { + state->locals_count = locals; state->saved_registers_mask |= (1U << REG_FUN_TABLE) | (1U << REG_LOCAL_1) | \ (1U << REG_LOCAL_2) | (1U << REG_LOCAL_3); - state->locals_count = locals; - emit_function_prologue(state, state->saved_registers_mask); + if (asm_rv32_allow_zcmp_opcodes()) { + emit_compressed_function_prologue(state, state->saved_registers_mask); + } else { + emit_function_prologue(state, state->saved_registers_mask); + } } void asm_rv32_exit(asm_rv32_t *state) { - emit_function_epilogue(state, state->saved_registers_mask); - // c.jr ra - asm_rv32_opcode_cjr(state, ASM_RV32_REG_RA); + if (asm_rv32_allow_zcmp_opcodes()) { + emit_compressed_function_epilogue(state, state->saved_registers_mask); + } else { + emit_function_epilogue(state, state->saved_registers_mask); + // c.jr ra + asm_rv32_opcode_cjr(state, ASM_RV32_REG_RA); + } } void asm_rv32_end_pass(asm_rv32_t *state) { @@ -557,32 +620,31 @@ void asm_rv32_emit_optimised_xor(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs) asm_rv32_opcode_xor(state, rd, rd, rs); } -static bool asm_rv32_allow_zba_opcodes(void) { - return asm_rv32_allowed_extensions() & RV32_EXT_ZBA; -} - +// WARNING: The scaled offset will be stored in REG_TEMP2. static void asm_rv32_fix_up_scaled_reg_reg_reg(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { assert(operation_size <= 2 && "Operation size value out of range."); if (operation_size > 0 && asm_rv32_allow_zba_opcodes()) { // sh{1,2}add rs1, rs2, rs1 - asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 1 << operation_size, 0x10, rs1, rs2, rs1)); + asm_rv32_emit_word_opcode(state, RV32_ENCODE_TYPE_R(0x33, 1 << operation_size, 0x10, REG_TEMP2, rs2, rs1)); } else { if (operation_size > 0) { - asm_rv32_opcode_cslli(state, rs2, operation_size); + asm_rv32_opcode_slli(state, REG_TEMP2, rs2, operation_size); + asm_rv32_opcode_cadd(state, REG_TEMP2, rs1); + } else { + asm_rv32_opcode_add(state, REG_TEMP2, rs1, rs2); } - asm_rv32_opcode_cadd(state, rs1, rs2); } } void asm_rv32_emit_load_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { asm_rv32_fix_up_scaled_reg_reg_reg(state, rs1, rs2, operation_size); - asm_rv32_emit_load_reg_reg_offset(state, rd, rs1, 0, operation_size); + asm_rv32_emit_load_reg_reg_offset(state, rd, REG_TEMP2, 0, operation_size); } void asm_rv32_emit_store_reg_reg_reg(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t operation_size) { asm_rv32_fix_up_scaled_reg_reg_reg(state, rs1, rs2, operation_size); - asm_rv32_emit_store_reg_reg_offset(state, rd, rs1, 0, operation_size); + asm_rv32_emit_store_reg_reg_offset(state, rd, REG_TEMP2, 0, operation_size); } void asm_rv32_meta_comparison_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd) { diff --git a/py/asmrv32.h b/py/asmrv32.h index 6f709daa11b..c25b1aa4e26 100644 --- a/py/asmrv32.h +++ b/py/asmrv32.h @@ -125,8 +125,9 @@ typedef struct _asm_rv32_t { enum { RV32_EXT_NONE = 0, RV32_EXT_ZBA = 1 << 0, + RV32_EXT_ZCMP = 1 << 1, - RV32_EXT_ALL = RV32_EXT_ZBA + RV32_EXT_ALL = RV32_EXT_ZBA | RV32_EXT_ZCMP }; typedef struct _asm_rv32_backend_options_t { @@ -196,6 +197,10 @@ void asm_rv32_end_pass(asm_rv32_t *state); ((rs & 0x07) << 7) | ((imm & 0x40) >> 1) | ((imm & 0x38) << 7) | \ ((imm & 0x04) << 4)) +#define RV32_ENCODE_TYPE_CMPP(op, ft6, ft2, rlist, imm) \ + ((op & 0x03) | ((ft6 & 0x3F) << 10) | ((ft2 & 0x03) << 8) | \ + ((rlist & 0x0F) << 4) | ((imm & 0x03) << 2)) + #define RV32_ENCODE_TYPE_CR(op, ft4, rs1, rs2) \ ((op & 0x03) | ((rs2 & 0x1F) << 2) | ((rs1 & 0x1F) << 7) | ((ft4 & 0x0F) << 12)) @@ -439,6 +444,18 @@ static inline void asm_rv32_opcode_cxor(asm_rv32_t *state, mp_uint_t rd, mp_uint asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CA(0x01, 0x23, 0x01, rd, rs)); } +// CM.POPRET {REG_LIST}, IMMEDIATE +static inline void asm_rv32_opcode_cmpopret(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { + // CMPP: 10111110 ... .. 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMPP(0x02, 0x2F, 0x02, reg_list, immediate)); +} + +// CM.PUSH {REG_LIST}, -IMMEDIATE +static inline void asm_rv32_opcode_cmpush(asm_rv32_t *state, mp_uint_t reg_list, mp_uint_t immediate) { + // CMPP: 10111000 .... .. 10 + asm_rv32_emit_halfword_opcode(state, RV32_ENCODE_TYPE_CMPP(0x02, 0x2E, 0x00, reg_list, immediate)); +} + // CSRRC RD, RS, IMMEDIATE static inline void asm_rv32_opcode_csrrc(asm_rv32_t *state, mp_uint_t rd, mp_uint_t rs, mp_int_t immediate) { // I: ............ ..... 011 ..... 1110011 @@ -710,7 +727,8 @@ static inline void asm_rv32_opcode_xori(asm_rv32_t *state, mp_uint_t rd, mp_uint } #define MICROPY_RV32_EXTENSIONS \ - (MICROPY_EMIT_RV32_ZBA ? RV32_EXT_ZBA : 0) + ((MICROPY_EMIT_RV32_ZBA ? RV32_EXT_ZBA : 0) | \ + (MICROPY_EMIT_RV32_ZCMP ? RV32_EXT_ZCMP : 0)) static inline uint8_t asm_rv32_allowed_extensions(void) { uint8_t extensions = MICROPY_RV32_EXTENSIONS; @@ -735,8 +753,8 @@ static inline uint8_t asm_rv32_allowed_extensions(void) { #define REG_TEMP2 ASM_RV32_REG_T3 #define REG_FUN_TABLE ASM_RV32_REG_S1 #define REG_LOCAL_1 ASM_RV32_REG_S3 -#define REG_LOCAL_2 ASM_RV32_REG_S4 -#define REG_LOCAL_3 ASM_RV32_REG_S5 +#define REG_LOCAL_2 ASM_RV32_REG_S2 +#define REG_LOCAL_3 ASM_RV32_REG_S4 #define REG_ZERO ASM_RV32_REG_ZERO void asm_rv32_meta_comparison_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd); @@ -804,7 +822,7 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_ #define ASM_STORE32_REG_REG_OFFSET(state, rd, rs, offset) asm_rv32_emit_store_reg_reg_offset(state, rd, rs, offset, 2) #define ASM_SUB_REG_REG(state, rd, rs) asm_rv32_opcode_sub(state, rd, rd, rs) #define ASM_XOR_REG_REG(state, rd, rs) asm_rv32_emit_optimised_xor(state, rd, rs) -#define ASM_CLR_REG(state, rd) +#define ASM_CLR_REG(state, rd) asm_rv32_emit_optimised_xor(state, rd, rd) #define ASM_LOAD8_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 0) #define ASM_LOAD16_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 1) #define ASM_LOAD32_REG_REG_REG(state, rd, rs1, rs2) asm_rv32_emit_load_reg_reg_reg(state, rd, rs1, rs2, 2) diff --git a/py/asmthumb.h b/py/asmthumb.h index cb786694f0b..229912c5385 100644 --- a/py/asmthumb.h +++ b/py/asmthumb.h @@ -466,26 +466,32 @@ void asm_thumb_b_rel12(asm_thumb_t *as, int rel); #define ASM_LOAD8_REG_REG_REG(as, reg_dest, reg_base, reg_index) asm_thumb_ldrb_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)) #define ASM_LOAD16_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ do { \ - asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 1); \ - asm_thumb_ldrh_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)); \ + asm_thumb_lsl_rlo_rlo_i5((as), REG_TEMP2, (reg_index), 1); \ + asm_thumb_ldrh_rlo_rlo_rlo((as), (reg_dest), (reg_base), REG_TEMP2); \ } while (0) #define ASM_LOAD32_REG_REG_REG(as, reg_dest, reg_base, reg_index) \ do { \ - asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 2); \ - asm_thumb_ldr_rlo_rlo_rlo((as), (reg_dest), (reg_base), (reg_index)); \ + asm_thumb_lsl_rlo_rlo_i5((as), REG_TEMP2, (reg_index), 2); \ + asm_thumb_ldr_rlo_rlo_rlo((as), (reg_dest), (reg_base), REG_TEMP2); \ } while (0) #define ASM_STORE8_REG_REG_REG(as, reg_val, reg_base, reg_index) asm_thumb_strb_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)) #define ASM_STORE16_REG_REG_REG(as, reg_val, reg_base, reg_index) \ do { \ + asm_thumb_op16((as), 0xB400 | (1 << reg_index)); \ asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 1); \ asm_thumb_strh_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)); \ + asm_thumb_op16((as), 0xBC00 | (1 << reg_index)); \ } while (0) #define ASM_STORE32_REG_REG_REG(as, reg_val, reg_base, reg_index) \ do { \ + asm_thumb_op16((as), 0xB400 | (1 << reg_index)); \ asm_thumb_lsl_rlo_rlo_i5((as), (reg_index), (reg_index), 2); \ asm_thumb_str_rlo_rlo_rlo((as), (reg_val), (reg_base), (reg_index)); \ + asm_thumb_op16((as), 0xBC00 | (1 << reg_index)); \ } while (0) +#define ASM_CLR_REG(as, reg_dest) asm_thumb_mov_rlo_i8((as), (reg_dest), 0) + #endif // GENERIC_ASM_API #endif // MICROPY_INCLUDED_PY_ASMTHUMB_H diff --git a/py/asmx64.h b/py/asmx64.h index 1e8cb0c905f..8e2c2679428 100644 --- a/py/asmx64.h +++ b/py/asmx64.h @@ -222,6 +222,8 @@ void asm_x64_call_ind(asm_x64_t *as, size_t fun_id, int temp_r32); #define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) #define ASM_STORE32_REG_REG_OFFSET(as, reg_src, reg_base, dword_offset) asm_x64_mov_r32_to_mem32((as), (reg_src), (reg_base), 4 * (dword_offset)) +#define ASM_CLR_REG(as, reg_dest) asm_x64_xor_r64_r64((as), (reg_dest), (reg_dest)) + #endif // GENERIC_ASM_API #endif // MICROPY_INCLUDED_PY_ASMX64_H diff --git a/py/asmx86.h b/py/asmx86.h index f5d37228a2f..3d95dbee733 100644 --- a/py/asmx86.h +++ b/py/asmx86.h @@ -217,6 +217,8 @@ void asm_x86_call_ind(asm_x86_t *as, size_t fun_id, mp_uint_t n_args, int temp_r #define ASM_STORE32_REG_REG(as, reg_src, reg_base) ASM_STORE32_REG_REG_OFFSET((as), (reg_src), (reg_base), 0) #define ASM_STORE32_REG_REG_OFFSET(as, reg_src, reg_base, dword_offset) asm_x86_mov_r32_to_mem32((as), (reg_src), (reg_base), 4 * (dword_offset)) +#define ASM_CLR_REG(as, reg_dest) asm_x86_xor_r32_r32((as), (reg_dest), (reg_dest)) + #endif // GENERIC_ASM_API #endif // MICROPY_INCLUDED_PY_ASMX86_H diff --git a/py/asmxtensa.c b/py/asmxtensa.c index bc3e717d9f3..18aea4ea29d 100644 --- a/py/asmxtensa.c +++ b/py/asmxtensa.c @@ -115,10 +115,12 @@ void asm_xtensa_exit(asm_xtensa_t *as) { } void asm_xtensa_entry_win(asm_xtensa_t *as, int num_locals) { - // jump over the constants - asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4); - mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte - as->const_table = (uint32_t *)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4); + if (as->num_const > 0) { + // jump over the constants + asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4); + mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte + as->const_table = (uint32_t *)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4); + } as->stack_adjust = 32 + ((((ASM_XTENSA_NUM_REGS_SAVED_WIN + num_locals) * WORD_SIZE) + 15) & ~15); asm_xtensa_op_entry(as, ASM_XTENSA_REG_A1, as->stack_adjust); diff --git a/py/asmxtensa.h b/py/asmxtensa.h index ed98c9d7305..1002511ddcc 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -465,6 +465,8 @@ void asm_xtensa_l32r(asm_xtensa_t *as, mp_uint_t reg, mp_uint_t label); asm_xtensa_op_s32i_n((as), (reg_val), (reg_base), 0); \ } while (0) +#define ASM_CLR_REG(as, reg_dest) asm_xtensa_op_movi_n((as), (reg_dest), 0) + #endif // GENERIC_ASM_API #endif // MICROPY_INCLUDED_PY_ASMXTENSA_H diff --git a/py/bc.c b/py/bc.c index 4caa022e3d8..fcd8d8e1337 100644 --- a/py/bc.c +++ b/py/bc.c @@ -104,7 +104,7 @@ static MP_NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expecte // CIRCUITPY-CHANGE: more specific mp_raise routine mp_raise_TypeError_varg( MP_ERROR_TEXT("%q() takes %d positional arguments but %d were given"), - mp_obj_fun_get_name(MP_OBJ_FROM_PTR(f)), expected, given); + mp_obj_fun_bc_get_name(f), expected, given); #endif } @@ -336,7 +336,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw mp_setup_code_state_helper(code_state, n_args, n_kw, args); } -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE // On entry code_state should be allocated somewhere (stack/heap) and // contain the following valid entries: // - code_state->fun_bc should contain a pointer to the function object diff --git a/py/builtinhelp.c b/py/builtinhelp.c index 9ab2a12d214..05b51ed4875 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -31,6 +31,7 @@ #include "genhdr/mpversion.h" #include "py/builtin.h" #include "py/mpconfig.h" +#include "py/objlist.h" #include "py/objmodule.h" #if MICROPY_PY_BUILTINS_HELP diff --git a/py/compile.c b/py/compile.c index c8704131234..e16e2f131e7 100644 --- a/py/compile.c +++ b/py/compile.c @@ -147,7 +147,7 @@ static const emit_inline_asm_method_table_t *emit_asm_table[] = { &emit_inline_thumb_method_table, &emit_inline_thumb_method_table, &emit_inline_xtensa_method_table, - NULL, + &emit_inline_xtensa_method_table, &emit_inline_rv32_method_table, }; @@ -3569,7 +3569,7 @@ void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool // TODO this can be improved by calculating it during SCOPE pass // but that requires some other structural changes to the asm emitters #if MICROPY_DYNAMIC_COMPILER - if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSA) + if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSA || mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSAWIN) #endif { compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE); diff --git a/py/dynruntime.mk b/py/dynruntime.mk index 030728cfc9a..3902acbd0b3 100644 --- a/py/dynruntime.mk +++ b/py/dynruntime.mk @@ -106,7 +106,7 @@ else ifeq ($(ARCH),rv32imc) # rv32imc CROSS = riscv64-unknown-elf- CFLAGS_ARCH += -march=rv32imac -mabi=ilp32 -mno-relax -# If Picolibc is available then select it explicitly. Ubuntu 22.04 ships its +# If Picolibc is available then select it explicitly. Ubuntu 24.04 ships its # bare metal RISC-V toolchain with Picolibc rather than Newlib, and the default # is "nosys" so a value must be provided. To avoid having per-distro # workarounds, always select Picolibc if available. @@ -120,6 +120,25 @@ endif MICROPY_FLOAT_IMPL ?= none +else ifeq ($(ARCH),rv64imc) + +# rv64imc +CROSS = riscv64-unknown-elf- +CFLAGS_ARCH += -march=rv64imac -mabi=lp64 -mno-relax +# If Picolibc is available then select it explicitly. Ubuntu 24.04 ships its +# bare metal RISC-V toolchain with Picolibc rather than Newlib, and the default +# is "nosys" so a value must be provided. To avoid having per-distro +# workarounds, always select Picolibc if available. +PICOLIBC_SPECS := $(shell $(CROSS)gcc --print-file-name=picolibc.specs) +ifneq ($(PICOLIBC_SPECS),picolibc.specs) +CFLAGS_ARCH += -specs=$(PICOLIBC_SPECS) +USE_PICOLIBC := 1 +PICOLIBC_ARCH := rv64imac +PICOLIBC_ABI := lp64 +endif + +MICROPY_FLOAT_IMPL ?= none + else $(error architecture '$(ARCH)' not supported) endif @@ -175,6 +194,9 @@ endif ifneq ($(MPY_EXTERN_SYM_FILE),) MPY_LD_FLAGS += --externs "$(realpath $(MPY_EXTERN_SYM_FILE))" endif +ifneq ($(ARCH_FLAGS),) +MPY_LD_FLAGS += --arch-flags "$(ARCH_FLAGS)" +endif CFLAGS += $(CFLAGS_EXTRA) diff --git a/py/emitglue.c b/py/emitglue.c index c6ad3e3bc38..e5c8b6a355a 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -98,7 +98,7 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #endif } -#if MICROPY_EMIT_MACHINE_CODE +#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, const void *fun_data, mp_uint_t fun_len, mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE @@ -113,7 +113,10 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, cons // Some architectures require flushing/invalidation of the I/D caches, // so that the generated native code which was created in data RAM will // be available for execution from instruction RAM. - #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB + // CIRCUITPY-CHANGE: cache-flush selection uses compiler defines (upstream d41b8dc52e), + // not MICROPY_EMIT_* emitter-enabled macros, so native .mpy loading works even when + // the matching on-board emitter is not compiled in. + #if defined(__thumb__) || defined(__thumb2__) // CIRCUITPY-CHANGE: prevent warning #if defined(__ICACHE_PRESENT) && __ICACHE_PRESENT == 1 // Flush D-cache, so the code emitted is stored in RAM. @@ -121,10 +124,10 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, cons // Invalidate I-cache, so the newly-created code is reloaded from RAM. SCB_InvalidateICache(); #endif - #elif MICROPY_EMIT_ARM + #elif defined(__arm__) #if (defined(__linux__) && defined(__GNUC__)) || __ARM_ARCH == 7 __builtin___clear_cache((void *)fun_data, (char *)fun_data + fun_len); - #elif defined(__arm__) + #else // Flush I-cache and D-cache. asm volatile ( "0:" @@ -134,7 +137,7 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, cons "mcr p15, 0, r0, c7, c7, 0\n" // invalidate I-cache and D-cache : : : "r0", "cc"); #endif - #elif (MICROPY_EMIT_RV32 || MICROPY_EMIT_INLINE_RV32) && defined(MP_HAL_CLEAN_DCACHE) + #elif defined(__riscv) && defined(MP_HAL_CLEAN_DCACHE) // Flush the D-cache. MP_HAL_CLEAN_DCACHE(fun_data, fun_len); #endif @@ -193,7 +196,7 @@ mp_obj_t mp_make_function_from_proto_fun(mp_proto_fun_t proto_fun, const mp_modu // def_kw_args must be MP_OBJ_NULL or a dict assert(def_args == NULL || def_args[1] == MP_OBJ_NULL || mp_obj_is_type(def_args[1], &mp_type_dict)); - #if MICROPY_MODULE_FROZEN_MPY + #if MICROPY_MODULE_FROZEN_MPY || MICROPY_PY_FUNCTION_ATTRS_CODE if (mp_proto_fun_is_bytecode(proto_fun)) { const uint8_t *bc = proto_fun; mp_obj_t fun = mp_obj_new_fun_bc(def_args, bc, context, NULL); @@ -219,7 +222,7 @@ mp_obj_t mp_make_function_from_proto_fun(mp_proto_fun_t proto_fun, const mp_modu // make the function, depending on the raw code kind mp_obj_t fun; switch (rc->kind) { - #if MICROPY_EMIT_NATIVE + #if MICROPY_ENABLE_NATIVE_CODE case MP_CODE_NATIVE_PY: fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children); // Check for a generator function, and if so change the type of the object diff --git a/py/emitglue.h b/py/emitglue.h index d19503b823f..674807910c1 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -83,7 +83,7 @@ typedef struct _mp_raw_code_t { #if MICROPY_PERSISTENT_CODE_SAVE uint32_t fun_data_len; // for mp_raw_code_save uint16_t n_children; - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE uint16_t prelude_offset; #endif #if MICROPY_PY_SYS_SETTRACE @@ -116,7 +116,7 @@ typedef struct _mp_raw_code_truncated_t { #if MICROPY_PERSISTENT_CODE_SAVE uint32_t fun_data_len; uint16_t n_children; - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE uint16_t prelude_offset; #endif #if MICROPY_PY_SYS_SETTRACE diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index d0eb3d566fc..c58754d47c3 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -35,6 +35,18 @@ #if MICROPY_EMIT_INLINE_XTENSA +#include "py/persistentcode.h" + +static inline bool emit_windowed_code() { + #if MICROPY_DYNAMIC_COMPILER + return mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSAWIN; + #elif MICROPY_EMIT_XTENSAWIN + return true; + #else + return false; + #endif +} + struct _emit_inline_asm_t { asm_xtensa_t as; uint16_t pass; @@ -73,11 +85,19 @@ static void emit_inline_xtensa_start_pass(emit_inline_asm_t *emit, pass_kind_t p memset(emit->label_lookup, 0, emit->max_num_labels * sizeof(qstr)); } mp_asm_base_start_pass(&emit->as.base, pass == MP_PASS_EMIT ? MP_ASM_PASS_EMIT : MP_ASM_PASS_COMPUTE); - asm_xtensa_entry(&emit->as, 0); + if (emit_windowed_code()) { + asm_xtensa_entry_win(&emit->as, 0); + } else { + asm_xtensa_entry(&emit->as, 0); + } } static void emit_inline_xtensa_end_pass(emit_inline_asm_t *emit, mp_uint_t type_sig) { - asm_xtensa_exit(&emit->as); + if (emit_windowed_code()) { + asm_xtensa_exit_win(&emit->as); + } else { + asm_xtensa_exit(&emit->as); + } asm_xtensa_end_pass(&emit->as); } @@ -168,57 +188,87 @@ static int get_arg_label(emit_inline_asm_t *emit, const char *op, mp_parse_node_ return 0; } -#define RRR (0) -#define RRI8 (1) -#define RRI8_B (2) +static const qstr_short_t BRANCH_OPCODE_NAMES[] = { + MP_QSTR_bnone, MP_QSTR_beq, MP_QSTR_blt, MP_QSTR_bltu, MP_QSTR_ball, + MP_QSTR_bbc, MP_QSTR_, MP_QSTR_, MP_QSTR_bany, MP_QSTR_bne, + MP_QSTR_bge, MP_QSTR_bgeu, MP_QSTR_bnall, MP_QSTR_bbs, +}; -typedef struct _opcode_table_3arg_t { +#define RRR_R0 (1 << 4) +#define RRR_R1 (2 << 4) +#define RRR_R2 (3 << 4) + +#define RRR (0) +#define RRI8 (1) +#define RRRN (2) + +static const struct opcode_entry_t { qstr_short_t name; - uint8_t type; - uint8_t a0 : 4; - uint8_t a1 : 4; -} opcode_table_3arg_t; - -static const opcode_table_3arg_t opcode_table_3arg[] = { - // arithmetic opcodes: reg, reg, reg - {MP_QSTR_and_, RRR, 0, 1}, - {MP_QSTR_or_, RRR, 0, 2}, - {MP_QSTR_xor, RRR, 0, 3}, - {MP_QSTR_add, RRR, 0, 8}, - {MP_QSTR_sub, RRR, 0, 12}, - {MP_QSTR_mull, RRR, 2, 8}, - {MP_QSTR_addx2, RRR, 0, 9}, - {MP_QSTR_addx4, RRR, 0, 10}, - {MP_QSTR_addx8, RRR, 0, 11}, - {MP_QSTR_subx2, RRR, 0, 13}, - {MP_QSTR_subx4, RRR, 0, 14}, - {MP_QSTR_subx8, RRR, 0, 15}, - {MP_QSTR_src, RRR, 1, 8}, - - // load/store/addi opcodes: reg, reg, imm - // upper nibble of type encodes the range of the immediate arg - {MP_QSTR_l8ui, RRI8 | 0x10, 2, 0}, - {MP_QSTR_l16ui, RRI8 | 0x30, 2, 1}, - {MP_QSTR_l32i, RRI8 | 0x50, 2, 2}, - {MP_QSTR_s8i, RRI8 | 0x10, 2, 4}, - {MP_QSTR_s16i, RRI8 | 0x30, 2, 5}, - {MP_QSTR_s32i, RRI8 | 0x50, 2, 6}, - {MP_QSTR_l16si, RRI8 | 0x30, 2, 9}, - {MP_QSTR_addi, RRI8 | 0x00, 2, 12}, - - // branch opcodes: reg, reg, label - {MP_QSTR_ball, RRI8_B, ASM_XTENSA_CC_ALL, 0}, - {MP_QSTR_bany, RRI8_B, ASM_XTENSA_CC_ANY, 0}, - {MP_QSTR_bbc, RRI8_B, ASM_XTENSA_CC_BC, 0}, - {MP_QSTR_bbs, RRI8_B, ASM_XTENSA_CC_BS, 0}, - {MP_QSTR_beq, RRI8_B, ASM_XTENSA_CC_EQ, 0}, - {MP_QSTR_bge, RRI8_B, ASM_XTENSA_CC_GE, 0}, - {MP_QSTR_bgeu, RRI8_B, ASM_XTENSA_CC_GEU, 0}, - {MP_QSTR_blt, RRI8_B, ASM_XTENSA_CC_LT, 0}, - {MP_QSTR_bltu, RRI8_B, ASM_XTENSA_CC_LTU, 0}, - {MP_QSTR_bnall, RRI8_B, ASM_XTENSA_CC_NALL, 0}, - {MP_QSTR_bne, RRI8_B, ASM_XTENSA_CC_NE, 0}, - {MP_QSTR_bnone, RRI8_B, ASM_XTENSA_CC_NONE, 0}, + uint16_t operands : 2; + uint16_t op2 : 4; + uint16_t op1 : 4; + uint16_t op0 : 4; + // 2 bits available here + uint32_t r : 6; + uint32_t s : 6; + uint32_t t : 6; + uint32_t shift : 3; + uint32_t kind : 2; + // 9 bits available here +} OPCODE_TABLE[] = { + { MP_QSTR_abs_, 2, 6, 0, 0, RRR_R0, 1, RRR_R1, 0, RRR }, + { MP_QSTR_add, 3, 8, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_add_n, 3, 0, 0, 10, RRR_R0, RRR_R1, RRR_R2, 0, RRRN }, + { MP_QSTR_addi, 3, 0, 0, 2, 12, RRR_R1, RRR_R0, 0, RRI8 }, + { MP_QSTR_addx2, 3, 9, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_addx4, 3, 10, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_addx8, 3, 11, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_and_, 3, 1, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_callx0, 1, 0, 0, 0, 0, RRR_R0, 12, 0, RRR }, + { MP_QSTR_jx, 1, 0, 0, 0, 0, RRR_R0, 10, 0, RRR }, + { MP_QSTR_l16si, 3, 0, 0, 2, 9, RRR_R1, RRR_R0, 3, RRI8 }, + { MP_QSTR_l16ui, 3, 0, 0, 2, 1, RRR_R1, RRR_R0, 3, RRI8 }, + { MP_QSTR_l32i, 3, 0, 0, 2, 2, RRR_R1, RRR_R0, 5, RRI8 }, + { MP_QSTR_l8ui, 3, 0, 0, 2, 0, RRR_R1, RRR_R0, 1, RRI8 }, + { MP_QSTR_mov, 2, 0, 0, 13, 0, RRR_R1, RRR_R0, 0, RRRN }, + { MP_QSTR_mov_n, 2, 0, 0, 13, 0, RRR_R1, RRR_R0, 0, RRRN }, + { MP_QSTR_mull, 3, 8, 2, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_neg, 2, 6, 0, 0, RRR_R0, 0, RRR_R1, 0, RRR }, + { MP_QSTR_nop, 0, 0, 0, 0, 2, 0, 15, 0, RRR }, + { MP_QSTR_nop_n, 0, 0, 0, 13, 15, 0, 3, 0, RRRN }, + { MP_QSTR_nsa, 2, 4, 0, 0, 14, RRR_R1, RRR_R0, 0, RRR }, + { MP_QSTR_nsau, 2, 4, 0, 0, 15, RRR_R1, RRR_R0, 0, RRR }, + { MP_QSTR_or_, 3, 2, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_ret, 0, 0, 0, 13, 15, 0, 0, 0, RRRN }, + { MP_QSTR_ret_n, 0, 0, 0, 13, 15, 0, 0, 0, RRRN }, + { MP_QSTR_s16i, 3, 0, 0, 2, 5, RRR_R1, RRR_R0, 3, RRI8 }, + { MP_QSTR_s32i, 3, 0, 0, 2, 6, RRR_R1, RRR_R0, 5, RRI8 }, + { MP_QSTR_s8i, 3, 0, 0, 2, 4, RRR_R1, RRR_R0, 1, RRI8 }, + { MP_QSTR_sll, 2, 10, 1, 0, RRR_R0, RRR_R1, 0, 0, RRR }, + { MP_QSTR_sra, 2, 11, 1, 0, RRR_R0, 0, RRR_R1, 0, RRR }, + { MP_QSTR_src, 3, 8, 1, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_srl, 2, 9, 1, 0, RRR_R0, 0, RRR_R1, 0, RRR }, + { MP_QSTR_ssa8b, 1, 4, 0, 0, 3, RRR_R0, 0, 0, RRR }, + { MP_QSTR_ssa8l, 1, 4, 0, 0, 2, RRR_R0, 0, 0, RRR }, + { MP_QSTR_ssl, 1, 4, 0, 0, 1, RRR_R0, 0, 0, RRR }, + { MP_QSTR_ssr, 1, 4, 0, 0, 0, RRR_R0, 0, 0, RRR }, + { MP_QSTR_sub, 3, 12, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_subx2, 3, 13, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_subx4, 3, 14, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_subx8, 3, 15, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + { MP_QSTR_xor, 3, 3, 0, 0, RRR_R0, RRR_R1, RRR_R2, 0, RRR }, + #if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES + { MP_QSTR_dsync, 0, 0, 0, 0, 2, 0, 3, 0, RRR }, + { MP_QSTR_esync, 0, 0, 0, 0, 2, 0, 2, 0, RRR }, + { MP_QSTR_extw, 0, 0, 0, 0, 2, 0, 13, 0, RRR }, + { MP_QSTR_ill, 0, 0, 0, 0, 2, 0, 0, 0, RRR }, + { MP_QSTR_ill_n, 0, 0, 0, 13, 15, 0, 6, 0, RRRN }, + { MP_QSTR_isync, 0, 0, 0, 0, 2, 0, 0, 0, RRR }, + { MP_QSTR_memw, 0, 0, 0, 0, 2, 0, 12, 0, RRR }, + { MP_QSTR_rer, 2, 4, 0, 0, 6, RRR_R1, RRR_R0, 0, RRR }, + { MP_QSTR_rsync, 0, 0, 0, 0, 2, 0, 1, 0, RRR }, + { MP_QSTR_wer, 2, 4, 0, 0, 7, RRR_R1, RRR_R0, 0, RRR }, + #endif }; // The index of the first four qstrs matches the CCZ condition value to be @@ -228,75 +278,60 @@ static const qstr_short_t BCCZ_OPCODES[] = { MP_QSTR_beqz_n, MP_QSTR_bnez_n }; -#if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES -typedef struct _single_opcode_t { - qstr_short_t name; - uint16_t value; -} single_opcode_t; - -static const single_opcode_t NOARGS_OPCODES[] = { - {MP_QSTR_dsync, 0x2030}, - {MP_QSTR_esync, 0x2020}, - {MP_QSTR_extw, 0x20D0}, - {MP_QSTR_ill, 0x0000}, - {MP_QSTR_isync, 0x2000}, - {MP_QSTR_memw, 0x20C0}, - {MP_QSTR_rsync, 0x2010}, -}; -#endif - static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_args, mp_parse_node_t *pn_args) { size_t op_len; const char *op_str = (const char *)qstr_data(op, &op_len); - if (n_args == 0) { - if (op == MP_QSTR_ret_n || op == MP_QSTR_ret) { - asm_xtensa_op_ret_n(&emit->as); - return; - } else if (op == MP_QSTR_nop) { - asm_xtensa_op24(&emit->as, 0x20F0); - return; - } else if (op == MP_QSTR_nop_n) { - asm_xtensa_op16(&emit->as, 0xF03D); - return; - } - #if MICROPY_EMIT_INLINE_XTENSA_UNCOMMON_OPCODES - for (size_t index = 0; index < MP_ARRAY_SIZE(NOARGS_OPCODES); index++) { - const single_opcode_t *opcode = &NOARGS_OPCODES[index]; - if (op == opcode->name) { - asm_xtensa_op24(&emit->as, opcode->value); - return; + for (size_t index = 0; index < MP_ARRAY_SIZE(OPCODE_TABLE); index++) { + const struct opcode_entry_t *entry = &OPCODE_TABLE[index]; + if (entry->name == op) { + if (n_args != entry->operands) { + goto unknown_op; } - } - #endif - goto unknown_op; + uint32_t opcode = ((entry->r & 0x0F) << 12) | ((entry->s & 0x0F) << 8) | ((entry->t & 0x0F) << 4) | entry->op0; + if (entry->kind == RRR) { + opcode |= (entry->op2 << 20) | (entry->op1 << 16); + } else if (entry->kind == RRI8) { + int min = 0; + int max = 0; + int shift = entry->shift; + if (shift > 0) { + shift >>= 1; + max = 0xFF << shift; + } else { + min = -128; + max = 127; + } + uint32_t immediate = get_arg_i(emit, op_str, pn_args[2], min, max); + opcode |= ((immediate >> shift) & 0xFF) << 16; + } + if (entry->r >= RRR_R0) { + opcode |= get_arg_reg(emit, op_str, pn_args[(entry->r >> 4) - 1]) << 12; + } + if (entry->s >= RRR_R0) { + opcode |= get_arg_reg(emit, op_str, pn_args[(entry->s >> 4) - 1]) << 8; + } + if (entry->t >= RRR_R0) { + opcode |= get_arg_reg(emit, op_str, pn_args[(entry->t >> 4) - 1]) << 4; + } + if (entry->kind == RRRN) { + assert((opcode >> 16) == 0 && "Stray bits in narrow opcode"); + asm_xtensa_op16(&emit->as, (uint16_t)(opcode & 0xFFFF)); + } else { + asm_xtensa_op24(&emit->as, opcode); + } + return; + } + } - } else if (n_args == 1) { - if (op == MP_QSTR_callx0) { - uint r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op_callx0(&emit->as, r0); - } else if (op == MP_QSTR_j) { + if (n_args == 1) { + if (op == MP_QSTR_j) { int label = get_arg_label(emit, op_str, pn_args[0]); asm_xtensa_j_label(&emit->as, label); - } else if (op == MP_QSTR_jx) { - uint r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op_jx(&emit->as, r0); - } else if (op == MP_QSTR_ssl) { - mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op_ssl(&emit->as, r0); - } else if (op == MP_QSTR_ssr) { - mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op_ssr(&emit->as, r0); } else if (op == MP_QSTR_ssai) { mp_uint_t sa = get_arg_i(emit, op_str, pn_args[0], 0, 31); asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 4, sa & 0x0F, (sa >> 4) & 0x01)); - } else if (op == MP_QSTR_ssa8b) { - mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 3, r0, 0)); - } else if (op == MP_QSTR_ssa8l) { - mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 2, r0, 0)); } else if (op == MP_QSTR_call0) { mp_uint_t label = get_arg_label(emit, op_str, pn_args[0]); asm_xtensa_call0(&emit->as, label); @@ -304,13 +339,10 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ } else if (op == MP_QSTR_fsync) { mp_uint_t imm3 = get_arg_i(emit, op_str, pn_args[0], 0, 7); asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 0, 2, 8 | imm3, 0)); - } else if (op == MP_QSTR_ill_n) { - asm_xtensa_op16(&emit->as, 0xF06D); #endif } else { goto unknown_op; } - } else if (n_args == 2) { uint r0 = get_arg_reg(emit, op_str, pn_args[0]); for (size_t index = 0; index < MP_ARRAY_SIZE(BCCZ_OPCODES); index++) { @@ -320,35 +352,10 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ return; } } - if (op == MP_QSTR_mov || op == MP_QSTR_mov_n) { - // we emit mov.n for both "mov" and "mov_n" opcodes - uint r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op_mov_n(&emit->as, r0, r1); - } else if (op == MP_QSTR_movi) { + if (op == MP_QSTR_movi) { // for convenience we emit l32r if the integer doesn't fit in movi uint32_t imm = get_arg_i(emit, op_str, pn_args[1], 0, 0); asm_xtensa_mov_reg_i32(&emit->as, r0, imm); - } else if (op == MP_QSTR_abs_) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 6, r0, 1, r1)); - } else if (op == MP_QSTR_neg) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 6, r0, 0, r1)); - } else if (op == MP_QSTR_sll) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 10, r0, r1, 0)); - } else if (op == MP_QSTR_sra) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 11, r0, 0, r1)); - } else if (op == MP_QSTR_srl) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 1, 9, r0, 0, r1)); - } else if (op == MP_QSTR_nsa) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 14, r1, r0)); - } else if (op == MP_QSTR_nsau) { - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, 0, 4, 15, r1, r0)); } else if (op == MP_QSTR_l32r) { mp_uint_t label = get_arg_label(emit, op_str, pn_args[1]); asm_xtensa_l32r(&emit->as, r0, label); @@ -377,44 +384,18 @@ static void emit_inline_xtensa_op(emit_inline_asm_t *emit, qstr op, mp_uint_t n_ { goto unknown_op; } - } else if (n_args == 3) { - // search table for 3 arg instructions - for (uint i = 0; i < MP_ARRAY_SIZE(opcode_table_3arg); i++) { - const opcode_table_3arg_t *o = &opcode_table_3arg[i]; - if (op == o->name) { - uint r0 = get_arg_reg(emit, op_str, pn_args[0]); - uint r1 = get_arg_reg(emit, op_str, pn_args[1]); - if (o->type == RRR) { - uint r2 = get_arg_reg(emit, op_str, pn_args[2]); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRR(0, o->a0, o->a1, r0, r1, r2)); - } else if (o->type == RRI8_B) { - int label = get_arg_label(emit, op_str, pn_args[2]); - asm_xtensa_bcc_reg_reg_label(&emit->as, o->a0, r0, r1, label); - } else { - int shift, min, max; - if ((o->type & 0xf0) == 0) { - shift = 0; - min = -128; - max = 127; - } else { - shift = (o->type & 0xf0) >> 5; - min = 0; - max = 0xff << shift; - } - uint32_t imm = get_arg_i(emit, op_str, pn_args[2], min, max); - asm_xtensa_op24(&emit->as, ASM_XTENSA_ENCODE_RRI8(o->a0, o->a1, r1, r0, (imm >> shift) & 0xff)); - } + for (size_t index = 0; index < MP_ARRAY_SIZE(BRANCH_OPCODE_NAMES); index++) { + if (BRANCH_OPCODE_NAMES[index] == op) { + int r0 = get_arg_reg(emit, op_str, pn_args[0]); + int r1 = get_arg_reg(emit, op_str, pn_args[1]); + int label = get_arg_label(emit, op_str, pn_args[2]); + asm_xtensa_bcc_reg_reg_label(&emit->as, index, r0, r1, label); return; } } - if (op == MP_QSTR_add_n) { - mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); - mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); - mp_uint_t r2 = get_arg_reg(emit, op_str, pn_args[2]); - asm_xtensa_op16(&emit->as, ASM_XTENSA_ENCODE_RRRN(10, r0, r1, r2)); - } else if (op == MP_QSTR_addi_n) { + if (op == MP_QSTR_addi_n) { mp_uint_t r0 = get_arg_reg(emit, op_str, pn_args[0]); mp_uint_t r1 = get_arg_reg(emit, op_str, pn_args[1]); mp_int_t imm4 = get_arg_i(emit, op_str, pn_args[2], -1, 15); diff --git a/py/emitnative.c b/py/emitnative.c index 6f12200f7e1..06c2f25b60c 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -324,17 +324,13 @@ struct _emit_t { ASM_T *as; }; -#ifndef REG_ZERO -#define REG_ZERO REG_TEMP0 -#define ASM_CLR_REG(state, rd) ASM_XOR_REG_REG(state, rd, rd) -#endif - -#if N_RV32 +#ifdef REG_ZERO #define ASM_MOV_LOCAL_MP_OBJ_NULL(as, local_num, reg_temp) \ ASM_MOV_LOCAL_REG(as, local_num, REG_ZERO) #else +#define REG_ZERO REG_TEMP0 #define ASM_MOV_LOCAL_MP_OBJ_NULL(as, local_num, reg_temp) \ - ASM_MOV_REG_IMM(as, reg_temp, (mp_uint_t)MP_OBJ_NULL); \ + ASM_CLR_REG(as, reg_temp); \ ASM_MOV_LOCAL_REG(as, local_num, reg_temp) #endif @@ -1187,7 +1183,7 @@ static void emit_native_leave_exc_stack(emit_t *emit, bool start_of_handler) { // Optimisation: PC is already cleared by global exc handler return; } - ASM_XOR_REG_REG(emit->as, REG_RET, REG_RET); + ASM_CLR_REG(emit->as, REG_RET); } else { // Found new active handler, get its PC ASM_MOV_REG_PCREL(emit->as, REG_RET, e->label); @@ -1284,8 +1280,7 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, start_label, true); } else { // Clear the unwind state - ASM_CLR_REG(emit->as, REG_ZERO); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_ZERO); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_ZERO); // clear nlr.ret_val, because it's passed to mp_native_raise regardless // of whether there was an exception or not @@ -1305,8 +1300,7 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true); // Clear PC of current code block, and jump there to resume execution - ASM_CLR_REG(emit->as, REG_ZERO); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_ZERO); + ASM_MOV_LOCAL_MP_OBJ_NULL(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_ZERO); ASM_JUMP_REG(emit->as, REG_LOCAL_1); // Global exception handler: check for valid exception handler @@ -1987,7 +1981,7 @@ static void emit_native_delete_attr(emit_t *emit, qstr qst) { vtype_kind_t vtype_base; emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base assert(vtype_base == VTYPE_PYOBJ); - ASM_XOR_REG_REG(emit->as, REG_ARG_3, REG_ARG_3); // arg3 = value (null for delete) + ASM_CLR_REG(emit->as, REG_ARG_3); // arg3 = value (null for delete) emit_call_with_qstr_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name emit_post(emit); } @@ -2133,7 +2127,7 @@ static void emit_native_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t exc // No finally, handle the jump ourselves // First, restore the exception handler address for the jump if (e < emit->exc_stack) { - ASM_XOR_REG_REG(emit->as, REG_RET, REG_RET); + ASM_CLR_REG(emit->as, REG_RET); } else { ASM_MOV_REG_PCREL(emit->as, REG_RET, e->label); } diff --git a/py/emitndebug.c b/py/emitndebug.c index e49c5cdbffa..2144d14e6b5 100644 --- a/py/emitndebug.c +++ b/py/emitndebug.c @@ -271,6 +271,9 @@ static void asm_debug_setcc_reg_reg_reg(asm_debug_t *as, int op, int reg1, int r #define ASM_STORE32_REG_REG(as, reg_src, reg_base) \ asm_debug_reg_reg(as, "store32", reg_src, reg_base) +#define ASM_CLR_REG(as, reg_dest) \ + asm_debug_reg(as, "clr", reg_dest) + // Word indices of REG_LOCAL_x in nlr_buf_t #define NLR_BUF_IDX_LOCAL_1 (5) // rbx diff --git a/py/gc.c b/py/gc.c index 757c27e3de6..38c169ccafd 100644 --- a/py/gc.c +++ b/py/gc.c @@ -147,6 +147,16 @@ #define CTB_SET(area, block) do { area->gc_collect_table_start[(block) / BLOCKS_PER_CTB] |= (1 << ((block) & 7)); } while (0) #define CTB_CLEAR(area, block) do { area->gc_collect_table_start[(block) / BLOCKS_PER_CTB] &= (~(1 << ((block) & 7))); } while (0) +#if MICROPY_PY_WEAKREF +// WTB = weakref table byte +// if set, then the corresponding block may have a weakref in MP_STATE_VM(mp_weakref_map). +#define BLOCKS_PER_WTB (8) + +#define WTB_GET(area, block) ((area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] >> ((block) & 7)) & 1) +#define WTB_SET(area, block) do { area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] |= (1 << ((block) & 7)); } while (0) +#define WTB_CLEAR(area, block) do { area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] &= (~(1 << ((block) & 7))); } while (0) +#endif + #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL #define GC_MUTEX_INIT() mp_thread_recursive_mutex_init(&MP_STATE_MEM(gc_mutex)) #define GC_ENTER() mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) @@ -211,10 +221,11 @@ static void gc_sweep_free_blocks(void); // TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { // CIRCUITPY-CHANGE: Updated calculation to include selective collect table - // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, C=collect table, P=pool; all in bytes): - // T = A + F + C + P + // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, C=collect table, W=weakref table, P=pool; all in bytes): + // T = A + F + C + W + P // F = A * BLOCKS_PER_ATB / BLOCKS_PER_FTB // C = A * BLOCKS_PER_ATB / BLOCKS_PER_CTB + // W = A * BLOCKS_PER_ATB / BLOCKS_PER_WTB // P = A * BLOCKS_PER_ATB * BYTES_PER_BLOCK size_t total_byte_len = (byte *)end - (byte *)start; @@ -230,6 +241,10 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { bits_per_block += MP_BITS_PER_BYTE / BLOCKS_PER_CTB; // Add bits for CTB #endif + #if MICROPY_PY_WEAKREF + bits_per_block += MP_BITS_PER_BYTE / BLOCKS_PER_WTB; // Add bits for WTB + #endif + bits_per_block += MP_BITS_PER_BYTE * BYTES_PER_BLOCK; // Add bits for the block itself // Calculate the allocation table size @@ -257,6 +272,12 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { next_table += gc_collect_table_byte_len; #endif + #if MICROPY_PY_WEAKREF + size_t gc_weakref_table_byte_len = (gc_pool_block_len + BLOCKS_PER_WTB - 1) / BLOCKS_PER_WTB; + area->gc_weakref_table_start = next_table; + next_table += gc_weakref_table_byte_len; + #endif + // Set pool pointers area->gc_pool_start = (byte *)end - gc_pool_block_len * BYTES_PER_BLOCK; area->gc_pool_end = end; @@ -292,6 +313,12 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { gc_collect_table_byte_len, gc_collect_table_byte_len * BLOCKS_PER_CTB); #endif + #if MICROPY_PY_WEAKREF + DEBUG_printf(" weakref table at %p, length " UINT_FMT " bytes, " + UINT_FMT " blocks\n", area->gc_weakref_table_start, + gc_weakref_table_byte_len, + gc_weakref_table_byte_len * BLOCKS_PER_WTB); + #endif DEBUG_printf(" pool at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", area->gc_pool_start, gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len); @@ -356,12 +383,16 @@ static size_t compute_heap_size(size_t total_blocks) { size_t atb_bytes = (total_blocks + BLOCKS_PER_ATB - 1) / BLOCKS_PER_ATB; size_t ftb_bytes = 0; size_t ctb_bytes = 0; + size_t wtb_bytes = 0; #if MICROPY_ENABLE_FINALISER ftb_bytes = (total_blocks + BLOCKS_PER_FTB - 1) / BLOCKS_PER_FTB; #endif #if MICROPY_ENABLE_SELECTIVE_COLLECT ctb_bytes = (total_blocks + BLOCKS_PER_CTB - 1) / BLOCKS_PER_CTB; #endif + #if MICROPY_PY_WEAKREF + wtb_bytes = (total_blocks + BLOCKS_PER_WTB - 1) / BLOCKS_PER_WTB; + #endif size_t pool_bytes = total_blocks * BYTES_PER_BLOCK; // Compute bytes needed to build a heap with total_blocks blocks. @@ -371,6 +402,7 @@ static size_t compute_heap_size(size_t total_blocks) { + ALLOC_TABLE_GAP_BYTE + ftb_bytes + ctb_bytes + + wtb_bytes + pool_bytes + BYTES_PER_BLOCK; // Extra block of bytes to account for end pointer alignment @@ -520,11 +552,22 @@ static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { && ptr < (void *)MP_STATE_MEM(area).gc_pool_end /* must be below end of pool */ \ ) -#ifndef TRACE_MARK +#ifdef TRACE_MARK +#error "TRACE_MARK is replaced by TRACE_MARK_R and TRACE_MARK_S" +#endif +// R for root pointer, S for subtree. +#ifndef TRACE_MARK_R #if DEBUG_PRINT -#define TRACE_MARK(block, ptr) DEBUG_printf("gc_mark(%p)\n", ptr) +#define TRACE_MARK_R(block, ptr) DEBUG_printf("gc_mark_r(%p)\n", ptr) #else -#define TRACE_MARK(block, ptr) +#define TRACE_MARK_R(block, ptr) +#endif +#endif +#ifndef TRACE_MARK_S +#if DEBUG_PRINT +#define TRACE_MARK_S(block, ptr) DEBUG_printf("gc_mark_s(%p)\n", ptr) +#else +#define TRACE_MARK_S(block, ptr) #endif #endif @@ -576,6 +619,7 @@ void gc_collect_root(void **ptrs, size_t len) { size_t block = BLOCK_FROM_PTR(area, ptr); if (ATB_GET_KIND(area, block) == AT_HEAD) { // An unmarked head: mark it, and mark all its children + TRACE_MARK_R(block, ptr); ATB_HEAD_TO_MARK(area, block); #if MICROPY_GC_SPLIT_HEAP gc_mark_subtree(area, block); @@ -649,7 +693,7 @@ static void MP_NO_INSTRUMENT PLACE_IN_ITCM(gc_mark_subtree)(size_t block) continue; } // An unmarked head. Mark it, and push it on gc stack. - TRACE_MARK(ptr_block, ptr); + TRACE_MARK_S(ptr_block, ptr); ATB_HEAD_TO_MARK(ptr_area, ptr_block); if (sp < MICROPY_ALLOC_GC_STACK_SIZE) { MP_STATE_MEM(gc_block_stack)[sp] = ptr_block; @@ -694,6 +738,9 @@ void gc_collect_end(void) { } MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; GC_EXIT(); + #if MICROPY_PY_WEAKREF + gc_weakref_sweep(); + #endif gc_perfetto_emit_heap_stats(); } @@ -720,12 +767,16 @@ static void gc_deal_with_stack_overflow(void) { // Run finalisers for all to-be-freed blocks static void gc_sweep_run_finalisers(void) { - #if MICROPY_ENABLE_FINALISER + #if MICROPY_ENABLE_FINALISER || MICROPY_PY_WEAKREF + #if MICROPY_ENABLE_FINALISER && MICROPY_PY_WEAKREF + MP_STATIC_ASSERT(BLOCKS_PER_FTB == BLOCKS_PER_WTB); + #endif for (const mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); // Small speed optimisation: skip over empty FTB blocks size_t ftb_end = area->gc_last_used_block / BLOCKS_PER_FTB; // index is inclusive for (size_t ftb_idx = 0; ftb_idx <= ftb_end; ftb_idx++) { + #if MICROPY_ENABLE_FINALISER byte ftb = area->gc_finaliser_table_start[ftb_idx]; size_t block = ftb_idx * BLOCKS_PER_FTB; while (ftb) { @@ -755,9 +806,26 @@ static void gc_sweep_run_finalisers(void) { ftb >>= 1; block++; } + #endif + #if MICROPY_PY_WEAKREF + byte wtb = area->gc_weakref_table_start[ftb_idx]; + block = ftb_idx * BLOCKS_PER_WTB; + while (wtb) { + MICROPY_GC_HOOK_LOOP(block); + if (wtb & 1) { // WTB_GET(area, block) shortcut + if (ATB_GET_KIND(area, block) == AT_HEAD) { + mp_obj_base_t *obj = (mp_obj_base_t *)PTR_FROM_BLOCK(area, block); + gc_weakref_about_to_be_freed(obj); + WTB_CLEAR(area, block); + } + } + wtb >>= 1; + block++; + } + #endif } } - #endif // MICROPY_ENABLE_FINALISER + #endif // MICROPY_ENABLE_FINALISER || MICROPY_PY_WEAKREF } // Free unmarked heads and their tails @@ -915,6 +983,25 @@ void gc_info(gc_info_t *info) { GC_EXIT(); } +#if MICROPY_PY_WEAKREF +// Mark the GC heap pointer as having a weakref. +void gc_weakref_mark(void *ptr) { + mp_state_mem_area_t *area; + #if MICROPY_GC_SPLIT_HEAP + area = gc_get_ptr_area(ptr); + assert(area); + #else + assert(VERIFY_PTR(ptr)); + area = &MP_STATE_MEM(area); + #endif + + size_t block = BLOCK_FROM_PTR(area, ptr); + assert(ATB_GET_KIND(area, block) == AT_HEAD); + + WTB_SET(area, block); +} +#endif + // CIRCUITPY-CHANGE: New function. // C code may be used when the VM heap isn't active. This function // allows that code to test if it is. It can use the outer pool if needed. @@ -1161,6 +1248,11 @@ void gc_free(void *ptr) { FTB_CLEAR(area, block); #endif + #if MICROPY_PY_WEAKREF + // Objects that have a weak reference should not be explicitly freed. + assert(!WTB_GET(area, block)); + #endif + #if MICROPY_GC_SPLIT_HEAP if (MP_STATE_MEM(gc_last_free_area) != area) { // We freed something but it isn't the current area. Reset the diff --git a/py/lexer.c b/py/lexer.c index 98a10c87b2e..22ff4d2cd3a 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -35,11 +35,8 @@ #if MICROPY_ENABLE_COMPILER #define TAB_SIZE (8) - -// TODO seems that CPython allows NULL byte in the input stream -// don't know if that's intentional or not, but we don't allow it - -#define MP_LEXER_EOF ((unichar)MP_READER_EOF) +#define MP_LEXER_EOF ('\0') +#define MP_LEXER_INVALID_BYTE ('\1') #define CUR_CHAR(lex) ((lex)->chr0) static bool is_end(mp_lexer_t *lex) { @@ -62,7 +59,11 @@ static bool is_char_or3(mp_lexer_t *lex, byte c1, byte c2, byte c3) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3; } -#if MICROPY_PY_FSTRINGS +#if MICROPY_PY_TSTRINGS +static bool is_char_or5(mp_lexer_t *lex, byte c1, byte c2, byte c3, byte c4, byte c5) { + return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3 || lex->chr0 == c4 || lex->chr0 == c5; +} +#elif MICROPY_PY_FSTRINGS static bool is_char_or4(mp_lexer_t *lex, byte c1, byte c2, byte c3, byte c4) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3 || lex->chr0 == c4; } @@ -111,7 +112,11 @@ static bool is_following_odigit(mp_lexer_t *lex) { static bool is_string_or_bytes(mp_lexer_t *lex) { return is_char_or(lex, '\'', '\"') - #if MICROPY_PY_FSTRINGS + #if MICROPY_PY_TSTRINGS + || (is_char_or5(lex, 'r', 'u', 'b', 'f', 't') && is_char_following_or(lex, '\'', '\"')) + || (((is_char_and(lex, 'r', 'f') || is_char_and(lex, 'f', 'r') || is_char_and(lex, 'r', 't') || is_char_and(lex, 't', 'r')) + && is_char_following_following_or(lex, '\'', '\"'))) + #elif MICROPY_PY_FSTRINGS || (is_char_or4(lex, 'r', 'u', 'b', 'f') && is_char_following_or(lex, '\'', '\"')) || (((is_char_and(lex, 'r', 'f') || is_char_and(lex, 'f', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) @@ -148,45 +153,51 @@ static void next_char(mp_lexer_t *lex) { lex->chr0 = lex->chr1; lex->chr1 = lex->chr2; - // and add the next byte from either the fstring args or the reader + // and add the next byte from either inject_chrs or the reader + mp_uint_t chr2; +fetch_next_byte: #if MICROPY_PY_FSTRINGS - if (lex->fstring_args_idx) { - // if there are saved chars, then we're currently injecting fstring args - if (lex->fstring_args_idx < lex->fstring_args.len) { - lex->chr2 = lex->fstring_args.buf[lex->fstring_args_idx++]; - } else { - // no more fstring arg bytes - lex->chr2 = '\0'; - } - - if (lex->chr0 == '\0') { - // consumed all fstring data, restore saved input queue - lex->chr0 = lex->chr0_saved; - lex->chr1 = lex->chr1_saved; - lex->chr2 = lex->chr2_saved; - // stop consuming fstring arg data - vstr_reset(&lex->fstring_args); - lex->fstring_args_idx = 0; + if (lex->inject_chrs_idx) { + // if there are saved chars, then we're currently injecting them + chr2 = lex->inject_chrs.buf[lex->inject_chrs_idx++]; + if (lex->inject_chrs_idx >= lex->inject_chrs.len) { + // consumed all injected characters, switch back to the input stream + vstr_reset(&lex->inject_chrs); + lex->inject_chrs_idx = 0; } } else #endif { - lex->chr2 = lex->reader.readbyte(lex->reader.data); + // get next byte from the reader + chr2 = lex->reader.readbyte(lex->reader.data); + + // convert stream mp_uint_t value to lexer uint8_t value: + // - MP_READER_EOF indicates end-of-stream, for which lexer uses MP_LEXER_EOF + // - MP_LEXER_EOF is not allowed in the input stream, as is converted to + // MP_LEXER_INVALID_BYTE so it's not interpreted as end-of-stream + // - all other byte values (1 through 255 inclusive) are passed through as-is + if (chr2 == MP_READER_EOF) { + chr2 = MP_LEXER_EOF; + } else if (chr2 == MP_LEXER_EOF) { + chr2 = MP_LEXER_INVALID_BYTE; + } } if (lex->chr1 == '\r') { // CR is a new line, converted to LF lex->chr1 = '\n'; - if (lex->chr2 == '\n') { + if (chr2 == '\n') { // CR LF is a single new line, throw out the extra LF - lex->chr2 = lex->reader.readbyte(lex->reader.data); + goto fetch_next_byte; } } // check if we need to insert a newline at end of file - if (lex->chr2 == MP_LEXER_EOF && lex->chr1 != MP_LEXER_EOF && lex->chr1 != '\n') { - lex->chr2 = '\n'; + if (chr2 == MP_LEXER_EOF && lex->chr1 != MP_LEXER_EOF && lex->chr1 != '\n') { + chr2 = '\n'; } + + lex->chr2 = chr2; } static void indent_push(mp_lexer_t *lex, size_t indent) { @@ -309,7 +320,7 @@ static bool get_hex(mp_lexer_t *lex, size_t num_digits, mp_uint_t *result) { return true; } -static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) { +static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring, bool is_tstring) { // get first quoting character char quote_char = '\''; if (is_char(lex, '\"')) { @@ -333,8 +344,7 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) #if MICROPY_PY_FSTRINGS if (is_fstring) { // assume there's going to be interpolation, so prep the injection data - // fstring_args_idx==0 && len(fstring_args)>0 means we're extracting the args. - // only when fstring_args_idx>0 will we consume the arg data + // len(fstring_args)>0 means we're extracting the args. // lex->fstring_args is reset when finished, so at this point there are two cases: // - lex->fstring_args is empty: start of a new f-string // - lex->fstring_args is non-empty: concatenation of adjacent f-strings @@ -343,20 +353,52 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) } } #endif + #if MICROPY_PY_TSTRINGS + if (is_tstring) { + if (vstr_len(&lex->fstring_args) == 0) { + vstr_add_byte(&lex->vstr, '('); + vstr_add_byte(&lex->vstr, '('); + for (size_t q = 0; q < num_quotes; ++q) { + vstr_add_byte(&lex->vstr, quote_char); + } + } + } + #endif + + #if MICROPY_PY_TSTRINGS + size_t tstring_num_interpolations = 0; + size_t end_of_format_index = 0; + size_t nested_formatting_in_tstring = 0; + bool nested_formatting_needs_fstring = false; + #endif while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { if (is_char(lex, quote_char)) { n_closing += 1; vstr_add_char(&lex->vstr, CUR_CHAR(lex)); + #if MICROPY_PY_TSTRINGS + } else if (is_tstring && is_char(lex, '\n')) { + // handle multi-line t-strings + vstr_add_byte(&lex->vstr, '\\'); + vstr_add_byte(&lex->vstr, 'n'); + #endif } else { n_closing = 0; #if MICROPY_PY_FSTRINGS - while (is_fstring && is_char(lex, '{')) { + while ((is_fstring || is_tstring) && is_char(lex, '{')) { + #if MICROPY_PY_TSTRINGS + if (nested_formatting_in_tstring) { + ++nested_formatting_in_tstring; + break; + } + #endif next_char(lex); if (is_char(lex, '{')) { // "{{" is passed through unchanged to be handled by str.format - vstr_add_byte(&lex->vstr, '{'); + if (!is_tstring) { + vstr_add_byte(&lex->vstr, '{'); + } next_char(lex); } else { // wrap each argument in (), e.g. @@ -393,23 +435,102 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) vstr_add_byte(&lex->fstring_args, c); next_char(lex); } + #if MICROPY_PY_TSTRINGS + bool was_debug = false; + #endif if (lex->fstring_args.buf[lex->fstring_args.len - 1] == '=') { // if the last character of the arg was '=', then inject "arg=" before the '{'. // f'{a=}' --> 'a={}'.format(a) vstr_add_strn(&lex->vstr, lex->fstring_args.buf + i, lex->fstring_args.len - i); // remove the trailing '=' lex->fstring_args.len--; + #if MICROPY_PY_TSTRINGS + was_debug = true; + #endif + } + #if MICROPY_PY_TSTRINGS + if (is_tstring) { + // truncate trailing spaces + while (lex->fstring_args.len && unichar_isspace(lex->fstring_args.buf[lex->fstring_args.len - 1])) { + lex->fstring_args.len--; + } + } + #endif + if (lex->fstring_args.len == i) { + // empty format, eg f'{}' + // (should apply to both f-strings and t-strings, needs test) + lex->tok_kind = MP_TOKEN_MALFORMED_FSTRING; } // close the paren-wrapped arg to .format(). vstr_add_byte(&lex->fstring_args, ')'); // comma-separate args to .format(). vstr_add_byte(&lex->fstring_args, ','); + #if MICROPY_PY_TSTRINGS + if (is_tstring) { + // start the interpolation part + + // duplicate expression to a string + vstr_add_byte(&lex->fstring_args, quote_char); + size_t nn = lex->fstring_args.len - i - 3; + for (size_t j = 0; j < nn; ++j) { + byte b = lex->fstring_args.buf[i + j]; + if (b == quote_char) { + vstr_add_byte(&lex->fstring_args, '\\'); + } else if (b == '\\') { + vstr_add_byte(&lex->fstring_args, '\\'); + } + vstr_add_byte(&lex->fstring_args, b); + } + vstr_add_byte(&lex->fstring_args, quote_char); + vstr_add_byte(&lex->fstring_args, ','); + + // start next part of string as next __template__ argument + for (size_t q = 0; q < num_quotes; ++q) { + vstr_add_byte(&lex->vstr, quote_char); + } + vstr_add_byte(&lex->vstr, ','); + for (size_t q = 0; q < num_quotes; ++q) { + vstr_add_byte(&lex->vstr, quote_char); + } + + // process conv and format spec + if (is_char(lex, '!')) { + next_char(lex); + vstr_add_byte(&lex->fstring_args, quote_char); + vstr_add_byte(&lex->fstring_args, CUR_CHAR(lex)); + next_char(lex); + vstr_add_byte(&lex->fstring_args, quote_char); + vstr_add_byte(&lex->fstring_args, ','); + } else if (was_debug && !is_char(lex, ':')) { + vstr_add_str(&lex->fstring_args, "'r',"); + } else { + vstr_add_str(&lex->fstring_args, "None,"); + } + + // start format str + if (is_char(lex, ':')) { + next_char(lex); + } + nested_formatting_in_tstring = 1; + end_of_format_index = lex->vstr.len; + } + #endif } vstr_add_byte(&lex->vstr, '{'); + goto continue_outer; } #endif - if (is_char(lex, '\\')) { + if (is_tstring && is_char(lex, '\\')) { + // it'll be reparsed as a string + vstr_add_byte(&lex->vstr, '\\'); + if (is_raw) { + vstr_add_byte(&lex->vstr, '\\'); + } else { + next_char(lex); + vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); + } + } else if (is_char(lex, '\\')) { next_char(lex); unichar c = CUR_CHAR(lex); if (is_raw) { @@ -417,11 +538,9 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) vstr_add_char(&lex->vstr, '\\'); } else { switch (c) { - // note: "c" can never be MP_LEXER_EOF because next_char - // always inserts a newline at the end of the input stream case '\n': - c = MP_LEXER_EOF; - break; // backslash escape the newline, just ignore it + // backslash escape the newline, just ignore it + goto continue_parsing_string_literal; case '\\': break; case '\'': @@ -492,33 +611,61 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) break; } } - if (c != MP_LEXER_EOF) { - #if MICROPY_PY_BUILTINS_STR_UNICODE - if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { - // Valid unicode character in a str object. - vstr_add_char(&lex->vstr, c); - } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { - // Valid byte in a bytes object. - vstr_add_byte(&lex->vstr, c); - } - #else - if (c < 0x100) { - // Without unicode everything is just added as an 8-bit byte. - vstr_add_byte(&lex->vstr, c); - } - #endif - else { - // Character out of range; this raises a generic SyntaxError. - lex->tok_kind = MP_TOKEN_INVALID; + #if MICROPY_PY_BUILTINS_STR_UNICODE + if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { + // Valid unicode character in a str object. + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { + // Valid byte in a bytes object. + vstr_add_byte(&lex->vstr, c); + } + #else + if (c < 0x100) { + // Without unicode everything is just added as an 8-bit byte. + vstr_add_byte(&lex->vstr, c); + } + #endif + else { + // Character out of range; this raises a generic SyntaxError. + lex->tok_kind = MP_TOKEN_INVALID; + } + #if MICROPY_PY_TSTRINGS + } else if (is_tstring && nested_formatting_in_tstring && is_char(lex, '}')) { + if (--nested_formatting_in_tstring > 0) { + nested_formatting_needs_fstring = true; + vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); + } else { + // finished the current interpolation + ++tstring_num_interpolations; + if (nested_formatting_needs_fstring) { + vstr_add_byte(&lex->fstring_args, 'f'); + nested_formatting_needs_fstring = false; } + vstr_add_byte(&lex->fstring_args, quote_char); + vstr_add_strn(&lex->fstring_args, lex->vstr.buf + end_of_format_index + 1, lex->vstr.len - end_of_format_index - 1); + lex->vstr.len = end_of_format_index; + vstr_add_byte(&lex->fstring_args, quote_char); + vstr_add_byte(&lex->fstring_args, ','); } + #endif } else { // Add the "character" as a byte so that we remain 8-bit clean. // This way, strings are parsed correctly whether or not they contain utf-8 chars. vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); + #if MICROPY_PY_TSTRINGS + if (is_tstring && is_char_and(lex, '}', '}')) { + next_char(lex); + } else if (is_tstring && is_char(lex, '}')) { + lex->tok_kind = MP_TOKEN_MALFORMED_FSTRING; + } + #endif } } + continue_parsing_string_literal: next_char(lex); + #if MICROPY_PY_FSTRINGS + continue_outer:; + #endif } // check we got the required end quotes @@ -526,8 +673,23 @@ static void parse_string_literal(mp_lexer_t *lex, bool is_raw, bool is_fstring) lex->tok_kind = MP_TOKEN_LONELY_STRING_OPEN; } - // cut off the end quotes from the token text - vstr_cut_tail_bytes(&lex->vstr, n_closing); + #if MICROPY_PY_TSTRINGS + if (is_tstring) { + if (nested_formatting_in_tstring > 0) { + lex->tok_kind = MP_TOKEN_MALFORMED_FSTRING; + } + + if (1 + tstring_num_interpolations * 4 > 255) { + // too many arguments for function call, so wrap interpolations in a tuple + vstr_ins_byte(&lex->fstring_args, 0, '('); + vstr_add_byte(&lex->fstring_args, ')'); + } + } else + #endif + { + // cut off the end quotes from the token text + vstr_cut_tail_bytes(&lex->vstr, n_closing); + } } // This function returns whether it has crossed a newline or not. @@ -559,23 +721,6 @@ static bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) { } void mp_lexer_to_next(mp_lexer_t *lex) { - #if MICROPY_PY_FSTRINGS - if (lex->fstring_args.len && lex->fstring_args_idx == 0) { - // moving onto the next token means the literal string is complete. - // switch into injecting the format args. - vstr_add_byte(&lex->fstring_args, ')'); - lex->chr0_saved = lex->chr0; - lex->chr1_saved = lex->chr1; - lex->chr2_saved = lex->chr2; - lex->chr0 = lex->fstring_args.buf[0]; - lex->chr1 = lex->fstring_args.buf[1]; - lex->chr2 = lex->fstring_args.buf[2]; - // we've already extracted 3 chars, but setting this non-zero also - // means we'll start consuming the fstring data - lex->fstring_args_idx = 3; - } - #endif - // start new token text vstr_reset(&lex->vstr); @@ -635,11 +780,16 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // MP_TOKEN_END is used to indicate that this is the first string token lex->tok_kind = MP_TOKEN_END; + #if MICROPY_PY_TSTRINGS + bool had_tstring = false; + #endif + // Loop to accumulate string/bytes literals do { // parse type codes bool is_raw = false; bool is_fstring = false; + bool is_tstring = false; mp_token_kind_t kind = MP_TOKEN_STRING; int n_char = 0; if (is_char(lex, 'u')) { @@ -659,11 +809,17 @@ void mp_lexer_to_next(mp_lexer_t *lex) { n_char = 2; } #if MICROPY_PY_FSTRINGS - if (is_char_following(lex, 'f')) { + else if (is_char_following(lex, 'f')) { is_fstring = true; n_char = 2; } #endif + #if MICROPY_PY_TSTRINGS + else if (is_char_following(lex, 't')) { + is_tstring = true; + n_char = 2; + } + #endif } #if MICROPY_PY_FSTRINGS else if (is_char(lex, 'f')) { @@ -675,6 +831,22 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } } #endif + #if MICROPY_PY_TSTRINGS + else if (is_char(lex, 't')) { + is_tstring = true; + n_char = 1; + if (is_char_following(lex, 'r')) { + is_raw = true; + n_char = 2; + } + } + #endif + + #if MICROPY_PY_TSTRINGS + if (is_tstring) { + had_tstring = true; + } + #endif // Set or check token kind if (lex->tok_kind == MP_TOKEN_END) { @@ -693,13 +865,52 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } // Parse the literal - parse_string_literal(lex, is_raw, is_fstring); + parse_string_literal(lex, is_raw, is_fstring, is_tstring); // Skip whitespace so we can check if there's another string following skip_whitespace(lex, true); } while (is_string_or_bytes(lex)); + #if MICROPY_PY_TSTRINGS + if (had_tstring) { + vstr_add_byte(&lex->vstr, ','); + vstr_add_byte(&lex->vstr, ')'); + vstr_add_byte(&lex->vstr, ','); + vstr_ins_strn(&lex->fstring_args, 0, lex->vstr.buf, lex->vstr.len); + if (lex->tok_kind > MP_TOKEN_MALFORMED_FSTRING) { + // next token is __template__ for the function + lex->tok_kind = MP_TOKEN_NAME; + vstr_reset(&lex->vstr); + vstr_add_str(&lex->vstr, "__template__"); + } + } + #endif + + #if MICROPY_PY_FSTRINGS + if (lex->fstring_args.len) { + // If there was an f-string then it's now complete. + // Switch into injecting the format args. + vstr_add_byte(&lex->fstring_args, ')'); + if (lex->inject_chrs_idx == 0) { + // switch from stream to inject_chrs + char *s = vstr_add_len(&lex->inject_chrs, 3); + s[0] = lex->chr0; + s[1] = lex->chr1; + s[2] = lex->chr2; + } else { + // already consuming from inject_chrs, rewind cached chars to insert new ones + assert(lex->inject_chrs_idx >= 3); + lex->inject_chrs_idx -= 3; + } + vstr_ins_strn(&lex->inject_chrs, lex->inject_chrs_idx, lex->fstring_args.buf, lex->fstring_args.len); + vstr_reset(&lex->fstring_args); + lex->chr0 = lex->inject_chrs.buf[lex->inject_chrs_idx++]; + lex->chr1 = lex->inject_chrs.buf[lex->inject_chrs_idx++]; + lex->chr2 = lex->inject_chrs.buf[lex->inject_chrs_idx++]; + } + #endif + } else if (is_head_of_identifier(lex)) { lex->tok_kind = MP_TOKEN_NAME; @@ -857,8 +1068,9 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { lex->indent_level = m_new(uint16_t, lex->alloc_indent_level); vstr_init(&lex->vstr, 32); #if MICROPY_PY_FSTRINGS + vstr_init(&lex->inject_chrs, 0); + lex->inject_chrs_idx = 0; vstr_init(&lex->fstring_args, 0); - lex->fstring_args_idx = 0; #endif // store sentinel for first indentation level @@ -915,6 +1127,7 @@ void mp_lexer_free(mp_lexer_t *lex) { lex->reader.close(lex->reader.data); vstr_clear(&lex->vstr); #if MICROPY_PY_FSTRINGS + vstr_clear(&lex->inject_chrs); vstr_clear(&lex->fstring_args); #endif m_del(uint16_t, lex->indent_level, lex->alloc_indent_level); diff --git a/py/lexer.h b/py/lexer.h index 6e6c3e8f23e..b1257d55948 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -162,10 +162,8 @@ typedef struct _mp_lexer_t { qstr source_name; // name of source mp_reader_t reader; // stream source - unichar chr0, chr1, chr2; // current cached characters from source - #if MICROPY_PY_FSTRINGS - unichar chr0_saved, chr1_saved, chr2_saved; // current cached characters from alt source - #endif + uint32_t chr0; // first cached byte from source (32-bits for efficient access) + uint8_t chr1, chr2; // subsequent cached bytes from source size_t line; // current source line size_t column; // current source column @@ -182,8 +180,9 @@ typedef struct _mp_lexer_t { mp_token_kind_t tok_kind; // token kind vstr_t vstr; // token data #if MICROPY_PY_FSTRINGS + vstr_t inject_chrs; // characters currently being injected into the stream + size_t inject_chrs_idx; // current index into inject_chrs vstr_t fstring_args; // extracted arguments to pass to .format() - size_t fstring_args_idx; // how many bytes of fstring_args have been read #endif } mp_lexer_t; diff --git a/py/misc.h b/py/misc.h index d5d7950574f..d85a8f34289 100644 --- a/py/misc.h +++ b/py/misc.h @@ -26,6 +26,7 @@ #ifndef MICROPY_INCLUDED_PY_MISC_H #define MICROPY_INCLUDED_PY_MISC_H +#include #include "py/mpconfig.h" // a mini library of useful types and functions @@ -264,30 +265,30 @@ void vstr_add_byte(vstr_t *vstr, byte v); void vstr_add_char(vstr_t *vstr, unichar chr); void vstr_add_str(vstr_t *vstr, const char *str); void vstr_add_strn(vstr_t *vstr, const char *str, size_t len); -void vstr_ins_byte(vstr_t *vstr, size_t byte_pos, byte b); -void vstr_ins_char(vstr_t *vstr, size_t char_pos, unichar chr); +char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len); +static inline void vstr_ins_byte(vstr_t *vstr, size_t byte_pos, byte b) { + char *s = vstr_ins_blank_bytes(vstr, byte_pos, 1); + *s = b; +} +static inline void vstr_ins_char(vstr_t *vstr, size_t char_pos, unichar chr) { + // TODO UNICODE + char *s = vstr_ins_blank_bytes(vstr, char_pos, 1); + *s = (char)chr; +} +static inline void vstr_ins_strn(vstr_t *vstr, size_t byte_pos, const char *str, size_t len) { + char *s = vstr_ins_blank_bytes(vstr, byte_pos, len); + memcpy(s, str, len); +} void vstr_cut_head_bytes(vstr_t *vstr, size_t bytes_to_cut); void vstr_cut_tail_bytes(vstr_t *vstr, size_t bytes_to_cut); void vstr_cut_out_bytes(vstr_t *vstr, size_t byte_pos, size_t bytes_to_cut); void vstr_printf(vstr_t *vstr, const char *fmt, ...); - -/** non-dynamic size-bounded variable buffer/string *************/ - -#define CHECKBUF(buf, max_size) char buf[max_size + 1]; size_t buf##_len = max_size; char *buf##_p = buf; -#define CHECKBUF_RESET(buf, max_size) buf##_len = max_size; buf##_p = buf; -#define CHECKBUF_APPEND(buf, src, src_len) \ - { size_t l = MIN(src_len, buf##_len); \ - memcpy(buf##_p, src, l); \ - buf##_len -= l; \ - buf##_p += l; } -#define CHECKBUF_APPEND_0(buf) { *buf##_p = 0; } -#define CHECKBUF_LEN(buf) (buf##_p - buf) - #ifdef va_start void vstr_vprintf(vstr_t *vstr, const char *fmt, va_list ap); #endif -// Debugging helpers +/** debugging helpers *******************************************/ + int DEBUG_printf(const char *fmt, ...); extern mp_uint_t mp_verbose_flag; @@ -444,11 +445,6 @@ static inline uint32_t mp_ctz(uint32_t x) { return _BitScanForward(&tz, x) ? tz : 0; } -// Workaround for 'warning C4127: conditional expression is constant'. -static inline bool mp_check(bool value) { - return value; -} - static inline uint32_t mp_popcount(uint32_t x) { return __popcnt(x); } @@ -457,7 +453,6 @@ static inline uint32_t mp_popcount(uint32_t x) { #define mp_clzl(x) __builtin_clzl(x) #define mp_clzll(x) __builtin_clzll(x) #define mp_ctz(x) __builtin_ctz(x) -#define mp_check(x) (x) #if __has_builtin(__builtin_popcount) #define mp_popcount(x) __builtin_popcount(x) #else @@ -592,4 +587,30 @@ static inline bool mp_sub_ll_overflow(long long int lhs, long long int rhs, long #define MP_SANITIZER_BUILD (MP_UBSAN || MP_ASAN) #endif +// halfword/word/longword swapping macros + +#if __has_builtin(__builtin_bswap16) +#define MP_BSWAP16(x) __builtin_bswap16(x) +#else +#define MP_BSWAP16(x) ((uint16_t)((((x) & 0xFF) << 8) | (((x) >> 8) & 0xFF))) +#endif + +#if __has_builtin(__builtin_bswap32) +#define MP_BSWAP32(x) __builtin_bswap32(x) +#else +#define MP_BSWAP32(x) \ + ((uint32_t)((((x) & 0xFF) << 24) | (((x) & 0xFF00) << 8) | \ + (((x) >> 8) & 0xFF00) | (((x) >> 24) & 0xFF))) +#endif + +#if __has_builtin(__builtin_bswap64) +#define MP_BSWAP64(x) __builtin_bswap64(x) +#else +#define MP_BSWAP64(x) \ + ((uint64_t)((((x) & 0xFF) << 56) | (((x) & 0xFF00) << 40) | \ + (((x) & 0xFF0000) << 24) | (((x) & 0xFF000000) << 8) | \ + (((x) >> 8) & 0xFF000000) | (((x) >> 24) & 0xFF0000) | \ + (((x) >> 40) & 0xFF00) | (((x) >> 56) & 0xFF))) +#endif + #endif // MICROPY_INCLUDED_PY_MISC_H diff --git a/py/modbuiltins.c b/py/modbuiltins.c index cdeacc25f71..186a79b7fac 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -607,6 +607,9 @@ MP_DEFINE_CONST_FUN_OBJ_0(mp_builtin_locals_obj, mp_builtin_locals); // These are defined in terms of MicroPython API functions right away MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_id_obj, mp_obj_id); MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_len_obj, mp_obj_len); +#if MICROPY_PY_TSTRINGS +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin___template___obj, 1, MP_OBJ_FUN_ARGS_MAX, mp_obj_new_template); +#endif static const mp_rom_map_elem_t mp_module_builtins_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_builtins) }, @@ -615,6 +618,9 @@ static const mp_rom_map_elem_t mp_module_builtins_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___build_class__), MP_ROM_PTR(&mp_builtin___build_class___obj) }, { MP_ROM_QSTR(MP_QSTR___import__), MP_ROM_PTR(&mp_builtin___import___obj) }, { MP_ROM_QSTR(MP_QSTR___repl_print__), MP_ROM_PTR(&mp_builtin___repl_print___obj) }, + #if MICROPY_PY_TSTRINGS + { MP_ROM_QSTR(MP_QSTR___template__), MP_ROM_PTR(&mp_builtin___template___obj) }, + #endif // built-in types { MP_ROM_QSTR(MP_QSTR_bool), MP_ROM_PTR(&mp_type_bool) }, diff --git a/py/modstring.c b/py/modstring.c new file mode 100644 index 00000000000..26e0c2d83cb --- /dev/null +++ b/py/modstring.c @@ -0,0 +1,56 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/obj.h" + +#if MICROPY_PY_TSTRINGS + +static const mp_rom_map_elem_t mp_module_string_templatelib_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_string_dot_templatelib) }, + { MP_ROM_QSTR(MP_QSTR_Template), MP_ROM_PTR(&mp_type_template) }, + { MP_ROM_QSTR(MP_QSTR_Interpolation), MP_ROM_PTR(&mp_type_interpolation) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_string_templatelib_globals, mp_module_string_templatelib_globals_table); + +static const mp_obj_module_t mp_module_string_templatelib = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_string_templatelib_globals, +}; + +static const mp_rom_map_elem_t mp_module_string_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_string) }, + { MP_ROM_QSTR(MP_QSTR_templatelib), MP_ROM_PTR(&mp_module_string_templatelib) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_string_globals, mp_module_string_globals_table); + +const mp_obj_module_t mp_module_string = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_string_globals, +}; + +MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_string, mp_module_string); + +#endif // MICROPY_PY_TSTRINGS diff --git a/py/modweakref.c b/py/modweakref.c new file mode 100644 index 00000000000..3360a4e2095 --- /dev/null +++ b/py/modweakref.c @@ -0,0 +1,314 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/gc.h" +#include "py/runtime.h" + +#if MICROPY_PY_WEAKREF + +// Macros to obfuscate a heap pointer as a small integer object. +#define PTR_TO_INT_OBJ(ptr) (MP_OBJ_NEW_SMALL_INT(((uintptr_t)ptr) >> 1)) +#define PTR_FROM_INT_OBJ(obj) ((void *)(MP_OBJ_SMALL_INT_VALUE((obj)) << 1)) + +// Macros to convert between a weak reference and a heap pointer. +#define WEAK_REFERENCE_FROM_HEAP_PTR(ptr) PTR_TO_INT_OBJ(ptr) +#define WEAK_REFERENCE_TO_HEAP_PTR(weak_ref) PTR_FROM_INT_OBJ(weak_ref) + +// Macros to manage ref-finalizer linked-list pointers. +// - mp_obj_ref_t is obfuscated as a small integer object so it's not traced by the GC. +// - mp_obj_finalize_t is stored as-is so it is traced by the GC. +#define REF_FIN_LIST_OBJ_IS_FIN(r) (!mp_obj_is_small_int((r))) +#define REF_FIN_LIST_OBJ_TO_PTR(r) ((mp_obj_ref_t *)(mp_obj_is_small_int((r)) ? PTR_FROM_INT_OBJ((r)) : MP_OBJ_TO_PTR((r)))) +#define REF_FIN_LIST_OBJ_FROM_REF(r) (PTR_TO_INT_OBJ((r))) +#define REF_FIN_LIST_OBJ_FROM_FIN(r) (MP_OBJ_FROM_PTR((r))) +#define REF_FIN_LIST_OBJ_TAIL (PTR_TO_INT_OBJ(NULL)) + +// weakref.ref() instance. +typedef struct _mp_obj_ref_t { + mp_obj_base_t base; + mp_obj_t ref_fin_next; + mp_obj_t obj_weak_ref; + mp_obj_t callback; +} mp_obj_ref_t; + +// weakref.finalize() instance. +// This is an extension of weakref.ref() and shares a lot of code with it. +typedef struct _mp_obj_finalize_t { + mp_obj_ref_t base; + size_t n_args; + size_t n_kw; + mp_obj_t *args; +} mp_obj_finalize_t; + +static const mp_obj_type_t mp_type_ref; +static const mp_obj_type_t mp_type_finalize; + +static mp_obj_t ref___del__(mp_obj_t self_in); + +void gc_weakref_about_to_be_freed(void *ptr) { + mp_obj_t idx = WEAK_REFERENCE_FROM_HEAP_PTR(ptr); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), idx, MP_MAP_LOOKUP); + if (elem != NULL) { + // Mark element as being freed. + elem->key = mp_const_none; + } +} + +void gc_weakref_sweep(void) { + mp_map_t *map = &MP_STATE_VM(mp_weakref_map); + for (size_t i = 0; i < map->alloc; i++) { + if (map->table[i].key == mp_const_none) { + // Element was just freed, so call all the registered callbacks. + --map->used; + map->table[i].key = MP_OBJ_SENTINEL; + mp_obj_ref_t *ref = REF_FIN_LIST_OBJ_TO_PTR(map->table[i].value); + map->table[i].value = MP_OBJ_NULL; + while (ref != NULL) { + // Invalidate the weak reference. + assert(ref->obj_weak_ref != mp_const_none); + ref->obj_weak_ref = mp_const_none; + + // Call any registered callbacks. + if (ref->callback != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (ref->base.type == &mp_type_ref) { + // weakref.ref() type. + mp_call_function_1(ref->callback, MP_OBJ_FROM_PTR(ref)); + } else { + // weakref.finalize() type. + mp_obj_finalize_t *fin = (mp_obj_finalize_t *)ref; + mp_call_function_n_kw(fin->base.callback, fin->n_args, fin->n_kw, fin->args); + } + nlr_pop(); + } else { + mp_printf(MICROPY_ERROR_PRINTER, "Unhandled exception in weakref callback:\n"); + mp_obj_print_exception(MICROPY_ERROR_PRINTER, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + } + + // Unlink the node. + mp_obj_ref_t *ref_fin_next = REF_FIN_LIST_OBJ_TO_PTR(ref->ref_fin_next); + ref->ref_fin_next = REF_FIN_LIST_OBJ_TAIL; + ref = ref_fin_next; + } + } + } +} + +static mp_obj_t mp_obj_ref_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + if (type == &mp_type_ref) { + // weakref.ref() type. + mp_arg_check_num(n_args, n_kw, 1, 2, false); + } else { + // weakref.finalize() type. + mp_arg_check_num(n_args, n_kw, 2, MP_OBJ_FUN_ARGS_MAX, true); + } + + // Validate the input object can have a weakref. + void *ptr = NULL; + if (mp_obj_is_obj(args[0])) { + ptr = MP_OBJ_TO_PTR(args[0]); + if (gc_nbytes(ptr) == 0) { + ptr = NULL; + } + } + if (ptr == NULL) { + mp_raise_TypeError(MP_ERROR_TEXT("not a heap object")); + } + + // Create or get the entry in mp_weakref_map corresponding to this object. + mp_obj_t obj_weak_reference = WEAK_REFERENCE_FROM_HEAP_PTR(ptr); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), obj_weak_reference, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + // This heap object does not have any existing weakref's, so initialise it. + elem->value = REF_FIN_LIST_OBJ_TAIL; + gc_weakref_mark(ptr); + } + + mp_obj_ref_t *self; + if (type == &mp_type_ref) { + // Create a new weakref.ref() object. + self = mp_obj_malloc_with_finaliser(mp_obj_ref_t, type); + // Link this new ref into the list of all refs/finalizers pointing to this object. + // To ensure it will *NOT* be traced by the GC (the user must manually hold onto it), + // store an integer version of the object after any weakref.finalize() objects (so + // the weakref.finalize() objects continue to be traced by the GC). + mp_obj_t *link = &elem->value; + while (REF_FIN_LIST_OBJ_IS_FIN(*link)) { + link = &REF_FIN_LIST_OBJ_TO_PTR(*link)->ref_fin_next; + } + self->ref_fin_next = *link; + *link = REF_FIN_LIST_OBJ_FROM_REF(self); + } else { + // Create a new weakref.finalize() object. + mp_obj_finalize_t *self_fin = mp_obj_malloc(mp_obj_finalize_t, type); + self_fin->n_args = n_args - 2; + self_fin->n_kw = n_kw; + size_t n_args_kw = self_fin->n_args + self_fin->n_kw * 2; + if (n_args_kw == 0) { + self_fin->args = NULL; + } else { + self_fin->args = m_new(mp_obj_t, n_args_kw); + memcpy(self_fin->args, args + 2, n_args_kw * sizeof(mp_obj_t)); + } + self = &self_fin->base; + // Link this new finalizer into the list of all refs/finalizers pointing to this object. + // To ensure it will be traced by the GC, store its pointer at the start of the list. + self->ref_fin_next = elem->value; + elem->value = REF_FIN_LIST_OBJ_FROM_FIN(self_fin); + } + + // Populate the object weak reference, and the callback. + self->obj_weak_ref = obj_weak_reference; + if (n_args > 1) { + self->callback = args[1]; + } else { + self->callback = mp_const_none; + } + + return MP_OBJ_FROM_PTR(self); +} + +static mp_obj_t mp_obj_ref_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_obj_ref_t *self = MP_OBJ_TO_PTR(self_in); + if (self->obj_weak_ref == mp_const_none) { + return mp_const_none; + } + if (self->base.type == &mp_type_ref) { + // weakref.ref() type. + return MP_OBJ_FROM_PTR(WEAK_REFERENCE_TO_HEAP_PTR(self->obj_weak_ref)); + } else { + // weakref.finalize() type. + mp_obj_finalize_t *self_fin = MP_OBJ_TO_PTR(self_in); + ref___del__(self_in); + return mp_call_function_n_kw(self_fin->base.callback, self_fin->n_args, self_fin->n_kw, self_fin->args); + } +} + +static mp_obj_t ref___del__(mp_obj_t self_in) { + mp_obj_ref_t *self = MP_OBJ_TO_PTR(self_in); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), self->obj_weak_ref, MP_MAP_LOOKUP); + if (elem != NULL) { + for (mp_obj_t *link = &elem->value; REF_FIN_LIST_OBJ_TO_PTR(*link) != NULL; link = &REF_FIN_LIST_OBJ_TO_PTR(*link)->ref_fin_next) { + if (self == REF_FIN_LIST_OBJ_TO_PTR(*link)) { + // Unlink and clear this node. + *link = self->ref_fin_next; + self->ref_fin_next = REF_FIN_LIST_OBJ_TAIL; + self->obj_weak_ref = mp_const_none; + break; + } + } + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(ref___del___obj, ref___del__); + +static mp_obj_t finalize_peek_detach_helper(mp_obj_t self_in, bool detach) { + mp_obj_finalize_t *self = MP_OBJ_TO_PTR(self_in); + if (self->base.obj_weak_ref == mp_const_none) { + return mp_const_none; + } + mp_obj_t tuple[4] = { + MP_OBJ_FROM_PTR(WEAK_REFERENCE_TO_HEAP_PTR(self->base.obj_weak_ref)), + self->base.callback, + mp_obj_new_tuple(self->n_args, self->args), + mp_obj_dict_make_new(&mp_type_dict, 0, self->n_kw, self->args + self->n_args), + }; + if (detach) { + ref___del__(self_in); + } + return mp_obj_new_tuple(MP_ARRAY_SIZE(tuple), tuple); +} + +static mp_obj_t finalize_peek(mp_obj_t self_in) { + return finalize_peek_detach_helper(self_in, false); +} +static MP_DEFINE_CONST_FUN_OBJ_1(finalize_peek_obj, finalize_peek); + +static mp_obj_t finalize_detach(mp_obj_t self_in) { + return finalize_peek_detach_helper(self_in, true); +} +static MP_DEFINE_CONST_FUN_OBJ_1(finalize_detach_obj, finalize_detach); + +static void mp_obj_finalize_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + if (dest[0] != MP_OBJ_NULL) { + // Store/delete attribute, unsupported. + return; + } + + if (attr == MP_QSTR_alive) { + mp_obj_finalize_t *self = MP_OBJ_TO_PTR(self_in); + dest[0] = mp_obj_new_bool(self->base.obj_weak_ref != mp_const_none); + return; + } else if (attr == MP_QSTR_peek) { + dest[0] = MP_OBJ_FROM_PTR(&finalize_peek_obj); + dest[1] = self_in; + } else if (attr == MP_QSTR_detach) { + dest[0] = MP_OBJ_FROM_PTR(&finalize_detach_obj); + dest[1] = self_in; + } +} + +static const mp_rom_map_elem_t ref_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&ref___del___obj) }, +}; +static MP_DEFINE_CONST_DICT(ref_locals_dict, ref_locals_dict_table); + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_ref, + MP_QSTR_ref, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_ref_make_new, + call, mp_obj_ref_call, + locals_dict, &ref_locals_dict + ); + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_finalize, + MP_QSTR_finalize, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_ref_make_new, + call, mp_obj_ref_call, + attr, mp_obj_finalize_attr + ); + +static const mp_rom_map_elem_t mp_module_weakref_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_weakref) }, + { MP_ROM_QSTR(MP_QSTR_ref), MP_ROM_PTR(&mp_type_ref) }, + { MP_ROM_QSTR(MP_QSTR_finalize), MP_ROM_PTR(&mp_type_finalize) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_weakref_globals, mp_module_weakref_globals_table); + +const mp_obj_module_t mp_module_weakref = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_weakref_globals, +}; + +MP_REGISTER_ROOT_POINTER(mp_map_t mp_weakref_map); +MP_REGISTER_MODULE(MP_QSTR_weakref, mp_module_weakref); + +#endif // MICROPY_PY_WEAKREF diff --git a/py/mpconfig.h b/py/mpconfig.h index 0e440066d78..846209e9412 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -49,7 +49,7 @@ // as well as a fallback to generate MICROPY_GIT_TAG if the git repo or tags // are unavailable. #define MICROPY_VERSION_MAJOR 1 -#define MICROPY_VERSION_MINOR 27 +#define MICROPY_VERSION_MINOR 28 #define MICROPY_VERSION_MICRO 0 #define MICROPY_VERSION_PRERELEASE 0 @@ -426,6 +426,11 @@ typedef uint64_t mp_uint_t; #define MICROPY_PERSISTENT_CODE_LOAD (0) #endif +// Whether to support loading of persistent native code +#ifndef MICROPY_PERSISTENT_CODE_LOAD_NATIVE +#define MICROPY_PERSISTENT_CODE_LOAD_NATIVE (MICROPY_EMIT_MACHINE_CODE) +#endif + // Whether to support saving of persistent code, i.e. for mpy-cross to // generate .mpy files. Enabling this enables additional metadata on raw code // objects which is also required for sys.settrace. @@ -446,7 +451,7 @@ typedef uint64_t mp_uint_t; // Whether generated code can persist independently of the VM/runtime instance // This is enabled automatically when needed by other features #ifndef MICROPY_PERSISTENT_CODE -#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) +#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_LOAD_NATIVE || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) #endif // Whether bytecode uses a qstr_table to map internal qstr indices in the bytecode @@ -522,6 +527,11 @@ typedef uint64_t mp_uint_t; #define MICROPY_EMIT_RV32_ZBA (0) #endif +// Whether to emit RISC-V RV32 Zcmp opcodes in native code +#ifndef MICROPY_EMIT_RV32_ZCMP +#define MICROPY_EMIT_RV32_ZCMP (0) +#endif + // Whether to enable the RISC-V RV32 inline assembler #ifndef MICROPY_EMIT_INLINE_RV32 #define MICROPY_EMIT_INLINE_RV32 (0) @@ -546,6 +556,10 @@ typedef uint64_t mp_uint_t; // Convenience definition for whether any native or inline assembler emitter is enabled #define MICROPY_EMIT_MACHINE_CODE (MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM) +// Convenience definition for whether native code has to be dealt with (either +// generated or loaded from a file). This does not cover inline asm code. +#define MICROPY_ENABLE_NATIVE_CODE (MICROPY_EMIT_NATIVE || MICROPY_PERSISTENT_CODE_LOAD_NATIVE) + /*****************************************************************************/ /* Compiler configuration */ @@ -1126,6 +1140,12 @@ typedef time_t mp_timestamp_t; #define MICROPY_STREAMS_POSIX_API (0) #endif +// Whether to delegate error raising to stream implementations using the +// MP_STREAM_RAISE_ERROR ioctl to support raising more detailed messages. +#ifndef MICROPY_STREAMS_DELEGATE_ERROR +#define MICROPY_STREAMS_DELEGATE_ERROR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + // Whether to process __all__ when importing all public symbols from a module. #ifndef MICROPY_MODULE___ALL__ #define MICROPY_MODULE___ALL__ (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_BASIC_FEATURES) @@ -1160,7 +1180,7 @@ typedef time_t mp_timestamp_t; // have __init__ methods. Instead, the top-level package's __init__ should // initialise all sub-packages. #ifndef MICROPY_MODULE_BUILTIN_SUBPACKAGES -#define MICROPY_MODULE_BUILTIN_SUBPACKAGES (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#define MICROPY_MODULE_BUILTIN_SUBPACKAGES (MICROPY_PY_TSTRINGS || MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif // Whether to support module-level __getattr__ (see PEP 562) @@ -1310,7 +1330,7 @@ typedef time_t mp_timestamp_t; // Whether to implement the __code__ attribute on functions, and function constructor #ifndef MICROPY_PY_FUNCTION_ATTRS_CODE -#define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_PY_MARSHAL || MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) #endif // Whether bound_method can just use == (feature disabled), or requires a call to @@ -1345,6 +1365,12 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_FSTRINGS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Support for template strings, t-strings (see PEP 750, Python 3.14+) +// Requires MICROPY_PY_FSTRINGS to be enabled. +#ifndef MICROPY_PY_TSTRINGS +#define MICROPY_PY_TSTRINGS (MICROPY_PY_FSTRINGS && MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#endif + // Support for assignment expressions with := (see PEP 572, Python 3.8+) #ifndef MICROPY_PY_ASSIGN_EXPR #define MICROPY_PY_ASSIGN_EXPR (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) @@ -1931,6 +1957,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_THREAD_RECURSIVE_MUTEX (MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL) #endif +// Whether to provide the "weakref" module. +#ifndef MICROPY_PY_WEAKREF +#define MICROPY_PY_WEAKREF (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Extended modules #ifndef MICROPY_PY_ASYNCIO @@ -2327,7 +2358,7 @@ typedef time_t mp_timestamp_t; // can be overridden if needed by defining both MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA // and MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA. #ifndef MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA -#if MICROPY_EMIT_MACHINE_CODE && MICROPY_PERSISTENT_CODE_LOAD +#if (MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) && MICROPY_PERSISTENT_CODE_LOAD // Pointer tracking is required when loading native code is enabled. #if defined(MP_PLAT_ALLOC_EXEC) || defined(MP_PLAT_COMMIT_EXEC) // If a port defined a custom allocator or commit function for native text, then the @@ -2348,7 +2379,7 @@ typedef time_t mp_timestamp_t; #define MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA (1) #define MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA (0) #endif -#else // MICROPY_EMIT_MACHINE_CODE && MICROPY_PERSISTENT_CODE_LOAD +#else // (MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) && MICROPY_PERSISTENT_CODE_LOAD // Pointer tracking not needed when loading native code is disabled. #define MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA (0) #define MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA (0) @@ -2462,7 +2493,7 @@ typedef time_t mp_timestamp_t; #ifndef MP_HTOBE16 #if MP_ENDIANNESS_LITTLE -#define MP_HTOBE16(x) ((uint16_t)((((x) & 0xff) << 8) | (((x) >> 8) & 0xff))) +#define MP_HTOBE16(x) MP_BSWAP16(x) #define MP_BE16TOH(x) MP_HTOBE16(x) #else #define MP_HTOBE16(x) (x) @@ -2472,7 +2503,7 @@ typedef time_t mp_timestamp_t; #ifndef MP_HTOBE32 #if MP_ENDIANNESS_LITTLE -#define MP_HTOBE32(x) ((uint32_t)((((x) & 0xff) << 24) | (((x) & 0xff00) << 8) | (((x) >> 8) & 0xff00) | (((x) >> 24) & 0xff))) +#define MP_HTOBE32(x) MP_BSWAP32(x) #define MP_BE32TOH(x) MP_HTOBE32(x) #else #define MP_HTOBE32(x) (x) diff --git a/py/mpstate.h b/py/mpstate.h index 7934e843f06..8519a1a32ad 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -122,6 +122,9 @@ typedef struct _mp_state_mem_area_t { #if MICROPY_ENABLE_SELECTIVE_COLLECT byte *gc_collect_table_start; #endif + #if MICROPY_PY_WEAKREF + byte *gc_weakref_table_start; + #endif byte *gc_pool_start; byte *gc_pool_end; diff --git a/py/nativeglue.c b/py/nativeglue.c index 3b072ba8c38..2613312e2b7 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -43,7 +43,7 @@ #define DEBUG_printf(...) (void)0 #endif -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE int mp_native_type_from_qstr(qstr qst) { switch (qst) { @@ -93,7 +93,7 @@ mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type) { #endif -#if MICROPY_EMIT_MACHINE_CODE +#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE // convert a native value to a MicroPython object based on type mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) { @@ -117,7 +117,7 @@ mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) { #endif -#if MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER +#if MICROPY_ENABLE_NATIVE_CODE && !MICROPY_DYNAMIC_COMPILER #if !MICROPY_PY_BUILTINS_SET mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items) { @@ -359,8 +359,8 @@ const mp_fun_table_t mp_fun_table = { &mp_stream_write_obj, }; -#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER +#elif MICROPY_ENABLE_NATIVE_CODE && MICROPY_DYNAMIC_COMPILER const int mp_fun_table; -#endif // MICROPY_EMIT_NATIVE +#endif // MICROPY_ENABLE_NATIVE_CODE diff --git a/py/nativeglue.h b/py/nativeglue.h index 01825ac60ed..f6ba600c99f 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -184,9 +184,9 @@ typedef struct _mp_fun_table_t { const mp_obj_fun_builtin_var_t *stream_write_obj; } mp_fun_table_t; -#if (MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME +#if (MICROPY_ENABLE_NATIVE_CODE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME extern const mp_fun_table_t mp_fun_table; -#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER +#elif MICROPY_ENABLE_NATIVE_CODE && MICROPY_DYNAMIC_COMPILER // In dynamic-compiler mode eliminate dependency on entries in mp_fun_table. // This only needs to be an independent pointer, content doesn't matter. extern const int mp_fun_table; diff --git a/py/obj.h b/py/obj.h index eb7143bd43f..e12cbff1ca2 100644 --- a/py/obj.h +++ b/py/obj.h @@ -866,6 +866,8 @@ extern const mp_obj_type_t mp_type_NoneType; extern const mp_obj_type_t mp_type_bool; extern const mp_obj_type_t mp_type_int; extern const mp_obj_type_t mp_type_str; +extern const mp_obj_type_t mp_type_template; +extern const mp_obj_type_t mp_type_interpolation; extern const mp_obj_type_t mp_type_bytes; extern const mp_obj_type_t mp_type_bytearray; extern const mp_obj_type_t mp_type_memoryview; @@ -1083,6 +1085,9 @@ mp_obj_t mp_obj_new_bytearray(size_t n, const void *items); // CIRCUITPY-CHANGE: new routine mp_obj_t mp_obj_new_bytearray_of_zeros(size_t n); mp_obj_t mp_obj_new_bytearray_by_ref(size_t n, void *items); +#if MICROPY_PY_TSTRINGS +mp_obj_t mp_obj_new_template(size_t n_args, const mp_obj_t *args); +#endif #if MICROPY_PY_BUILTINS_FLOAT mp_obj_t mp_obj_new_int_from_float(mp_float_t val); mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag); @@ -1260,21 +1265,6 @@ mp_obj_t mp_obj_complex_binary_op(mp_binary_op_t op, mp_float_t lhs_real, mp_flo #define mp_obj_is_float(o) (false) #endif -// tuple -void mp_obj_tuple_get(mp_obj_t self_in, size_t *len, mp_obj_t **items); -void mp_obj_tuple_del(mp_obj_t self_in); -mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); - -// list -// CIRCUITPY-CHANGE: public routine -mp_obj_t mp_obj_list_clear(mp_obj_t self_in); -mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); -mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); -void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items); -void mp_obj_list_set_len(mp_obj_t self_in, size_t len); -void mp_obj_list_store(mp_obj_t self_in, mp_obj_t index, mp_obj_t value); -mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); - // dict typedef struct _mp_obj_dict_t { mp_obj_base_t base; @@ -1331,7 +1321,6 @@ typedef struct _mp_obj_fun_builtin_var_t { } fun; } mp_obj_fun_builtin_var_t; -qstr mp_obj_fun_get_name(mp_const_obj_t fun); mp_obj_t mp_identity(mp_obj_t self); MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj); @@ -1346,9 +1335,6 @@ typedef struct _mp_obj_module_t { mp_obj_base_t base; mp_obj_dict_t *globals; } mp_obj_module_t; -static inline mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t module) { - return ((mp_obj_module_t *)MP_OBJ_TO_PTR(module))->globals; -} // staticmethod and classmethod types; defined here so we can make const versions // this structure is used for instances of both staticmethod and classmethod diff --git a/py/objarray.c b/py/objarray.c index 1e259a20ac8..f22cbb9fbe8 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -211,7 +211,10 @@ static mp_obj_t bytearray_make_new(const mp_obj_type_t *type_in, size_t n_args, // 1 arg, an integer: construct a blank bytearray of that length mp_uint_t len = mp_obj_get_int(args[0]); mp_obj_array_t *o = array_new(BYTEARRAY_TYPECODE, len); + // If this config is set then the GC clears all memory, so we don't need to. + #if !MICROPY_GC_CONSERVATIVE_CLEAR memset(o->items, 0, len); + #endif return MP_OBJ_FROM_PTR(o); } else { // 1 arg: construct the bytearray from that diff --git a/py/objexcept.c b/py/objexcept.c index 6a6b1e4d4a7..6a2fecd51e5 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -610,7 +610,14 @@ bool mp_obj_is_exception_type(mp_obj_t self_in) { // return true if the given object is an instance of an exception type bool mp_obj_is_exception_instance(mp_obj_t self_in) { - return mp_obj_is_exception_type(MP_OBJ_FROM_PTR(mp_obj_get_type(self_in))); + if (mp_obj_is_native_exception_instance(self_in)) { + return true; + } + if (!mp_obj_is_exception_type(MP_OBJ_FROM_PTR(mp_obj_get_type(self_in)))) { + return false; + } + mp_obj_instance_t *self = MP_OBJ_TO_PTR(self_in); + return self->subobj[0] != MP_OBJ_FROM_PTR((void *)&mp_native_base_init_wrapper_obj); } // Return true if exception (type or instance) is a subclass of given diff --git a/py/objfun.c b/py/objfun.c index 34565cf6336..6392a6c8730 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -135,11 +135,10 @@ MP_DEFINE_CONST_OBJ_TYPE( /******************************************************************************/ /* byte code functions */ -qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { - const mp_obj_fun_bc_t *fun = MP_OBJ_TO_PTR(fun_in); +qstr mp_obj_fun_bc_get_name(const mp_obj_fun_bc_t *fun) { const byte *bc = fun->bytecode; - #if MICROPY_EMIT_NATIVE + #if MICROPY_ENABLE_NATIVE_CODE if (fun->base.type == &mp_type_fun_native || fun->base.type == &mp_type_native_gen_wrap) { bc = mp_obj_fun_native_get_prelude_ptr(fun); } @@ -185,7 +184,7 @@ static void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t (void)kind; mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(o_in); // CIRCUITPY-CHANGE: %p already prints "0x", so don't include it explicitly. - mp_printf(print, "", mp_obj_fun_get_name(o_in), o); + mp_printf(print, "", mp_obj_fun_bc_get_name(o), o); } #endif @@ -364,21 +363,33 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { // not load attribute return; } + const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); if (attr == MP_QSTR___name__) { - dest[0] = MP_OBJ_NEW_QSTR(mp_obj_fun_get_name(self_in)); + dest[0] = MP_OBJ_NEW_QSTR(mp_obj_fun_bc_get_name(self)); } if (attr == MP_QSTR___globals__) { - mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } #if MICROPY_PY_FUNCTION_ATTRS_CODE if (attr == MP_QSTR___code__) { - const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - if ((self->base.type == &mp_type_fun_bc - || self->base.type == &mp_type_gen_wrap) - && self->child_table == NULL) { + if (self->base.type == &mp_type_fun_bc + || self->base.type == &mp_type_gen_wrap) { #if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC - dest[0] = mp_obj_new_code(self->context->constants, self->bytecode); + #if MICROPY_PERSISTENT_CODE_SAVE + #error "MICROPY_PERSISTENT_CODE_SAVE can't be enabled at this level of MICROPY_PY_BUILTINS_CODE" + #endif + mp_proto_fun_t proto_fun; + if (self->child_table != NULL) { + mp_raw_code_truncated_t *rc = m_new0(mp_raw_code_truncated_t, 1); + rc->kind = MP_CODE_BYTECODE; + rc->is_generator = self->base.type == &mp_type_gen_wrap; + rc->fun_data = self->bytecode; + rc->children = (mp_raw_code_t **)self->child_table; + proto_fun = rc; + } else { + proto_fun = self->bytecode; + } + dest[0] = mp_obj_new_code(self->context->constants, proto_fun); #else dest[0] = mp_obj_new_code(self->context, self->rc, true); #endif @@ -448,7 +459,7 @@ mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_ /******************************************************************************/ /* native functions */ -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE // CIRCUITPY-CHANGE: PLACE_IN_ITCM static mp_obj_t PLACE_IN_ITCM(fun_native_call)(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { @@ -478,13 +489,9 @@ MP_DEFINE_CONST_OBJ_TYPE( call, fun_native_call ); -#endif // MICROPY_EMIT_NATIVE - /******************************************************************************/ /* viper functions */ -#if MICROPY_EMIT_NATIVE - static mp_obj_t fun_viper_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { mp_cstack_check(); mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); @@ -499,7 +506,7 @@ MP_DEFINE_CONST_OBJ_TYPE( call, fun_viper_call ); -#endif // MICROPY_EMIT_NATIVE +#endif // MICROPY_ENABLE_NATIVE_CODE /******************************************************************************/ /* inline assembler functions */ diff --git a/py/objfun.h b/py/objfun.h index 4059343983c..0fcc25ff09c 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -51,9 +51,10 @@ typedef struct _mp_obj_fun_asm_t { } mp_obj_fun_asm_t; mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +qstr mp_obj_fun_bc_get_name(const mp_obj_fun_bc_t *fun); void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); diff --git a/py/objgenerator.c b/py/objgenerator.c index de063d38458..83cf6715bbc 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -114,7 +114,7 @@ MP_DEFINE_CONST_OBJ_TYPE( /******************************************************************************/ // native generator wrapper -#if MICROPY_EMIT_NATIVE +#if MICROPY_ENABLE_NATIVE_CODE // Based on mp_obj_gen_instance_t. typedef struct _mp_obj_gen_instance_native_t { @@ -184,7 +184,7 @@ MP_DEFINE_CONST_OBJ_TYPE( ); #endif -#endif // MICROPY_EMIT_NATIVE +#endif // MICROPY_ENABLE_NATIVE_CODE /******************************************************************************/ /* generator instance */ @@ -192,7 +192,7 @@ MP_DEFINE_CONST_OBJ_TYPE( static void gen_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); - mp_printf(print, "", mp_obj_fun_get_name(MP_OBJ_FROM_PTR(self->code_state.fun_bc)), self); + mp_printf(print, "", mp_obj_fun_bc_get_name(self->code_state.fun_bc), self); } // CIRCUITPY-CHANGE @@ -200,7 +200,7 @@ static void gen_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_pri static void coro_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); - mp_printf(print, "", mp_obj_fun_get_name(MP_OBJ_FROM_PTR(self->code_state.fun_bc)), self); + mp_printf(print, "", mp_obj_fun_bc_get_name(self->code_state.fun_bc), self); } #endif @@ -237,7 +237,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ // If the generator is started, allow sending a value. void *state_start = self->code_state.state - 1; - #if MICROPY_EMIT_NATIVE + #if MICROPY_ENABLE_NATIVE_CODE if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { state_start = ((mp_obj_gen_instance_native_t *)self)->code_state.state - 1; } @@ -259,7 +259,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ mp_vm_return_kind_t ret_kind; - #if MICROPY_EMIT_NATIVE + #if MICROPY_ENABLE_NATIVE_CODE if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { // A native generator. typedef uintptr_t (*mp_fun_native_gen_t)(void *, mp_obj_t); @@ -297,7 +297,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_EXCEPTION: { self->code_state.ip = 0; - #if MICROPY_EMIT_NATIVE + #if MICROPY_ENABLE_NATIVE_CODE if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) { *ret_val = ((mp_obj_gen_instance_native_t *)self)->code_state.state[0]; } else diff --git a/py/objlist.c b/py/objlist.c index a29fc51b104..3edf1e4244b 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -529,27 +529,6 @@ mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items) { return MP_OBJ_FROM_PTR(o); } -void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) { - // CIRCUITPY-CHANGE - mp_obj_list_t *self = native_list(self_in); - *len = self->len; - *items = self->items; -} - -void mp_obj_list_set_len(mp_obj_t self_in, size_t len) { - // trust that the caller knows what it's doing - // TODO realloc if len got much smaller than alloc - mp_obj_list_t *self = MP_OBJ_TO_PTR(self_in); - self->len = len; -} - -void mp_obj_list_store(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { - // CIRCUITPY-CHANGE - mp_obj_list_t *self = native_list(self_in); - size_t i = mp_get_index(self->base.type, self->len, index, false); - self->items[i] = value; -} - /******************************************************************************/ /* list iterator */ @@ -581,3 +560,22 @@ mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur, mp_obj_iter_buf_t * o->cur = cur; return MP_OBJ_FROM_PTR(o); } + +mp_obj_list_t *mp_obj_list_optional_arg(mp_obj_t arg_in, size_t min_len) { + if (arg_in == MP_OBJ_NULL || arg_in == mp_const_none) { + return MP_OBJ_TO_PTR(mp_obj_new_list(min_len, NULL)); + } else { + return mp_obj_list_ensure(arg_in, min_len); + } +} + +mp_obj_list_t *mp_obj_list_ensure(mp_obj_t in, size_t min_len) { + if (!mp_obj_is_type(in, &mp_type_list)) { + mp_raise_TypeError(NULL); + } + mp_obj_list_t *list = MP_OBJ_TO_PTR(in); + if (list->len < min_len) { + mp_raise_ValueError(NULL); + } + return list; +} diff --git a/py/objlist.h b/py/objlist.h index 3fd49baf29f..6d5b3107443 100644 --- a/py/objlist.h +++ b/py/objlist.h @@ -37,8 +37,41 @@ typedef struct _mp_obj_list_t { void mp_obj_list_init(mp_obj_list_t *o, size_t n); mp_obj_t mp_obj_list_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *args); +mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); +mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); +mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); +// CIRCUITPY-CHANGE: public routine +mp_obj_t mp_obj_list_clear(mp_obj_t self_in); // CIRCUITPY-CHANGE: new public functions mp_obj_t mp_obj_list_pop(mp_obj_list_t *self, size_t index); void mp_obj_list_insert(mp_obj_list_t *self, size_t index, mp_obj_t obj); +static inline void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) { + // CIRCUITPY-CHANGE: handle subclassing + mp_obj_list_t *self = (mp_obj_list_t *)MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, MP_OBJ_FROM_PTR(&mp_type_list))); + *len = self->len; + *items = self->items; +} + +static inline void mp_obj_list_set_len(mp_obj_t self_in, size_t len) { + // trust that the caller knows what it's doing + // TODO realloc if len got much smaller than alloc + mp_obj_list_t *self = (mp_obj_list_t *)MP_OBJ_TO_PTR(self_in); + self->len = len; +} + +static inline void mp_obj_list_store(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { + // CIRCUITPY-CHANGE: handle subclassing + mp_obj_list_t *self = (mp_obj_list_t *)MP_OBJ_TO_PTR(mp_obj_cast_to_native_base(self_in, MP_OBJ_FROM_PTR(&mp_type_list))); + size_t i = mp_get_index(self->base.type, self->len, index, false); + self->items[i] = value; +} + +// Helper function for pattern of an optional argument which can be a list of a specified size, and is +// allocated on-demand otherwise +mp_obj_list_t *mp_obj_list_optional_arg(mp_obj_t arg_in, size_t min_len); + +// Ensure provided object is a list of minimum length min_len. Raises TypeError & ValueError otherwise. +mp_obj_list_t *mp_obj_list_ensure(mp_obj_t in, size_t min_len); + #endif // MICROPY_INCLUDED_PY_OBJLIST_H diff --git a/py/objmodule.h b/py/objmodule.h index 221392ccce4..4724270a11c 100644 --- a/py/objmodule.h +++ b/py/objmodule.h @@ -47,4 +47,8 @@ mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible); void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values); +static inline mp_obj_dict_t *mp_obj_module_get_globals(mp_obj_t module) { + return ((mp_obj_module_t *)MP_OBJ_TO_PTR(module))->globals; +} + #endif // MICROPY_INCLUDED_PY_OBJMODULE_H diff --git a/py/objstr.c b/py/objstr.c index bd650bbe18a..d8dd61d04af 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -75,6 +75,14 @@ static void check_is_str_or_bytes(mp_obj_t self_in) { mp_check_self(mp_obj_is_str_or_bytes(self_in)); } +static mp_obj_t make_empty_str_of_type(const mp_obj_type_t *type) { + if (type == &mp_type_str) { + return MP_OBJ_NEW_QSTR(MP_QSTR_); // empty str + } else { + return mp_const_empty_bytes; + } +} + static const byte *get_substring_data(const mp_obj_t obj, size_t n_args, const mp_obj_t *args, size_t *len) { // Get substring data from obj, using args[0,1] to specify start and end indices. GET_STR_DATA_LEN(obj, str, str_len); @@ -289,7 +297,10 @@ static mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size } vstr_t vstr; vstr_init_len(&vstr, len); + // If this config is set then the GC clears all memory, so we don't need to. + #if !MICROPY_GC_CONSERVATIVE_CLEAR memset(vstr.buf, 0, len); + #endif return mp_obj_new_bytes_from_vstr(&vstr); } @@ -395,11 +406,7 @@ mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return MP_OBJ_NULL; // op not supported } if (n <= 0) { - if (lhs_type == &mp_type_str) { - return MP_OBJ_NEW_QSTR(MP_QSTR_); // empty str - } else { - return mp_const_empty_bytes; - } + return make_empty_str_of_type(lhs_type); } // CIRCUITPY-CHANGE: more careful checking of length size_t new_len = mp_seq_multiply_len(lhs_len, n); @@ -920,11 +927,7 @@ static mp_obj_t str_uni_strip(int type, size_t n_args, const mp_obj_t *args) { if (!first_good_char_pos_set) { // string is all whitespace, return '' - if (self_type == &mp_type_str) { - return MP_OBJ_NEW_QSTR(MP_QSTR_); - } else { - return mp_const_empty_bytes; - } + return make_empty_str_of_type(self_type); } assert(last_good_char_pos >= first_good_char_pos); @@ -1858,15 +1861,9 @@ static mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, int direction) { } mp_obj_t result[3]; - if (self_type == &mp_type_str) { - result[0] = MP_OBJ_NEW_QSTR(MP_QSTR_); - result[1] = MP_OBJ_NEW_QSTR(MP_QSTR_); - result[2] = MP_OBJ_NEW_QSTR(MP_QSTR_); - } else { - result[0] = mp_const_empty_bytes; - result[1] = mp_const_empty_bytes; - result[2] = mp_const_empty_bytes; - } + result[0] = make_empty_str_of_type(self_type); + result[1] = make_empty_str_of_type(self_type); + result[2] = make_empty_str_of_type(self_type); if (direction > 0) { result[0] = self_in; @@ -2023,7 +2020,7 @@ mp_obj_t mp_obj_bytes_hex(size_t n_args, const mp_obj_t *args, const mp_obj_type // Code below assumes non-zero buffer length when computing size with // separator, so handle the zero-length case here. if (bufinfo.len == 0) { - return mp_const_empty_bytes; + return make_empty_str_of_type(type); } vstr_t vstr; diff --git a/py/objtemplate.c b/py/objtemplate.c new file mode 100644 index 00000000000..86451350a21 --- /dev/null +++ b/py/objtemplate.c @@ -0,0 +1,395 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Koudai Aono + * Copyright (c) 2026 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/runtime.h" + +#if MICROPY_PY_TSTRINGS + +typedef struct _mp_obj_template_t { + mp_obj_base_t base; + mp_obj_t strings; + mp_obj_t interpolations; +} mp_obj_template_t; + +typedef struct _mp_obj_interpolation_t { + mp_obj_base_t base; + mp_obj_t value; + mp_obj_t expression; + mp_obj_t conversion; + mp_obj_t format_spec; +} mp_obj_interpolation_t; + +static mp_obj_t mp_obj_new_interpolation(mp_obj_t value, mp_obj_t expr, mp_obj_t conv, mp_obj_t spec); + +static mp_obj_t mp_obj_template_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 0, MP_OBJ_FUN_ARGS_MAX, false); + + mp_obj_t strings_obj; + mp_obj_t interpolations_obj; + + if (n_args == 0) { + mp_obj_t empty = MP_OBJ_NEW_QSTR(MP_QSTR_); + strings_obj = mp_obj_new_tuple(1, &empty); + interpolations_obj = mp_obj_new_tuple(0, NULL); + } else { + size_t n_interpolations = 0; + size_t n_str_args = 0; + for (size_t i = 0; i < n_args; i++) { + if (mp_obj_is_exact_type(args[i], &mp_type_interpolation)) { + n_interpolations++; + } else if (mp_obj_is_str(args[i])) { + n_str_args++; + } else { + mp_raise_TypeError(MP_ERROR_TEXT("expected str or Interpolation")); + } + } + + if (n_interpolations == 0) { + if (n_str_args == 1) { + strings_obj = mp_obj_new_tuple(1, &args[0]); + } else { + size_t total_len = 0; + for (size_t i = 0; i < n_args; i++) { + size_t str_len; + (void)mp_obj_str_get_data(args[i], &str_len); + total_len += str_len; + } + vstr_t vstr; + vstr_init(&vstr, total_len); + for (size_t i = 0; i < n_args; i++) { + size_t str_len; + const char *str_data = mp_obj_str_get_data(args[i], &str_len); + vstr_add_strn(&vstr, str_data, str_len); + } + mp_obj_t str_items[1]; + str_items[0] = mp_obj_new_str_from_vstr(&vstr); + strings_obj = mp_obj_new_tuple(1, str_items); + } + interpolations_obj = mp_obj_new_tuple(0, NULL); + } else { + size_t n_strings = n_interpolations + 1; + mp_obj_tuple_t *strings_tuple = mp_obj_malloc_var(mp_obj_tuple_t, items, mp_obj_t, n_strings, &mp_type_tuple); + mp_obj_tuple_t *interpolations_tuple = mp_obj_malloc_var(mp_obj_tuple_t, items, mp_obj_t, n_interpolations, &mp_type_tuple); + strings_tuple->len = n_strings; + interpolations_tuple->len = n_interpolations; + + size_t string_idx = 0; + size_t interp_idx = 0; + mp_obj_t current_str = MP_OBJ_NULL; + bool current_vstr_active = false; + vstr_t current_vstr = {0}; + + for (size_t i = 0; i <= n_args; i++) { + if (i == n_args || mp_obj_is_exact_type(args[i], &mp_type_interpolation)) { + mp_obj_t out_str; + if (current_vstr_active) { + out_str = mp_obj_new_str_from_vstr(¤t_vstr); + current_vstr_active = false; + } else if (current_str != MP_OBJ_NULL) { + out_str = current_str; + } else { + out_str = MP_OBJ_NEW_QSTR(MP_QSTR_); + } + strings_tuple->items[string_idx++] = out_str; + current_str = MP_OBJ_NULL; + if (i < n_args) { + interpolations_tuple->items[interp_idx++] = args[i]; + } + } else { + size_t str_len; + const char *str_data = mp_obj_str_get_data(args[i], &str_len); + if (current_vstr_active) { + vstr_add_strn(¤t_vstr, str_data, str_len); + } else if (current_str == MP_OBJ_NULL) { + current_str = args[i]; + } else { + size_t prev_len; + const char *prev_data = mp_obj_str_get_data(current_str, &prev_len); + vstr_init(¤t_vstr, prev_len + str_len); + vstr_add_strn(¤t_vstr, prev_data, prev_len); + vstr_add_strn(¤t_vstr, str_data, str_len); + current_vstr_active = true; + current_str = MP_OBJ_NULL; + } + } + } + + strings_obj = MP_OBJ_FROM_PTR(strings_tuple); + interpolations_obj = MP_OBJ_FROM_PTR(interpolations_tuple); + } + } + + mp_obj_template_t *self = mp_obj_malloc(mp_obj_template_t, type); + self->strings = strings_obj; + self->interpolations = interpolations_obj; + return MP_OBJ_FROM_PTR(self); +} + +static void mp_obj_template_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + (void)kind; + mp_obj_template_t *self = MP_OBJ_TO_PTR(self_in); + mp_printf(print, "%q(%q=", (qstr)MP_QSTR_Template, (qstr)MP_QSTR_strings); + mp_obj_print_helper(print, self->strings, PRINT_REPR); + mp_printf(print, ", %q=", (qstr)MP_QSTR_interpolations); + mp_obj_print_helper(print, self->interpolations, PRINT_REPR); + mp_print_str(print, ")"); +} + +static mp_obj_t mp_obj_template_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { + mp_obj_template_t *lhs = MP_OBJ_TO_PTR(lhs_in); + + switch (op) { + case MP_BINARY_OP_ADD: { + if (!mp_obj_is_exact_type(rhs_in, &mp_type_template)) { + return MP_OBJ_NULL; // op not supported + } + + mp_obj_template_t *rhs = MP_OBJ_TO_PTR(rhs_in); + + mp_obj_tuple_t *lhs_strings = MP_OBJ_TO_PTR(lhs->strings); + mp_obj_tuple_t *lhs_interps = MP_OBJ_TO_PTR(lhs->interpolations); + mp_obj_tuple_t *rhs_strings = MP_OBJ_TO_PTR(rhs->strings); + mp_obj_tuple_t *rhs_interps = MP_OBJ_TO_PTR(rhs->interpolations); + + size_t new_strings_len = lhs_strings->len + rhs_strings->len - 1; + size_t new_interps_len = lhs_interps->len + rhs_interps->len; + + // Create tuples directly to avoid GC issues. + mp_obj_tuple_t *new_strings_tuple = mp_obj_malloc_var(mp_obj_tuple_t, items, mp_obj_t, new_strings_len, &mp_type_tuple); + mp_obj_tuple_t *new_interps_tuple = mp_obj_malloc_var(mp_obj_tuple_t, items, mp_obj_t, new_interps_len, &mp_type_tuple); + new_strings_tuple->len = new_strings_len; + new_interps_tuple->len = new_interps_len; + + // Copy all but the last string from lhs. + for (size_t i = 0; i < lhs_strings->len - 1; i++) { + new_strings_tuple->items[i] = lhs_strings->items[i]; + } + + // Merge last string from lhs with first string from rhs. + size_t lhs_last_len, rhs_first_len; + const char *lhs_last_str = mp_obj_str_get_data(lhs_strings->items[lhs_strings->len - 1], &lhs_last_len); + const char *rhs_first_str = mp_obj_str_get_data(rhs_strings->items[0], &rhs_first_len); + + vstr_t vstr; + vstr_init(&vstr, lhs_last_len + rhs_first_len); + vstr_add_strn(&vstr, lhs_last_str, lhs_last_len); + vstr_add_strn(&vstr, rhs_first_str, rhs_first_len); + new_strings_tuple->items[lhs_strings->len - 1] = mp_obj_new_str_from_vstr(&vstr); + + // Copy remaining strings from rhs. + for (size_t i = 1; i < rhs_strings->len; i++) { + new_strings_tuple->items[lhs_strings->len - 1 + i] = rhs_strings->items[i]; + } + + // Copy interpolations from both sides. + for (size_t i = 0; i < lhs_interps->len; i++) { + new_interps_tuple->items[i] = lhs_interps->items[i]; + } + + for (size_t i = 0; i < rhs_interps->len; i++) { + new_interps_tuple->items[lhs_interps->len + i] = rhs_interps->items[i]; + } + + mp_obj_template_t *result = mp_obj_malloc(mp_obj_template_t, &mp_type_template); + result->strings = MP_OBJ_FROM_PTR(new_strings_tuple); + result->interpolations = MP_OBJ_FROM_PTR(new_interps_tuple); + return MP_OBJ_FROM_PTR(result); + } + + default: + return MP_OBJ_NULL; // op not supported + } +} + +static void mp_obj_template_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + mp_obj_template_t *self = MP_OBJ_TO_PTR(self_in); + + if (dest[0] == MP_OBJ_NULL) { + // Load attribute. + if (attr == MP_QSTR_strings) { + dest[0] = self->strings; + } else if (attr == MP_QSTR_interpolations) { + dest[0] = self->interpolations; + } else if (attr == MP_QSTR_values) { + mp_obj_tuple_t *interps = MP_OBJ_TO_PTR(self->interpolations); + mp_obj_tuple_t *values_tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(interps->len, NULL)); + for (size_t i = 0; i < interps->len; i++) { + mp_obj_interpolation_t *interp = MP_OBJ_TO_PTR(interps->items[i]); + values_tuple->items[i] = interp->value; + } + dest[0] = MP_OBJ_FROM_PTR(values_tuple); + } + } +} + +typedef struct _mp_obj_template_iter_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + mp_obj_t template; + size_t index; +} mp_obj_template_iter_t; + +static mp_obj_t template_iter_iternext(mp_obj_t self_in) { + mp_obj_template_iter_t *self = MP_OBJ_TO_PTR(self_in); + mp_obj_template_t *tmpl = MP_OBJ_TO_PTR(self->template); + mp_obj_tuple_t *strings = MP_OBJ_TO_PTR(tmpl->strings); + mp_obj_tuple_t *interps = MP_OBJ_TO_PTR(tmpl->interpolations); + + while (self->index < strings->len + interps->len) { + if ((self->index & 1) == 0) { + // A string. + mp_obj_t str_obj = strings->items[self->index++ / 2]; + size_t str_len; + mp_obj_str_get_data(str_obj, &str_len); + if (str_len > 0) { + return str_obj; + } + } else { + // An interpolation. + return interps->items[self->index++ / 2]; + } + } + + return MP_OBJ_STOP_ITERATION; +} + +static mp_obj_t mp_obj_template_iter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_template_iter_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_template_iter_t *iter = (mp_obj_template_iter_t *)iter_buf; + iter->base.type = &mp_type_polymorph_iter; + iter->iternext = template_iter_iternext; + iter->template = self_in; + iter->index = 0; + return MP_OBJ_FROM_PTR(iter); +} + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_template, + MP_QSTR_Template, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_template_make_new, + print, mp_obj_template_print, + binary_op, mp_obj_template_binary_op, + attr, mp_obj_template_attr, + iter, mp_obj_template_iter + ); + +mp_obj_t mp_obj_new_template(size_t n_args, const mp_obj_t *args) { + mp_obj_template_t *o = mp_obj_malloc(mp_obj_template_t, &mp_type_template); + o->strings = args[0]; + if (n_args == 2) { + // Unpack interpolations from second argument (which is a tuple). + mp_obj_t *iargs; + mp_obj_get_array(args[1], &n_args, &iargs); + args = iargs; + } else { + // Unpack interpolations directly from arguments. + --n_args; + ++args; + } + size_t n_interpolations = n_args / 4; + mp_obj_tuple_t *interpolations = MP_OBJ_TO_PTR(mp_obj_new_tuple(n_interpolations, NULL)); + for (size_t i = 0; i < n_interpolations; ++i) { + interpolations->items[i] = mp_obj_new_interpolation(args[i * 4], args[i * 4 + 1], args[i * 4 + 2], args[i * 4 + 3]); + } + o->interpolations = MP_OBJ_FROM_PTR(interpolations); + return MP_OBJ_FROM_PTR(o); +} + +///////////////////////////////////////////////////////////////// + +static mp_obj_t mp_obj_interpolation_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_value, ARG_expression, ARG_conversion, ARG_format_spec }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_value, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_expression, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR_)} }, + { MP_QSTR_conversion, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, + { MP_QSTR_format_spec, MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR_)} }, + }; + + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); + + mp_obj_interpolation_t *self = mp_obj_malloc(mp_obj_interpolation_t, &mp_type_interpolation); + self->value = args[ARG_value].u_obj; + self->expression = args[ARG_expression].u_obj; + self->conversion = args[ARG_conversion].u_obj; + self->format_spec = args[ARG_format_spec].u_obj; + + return MP_OBJ_FROM_PTR(self); +} + +static void mp_obj_interpolation_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { + (void)kind; + mp_obj_interpolation_t *self = MP_OBJ_TO_PTR(self_in); + mp_printf(print, "%q(", (qstr)MP_QSTR_Interpolation); + mp_obj_print_helper(print, self->value, PRINT_REPR); + mp_print_str(print, ", "); + mp_obj_print_helper(print, self->expression, PRINT_REPR); + mp_print_str(print, ", "); + mp_obj_print_helper(print, self->conversion, PRINT_REPR); + mp_print_str(print, ", "); + mp_obj_print_helper(print, self->format_spec, PRINT_REPR); + mp_print_str(print, ")"); +} + +static void mp_obj_interpolation_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + mp_obj_interpolation_t *self = MP_OBJ_TO_PTR(self_in); + + if (dest[0] == MP_OBJ_NULL) { + // load attribute + if (attr == MP_QSTR_value) { + dest[0] = self->value; + } else if (attr == MP_QSTR_expression) { + dest[0] = self->expression; + } else if (attr == MP_QSTR_conversion) { + dest[0] = self->conversion; + } else if (attr == MP_QSTR_format_spec) { + dest[0] = self->format_spec; + } + } +} + +static mp_obj_t mp_obj_new_interpolation(mp_obj_t value, mp_obj_t expression, mp_obj_t conversion, mp_obj_t format_spec) { + mp_obj_interpolation_t *o = mp_obj_malloc(mp_obj_interpolation_t, &mp_type_interpolation); + o->value = value; + o->expression = expression; + o->conversion = conversion; + o->format_spec = format_spec; + return MP_OBJ_FROM_PTR(o); +} + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_interpolation, + MP_QSTR_Interpolation, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_interpolation_make_new, + print, mp_obj_interpolation_print, + attr, mp_obj_interpolation_attr + ); + +#endif // MICROPY_PY_TSTRINGS diff --git a/py/objtuple.c b/py/objtuple.c index b2ccbbb2990..fcfb8e0aa1f 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -264,19 +264,6 @@ mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items) { return MP_OBJ_FROM_PTR(o); } -void mp_obj_tuple_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) { - assert(mp_obj_is_tuple_compatible(self_in)); - mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); - *len = self->len; - *items = &self->items[0]; -} - -void mp_obj_tuple_del(mp_obj_t self_in) { - assert(mp_obj_is_type(self_in, &mp_type_tuple)); - mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); - m_del_var(mp_obj_tuple_t, items, mp_obj_t, self->len, self); -} - /******************************************************************************/ /* tuple iterator */ diff --git a/py/objtuple.h b/py/objtuple.h index 783522a6222..f0b081faad6 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -28,6 +28,9 @@ #include "py/obj.h" +// type check is done on getiter method to allow tuple, namedtuple, attrtuple +#define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) + typedef struct _mp_obj_tuple_t { mp_obj_base_t base; size_t len; @@ -49,6 +52,13 @@ mp_obj_t mp_obj_tuple_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs); mp_obj_t mp_obj_tuple_subscr(mp_obj_t base, mp_obj_t index, mp_obj_t value); mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf); +static inline void mp_obj_tuple_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) { + assert(mp_obj_is_tuple_compatible(self_in)); + mp_obj_tuple_t *self = (mp_obj_tuple_t *)MP_OBJ_TO_PTR(self_in); + *len = self->len; + *items = &self->items[0]; +} + extern const mp_obj_type_t mp_type_attrtuple; // CIRCUITPY-CHANGE @@ -68,7 +78,4 @@ void mp_obj_attrtuple_print_helper(const mp_print_t *print, const qstr *fields, mp_obj_t mp_obj_new_attrtuple(const qstr *fields, size_t n, const mp_obj_t *items); -// type check is done on getiter method to allow tuple, namedtuple, attrtuple -#define mp_obj_is_tuple_compatible(o) (MP_OBJ_TYPE_GET_SLOT_OR_NULL(mp_obj_get_type(o), iter) == mp_obj_tuple_getiter) - #endif // MICROPY_INCLUDED_PY_OBJTUPLE_H diff --git a/py/objtype.c b/py/objtype.c index b1a984b50e0..e0b35a55d33 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -112,7 +112,7 @@ static mp_obj_t native_base_init_wrapper(size_t n_args, const mp_obj_t *pos_args return mp_const_none; } -static MP_DEFINE_CONST_FUN_OBJ_KW(native_base_init_wrapper_obj, 1, native_base_init_wrapper); +MP_DEFINE_CONST_FUN_OBJ_KW(mp_native_base_init_wrapper_obj, 1, native_base_init_wrapper); #if !MICROPY_CPYTHON_COMPAT static @@ -126,7 +126,7 @@ mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_ // object. It doesn't matter which object, so long as it can be uniquely // distinguished from a native class that is initialised. if (num_native_bases != 0) { - o->subobj[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + o->subobj[0] = MP_OBJ_FROM_PTR(&mp_native_base_init_wrapper_obj); } return o; } @@ -137,7 +137,7 @@ mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_ // code so it must call this method to ensure that the given object has been __init__'d and is // valid. void mp_obj_assert_native_inited(mp_obj_t native_object) { - if (native_object == MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj)) { + if (native_object == MP_OBJ_FROM_PTR(&mp_native_base_init_wrapper_obj)) { mp_raise_NotImplementedError(MP_ERROR_TEXT("Call super().__init__() before accessing native object.")); } } @@ -395,7 +395,7 @@ static mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_arg // If the type had a native base that was not explicitly initialised // (constructed) by the Python __init__() method then construct it now. - if (native_base != NULL && o->subobj[0] == MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj)) { + if (native_base != NULL && o->subobj[0] == MP_OBJ_FROM_PTR(&mp_native_base_init_wrapper_obj)) { o->subobj[0] = MP_OBJ_TYPE_GET_SLOT(native_base, make_new)(native_base, n_args, n_kw, args); } @@ -1447,7 +1447,7 @@ static void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] != MP_OBJ_NULL) { if (dest[0] == MP_OBJ_SENTINEL) { // Looked up native __init__ so defer to it - dest[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + dest[0] = MP_OBJ_FROM_PTR(&mp_native_base_init_wrapper_obj); dest[1] = self->obj; // CIRCUITPY-CHANGE: better support for properties } else { diff --git a/py/objtype.h b/py/objtype.h index 07a8e1cffd9..098b7699da9 100644 --- a/py/objtype.h +++ b/py/objtype.h @@ -55,4 +55,7 @@ mp_obj_t mp_obj_instance_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf); // CIRCUITPY-CHANGE: addition void mp_obj_assert_native_inited(mp_obj_t native_object); +// upstream v1.28: exposed so py/objexcept.c can reference it directly +MP_DECLARE_CONST_FUN_OBJ_KW(mp_native_base_init_wrapper_obj); + #endif // MICROPY_INCLUDED_PY_OBJTYPE_H diff --git a/py/objzip.c b/py/objzip.c index dd2b39ee071..300448b665f 100644 --- a/py/objzip.c +++ b/py/objzip.c @@ -50,19 +50,20 @@ static mp_obj_t zip_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ static mp_obj_t zip_iternext(mp_obj_t self_in) { mp_check_self(mp_obj_is_type(self_in, &mp_type_zip)); mp_obj_zip_t *self = MP_OBJ_TO_PTR(self_in); - if (self->n_iters == 0) { - return MP_OBJ_STOP_ITERATION; - } - mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(self->n_iters, NULL)); - + mp_obj_tuple_t *tuple = NULL; for (size_t i = 0; i < self->n_iters; i++) { mp_obj_t next = mp_iternext(self->iters[i]); if (next == MP_OBJ_STOP_ITERATION) { - mp_obj_tuple_del(MP_OBJ_FROM_PTR(tuple)); return MP_OBJ_STOP_ITERATION; } + if (tuple == NULL) { + tuple = MP_OBJ_TO_PTR(mp_obj_new_tuple(self->n_iters, NULL)); + } tuple->items[i] = next; } + if (tuple == NULL) { + return MP_OBJ_STOP_ITERATION; + } return MP_OBJ_FROM_PTR(tuple); } diff --git a/py/persistentcode.c b/py/persistentcode.c index 5c3de3174bc..7fea1465975 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -76,7 +76,7 @@ typedef struct _bytecode_prelude_t { static int read_byte(mp_reader_t *reader); static size_t read_uint(mp_reader_t *reader); -#if MICROPY_EMIT_MACHINE_CODE +#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA @@ -229,7 +229,7 @@ static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *dat static mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE if (obj_type == MP_PERSISTENT_OBJ_FUN_TABLE) { return MP_OBJ_FROM_PTR(&mp_fun_table); } else @@ -302,14 +302,14 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co bool has_children = !!(kind_len & 4); size_t fun_data_len = kind_len >> 3; - #if !MICROPY_EMIT_MACHINE_CODE + #if !(MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) if (kind != MP_CODE_BYTECODE) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } #endif uint8_t *fun_data = NULL; - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE size_t prelude_offset = 0; mp_uint_t native_scope_flags = 0; mp_uint_t native_n_pos_args = 0; @@ -329,7 +329,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co read_bytes(reader, fun_data, fun_data_len); } - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE } else { // Allocate memory for native data and load it size_t fun_alloc; @@ -356,7 +356,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co size_t n_children = 0; mp_raw_code_t **children = NULL; - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE // Load optional BSS/rodata for viper. uint8_t *rodata = NULL; uint8_t *bss = NULL; @@ -411,7 +411,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #endif scope_flags); - #if MICROPY_EMIT_MACHINE_CODE + #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE } else { const uint8_t *prelude_ptr = NULL; #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE @@ -856,8 +856,28 @@ static mp_opcode_t mp_opcode_decode(const uint8_t *ip) { return op; } -mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode) { - const uint8_t *fun_data = bytecode; +typedef struct _mp_raw_code_simplified_t { + const uint8_t *fun_data; + struct _mp_raw_code_simplified_t *children; + size_t fun_data_len; + size_t n_children; +} mp_raw_code_simplified_t; + +static void proto_fun_to_raw_code_simplified(const void *proto_fun, bit_vector_t *qstr_table_used, bit_vector_t *obj_table_used, mp_raw_code_simplified_t *rcs) { + const uint8_t *fun_data; + mp_raw_code_t **children; + if (mp_proto_fun_is_bytecode(proto_fun)) { + fun_data = proto_fun; + children = NULL; + } else { + const mp_raw_code_t *rc = proto_fun; + if (rc->kind != MP_CODE_BYTECODE) { + mp_raise_ValueError(MP_ERROR_TEXT("function must be bytecode")); + } + fun_data = rc->fun_data; + children = rc->children; + } + const uint8_t *fun_data_top = fun_data + gc_nbytes(fun_data); // Extract function information. @@ -865,20 +885,12 @@ mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, cons MP_BC_PRELUDE_SIG_DECODE(ip); MP_BC_PRELUDE_SIZE_DECODE(ip); - // Track the qstrs used by the function. - bit_vector_t qstr_table_used; - bit_vector_init(&qstr_table_used); - - // Track the objects used by the function. - bit_vector_t obj_table_used; - bit_vector_init(&obj_table_used); - const byte *ip_names = ip; mp_uint_t simple_name = mp_decode_uint(&ip_names); - bit_vector_set(&qstr_table_used, simple_name); + bit_vector_set(qstr_table_used, simple_name); for (size_t i = 0; i < n_pos_args + n_kwonly_args; ++i) { mp_uint_t arg_name = mp_decode_uint(&ip_names); - bit_vector_set(&qstr_table_used, arg_name); + bit_vector_set(qstr_table_used, arg_name); } // Skip pass source code info and cell info. @@ -886,20 +898,74 @@ mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, cons ip += n_info + n_cell; // Decode bytecode. + size_t n_children = 0; while (ip < fun_data_top) { mp_opcode_t op = mp_opcode_decode(ip); if (op.opcode == MP_BC_BASE_RESERVED) { // End of opcodes. fun_data_top = ip; - } else if (op.opcode == MP_BC_LOAD_CONST_OBJ) { - bit_vector_set(&obj_table_used, op.arg); } else if (op.format == MP_BC_FORMAT_QSTR) { - bit_vector_set(&qstr_table_used, op.arg); + bit_vector_set(qstr_table_used, op.arg); + } else if (op.opcode == MP_BC_LOAD_CONST_OBJ) { + bit_vector_set(obj_table_used, op.arg); + } else if (op.opcode == MP_BC_MAKE_FUNCTION + || op.opcode == MP_BC_MAKE_FUNCTION_DEFARGS + || op.opcode == MP_BC_MAKE_CLOSURE + || op.opcode == MP_BC_MAKE_CLOSURE_DEFARGS) { + if ((mp_uint_t)op.arg + 1 > n_children) { + n_children = (mp_uint_t)op.arg + 1; + } } ip += op.size; } - mp_uint_t fun_data_len = fun_data_top - fun_data; + rcs->fun_data = fun_data; + rcs->fun_data_len = fun_data_top - fun_data; + rcs->n_children = n_children; + rcs->children = NULL; + + if (n_children) { + rcs->children = m_new(mp_raw_code_simplified_t, n_children); + for (size_t i = 0; i < n_children; ++i) { + proto_fun_to_raw_code_simplified(children[i], qstr_table_used, obj_table_used, &rcs->children[i]); + } + } +} + +static void save_raw_code_simplified(mp_print_t *print, const mp_raw_code_simplified_t *rcs) { + // Save function kind and data length. + mp_print_uint(print, rcs->fun_data_len << 3 | (rcs->n_children != 0) << 2); + + // Save function code. + mp_print_bytes(print, rcs->fun_data, rcs->fun_data_len); + + // Save (and free) children. + if (rcs->n_children) { + mp_print_uint(print, rcs->n_children); + for (size_t i = 0; i < rcs->n_children; ++i) { + save_raw_code_simplified(print, &rcs->children[i]); + } + m_del(mp_raw_code_simplified_t, rcs->children, rcs->n_children); + } +} + +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, mp_proto_fun_t proto_fun) { + // Track the qstrs used by the function. + bit_vector_t qstr_table_used; + bit_vector_init(&qstr_table_used); + + // Track the objects used by the function. + bit_vector_t obj_table_used; + bit_vector_init(&obj_table_used); + + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_FULL + // Make sure the filename appears in the qstr table. + bit_vector_set(&qstr_table_used, 0); + #endif + + // Convert function into a simplified raw code tree. + mp_raw_code_simplified_t rcs; + proto_fun_to_raw_code_simplified(proto_fun, &qstr_table_used, &obj_table_used, &rcs); mp_print_t print; vstr_t vstr; @@ -934,11 +1000,8 @@ mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, cons bit_vector_clear(&qstr_table_used); bit_vector_clear(&obj_table_used); - // Save function kind and data length. - mp_print_uint(&print, fun_data_len << 3); - - // Save function code. - mp_print_bytes(&print, fun_data, fun_data_len); + // Save the bytecode data (also free the simplified raw code tree at the same time). + save_raw_code_simplified(&print, &rcs); // Create and return bytes representing the .mpy data. return mp_obj_new_bytes_from_vstr(&vstr); diff --git a/py/persistentcode.h b/py/persistentcode.h index 85668e608c8..9a9d8605641 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -53,31 +53,44 @@ #define MPY_FEATURE_DECODE_ARCH(feat) (((feat) >> 2) & 0x2F) // Define the host architecture -#if MICROPY_EMIT_X86 - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86) -#elif MICROPY_EMIT_X64 - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X64) -#elif MICROPY_EMIT_THUMB - #if defined(__thumb2__) - #if defined(__ARM_FP) && (__ARM_FP & 8) == 8 - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMDP) - #elif defined(__ARM_FP) && (__ARM_FP & 4) == 4 - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMSP) +#if MICROPY_PERSISTENT_CODE_LOAD_NATIVE + #if defined(__i386__) || defined(_M_IX86) + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86) + #elif defined(__x86_64__) || defined(_M_X64) + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X64) + #elif defined(__thumb2__) || defined(__thumb__) + #if defined(__thumb2__) + #if defined(__ARM_FP) && (__ARM_FP & 8) == 8 + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMDP) + #elif defined(__ARM_FP) && (__ARM_FP & 4) == 4 + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMSP) + #else + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EM) + #endif #else - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EM) + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6M) + #endif + #define MPY_FEATURE_ARCH_TEST(x) (MP_NATIVE_ARCH_ARMV6M <= (x) && (x) <= MPY_FEATURE_ARCH) + #elif defined(__arm__) + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6) + #elif defined(__xtensa__) + #include + #if XCHAL_HAVE_WINDOWED + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSAWIN) + #else + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSA) + #endif + #elif defined(__riscv) + #if __riscv_xlen == 32 + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_RV32IMC) + #elif __riscv_xlen == 64 + #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_RV64IMC) + #else + #error "Unsupported RISC-V architecture." #endif #else - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6M) + #error "Unsupported native architecture." #endif - #define MPY_FEATURE_ARCH_TEST(x) (MP_NATIVE_ARCH_ARMV6M <= (x) && (x) <= MPY_FEATURE_ARCH) -#elif MICROPY_EMIT_ARM - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6) -#elif MICROPY_EMIT_XTENSA - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSA) -#elif MICROPY_EMIT_XTENSAWIN - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSAWIN) -#elif MICROPY_EMIT_RV32 - #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_RV32IMC) #else #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_NONE) #endif @@ -131,7 +144,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *ctx); void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename); -mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode); +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, mp_proto_fun_t proto_fun); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); diff --git a/py/profile.c b/py/profile.c index b5a0c54728c..0523a1806e7 100644 --- a/py/profile.c +++ b/py/profile.c @@ -173,7 +173,7 @@ static mp_obj_t mp_prof_callback_invoke(mp_obj_t callback, prof_callback_args_t mp_prof_is_executing = false; if (MP_STATE_THREAD(mp_pending_exception) != MP_OBJ_NULL) { - mp_handle_pending(true); + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); } return top; } diff --git a/py/py.cmake b/py/py.cmake index b99d2d24b5d..6888c170214 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -49,10 +49,12 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/modio.c ${MICROPY_PY_DIR}/modmath.c ${MICROPY_PY_DIR}/modmicropython.c + ${MICROPY_PY_DIR}/modstring.c ${MICROPY_PY_DIR}/modstruct.c ${MICROPY_PY_DIR}/modsys.c ${MICROPY_PY_DIR}/modthread.c ${MICROPY_PY_DIR}/moderrno.c + ${MICROPY_PY_DIR}/modweakref.c ${MICROPY_PY_DIR}/mpprint.c ${MICROPY_PY_DIR}/mpstate.c ${MICROPY_PY_DIR}/mpz.c @@ -106,6 +108,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/objstr.c ${MICROPY_PY_DIR}/objstringio.c ${MICROPY_PY_DIR}/objstrunicode.c + ${MICROPY_PY_DIR}/objtemplate.c # CIRCUITPY-CHANGE: add objtraceback.c ${MICROPY_PY_DIR}/objtraceback.c ${MICROPY_PY_DIR}/objtuple.c diff --git a/py/py.mk b/py/py.mk index 21be07c794f..f5365ed6f78 100644 --- a/py/py.mk +++ b/py/py.mk @@ -198,6 +198,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ objstr.o \ objstrunicode.o \ objstringio.o \ + objtemplate.o \ objtraceback.o \ objtuple.o \ objtype.o \ @@ -217,10 +218,12 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ modmath.o \ modcmath.o \ modmicropython.o \ + modstring.o \ modstruct.o \ modsys.o \ moderrno.o \ modthread.o \ + modweakref.o \ vm.o \ bc.o \ showbc.o \ diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 01e784efc3a..afa57afa248 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -68,3 +68,7 @@ Q(utf-8) #if MICROPY_MODULE_FROZEN Q(.frozen) #endif + +#if MICROPY_PY_TSTRINGS +Q(string.templatelib) +#endif diff --git a/py/runtime.c b/py/runtime.c index 81abc3de875..51d536efd86 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -134,7 +134,7 @@ void mp_init(void) { MP_STATE_VM(mp_module_builtins_override_dict) = NULL; #endif - #if MICROPY_EMIT_MACHINE_CODE && (MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA) + #if (MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) && (MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA) MP_STATE_VM(persistent_code_root_pointers) = MP_OBJ_NULL; #endif @@ -191,6 +191,10 @@ void mp_init(void) { MP_STATE_VM(usbd) = MP_OBJ_NULL; #endif + #if MICROPY_PY_WEAKREF + mp_map_init(&MP_STATE_VM(mp_weakref_map), 0); + #endif + #if MICROPY_PY_THREAD_GIL mp_thread_mutex_init(&MP_STATE_VM(gil_mutex)); #endif @@ -1604,7 +1608,7 @@ mp_obj_t mp_make_raise_obj(mp_obj_t o) { } if (mp_obj_is_exception_instance(o)) { - // o is an instance of an exception, so use it as the exception + // o is a fully-constructed instance of an exception, so use it as the exception return o; } else { // o cannot be used as an exception, so return a type error (which will be raised by the caller) @@ -1689,19 +1693,19 @@ MP_NOINLINE mp_obj_t mp_import_from(mp_obj_t module, qstr name) { void mp_import_all(mp_obj_t module) { DEBUG_printf("import all %p\n", module); - mp_map_t *map = &mp_obj_module_get_globals(module)->map; + mp_obj_t dest[2]; #if MICROPY_MODULE___ALL__ - mp_map_elem_t *elem = mp_map_lookup(map, MP_OBJ_NEW_QSTR(MP_QSTR___all__), MP_MAP_LOOKUP); - if (elem != NULL) { + + mp_load_method_maybe(module, MP_QSTR___all__, dest); + if (dest[0] != MP_OBJ_NULL) { // When __all__ is defined, we must explicitly load all specified // symbols, possibly invoking the module __getattr__ function size_t len; mp_obj_t *items; - mp_obj_get_array(elem->value, &len, &items); + mp_obj_get_array(dest[0], &len, &items); for (size_t i = 0; i < len; i++) { qstr qname = mp_obj_str_get_qstr(items[i]); - mp_obj_t dest[2]; mp_load_method(module, qname, dest); mp_store_name(qname, dest[0]); } @@ -1709,8 +1713,19 @@ void mp_import_all(mp_obj_t module) { } #endif + #if MICROPY_CPYTHON_COMPAT + // Load the dict from the module. In MicroPython, if __dict__ is + // available then it always returns a native mp_obj_dict_t instance. + mp_load_method(module, MP_QSTR___dict__, dest); + #else + // Without MICROPY_CPYTHON_COMPAT __dict__ is not available, so just + // assume the given module is actually an mp_obj_module_t instance. + dest[0] = MP_OBJ_FROM_PTR(mp_obj_module_get_globals(module)); + #endif + // By default, the set of public names includes all names found in the module's // namespace which do not begin with an underscore character ('_') + mp_map_t *map = mp_obj_dict_get_map(dest[0]); for (size_t i = 0; i < map->alloc; i++) { if (mp_map_slot_is_filled(map, i)) { // Entry in module global scope may be generated programmatically diff --git a/py/runtime.h b/py/runtime.h index 22ed9c3eaf9..38fb05db568 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -59,10 +59,11 @@ typedef enum { MP_ARG_KW_ONLY = 0x200, } mp_arg_flag_t; +// These first two enum values match the original signature of `mp_handle_pending(bool)`. typedef enum { + MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS = false, + MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS = true, MP_HANDLE_PENDING_CALLBACKS_ONLY, - MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS, - MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS, } mp_handle_pending_behaviour_t; typedef union _mp_arg_val_t { @@ -114,13 +115,7 @@ void mp_sched_keyboard_interrupt(void); void mp_sched_vm_abort(void); #endif -void mp_handle_pending_internal(mp_handle_pending_behaviour_t behavior); - -static inline void mp_handle_pending(bool raise_exc) { - mp_handle_pending_internal(raise_exc ? - MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS : - MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS); -} +void mp_handle_pending(mp_handle_pending_behaviour_t behavior); #if MICROPY_ENABLE_SCHEDULER void mp_sched_lock(void); diff --git a/py/scheduler.c b/py/scheduler.c index ee9cd96b331..e75927dbe0b 100644 --- a/py/scheduler.c +++ b/py/scheduler.c @@ -222,7 +222,7 @@ MP_REGISTER_ROOT_POINTER(mp_sched_item_t sched_queue[MICROPY_SCHEDULER_DEPTH]); // Called periodically from the VM or from "waiting" code (e.g. sleep) to // process background tasks and pending exceptions (e.g. KeyboardInterrupt). -void mp_handle_pending_internal(mp_handle_pending_behaviour_t behavior) { +void mp_handle_pending(mp_handle_pending_behaviour_t behavior) { bool handle_exceptions = (behavior != MP_HANDLE_PENDING_CALLBACKS_ONLY); bool raise_exceptions = (behavior == MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); @@ -272,7 +272,7 @@ void mp_event_handle_nowait(void) { #else // Process any port layer (non-blocking) events. MICROPY_INTERNAL_EVENT_HOOK; - mp_handle_pending(true); + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); #endif } diff --git a/py/stream.c b/py/stream.c index 520422cc92f..b14a34d2658 100644 --- a/py/stream.c +++ b/py/stream.c @@ -107,6 +107,17 @@ const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) { mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("stream operation not supported")); } +static MP_NORETURN void mp_stream_raise_error(mp_obj_t stream, int error) { + #if MICROPY_STREAMS_DELEGATE_ERROR + const mp_stream_p_t *stream_p = mp_get_stream(stream); + if (stream_p->ioctl != NULL) { + int err; + stream_p->ioctl(stream, MP_STREAM_RAISE_ERROR, error, &err); + } + #endif + mp_raise_OSError(error); +} + static mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) { // What to do if sz < -1? Python docs don't specify this case. // CPython does a readall, but here we silently let negatives through, @@ -150,7 +161,7 @@ static mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl } break; } - mp_raise_OSError(error); + mp_stream_raise_error(args[0], error); } if (out_sz < more_bytes) { @@ -218,7 +229,7 @@ static mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte fl // this as EOF. return mp_const_none; } - mp_raise_OSError(error); + mp_stream_raise_error(args[0], error); } else { vstr.len = out_sz; if (stream_p->is_text) { @@ -249,7 +260,7 @@ mp_obj_t mp_stream_write(mp_obj_t self_in, const void *buf, size_t len, byte fla // no single byte could be readily written to it." return mp_const_none; } - mp_raise_OSError(error); + mp_stream_raise_error(self_in, error); } else { return MP_OBJ_NEW_SMALL_INT(out_sz); } @@ -326,7 +337,7 @@ static mp_obj_t stream_readall(mp_obj_t self_in) { } break; } - mp_raise_OSError(error); + mp_stream_raise_error(self_in, error); } if (out_sz == 0) { break; @@ -384,7 +395,7 @@ static mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) goto done; } } - mp_raise_OSError(error); + mp_stream_raise_error(args[0], error); } if (out_sz == 0) { done: @@ -434,7 +445,7 @@ mp_obj_t mp_stream_close(mp_obj_t stream) { int error; mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); if (res == MP_STREAM_ERROR) { - mp_raise_OSError(error); + mp_stream_raise_error(stream, error); } return mp_const_none; } @@ -462,7 +473,7 @@ static mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { int error; mp_off_t res = mp_stream_seek(args[0], offset, whence, &error); if (res == (mp_off_t)-1) { - mp_raise_OSError(error); + mp_stream_raise_error(args[0], error); } // TODO: Could be uint64 @@ -484,7 +495,7 @@ mp_obj_t mp_stream_flush(mp_obj_t self) { int error; mp_uint_t res = stream_p->ioctl(self, MP_STREAM_FLUSH, 0, &error); if (res == MP_STREAM_ERROR) { - mp_raise_OSError(error); + mp_stream_raise_error(self, error); } return mp_const_none; } @@ -505,7 +516,7 @@ static mp_obj_t stream_ioctl(size_t n_args, const mp_obj_t *args) { int error; mp_uint_t res = stream_p->ioctl(args[0], mp_obj_get_int(args[1]), val, &error); if (res == MP_STREAM_ERROR) { - mp_raise_OSError(error); + mp_stream_raise_error(args[0], error); } return mp_obj_new_int(res); diff --git a/py/stream.h b/py/stream.h index 6e60e849b5f..153800c3808 100644 --- a/py/stream.h +++ b/py/stream.h @@ -27,6 +27,7 @@ #ifndef MICROPY_INCLUDED_PY_STREAM_H #define MICROPY_INCLUDED_PY_STREAM_H +#include "py/mpconfig.h" #include "py/obj.h" // CIRCUITPY-CHANGE #include "py/proto.h" @@ -35,17 +36,20 @@ #define MP_STREAM_ERROR ((mp_uint_t)-1) // Stream ioctl request codes -#define MP_STREAM_FLUSH (1) -#define MP_STREAM_SEEK (2) -#define MP_STREAM_POLL (3) -#define MP_STREAM_CLOSE (4) -#define MP_STREAM_TIMEOUT (5) // Get/set timeout (single op) -#define MP_STREAM_GET_OPTS (6) // Get stream options -#define MP_STREAM_SET_OPTS (7) // Set stream options -#define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options -#define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options -#define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file -#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file +#define MP_STREAM_FLUSH (1) +#define MP_STREAM_SEEK (2) +#define MP_STREAM_POLL (3) +#define MP_STREAM_CLOSE (4) +#define MP_STREAM_TIMEOUT (5) // Get/set timeout (single op) +#define MP_STREAM_GET_OPTS (6) // Get stream options +#define MP_STREAM_SET_OPTS (7) // Set stream options +#define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options +#define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options +#define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file +#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file +#if MICROPY_STREAMS_DELEGATE_ERROR +#define MP_STREAM_RAISE_ERROR (12) // Raise an error with detailed error string +#endif // These poll ioctl values are compatible with Linux #define MP_STREAM_POLL_RD (0x0001) @@ -72,8 +76,12 @@ struct mp_stream_seek_t { typedef struct _mp_stream_p_t { // CIRCUITPY-CHANGE MP_PROTOCOL_HEAD - // On error, functions should return MP_STREAM_ERROR and fill in *errcode (values - // are implementation-dependent, but will be exposed to user, e.g. via exception). + // On error, functions should return MP_STREAM_ERROR and fill in *errcode + // (values are are implementation-dependent, but will be exposed to user). + // If MICROPY_STREAMS_DELEGATE_ERROR is enabled and ioctl is not null the + // stream will receive a MP_STREAM_RAISE_ERROR ioctl request with the arg + // containing the errcode, so it may raise a more detailed error. If that + // ioctl returns without raising, an OSError with the errcode will raised. mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode); mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode); mp_uint_t (*ioctl)(mp_obj_t obj, mp_uint_t request, uintptr_t arg, int *errcode); diff --git a/py/vm.c b/py/vm.c index 860aa36e397..fcb6d9f184e 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1380,7 +1380,7 @@ unwind_jump:; MICROPY_VM_HOOK_LOOP // Check for pending exceptions or scheduled tasks to run. - // Note: it's safe to just call mp_handle_pending(true), but + // Note: it's safe to just call mp_handle_pending(...), but // we can inline the check for the common case where there is // neither. if ( @@ -1402,7 +1402,7 @@ unwind_jump:; #endif ) { MARK_EXC_IP_SELECTIVE(); - mp_handle_pending(true); + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); } #if MICROPY_PY_THREAD_GIL diff --git a/py/vstr.c b/py/vstr.c index 522509d0d05..95607d11bb8 100644 --- a/py/vstr.c +++ b/py/vstr.c @@ -187,33 +187,21 @@ void vstr_add_strn(vstr_t *vstr, const char *str, size_t len) { vstr->len += len; } -static char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len) { +char *vstr_ins_blank_bytes(vstr_t *vstr, size_t byte_pos, size_t byte_len) { size_t l = vstr->len; if (byte_pos > l) { byte_pos = l; } - if (byte_len > 0) { - // ensure room for the new bytes - vstr_ensure_extra(vstr, byte_len); - // copy up the string to make room for the new bytes - memmove(vstr->buf + byte_pos + byte_len, vstr->buf + byte_pos, l - byte_pos); - // increase the length - vstr->len += byte_len; - } + // ensure room for the new bytes + vstr_ensure_extra(vstr, byte_len); + // copy up the string to make room for the new bytes + memmove(vstr->buf + byte_pos + byte_len, vstr->buf + byte_pos, l - byte_pos); + // increase the length + vstr->len += byte_len; + // return a pointer to the location to insert new bytes at return vstr->buf + byte_pos; } -void vstr_ins_byte(vstr_t *vstr, size_t byte_pos, byte b) { - char *s = vstr_ins_blank_bytes(vstr, byte_pos, 1); - *s = b; -} - -void vstr_ins_char(vstr_t *vstr, size_t char_pos, unichar chr) { - // TODO UNICODE - char *s = vstr_ins_blank_bytes(vstr, char_pos, 1); - *s = chr; -} - void vstr_cut_head_bytes(vstr_t *vstr, size_t bytes_to_cut) { vstr_cut_out_bytes(vstr, 0, bytes_to_cut); } diff --git a/pyproject.toml b/pyproject.toml index 5f84ca104ac..e02375fd616 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,17 +32,29 @@ extend-exclude = [ # Exclude third-party code, and exclude the following tests: # basics: needs careful attention before applying automatic formatting # repl_: not real python files +# tstring: ruff does not support template strings # viper_args: uses f(*) format.exclude = [ "tests/*/repl_*.py", "tests/basics/*.py", "tests/cmdline/cmd_compile_only_error.py", + "tests/feature_check/tstring.py", + "tests/micropython/heapalloc_fail_tstring.py", "tests/micropython/test_normalize_newlines.py", "tests/micropython/viper_args.py", ] lint.extend-select = [ "C9", "PLC" ] lint.exclude = [ # Ruff finds Python SyntaxError in these files + "tests/basics/string_module_tstring.py", + "tests/basics/string_tstring_basic.py", + "tests/basics/string_tstring_basic1.py", + "tests/basics/string_tstring_constructor.py", + "tests/basics/string_tstring_errors1.py", + "tests/basics/string_tstring_format1.py", + "tests/basics/string_tstring_interpolation1.py", + "tests/basics/string_tstring_operations.py", + "tests/basics/string_tstring_parser1.py", "tests/cmdline/cmd_compile_only_error.py", "tests/cmdline/repl_autocomplete.py", "tests/cmdline/repl_autocomplete_underscore.py", @@ -54,6 +66,8 @@ lint.exclude = [ "tests/cmdline/repl_words_move.py", "tests/feature_check/repl_emacs_check.py", "tests/feature_check/repl_words_move_check.py", + "tests/feature_check/tstring.py", + "tests/micropython/heapalloc_fail_tstring.py", "tests/micropython/viper_args.py", ] lint.extend-ignore = [ diff --git a/shared/readline/readline.c b/shared/readline/readline.c index 12acf158978..23ff6c56d29 100644 --- a/shared/readline/readline.c +++ b/shared/readline/readline.c @@ -114,6 +114,7 @@ typedef struct _readline_t { // CIRCUITPY-CHANGE uint8_t utf8_cont_chars; char escape_seq_buf[1]; + char last_nl; #if MICROPY_REPL_AUTO_INDENT uint8_t auto_indent_state; #endif @@ -151,6 +152,17 @@ static size_t cursor_count_word(int forward) { } #endif +// Function returns true if newline character shall be processed. +static bool process_nl(int c) { + if ((c == '\r' || c == '\n') && (rl.last_nl == 0 || rl.last_nl == c)) { + rl.last_nl = c; + return true; + } + + rl.last_nl = 0; + return false; +} + int readline_process_char(int c) { // CIRCUITPY-CHANGE size_t last_line_len = utf8_charlen((byte *)rl.line->buf, rl.line->len); @@ -221,7 +233,7 @@ int readline_process_char(int c) { } else if (c == CHAR_CTRL_W) { goto backward_kill_word; #endif - } else if (c == '\r') { + } else if (process_nl(c)) { // newline mp_hal_stdout_tx_str("\r\n"); readline_push_history(vstr_null_terminated_str(rl.line) + rl.orig_line_len); diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index bc5f89bdc81..2b57830abfc 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -168,7 +168,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input } } mp_hal_set_interrupt_char(-1); // disable interrupt - mp_handle_pending(true); // handle any pending exceptions (and any callbacks) + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_EXCEPTIONS); // handle any pending exceptions (and any callbacks) nlr_pop(); ret = PYEXEC_NORMAL_EXIT; if (exec_flags & EXEC_FLAG_PRINT_EOF) { @@ -177,7 +177,7 @@ static int parse_compile_execute(const void *source, mp_parse_input_kind_t input } else { // uncaught exception mp_hal_set_interrupt_char(-1); // disable interrupt - mp_handle_pending(false); // clear any pending exceptions (and run any callbacks) + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS); // clear any pending exceptions (and run any callbacks) if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) { const mp_reader_t *reader = source; @@ -740,7 +740,7 @@ int pyexec_friendly_repl(void) { ret = readline(&line, mp_repl_get_ps1()); } else { // Uncaught exception - mp_handle_pending(false); // clear any pending exceptions (and run any callbacks) + mp_handle_pending(MP_HANDLE_PENDING_CALLBACKS_AND_CLEAR_EXCEPTIONS); // clear any pending exceptions (and run any callbacks) // Print exceptions but stay in the REPL. There are very few delayed // exceptions. The WatchDogTimer can raise one though. diff --git a/tests/basics/builtin_str_hex.py b/tests/basics/builtin_str_hex.py index 9455883012c..f77fe7c4fa9 100644 --- a/tests/basics/builtin_str_hex.py +++ b/tests/basics/builtin_str_hex.py @@ -3,6 +3,7 @@ raise SystemExit for x in ( + b"", b"\x00\x01\x02\x03\x04\x05\x06\x07", b"\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", b"\x7f\x80\xff", @@ -35,5 +36,5 @@ ): try: print(bytes.fromhex(x)) - except ValueError as e: - print("ValueError:", e) + except ValueError: + print("ValueError", x) diff --git a/tests/basics/builtin_str_hex.py.exp b/tests/basics/builtin_str_hex.py.exp deleted file mode 100644 index 0309cad02d1..00000000000 --- a/tests/basics/builtin_str_hex.py.exp +++ /dev/null @@ -1,39 +0,0 @@ -0001020304050607 -0001020304050607 -0001020304050607 -00:01:02:03:04:05:06:07 -00:01:02:03:04:05:06:07 -00:01:02:03:04:05:06:07 -08090a0b0c0d0e0f -08090a0b0c0d0e0f -08090a0b0c0d0e0f -08:09:0a:0b:0c:0d:0e:0f -08:09:0a:0b:0c:0d:0e:0f -08:09:0a:0b:0c:0d:0e:0f -7f80ff -7f80ff -7f80ff -7f:80:ff -7f:80:ff -7f:80:ff -313233344142434461626364 -313233344142434461626364 -313233344142434461626364 -31:32:33:34:41:42:43:44:61:62:63:64 -31:32:33:34:41:42:43:44:61:62:63:64 -31:32:33:34:41:42:43:44:61:62:63:64 -b'\x00\x01\x02\x03\x04\x05\x06\x07' -b'\x08\t\n\x0b\x0c\r\x0e\x0f' -b'\x7f\x80\xff' -b'1234ABCDabcd' -b'\xab\xcd\xef' -b'\xab\xcd\xef' -b'\xab\xcd\xef' -b'\xab\xcd\xef' -ValueError: non-hex digit -ValueError: non-hex digit -ValueError: non-hex digit -ValueError: non-hex digit -ValueError: non-hex digit -ValueError: non-hex digit -ValueError: non-hex digit diff --git a/tests/basics/fun_code.py b/tests/basics/fun_code.py index 59e1f7ec048..841357e8aad 100644 --- a/tests/basics/fun_code.py +++ b/tests/basics/fun_code.py @@ -34,3 +34,8 @@ def f(): ftype(f.__code__, None) except TypeError: print("TypeError") + +# Test __code__ on functions with children functions. +code = (lambda: (lambda: a)).__code__ +print(ftype(code, {"a": 1})()()) +print(ftype(code, {"a": 2})()()) diff --git a/tests/basics/fun_code_micropython.py b/tests/basics/fun_code_micropython.py deleted file mode 100644 index 2c319a2db8c..00000000000 --- a/tests/basics/fun_code_micropython.py +++ /dev/null @@ -1,19 +0,0 @@ -# Test MicroPython-specific restrictions of function.__code__ attribute. - -try: - (lambda: 0).__code__ -except AttributeError: - print("SKIP") - raise SystemExit - - -def f_with_children(): - def g(): - pass - - -# Can't access __code__ when function has children. -try: - f_with_children.__code__ -except AttributeError: - print("AttributeError") diff --git a/tests/basics/fun_code_micropython.py.exp b/tests/basics/fun_code_micropython.py.exp deleted file mode 100644 index d169edffb4c..00000000000 --- a/tests/basics/fun_code_micropython.py.exp +++ /dev/null @@ -1 +0,0 @@ -AttributeError diff --git a/tests/basics/import_star_nonmodule.py b/tests/basics/import_star_nonmodule.py new file mode 100644 index 00000000000..8a98ef26ce5 --- /dev/null +++ b/tests/basics/import_star_nonmodule.py @@ -0,0 +1,65 @@ +# Test "from x import *" where x is something other than a module. + +import sys + +try: + next(iter([]), 42) +except TypeError: + # Two-argument version of next() not supported. We are probably not at + # MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES which is needed for "import *". + print("SKIP") + raise SystemExit + +print("== test with a class as a module ==") + + +class M: + x = "a1" + + def __init__(self): + self.x = "a2" + + +sys.modules["mod"] = M +from mod import * + +print(x) + +sys.modules["mod"] = M() +from mod import * + +print(x) + +print("== test with a class as a module that overrides __all__ ==") + + +class M: + __all__ = ("y",) + x = "b1" + y = "b2" + + def __init__(self): + self.__all__ = ("x",) + self.x = "b3" + self.y = "b4" + + +sys.modules["mod"] = M +x = None +from mod import * + +print(x, y) + +sys.modules["mod"] = M() +from mod import * + +print(x, y) + +print("== test with objects that don't have a __dict__ ==") + +sys.modules["mod"] = 1 +try: + from mod import * + # MicroPython raises AttributeError, CPython raises ImportError. +except (AttributeError, ImportError): + print("ImportError") diff --git a/tests/basics/lexer.py b/tests/basics/lexer.py index addb8a13df3..dfdc0b9900d 100644 --- a/tests/basics/lexer.py +++ b/tests/basics/lexer.py @@ -91,3 +91,11 @@ def a(x): eval("01") except SyntaxError: print("SyntaxError") + +# Bytes 0-8 inclusive are not allowed in input stream. +# Earlier CPython (eg 3.10.12) raises ValueError, later CPython (eg 3.11.14) raises SyntaxError. +for invalid_byte_value in range(0, 10): + try: + print(eval(b"123" + bytes([invalid_byte_value]))) + except (ValueError, SyntaxError): + print("byte {}: SyntaxError".format(invalid_byte_value)) diff --git a/tests/basics/string_fstring.py b/tests/basics/string_fstring.py index d94cc0cd3e6..daa687dbddb 100644 --- a/tests/basics/string_fstring.py +++ b/tests/basics/string_fstring.py @@ -31,6 +31,8 @@ def foo(a, b): # PEP-0498 specifies that handling of double braces '{{' or '}}' should # behave like str.format. +print(f'{{') +print(f'}}') print(f'{{}}') print(f'{{{4*10}}}', '{40}') @@ -79,3 +81,14 @@ def foo(a, b): # Raw f-strings. print(rf"\r\a\w {'f'} \s\t\r\i\n\g") print(fr"\r{x}") + +# Format specifiers with nested replacement fields +space = 5 +prec = 2 +print(f"{3.14:{space}.{prec}}") + +space_prec = "5.2" +print(f"{3.14:{space_prec}}") + +radix = "x" +print(f"{314:{radix}}") diff --git a/tests/basics/string_fstring_nested.py b/tests/basics/string_fstring_nested.py new file mode 100644 index 00000000000..82de0bf6470 --- /dev/null +++ b/tests/basics/string_fstring_nested.py @@ -0,0 +1,9 @@ +# Test nesting of f-strings within f-strings. + +x = 1 + +# 2-level nesting, with padding. +print(f"a{f'b{x:2}c':>5}d") + +# 4-level nesting using the different styles of quotes. +print(f"""a{f'''b{f"c{f'd{x}e'}f"}g'''}h""") diff --git a/tests/basics/string_fstring_nested_py312.py b/tests/basics/string_fstring_nested_py312.py new file mode 100644 index 00000000000..aa731b1637d --- /dev/null +++ b/tests/basics/string_fstring_nested_py312.py @@ -0,0 +1,7 @@ +# Test nesting of f-strings within f-strings. +# These test rely on Python 3.12+ to use the same quote style for nesting. + +x = 1 + +# 8-level nesting using the same quote style. +print(f"a{f"b{f"c{f"d{f"e{f"f{f"g{f"h{x}i"}j"}k"}l"}m"}n"}o"}p") diff --git a/tests/basics/string_fstring_nested_py312.py.exp b/tests/basics/string_fstring_nested_py312.py.exp new file mode 100644 index 00000000000..1d624933dff --- /dev/null +++ b/tests/basics/string_fstring_nested_py312.py.exp @@ -0,0 +1 @@ +abcdefgh1ijklmnop diff --git a/tests/basics/string_module_tstring.py b/tests/basics/string_module_tstring.py new file mode 100644 index 00000000000..83809926926 --- /dev/null +++ b/tests/basics/string_module_tstring.py @@ -0,0 +1,6 @@ +# Test basic templatelib functionality. +# This test requires t-strings support. + +from string.templatelib import Template + +print("templatelib", isinstance(t"hi", Template)) diff --git a/tests/basics/string_module_tstring.py.exp b/tests/basics/string_module_tstring.py.exp new file mode 100644 index 00000000000..26dba98f403 --- /dev/null +++ b/tests/basics/string_module_tstring.py.exp @@ -0,0 +1 @@ +templatelib True diff --git a/tests/basics/string_tstring_basic.py b/tests/basics/string_tstring_basic.py new file mode 100644 index 00000000000..e23a3f06595 --- /dev/null +++ b/tests/basics/string_tstring_basic.py @@ -0,0 +1,338 @@ +from string.templatelib import Template, Interpolation + +print("=== Basic functionality ===") +t = t"Hello World" +print(type(t).__name__) + +name = "World" +t2 = t"Hello {name}" +print(f"Strings: {t2.strings}") +print(f"Value: {t2.interpolations[0].value}") +print(f"str(): {str(t2)}") + +t_raw = rt"Path: C:\test\{name}" +print(f"Raw: '{t_raw.strings[0]}'") + +t_tr = tr"Path: C:\test\{name}" +print(f"tr: '{t_tr.strings[0]}'") + +print("\n=== Parser tests ===") +data = {"a": {"b": [1, 2, 3]}} +t_complex = t"{data['a']['b'][0]}" +print(f"Complex: {str(t_complex)}") + +print(f"None: {str(t'{None}')}") +print(f"True: {str(t'{True}')}") +print(f"False: {str(t'{False}')}") +print(f"Ellipsis: {str(t'{...}')}") + +obj = type('Obj', (), { + '__getattr__': lambda s, n: s, + '__getitem__': lambda s, k: s, + '__call__': lambda s, *a: 42, + '__str__': lambda s: "42" +})() +print(f"Deep nest: {str(t'{obj.a.b[0].c()}')}") + +print("\n=== Conversions and formatting ===") +val = {"key": "value"} +print(f"repr: {str(t'{val!r}')}") +print(f"str: {str(t'{val!s}')}") +# print(f"ascii: {str(t'{val!a}')}") + +print(f"Width: '{str(t'{42:10d}')}'") +print(f"Precision: {str(t'{3.14159:.2f}')}") + +x = 42 +t_debug = t"{x=}" +assert t_debug.strings == ("x=", "") +assert t_debug.interpolations[0].expression == "x" +assert t_debug.interpolations[0].conversion == "r" +assert t_debug.interpolations[0].value == 42 +print(f"Debug: {str(t_debug)}") + +y = 10 +t_debug2 = t"{x + y=}" +assert t_debug2.strings == ("x + y=", "") +assert t_debug2.interpolations[0].expression == "x + y" +assert t_debug2.interpolations[0].conversion == "r" +assert t_debug2.interpolations[0].value == 52 + +pi = 3.14159 +t_debug3 = t"{pi=:.2f}" +assert t_debug3.strings == ("pi=", "") +assert t_debug3.interpolations[0].expression == "pi" +assert t_debug3.interpolations[0].conversion is None +assert t_debug3.interpolations[0].format_spec == ".2f" +assert t_debug3.interpolations[0].value == 3.14159 + +print("\n=== Constructor tests ===") +t_empty = Template() +print(f"Empty: {t_empty.strings}") + +t_single = Template("single") +print(f"Single: {t_single.strings}") + +t_multi = Template("a", "b", "c") +print(f"Multi: {t_multi.strings}") + +i1 = Interpolation(1, "x") +i2 = Interpolation(2, "y") +t_mixed = Template("start", i1, "middle", i2, "end") +print(f"Mixed: strings={t_mixed.strings}, values={t_mixed.values}") + +print("\n=== Operations ===") +t1 = t"Hello" +t2 = t" World" +t_concat = t1 + t2 +print(f"Concat: '{str(t_concat)}'") + +items = list(t"a{1}b{2}c") +print(f"Iterator: {[type(x).__name__ for x in items]}") + +t_attr = t"test{42}" +print(f"strings attr: {t_attr.strings}") +print(f"interpolations attr: {len(t_attr.interpolations)}") +print(f"values attr: {t_attr.values}") + +print(f"repr: {repr(t2)[:50]}...") + +print("\n=== Escaped braces evaluation ===") +t_escaped = Template("{", Interpolation(42, "x"), "}", Interpolation(42, "x"), "{{", Interpolation(42, "x"), "}}") +print(f"Escaped eval: '{str(t_escaped)}'") + +t_braces = Template("{{hello}}", Interpolation(1, "a"), " {", Interpolation(2, "b"), "} ", Interpolation(3, "c"), "{{world}}") +print(f"Braces in strings: '{str(t_braces)}'") + +print("\n=== Memory stress test ===") +for n in [10, 20, 30]: + args = [] + for i in range(n): + args.append("s") + args.append(Interpolation(i, f"var{i}")) + args.append("s") + t_mem = Template(*args) + result = str(t_mem) + print(f"Memory test [{n}]: {len(result)} chars") + +large_args = [] +for i in range(20): + large_args.append("") + large_args.append(Interpolation(i, f"v{i}")) +large_args.append("") +t_large = Template(*large_args) +print(f"Large values: {len(t_large.values)} values") + +print("\n=== Nested quotes and braces tests ===") +t_nested1 = t"{"{}"}" +print(f"Nested quotes 1: {str(t_nested1)}") +print(f" Value: {t_nested1.interpolations[0].value}") +print(f" Expression: {t_nested1.interpolations[0].expression}") + +t_nested2 = t"{'hello'}" +print(f"Nested quotes 2: {str(t_nested2)}") +print(f" Value: {t_nested2.interpolations[0].value}") + +t_nested3 = t'{"world"}' +print(f"Nested quotes 3: {str(t_nested3)}") +print(f" Value: {t_nested3.interpolations[0].value}") + +t_nested4 = t"{['a', 'b', 'c'][1]}" +print(f"Nested quotes 4: {str(t_nested4)}") +print(f" Value: {t_nested4.interpolations[0].value}") + +d = {"key": "value", "x": 123} +t_nested5 = t"{d['key']}" +print(f"Nested quotes 5: {str(t_nested5)}") +print(f" Value: {t_nested5.interpolations[0].value}") + +t_escaped1 = t'{{""}}' +print(f"Escaped braces 1: {str(t_escaped1)}") +print(f" Strings: {t_escaped1.strings}") + +t_escaped2 = t"{{}}}}" +print(f"Escaped braces 2: {str(t_escaped2)}") +print(f" Strings: {t_escaped2.strings}") + +x = "test" +t_mixed = t"{{before}} {x} {{after}}" +print(f"Mixed escaped: {str(t_mixed)}") +print(f" Strings: {t_mixed.strings}") +print(f" Value: {t_mixed.interpolations[0].value}") + +t_escape = t"{'\n\t'}" +print(f"Escape sequences: {str(t_escape)}") +print(f" Value repr: {repr(t_escape.interpolations[0].value)}") + +inner = "{}" +t_nested_expr = t"{inner}" +print(f"Nested expr: {str(t_nested_expr)}") +print(f" Value: {t_nested_expr.interpolations[0].value}") + +print("\n=== Interpolation attribute tests ===") +i_basic = Interpolation(42, "x") +print(f"Basic conversion: {i_basic.conversion}") +print(f"Basic format_spec: {i_basic.format_spec}") + +i_with_conv = Interpolation(42, "x", "s") +print(f"With conversion: {i_with_conv.conversion}") + +i_with_fmt = Interpolation(42, "x", None, ":>10") +print(f"With format_spec: {i_with_fmt.format_spec}") + +i_full = Interpolation(42, "x", "r", ":>10") +print(f"Full conversion: {i_full.conversion}") +print(f"Full format_spec: {i_full.format_spec}") + +t_conv = t"{42!s}" +print(f"Template conversion: {t_conv.interpolations[0].conversion}") + +t_fmt = t"{42:>10}" +print(f"Template format_spec: {t_fmt.interpolations[0].format_spec}") + +t_both = t"{42!r:>10}" +print(f"Both conversion: {t_both.interpolations[0].conversion}") +print(f"Both format_spec: {t_both.interpolations[0].format_spec}") + +print("\n=== Escape sequence tests ===") +print(repr(t"Line1\nLine2")) +print(repr(t"Path\\to\\file")) +print(repr(t"It\'s working")) +print(repr(t"She said \"Hello\"")) +print(repr(t"Bell\a")) +print(repr(t"Back\bspace")) +print(repr(t"Tab\there")) +print(repr(t"Vertical\vtab")) +print(repr(t"Form\ffeed")) +print(repr(t"Carriage\rreturn")) + +# Valid hex escapes +print(repr(t"\x41")) +print(repr(t"\x7F")) + +# Valid octal escapes +print(repr(t"\0")) +print(repr(t"\7")) +print(repr(t"\77")) +print(repr(t"\101")) + +print(repr(t"\1a")) +print(repr(t"\123c")) + +# Line continuation +print(repr(t"First \ +second")) +print(repr(t"One\ +Two\ +Three")) + +# Multiple escapes +print(repr(t"\n\t\r")) + +# Raw strings +print(repr(rt"\n\t\x41")) +print(repr(rt"C:\new\test")) + +# Triple quoted strings +print(repr(t"""\n\t\x41""")) +print(repr(rt"""\n\t\x41""")) + +print("\n=== Triple quote edge cases ===") +doc = '''This is a +multi-line +docstring''' +result1 = t"""Documentation: +{doc} +End of doc""" +print(f"Triple in interpolation: {str(result1)}") +print(f" Value: {repr(result1.interpolations[0].value)}") + +data = { + "users": [ + {"name": "Alice", "msg": 'Say "Hello"'}, + {"name": "Bob", "msg": "It's great!"} + ] +} +complex_nested = t"""Users: +{data['users'][0]['name']}: {data['users'][0]['msg']} +{data['users'][1]['name']}: {data['users'][1]['msg']}""" +print(f"Complex nested: {str(complex_nested)}") +print(f" Values: {complex_nested.values}") + +import os +path_sep = os.sep +raw_with_interp = rt"""Path: C:\Users\{path_sep}Documents +Raw newline: \n +Raw tab: \t""" +print(f"Raw with interpolation: {str(raw_with_interp)}") +print(f" String 0 repr: {repr(raw_with_interp.strings[0])}") + +long_line = "x" * 50 +long_triple = t"""Start +{long_line} +End""" +print(f"Long line: strings={long_triple.strings}") +print(f" Value length: {len(long_triple.values[0])}") + +inner = t"inner {42}" +outer = t"""Outer template: +{inner} +End""" +print(f"Nested templates: {str(outer)}") +print(f" Inner value: {outer.values[0]}") + +formatted = t"""Math constants: +Pi: {314:.2f} +E: {271:.1e} +Sqrt(2): {141:.0f}""" +print(f"Formatted values: {repr(formatted)}") +print(f" Interpolation count: {len(formatted.interpolations)}") +print(f" Format specs: {[i.format_spec for i in formatted.interpolations]}") + +x, y, z = 1, 2, 3 +debug_complex = t"""Debug info: +{x + y=} {x * y * z=} {x < y < z=}""" +print(f"Debug complex: {repr(debug_complex)}") +print(f" Expressions: {[i.expression for i in debug_complex.interpolations]}") + +part1 = t"""Part 1 +with newline""" +part2 = t'''Part 2 +also multiline''' +concatenated = part1 + part2 +print(f"Concatenated triple: {repr(concatenated)}") +print(f" Strings: {concatenated.strings}") + +print("\n=== PEP 701 brace compliance ===") +t_brace1 = t"{{" +print(f"Escaped {{{{: {t_brace1.strings}") +assert t_brace1.strings == ('{',) + +t_brace2 = t"}}" +print(f"Escaped }}}}: {t_brace2.strings}") +assert t_brace2.strings == ('}',) + +t_brace3 = t"test{{escape}}here" +print(f"Mixed: {t_brace3.strings}") +assert t_brace3.strings == ('test{escape}here',) + +print("\n=== Format spec scope resolution ===") +def test_format_scope(): + width = 10 + x = 42 + t_scope = t"{x:{width}}" + print(f"Local vars: width={width}, x={x}") + print(f"Format spec: {t_scope.interpolations[0].format_spec}") + return t_scope.interpolations[0].format_spec == '10' + +result = test_format_scope() +print(f"Scope test: {'PASS' if result else 'FAIL'}") +assert result, "Format spec scope resolution failed" + +precision = 2 +value = 3.14159 +t_prec = t"{value:.{precision}f}" +print(f"Precision: format_spec={t_prec.interpolations[0].format_spec}") +assert t_prec.interpolations[0].format_spec == '.2f' + +print("\nBasic tests completed!") diff --git a/tests/basics/string_tstring_basic.py.exp b/tests/basics/string_tstring_basic.py.exp new file mode 100644 index 00000000000..4fcb3a75834 --- /dev/null +++ b/tests/basics/string_tstring_basic.py.exp @@ -0,0 +1,141 @@ +=== Basic functionality === +Template +Strings: ('Hello ', '') +Value: World +str(): Template(strings=('Hello ', ''), interpolations=(Interpolation('World', 'name', None, ''),)) +Raw: 'Path: C:\test\' +tr: 'Path: C:\test\' + +=== Parser tests === +Complex: Template(strings=('', ''), interpolations=(Interpolation(1, "data['a']['b'][0]", None, ''),)) +None: Template(strings=('', ''), interpolations=(Interpolation(None, 'None', None, ''),)) +True: Template(strings=('', ''), interpolations=(Interpolation(True, 'True', None, ''),)) +False: Template(strings=('', ''), interpolations=(Interpolation(False, 'False', None, ''),)) +Ellipsis: Template(strings=('', ''), interpolations=(Interpolation(Ellipsis, '...', None, ''),)) +Deep nest: Template(strings=('', ''), interpolations=(Interpolation(42, 'obj.a.b[0].c()', None, ''),)) + +=== Conversions and formatting === +repr: Template(strings=('', ''), interpolations=(Interpolation({'key': 'value'}, 'val', 'r', ''),)) +str: Template(strings=('', ''), interpolations=(Interpolation({'key': 'value'}, 'val', 's', ''),)) +Width: 'Template(strings=('', ''), interpolations=(Interpolation(42, '42', None, '10d'),))' +Precision: Template(strings=('', ''), interpolations=(Interpolation(3.14159, '3.14159', None, '.2f'),)) +Debug: Template(strings=('x=', ''), interpolations=(Interpolation(42, 'x', 'r', ''),)) + +=== Constructor tests === +Empty: ('',) +Single: ('single',) +Multi: ('abc',) +Mixed: strings=('start', 'middle', 'end'), values=(1, 2) + +=== Operations === +Concat: 'Template(strings=('Hello World',), interpolations=())' +Iterator: ['str', 'Interpolation', 'str', 'Interpolation', 'str'] +strings attr: ('test', '') +interpolations attr: 1 +values attr: (42,) +repr: Template(strings=(' World',), interpolations=())... + +=== Escaped braces evaluation === +Escaped eval: 'Template(strings=('{', '}', '{{', '}}'), interpolations=(Interpolation(42, 'x', None, ''), Interpolation(42, 'x', None, ''), Interpolation(42, 'x', None, '')))' +Braces in strings: 'Template(strings=('{{hello}}', ' {', '} ', '{{world}}'), interpolations=(Interpolation(1, 'a', None, ''), Interpolation(2, 'b', None, ''), Interpolation(3, 'c', None, '')))' + +=== Memory stress test === +Memory test [10]: 450 chars +Memory test [20]: 880 chars +Memory test [30]: 1310 chars +Large values: 20 values + +=== Nested quotes and braces tests === +Nested quotes 1: Template(strings=('', ''), interpolations=(Interpolation('{}', '"{}"', None, ''),)) + Value: {} + Expression: "{}" +Nested quotes 2: Template(strings=('', ''), interpolations=(Interpolation('hello', "'hello'", None, ''),)) + Value: hello +Nested quotes 3: Template(strings=('', ''), interpolations=(Interpolation('world', '"world"', None, ''),)) + Value: world +Nested quotes 4: Template(strings=('', ''), interpolations=(Interpolation('b', "['a', 'b', 'c'][1]", None, ''),)) + Value: b +Nested quotes 5: Template(strings=('', ''), interpolations=(Interpolation('value', "d['key']", None, ''),)) + Value: value +Escaped braces 1: Template(strings=('{""}',), interpolations=()) + Strings: ('{""}',) +Escaped braces 2: Template(strings=('{}}',), interpolations=()) + Strings: ('{}}',) +Mixed escaped: Template(strings=('{before} ', ' {after}'), interpolations=(Interpolation('test', 'x', None, ''),)) + Strings: ('{before} ', ' {after}') + Value: test +Escape sequences: Template(strings=('', ''), interpolations=(Interpolation('\n\t', "'\\n\\t'", None, ''),)) + Value repr: '\n\t' +Nested expr: Template(strings=('', ''), interpolations=(Interpolation('{}', 'inner', None, ''),)) + Value: {} + +=== Interpolation attribute tests === +Basic conversion: None +Basic format_spec: +With conversion: s +With format_spec: :>10 +Full conversion: r +Full format_spec: :>10 +Template conversion: s +Template format_spec: >10 +Both conversion: r +Both format_spec: >10 + +=== Escape sequence tests === +Template(strings=('Line1\nLine2',), interpolations=()) +Template(strings=('Path\\to\\file',), interpolations=()) +Template(strings=("It's working",), interpolations=()) +Template(strings=('She said "Hello"',), interpolations=()) +Template(strings=('Bell\x07',), interpolations=()) +Template(strings=('Back\x08space',), interpolations=()) +Template(strings=('Tab\there',), interpolations=()) +Template(strings=('Vertical\x0btab',), interpolations=()) +Template(strings=('Form\x0cfeed',), interpolations=()) +Template(strings=('Carriage\rreturn',), interpolations=()) +Template(strings=('A',), interpolations=()) +Template(strings=('\x7f',), interpolations=()) +Template(strings=('\x00',), interpolations=()) +Template(strings=('\x07',), interpolations=()) +Template(strings=('?',), interpolations=()) +Template(strings=('A',), interpolations=()) +Template(strings=('\x01a',), interpolations=()) +Template(strings=('Sc',), interpolations=()) +Template(strings=('First second',), interpolations=()) +Template(strings=('OneTwoThree',), interpolations=()) +Template(strings=('\n\t\r',), interpolations=()) +Template(strings=('\\n\\t\\x41',), interpolations=()) +Template(strings=('C:\\new\\test',), interpolations=()) +Template(strings=('\n\tA',), interpolations=()) +Template(strings=('\\n\\t\\x41',), interpolations=()) + +=== Triple quote edge cases === +Triple in interpolation: Template(strings=('Documentation:\n', '\nEnd of doc'), interpolations=(Interpolation('This is a\nmulti-line\ndocstring', 'doc', None, ''),)) + Value: 'This is a\nmulti-line\ndocstring' +Complex nested: Template(strings=('Users:\n', ': ', '\n', ': ', ''), interpolations=(Interpolation('Alice', "data['users'][0]['name']", None, ''), Interpolation('Say "Hello"', "data['users'][0]['msg']", None, ''), Interpolation('Bob', "data['users'][1]['name']", None, ''), Interpolation("It's great!", "data['users'][1]['msg']", None, ''))) + Values: ('Alice', 'Say "Hello"', 'Bob', "It's great!") +Raw with interpolation: Template(strings=('Path: C:\\Users\\', 'Documents\nRaw newline: \\n\nRaw tab: \\t'), interpolations=(Interpolation('/', 'path_sep', None, ''),)) + String 0 repr: 'Path: C:\\Users\\' +Long line: strings=('Start\n', '\nEnd') + Value length: 50 +Nested templates: Template(strings=('Outer template:\n', '\nEnd'), interpolations=(Interpolation(Template(strings=('inner ', ''), interpolations=(Interpolation(42, '42', None, ''),)), 'inner', None, ''),)) + Inner value: Template(strings=('inner ', ''), interpolations=(Interpolation(42, '42', None, ''),)) +Formatted values: Template(strings=('Math constants:\nPi: ', '\nE: ', '\nSqrt(2): ', ''), interpolations=(Interpolation(314, '314', None, '.2f'), Interpolation(271, '271', None, '.1e'), Interpolation(141, '141', None, '.0f'))) + Interpolation count: 3 + Format specs: ['.2f', '.1e', '.0f'] +Debug complex: Template(strings=('Debug info:\nx + y=', ' x * y * z=', ' x < y < z=', ''), interpolations=(Interpolation(3, 'x + y', 'r', ''), Interpolation(6, 'x * y * z', 'r', ''), Interpolation(True, 'x < y < z', 'r', ''))) + Expressions: ['x + y', 'x * y * z', 'x < y < z'] +Concatenated triple: Template(strings=('Part 1\nwith newlinePart 2\nalso multiline',), interpolations=()) + Strings: ('Part 1\nwith newlinePart 2\nalso multiline',) + +=== PEP 701 brace compliance === +Escaped {{: ('{',) +Escaped }}: ('}',) +Mixed: ('test{escape}here',) + +=== Format spec scope resolution === +Local vars: width=10, x=42 +Format spec: 10 +Scope test: PASS +Precision: format_spec=.2f + +Basic tests completed! diff --git a/tests/basics/string_tstring_basic1.py b/tests/basics/string_tstring_basic1.py new file mode 100644 index 00000000000..89ee3e0d2d4 --- /dev/null +++ b/tests/basics/string_tstring_basic1.py @@ -0,0 +1,131 @@ +print("=== Parser error tests ===") +try: + exec('t_empty = t"{}"') + print(f"Empty expr: {t_empty.interpolations[0].value}") +except SyntaxError as e: + print(f"Empty expr: SyntaxError - {e}") + +# Whitespace in expression (Python semantics are to strip trailing) +t_ws = t"{ 42 }" +print(f"Whitespace: {str(t_ws)}") + +print("\n=== Error cases ===") +try: + exec('t"{@}"') +except SyntaxError: + print("Invalid syntax: SyntaxError") + +try: + long_expr = "x" * 10001 + exec(f't"{{{long_expr}}}"') +except (ValueError, SyntaxError, RuntimeError, NameError): + print("Long expr: Error") + +try: + exec('t"hello" "world"') +except SyntaxError: + print("Mixed concat: SyntaxError") + +try: + exec('bt"test"') +except SyntaxError: + print("bt prefix: SyntaxError") + +print("\n=== Escape sequence coverage tests ===") + +def expect_invalid_escape(label, expr): + try: + eval(expr) + except SyntaxError: + print(f"{label}: SyntaxError") + else: + print(f"{label}: ERROR (expected SyntaxError)") + +# Deprecated escape sequences (CPython warns, MicroPython accepts as literal) +print(repr(t"\8")) +print(repr(t"\9")) + +# Unknown escape chars +print(repr(t"\z")) +print(repr(t"\k")) + +# Invalid escapes +expect_invalid_escape("Invalid \\x escape", 't"\\xGG"') +expect_invalid_escape("Invalid \\u escape", 't"\\uGGGG"') +expect_invalid_escape("Invalid \\U escape", 't"\\UGGGGGGGG"') + +# Unicode escapes (display format differs) +print(repr(t"\x00\x01\xFF")) +print(repr(t"\u0041")) +print(repr(t"\u03B1")) +print(repr(t"\u2764")) +print(repr(t"\U00000041")) +print(repr(t"\U0001F600")) +print(repr(t"\x41\u0042\103")) + +# Unicode in triple-quoted strings +unicode_test = t"""Unicode test: +Emoji: {'\U0001f40d'} +Special: {'\u03b1 \u03b2 \u03b3'}""" +print(f"Unicode: {str(unicode_test)}") + +print("\n=== Trailing whitespace preservation (PEP 750) ===") +x = 42 +tmpl_trail = t"{x }" +expr = tmpl_trail.interpolations[0].expression +print(f"Expression with trailing spaces: |{expr}|") +assert expr == "x", f"Expected 'x' but got '{expr}'" +assert len(expr) == 1, f"Expected length 1 but got {len(expr)}" + +tmpl_both = t"{ x }" +expr2 = tmpl_both.interpolations[0].expression +print(f"Expression with both spaces: |{expr2}|") +assert expr2 == " x", f"Expected ' x' but got '{expr2}'" +assert len(expr2) == 4, f"Expected length 4 but got {len(expr2)}" + +tmpl_lead = t"{ x}" +expr3 = tmpl_lead.interpolations[0].expression +print(f"Expression with leading spaces: |{expr3}|") +assert expr3 == " x", f"Expected ' x' but got '{expr3}'" + +# Debug specifiers: leading space preserved, trailing space stripped from expression +# PEP 750: "Whitespace is preserved in the debug specifier" (in strings part) +z = 99 +t_debug4 = t"{z =}" +assert t_debug4.strings == ("z =", "") +assert t_debug4.interpolations[0].expression == "z" # Trailing space stripped +assert t_debug4.interpolations[0].conversion == "r" + +t_debug5 = t"{ z=}" +assert t_debug5.strings == (" z=", "") +assert t_debug5.interpolations[0].expression == " z" # Leading space preserved +assert t_debug5.interpolations[0].conversion == "r" + +t_debug6 = t"{ z =}" +assert t_debug6.strings == (" z =", "") +assert t_debug6.interpolations[0].expression == " z" # Leading preserved, trailing stripped +assert t_debug6.interpolations[0].conversion == "r" + +print("Trailing whitespace: PASS") + +print("\n=== Error message tests ===") +try: + exec('t"}"') +except SyntaxError: + print("Lone }} rejected (correct)") + +try: + exec('t"{"') +except SyntaxError: + print("Unterminated {{ rejected (correct)") + +print("\n=== Triple quote empty interpolation ===") +try: + exec('''empty_interp = t"""Start +{} +End"""''') + print(f"Empty interpolation: {str(empty_interp)}") +except Exception as e: + print(f"Empty interpolation error: {type(e).__name__}: {e}") + +print("\nIncompatible tests completed!") diff --git a/tests/basics/string_tstring_basic1.py.exp b/tests/basics/string_tstring_basic1.py.exp new file mode 100644 index 00000000000..52fc6f6c94c --- /dev/null +++ b/tests/basics/string_tstring_basic1.py.exp @@ -0,0 +1,41 @@ +=== Parser error tests === +Empty expr: SyntaxError - malformed f-string +Whitespace: Template(strings=('', ''), interpolations=(Interpolation(42, ' 42', None, ''),)) + +=== Error cases === +Invalid syntax: SyntaxError +Long expr: Error +Mixed concat: SyntaxError +bt prefix: SyntaxError + +=== Escape sequence coverage tests === +Template(strings=('\\8',), interpolations=()) +Template(strings=('\\9',), interpolations=()) +Template(strings=('\\z',), interpolations=()) +Template(strings=('\\k',), interpolations=()) +Invalid \x escape: SyntaxError +Invalid \u escape: SyntaxError +Invalid \U escape: SyntaxError +Template(strings=('\x00\x01\xff',), interpolations=()) +Template(strings=('A',), interpolations=()) +Template(strings=('\u03b1',), interpolations=()) +Template(strings=('\u2764',), interpolations=()) +Template(strings=('A',), interpolations=()) +Template(strings=('\U0001f600',), interpolations=()) +Template(strings=('ABC',), interpolations=()) +Unicode: Template(strings=('Unicode test:\nEmoji: ', '\nSpecial: ', ''), interpolations=(Interpolation('\U0001f40d', "'\\U0001f40d'", None, ''), Interpolation('\u03b1 \u03b2 \u03b3', "'\\u03b1 \\u03b2 \\u03b3'", None, ''))) + +=== Trailing whitespace preservation (PEP 750) === +Expression with trailing spaces: |x| +Expression with both spaces: | x| +Expression with leading spaces: | x| +Trailing whitespace: PASS + +=== Error message tests === +Lone }} rejected (correct) +Unterminated {{ rejected (correct) + +=== Triple quote empty interpolation === +Empty interpolation error: SyntaxError: malformed f-string + +Incompatible tests completed! diff --git a/tests/basics/string_tstring_constructor.py b/tests/basics/string_tstring_constructor.py new file mode 100644 index 00000000000..dc4a09a9578 --- /dev/null +++ b/tests/basics/string_tstring_constructor.py @@ -0,0 +1,112 @@ +from string.templatelib import Template, Interpolation + +print("=== Constructor basic usage ===") +t = Template("hello ", Interpolation(42, "x"), "world") +print(f"Template repr: {repr(t)}") + +t_varargs = Template("Hello ", Interpolation("World", "name"), "!") +print(f"Varargs constructor: strings={t_varargs.strings}, values={t_varargs.values}") + +t_concat = Template("A", "B", Interpolation(1, "value"), "C", "D") +print(f"Varargs merged strings: {t_concat.strings}") + +t_leading = Template(Interpolation(1, "x"), " tail") +print(f"Leading interpolation strings: {t_leading.strings}") + +t_trailing = Template("head ", Interpolation(2, "y")) +print(f"Trailing interpolation strings: {t_trailing.strings}") + +t_interps_only = Template(Interpolation(1, "x"), Interpolation(2, "y")) +print(f"Interpolation only strings: {t_interps_only.strings}") + +print("\n=== Special cases ===") + +i = Interpolation(42, "x", "s", ":>10") +try: + i.value = 100 +except AttributeError: + print("Interp read-only: AttributeError") + +t_ws_trim = Template("", Interpolation(None, " ", None, ""), "") +print(f"Whitespace trim: '{t_ws_trim}'") + +t_debug = Template("", Interpolation(42, "x=", None, ""), "") +print(f"Debug =: {t_debug}") + + +class Custom: + def __repr__(self): + return "CustomRepr" + + def __str__(self): + return "CustomStr" + + +obj = Custom() +print(f"Custom !r: {t'{obj!r}'}") +print(f"Custom !s: {t'{obj!s}'}") + +t_empty_start = Template("", Interpolation(1, "1"), "text") +print(f"Empty start iter: {[type(x).__name__ for x in t_empty_start]}") + +t_iter_edge = Template("", Interpolation(1, "1"), "", Interpolation(2, "2"), "") +iter_items = [] +for item in t_iter_edge: + iter_items.append(type(item).__name__) +print(f"Iterator edge: {iter_items}") + +print("\n=== Values property ===") +for n in range(7): + args = [] + for i in range(n): + args.append("") + args.append(Interpolation(i, str(i))) + args.append("") + t = Template(*args) + print(f"Values[{n}]: {t.values}") + +print("\n=== Multiple consecutive strings ===") +try: + t = Template("first", "second", "third", Interpolation(42, "x"), "fourth", "fifth") + if t.strings == ("firstsecondthird", "fourthfifth"): + print("Multiple strings concatenated: OK") + else: + print(f"Multiple strings: strings={t.strings}") +except Exception as e: + print(f"Multiple strings error: {e}") + +print("\n=== Template() constructor with many interpolations ===") +try: + exprs = ["a", "b", "c", "d", "e"] + interps = [Interpolation(i, exprs[i % len(exprs)]) for i in range(20)] + strings = [""] * 21 + t = Template(*strings, *interps) + print(f"Template() constructor: OK ({len(t.interpolations)} interpolations)") +except Exception as e: + print(f"Template() constructor: {type(e).__name__}") + +print("\n=== vstr string concatenation ===") +try: + t1 = Template("part1", "part2", "part3", "part4", Interpolation(1, "x"), "end") + result = str(t1) + print(f"vstr concat: '{result}'") +except Exception as e: + print(f"vstr concat error: {e}") + +print("\n=== High byte handling ===") +try: + result = t"\x7f\x80\x81\xfe\xff" + first_str = result.strings[0] + print( + f"High bytes: len={len(first_str)}, first=0x{ord(first_str[0]):02x}, last=0x{ord(first_str[-1]):02x}" + ) +except Exception as e: + print(f"High bytes error: {e}") + +try: + result = t"\200\201\377" + print(f"Octal high bytes: OK, len={len(result.strings[0])}") +except Exception as e: + print(f"Octal high bytes error: {e}") + +print("\nConstructor tests completed!") diff --git a/tests/basics/string_tstring_constructor.py.exp b/tests/basics/string_tstring_constructor.py.exp new file mode 100644 index 00000000000..e7d6ea90a9b --- /dev/null +++ b/tests/basics/string_tstring_constructor.py.exp @@ -0,0 +1,40 @@ +=== Constructor basic usage === +Template repr: Template(strings=('hello ', 'world'), interpolations=(Interpolation(42, 'x', None, ''),)) +Varargs constructor: strings=('Hello ', '!'), values=('World',) +Varargs merged strings: ('AB', 'CD') +Leading interpolation strings: ('', ' tail') +Trailing interpolation strings: ('head ', '') +Interpolation only strings: ('', '', '') + +=== Special cases === +Interp read-only: AttributeError +Whitespace trim: 'Template(strings=('', ''), interpolations=(Interpolation(None, ' ', None, ''),))' +Debug =: Template(strings=('', ''), interpolations=(Interpolation(42, 'x=', None, ''),)) +Custom !r: Template(strings=('', ''), interpolations=(Interpolation(CustomRepr, 'obj', 'r', ''),)) +Custom !s: Template(strings=('', ''), interpolations=(Interpolation(CustomRepr, 'obj', 's', ''),)) +Empty start iter: ['Interpolation', 'str'] +Iterator edge: ['Interpolation', 'Interpolation'] + +=== Values property === +Values[0]: () +Values[1]: (0,) +Values[2]: (0, 1) +Values[3]: (0, 1, 2) +Values[4]: (0, 1, 2, 3) +Values[5]: (0, 1, 2, 3, 4) +Values[6]: (0, 1, 2, 3, 4, 5) + +=== Multiple consecutive strings === +Multiple strings concatenated: OK + +=== Template() constructor with many interpolations === +Template() constructor: OK (20 interpolations) + +=== vstr string concatenation === +vstr concat: 'Template(strings=('part1part2part3part4', 'end'), interpolations=(Interpolation(1, 'x', None, ''),))' + +=== High byte handling === +High bytes: len=5, first=0x7f, last=0xff +Octal high bytes: OK, len=3 + +Constructor tests completed! diff --git a/tests/basics/string_tstring_constructor1.py b/tests/basics/string_tstring_constructor1.py new file mode 100644 index 00000000000..1593efa9d79 --- /dev/null +++ b/tests/basics/string_tstring_constructor1.py @@ -0,0 +1,24 @@ +# NOTE: Error messages are shortened in MicroPython to avoid stack overflow +# during ROM compression on constrained platforms (Windows x86, ASan). +# CPython 3.14 message: "Template.__new__ *args need to be of type 'str' or 'Interpolation', got int" +# MicroPython message: "Template.__new__ args must be str or Interpolation, got 'int'" + +from string.templatelib import Template + +print("=== Constructor error messages ===") +try: + Template(strings=("test",)) +except TypeError as e: + print(f"Keyword args: {e}") + +try: + Template("hello", 42, "world") +except TypeError as e: + print(f"Invalid type: {e}") + +try: + Template("a", 42, "b") +except TypeError as e: + print(f"Invalid type in varargs: {e}") + +print("\nConstructor error tests completed!") diff --git a/tests/basics/string_tstring_constructor1.py.exp b/tests/basics/string_tstring_constructor1.py.exp new file mode 100644 index 00000000000..932529c97b4 --- /dev/null +++ b/tests/basics/string_tstring_constructor1.py.exp @@ -0,0 +1,6 @@ +=== Constructor error messages === +Keyword args: function doesn't take keyword arguments +Invalid type: expected str or Interpolation +Invalid type in varargs: expected str or Interpolation + +Constructor error tests completed! diff --git a/tests/basics/string_tstring_errors1.py b/tests/basics/string_tstring_errors1.py new file mode 100644 index 00000000000..7a0c737ef67 --- /dev/null +++ b/tests/basics/string_tstring_errors1.py @@ -0,0 +1,246 @@ +from string.templatelib import Template, Interpolation + +print("\n=== Edge cases ===") +t_empty = Template() +print(f"Empty template: '{t_empty}'") + +t_empty_strs = Template("", Interpolation(1, "a"), "", Interpolation(2, "b"), "") +print(f"Empty strings: {list(t_empty_strs)}") + +t_adj = t"{1}{2}{3}" +print(f"Adjacent: '{t_adj}'") + +t_single = Template("only") +print(f"Single iter: {list(t_single)}") + +t_self = t"test" +print(f"Self+self: '{(t_self + t_self)}'") + +print("\n=== Additional coverage tests ===") + +t_str_literal = t"{'hello'}" +print(f"String literal: {t_str_literal}") + +path = "/usr/local/bin" +count = 42 +raw_path = rt"Path: {path}\n" +print(f"Raw path strings: {raw_path.strings}, value={raw_path.interpolations[0].value}") + +raw_regex = rt"Regex: \\d+{count}" +print(f"Raw regex strings: {raw_regex.strings}, value={raw_regex.interpolations[0].value}") + +try: + t_str_expr = t'{"test"}' + print(f"String expr: '{t_str_expr}'") +except Exception as e: + print(f"String expr error: {e}") + +def raise_error(): + raise ValueError("Special error") + +try: + t_exc = t"{raise_error()}" + print(t_exc) +except ValueError as e: + print(f"Re-raised exception: {e}") + +try: + large_str = "x" * 100000 + exec(f'very_long_name_{large_str} = t"test"') +except (ValueError, MemoryError, SyntaxError, RuntimeError) as e: + print("Large template: SyntaxError") + +try: + exec('''t_triple = t"""Triple "quoted" string"""''') + print(f"Triple quoted: '{t_triple}'") +except Exception as e: + print(f"Triple quoted error: {e}") + +try: + exec(r'''t_raw_triple = rt"""Raw triple\n{42}"""''') + print(f"Raw triple: '{t_raw_triple}'") +except Exception as e: + print(f"Raw triple error: {e}") + +t_concat1 = t"a{1}b" +t_concat2 = t"c{2}d{3}e" +t_concat3 = t_concat1 + t_concat2 +print(f"Complex concat: strings={t_concat3.strings}, values={t_concat3.values}") + +t_empty = Template() +t_nonempty = t"test{42}" +t_concat_empty = t_empty + t_nonempty +print(f"Empty concat: '{t_concat_empty}'") + +t_self_interp = t"x{1}y" +t_self_concat = t_self_interp + t_self_interp +print(f"Self interp concat: values={t_self_concat.values}") + +try: + exec('t"unterminated') +except SyntaxError as e: + print(f"Lonely string: {type(e).__name__}") + +try: + t_edge = t"" + print(f"Empty t-string: '{t_edge}'") +except Exception as e: + print(f"Empty t-string error: {e}") + + +print("\n=== High byte handling ===") +try: + result = t'\x7F\x80\x81\xFE\xFF' + first_str = result.strings[0] + print(f"High bytes: len={len(first_str)}, first=0x{ord(first_str[0]):02x}, last=0x{ord(first_str[-1]):02x}") +except Exception as e: + print(f"High bytes error: {e}") + +try: + result = t'\200\201\377' + print(f"Octal high bytes: OK, len={len(result.strings[0])}") +except Exception as e: + print(f"Octal high bytes error: {e}") + +print("\n=== Deep nesting test ===") +try: + nested = "1" + " + 1" * 150 + code = f't"{{{nested}}}"' + exec(code) + print("Deep nesting: OK") +except Exception as e: + print(f"Deep nesting error: {type(e).__name__}") + +print("\n=== Trailing whitespace in expression ===") +try: + x = 42 + code = 't"{x }"' + result = eval(code) + print(f"Trailing whitespace: OK") +except Exception as e: + print(f"Trailing whitespace error: {e}") + +print("\n=== Single closing brace ===") +try: + exec('t"test }"') + print("ERROR: Single } should have raised SyntaxError") +except SyntaxError as e: + print(f"Single }}: SyntaxError - {e}") + +print("\n=== Escape in string within interpolation ===") + +try: + compile('x = 1; t"{x=:"', '', 'exec') +except SyntaxError: + print('Debug unclosed colon: SyntaxError') + +try: + compile('x = 1; t"{x=!"', '', 'exec') +except SyntaxError: + print('Debug unclosed exclaim: SyntaxError') + +try: + compile('t"prefix{incomplete"', '', 'exec') +except SyntaxError as e: + print('Unclosed brace literal: SyntaxError') + +try: + import sys + if hasattr(sys.implementation, '_mpy'): + compile('f"{x!invalid}"', '', 'exec') +except SyntaxError: + print('Malformed f-string: SyntaxError') + +# NOTE: Error messages are shortened in MicroPython to avoid stack overflow +# during ROM compression on constrained platforms (Windows x86, ASan). +# CPython 3.14 message: "Template.__new__ *args need to be of type 'str' or 'Interpolation', got int" +# MicroPython message: "Template.__new__ args must be str or Interpolation, got 'int'" + +try: + t = Template("string", 123) +except TypeError as e: + print('Type error int:', e) + +try: + t = Template("a", "b", 3.14) +except TypeError: + print('Type error float: TypeError') + +try: + compile('t"text { more text"', '', 'exec') +except SyntaxError as e: + print('Unmatched brace:', e) + +try: + compile('x=1; t"{x= {nested}}"', '', 'exec') +except SyntaxError: + print('Nested debug braces: SyntaxError') + +try: + compile('t"{x!}"', '', 'exec') +except SyntaxError: + print('Missing conversion: SyntaxError') + +try: + compile('t"{x!z}"', '', 'exec') +except SyntaxError as e: + print('Invalid conversion:', e) + +try: + compile('t"{x!r' + chr(0x0b) + ':10}"', '', 'exec') +except SyntaxError as e: + print('Vertical tab:', e) + +try: + compile('t"{x!r@:10}"', '', 'exec') +except SyntaxError as e: + print('Invalid char after conversion:', e) + +# MicroPython doesn't allow space after conversion. +# try: +# x = 'test' +# exec('result = t"{x!r :10}"') +# print('Space after conversion: OK') +# except Exception as e: +# print(f'Unexpected error: {e}') + +try: + x = 'test' + exec('result = t"{x!s\\t:10}"') + print('Tab after conversion: OK') +except Exception as e: + print(f'Unexpected error: {e}') + +try: + x = 'test' + exec('result = t"{x!r\\n:10}"') + print('Newline after conversion: OK') +except Exception as e: + print(f'Unexpected error: {e}') + +try: + x = 'test' + exec('result = t"{x!r\\r:10}"') + print('CR after conversion: OK') +except Exception as e: + print(f'Unexpected error: {e}') + +try: + x = 'test' + exec('result = t"{x!s\\f:10}"') + print('Form-feed after conversion: OK') +except Exception as e: + print(f'Unexpected error: {e}') + +print("\n=== Mixed prefixes ===") +for src in ('ft"{x}"', 'tf"{x}"', 'frt"{x}"', 'rtf"{x}"', 'trf"{x}"'): + try: + exec(src) + print(src.split('"')[0] + ": OK (BUG!)") + except SyntaxError as e: + prefix = src.split('"')[0] + # Check if we get the specific error message or generic one + if "'f' and 't' prefixes are incompatible" in str(e): + print(f"{prefix}: incompatible prefixes") + else: + print(f"{prefix}: invalid syntax") diff --git a/tests/basics/string_tstring_errors1.py.exp b/tests/basics/string_tstring_errors1.py.exp new file mode 100644 index 00000000000..f8d993770d1 --- /dev/null +++ b/tests/basics/string_tstring_errors1.py.exp @@ -0,0 +1,60 @@ + +=== Edge cases === +Empty template: 'Template(strings=('',), interpolations=())' +Empty strings: [Interpolation(1, 'a', None, ''), Interpolation(2, 'b', None, '')] +Adjacent: 'Template(strings=('', '', '', ''), interpolations=(Interpolation(1, '1', None, ''), Interpolation(2, '2', None, ''), Interpolation(3, '3', None, '')))' +Single iter: ['only'] +Self+self: 'Template(strings=('testtest',), interpolations=())' + +=== Additional coverage tests === +String literal: Template(strings=('', ''), interpolations=(Interpolation('hello', "'hello'", None, ''),)) +Raw path strings: ('Path: ', '\\n'), value=/usr/local/bin +Raw regex strings: ('Regex: \\\\d+', ''), value=42 +String expr: 'Template(strings=('', ''), interpolations=(Interpolation('test', '"test"', None, ''),))' +Re-raised exception: Special error +Large template: SyntaxError +Triple quoted: 'Template(strings=('Triple "quoted" string',), interpolations=())' +Raw triple: 'Template(strings=('Raw triple\\n', ''), interpolations=(Interpolation(42, '42', None, ''),))' +Complex concat: strings=('a', 'bc', 'd', 'e'), values=(1, 2, 3) +Empty concat: 'Template(strings=('test', ''), interpolations=(Interpolation(42, '42', None, ''),))' +Self interp concat: values=(1, 1) +Lonely string: SyntaxError +Empty t-string: 'Template(strings=('',), interpolations=())' + +=== High byte handling === +High bytes: len=5, first=0x7f, last=0xff +Octal high bytes: OK, len=3 + +=== Deep nesting test === +Deep nesting: OK + +=== Trailing whitespace in expression === +Trailing whitespace: OK + +=== Single closing brace === +Single }: SyntaxError - malformed f-string + +=== Escape in string within interpolation === +Debug unclosed colon: SyntaxError +Debug unclosed exclaim: SyntaxError +Unclosed brace literal: SyntaxError +Malformed f-string: SyntaxError +Type error int: expected str or Interpolation +Type error float: TypeError +Unmatched brace: malformed f-string +Nested debug braces: SyntaxError +Missing conversion: SyntaxError +Invalid conversion: invalid syntax +Vertical tab: invalid syntax +Invalid char after conversion: invalid syntax +Unexpected error: invalid syntax +Unexpected error: invalid syntax +Unexpected error: invalid syntax +Unexpected error: invalid syntax + +=== Mixed prefixes === +ft: invalid syntax +tf: invalid syntax +frt: invalid syntax +rtf: invalid syntax +trf: invalid syntax diff --git a/tests/basics/string_tstring_format1.py b/tests/basics/string_tstring_format1.py new file mode 100644 index 00000000000..54fb0f1e76b --- /dev/null +++ b/tests/basics/string_tstring_format1.py @@ -0,0 +1,186 @@ +print("\n=== Format spec edge cases ===") +print(f"Empty fmt: {t'{42:}'}") +print(f"Width: '{t'{42:10}'}'") +print(f"Conv+fmt: '{t'{42!r:>10}'}'") + +width = 10 +print(f"Interp fmt: '{t'{3.14:{width}.2f}'}'") + +try: + t_escaped = t"Hello {{name}} and {{{{value}}}}" + print(f"Escaped braces: '{t_escaped}'") +except Exception as e: + print(f"Escaped braces error: {e}") + +try: + t_conv_fmt = t"{42!r:}" + print(f"Conv empty fmt: '{t_conv_fmt}'") +except Exception as e: + print(f"Conv fmt error: {e}") + +print("\n=== Format spec edge cases ===") +try: + x = 42 + result = t'{x!r:0>+#10.5}' + print(f"Full format: '{result}'") +except Exception as e: + print(f"Full format error: {e}") + +print("\n=== Format spec with special characters ===") +try: + x = 42 + result = t"{x:!r}" + print(f"Format spec '!r': {result.interpolations[0].format_spec == '!r'}") + print(f"Conversion is None: {result.interpolations[0].conversion is None}") +except Exception as e: + print(f"Format spec error: {type(e).__name__}") + +print("\nCoverage tests completed!") +print("\n=== Debug format edge cases ===") +try: + x = 42 + result = t'{x=!r}' + print(f"Debug with conv: '{result}'") +except SyntaxError as e: + print(f"Debug conv: SyntaxError - {e}") + + +print("\n=== Debug specifier tests ===") +try: + value = 42 + t1 = t'{value=}' + print(f"Debug basic: strings={t1.strings}, conv={t1.interpolations[0].conversion}") + + t2 = t'{value=!s}' + print(f"Debug !s: strings={t2.strings}, conv={t2.interpolations[0].conversion}") + + t3 = t'{value=:>10}' + print(f"Debug fmt: strings={t3.strings}, fmt_spec='{t3.interpolations[0].format_spec}'") +except Exception as e: + print(f"Debug format error: {e}") + + +print("\n=== Large number of interpolations ===") +for N in (62, 63, 64, 65): + x = 1 + code = 'result = str(t"' + '{x}' * N + '")' + exec(code) + expected = "Template(strings=" + repr(tuple("" for _ in range(N + 1))) + ", interpolations=(" + ", ".join("Interpolation(1, 'x', None, '')" for _ in range(N)) + "))" + print(N, result == expected) + +print("\n=== Malformed format specs (valid in CPython) ===") +x = 42 +value = 42 +result = t'{x:10!r}' +print(f"Malformed 1: format_spec: '{result.interpolations[0].format_spec}', conversion: {result.interpolations[0].conversion}") + +result = t'{value:^10!s}' +print(f"Malformed 2: format_spec: '{result.interpolations[0].format_spec}', conversion: {result.interpolations[0].conversion}") + +result = t'{x:>!10}' +print(f"Malformed 3: format_spec: '{result.interpolations[0].format_spec}', conversion: {result.interpolations[0].conversion}") + +try: + exec(r"t'\U00110000'") + print("ERROR: Should have failed") +except SyntaxError as e: + print(f"Invalid unicode: SyntaxError - {e}") + +class CustomException(Exception): + pass + +class BadClass: + def __getattr__(self, name): + raise CustomException("Test exception") + +try: + bad_obj = BadClass() + result = t"{bad_obj.attr}" + print("ERROR: Should have raised exception") +except CustomException as e: + print(f"Custom exception re-raised: {e}") +except Exception as e: + print(f"Other exception: {type(e).__name__} - {e}") + +print("\nNULL lexer: Tested in heapalloc_fail_tstring.py") + +try: + exec(r"t'\N{LATIN SMALL LETTER A}'") + print("ERROR: \\N{{}} should not be supported") +except (SyntaxError, NotImplementedError) as e: + print(f"\\N{{}} escape: SyntaxError") + +try: + result = rt'\u0041\U00000042' + print(f"Raw Unicode escapes: '{result.strings[0]}'") +except Exception as e: + print(f"Raw Unicode error: {e}") + +try: + code_with_crlf = 't"line1\\r\\nline2"' + exec(code_with_crlf) + print("CR LF handling: OK") +except Exception as e: + print(f"CR LF error: {e}") + +try: + code = "" + for i in range(20): + code += " " * i + "if True:\n" + code += " " * 20 + "x = t'deep indent'" + exec(code) + print("Deep indent: OK") +except Exception as e: + print(f"Deep indent error: {e}") + +try: + exec("t'test\\") + print("ERROR: Trailing backslash should fail") +except SyntaxError as e: + print(f"Trailing backslash: SyntaxError") + +print("\n=== Empty format spec node ===") +try: + x = 42 + result = t'{x:}' + print(f"Empty format after colon: OK") +except Exception as e: + print(f"Empty format spec error: {e}") + + +print("\n=== Format spec with special characters (coverage) ===") +try: + align = "<" + width = 5 + value = "test" + result = t'{value:{"^"*width}}' + print(f"Double quotes expr: format_spec={repr(result.interpolations[0].format_spec)}") +except Exception as e: + print(f"Double quotes expr error: {type(e).__name__}: {e}") + +try: + align = "<" + width = 5 + value = "test" + result = rt"{value:{align}{width}\\}" + print(f"Backslash raw: format_spec={repr(result.interpolations[0].format_spec)}") +except Exception as e: + print(f"Backslash raw error: {type(e).__name__}: {e}") + +try: + align = "<" + width = 5 + value = "test" + result = t"{value:{align}{width}\\}" + print(f"Backslash non-raw: format_spec={repr(result.interpolations[0].format_spec)}") +except Exception as e: + print(f"Backslash non-raw error: {type(e).__name__}: {e}") + +try: + align = "<" + width = 5 + value = "test" + result = t"{value:{align}{width}\n\t\r}" + print(f"Multiple escapes: format_spec={repr(result.interpolations[0].format_spec)}") +except Exception as e: + print(f"Multiple escapes error: {type(e).__name__}: {e}") diff --git a/tests/basics/string_tstring_format1.py.exp b/tests/basics/string_tstring_format1.py.exp new file mode 100644 index 00000000000..601087ccfd3 --- /dev/null +++ b/tests/basics/string_tstring_format1.py.exp @@ -0,0 +1,54 @@ + +=== Format spec edge cases === +Empty fmt: Template(strings=('', ''), interpolations=(Interpolation(42, '42', None, ''),)) +Width: 'Template(strings=('', ''), interpolations=(Interpolation(42, '42', None, '10'),))' +Conv+fmt: 'Template(strings=('', ''), interpolations=(Interpolation(42, '42', 'r', '>10'),))' +Interp fmt: 'Template(strings=('', ''), interpolations=(Interpolation(3.14, '3.14', None, '10.2f'),))' +Escaped braces: 'Template(strings=('Hello {name} and {{value}}',), interpolations=())' +Conv empty fmt: 'Template(strings=('', ''), interpolations=(Interpolation(42, '42', 'r', ''),))' + +=== Format spec edge cases === +Full format: 'Template(strings=('', ''), interpolations=(Interpolation(42, 'x', 'r', '0>+#10.5'),))' + +=== Format spec with special characters === +Format spec '!r': True +Conversion is None: True + +Coverage tests completed! + +=== Debug format edge cases === +Debug with conv: 'Template(strings=('x=', ''), interpolations=(Interpolation(42, 'x', 'r', ''),))' + +=== Debug specifier tests === +Debug basic: strings=('value=', ''), conv=r +Debug !s: strings=('value=', ''), conv=s +Debug fmt: strings=('value=', ''), fmt_spec='>10' + +=== Large number of interpolations === +62 True +63 True +64 True +65 True + +=== Malformed format specs (valid in CPython) === +Malformed 1: format_spec: '10!r', conversion: None +Malformed 2: format_spec: '^10!s', conversion: None +Malformed 3: format_spec: '>!10', conversion: None +Invalid unicode: SyntaxError - invalid syntax +Custom exception re-raised: Test exception + +NULL lexer: Tested in heapalloc_fail_tstring.py +\N{} escape: SyntaxError +Raw Unicode escapes: '\u0041\U00000042' +CR LF handling: OK +Deep indent: OK +Trailing backslash: SyntaxError + +=== Empty format spec node === +Empty format after colon: OK + +=== Format spec with special characters (coverage) === +Double quotes expr: format_spec='^^^^^' +Backslash raw: format_spec='<5\\\\' +Backslash non-raw: format_spec='<5\\' +Multiple escapes: format_spec='<5\n\t\r' diff --git a/tests/basics/string_tstring_interpolation1.py b/tests/basics/string_tstring_interpolation1.py new file mode 100644 index 00000000000..da9c469d2b3 --- /dev/null +++ b/tests/basics/string_tstring_interpolation1.py @@ -0,0 +1,72 @@ +print("\n=== Bracket/paren depth tracking ===") +d = {'a': 1, 'b:c': 2} +items = [10, 20, 30] +colon_key = t"{d['b:c']}" +print(f"Colon in key: expr={colon_key.interpolations[0].expression}, value={colon_key.interpolations[0].value}") + +slice_expr = t"{items[1:3]}" +print(f"Slice colon: expr={slice_expr.interpolations[0].expression}, value={slice_expr.interpolations[0].value}") + +def pair(x, y=':'): + return (x, y) + +func_expr = t"{pair(1, 2)}" +print(f"Function call: expr={func_expr.interpolations[0].expression}, value={func_expr.interpolations[0].value}") + +default_expr = t"{pair(3)}" +print(f"Default arg colon: expr={default_expr.interpolations[0].expression}, value={default_expr.interpolations[0].value}") + +matrix = [[1, 2], [3, 4]] +nested_expr = t"{matrix[0][1]}" +print(f"Nested brackets: expr={nested_expr.interpolations[0].expression}, value={nested_expr.interpolations[0].value}") + + +print("\n=== Inline template literal tests ===") + +try: + result = t"outer { t'' }" + print(f"Empty nested: {type(result.values[0]).__name__}") +except Exception as e: + print(f"Empty nested: FAIL - {type(e).__name__}: {e}") + +try: + result = t"outer { t'inner' }" + print(f"With content: {type(result.values[0]).__name__}") +except Exception as e: + print(f"With content: FAIL - {type(e).__name__}: {e}") + +try: + result = t"result: { t'a' + t'b' }" + print(f"Concatenation: {type(result.values[0]).__name__}") +except Exception as e: + print(f"Concatenation: FAIL - {type(e).__name__}: {e}") + +try: + result = t'Single { t"double" }' + print(f"Mixed quotes: {type(result.values[0]).__name__}") +except Exception as e: + print(f"Mixed quotes: FAIL - {type(e).__name__}: {e}") + +try: + x = 42 + result = t"outer { t'inner {x}' }" + print(f"Nested interp: {type(result.values[0]).__name__}") +except Exception as e: + print(f"Nested interp: FAIL - {type(e).__name__}: {e}") + +print("\n=== Backslashes not allowed in expressions (PEP 498/750) ===") +try: + code = r"result = t'{\"test\"value\"}'" + exec(code) + print(f"ERROR: Backslash in expression should raise SyntaxError") +except SyntaxError as e: + print(f"Backslash in expression: SyntaxError") + +try: + code = r"result = t'{\"\\n\"}'" + exec(code) + print(f"ERROR: Backslash in expression should raise SyntaxError") +except SyntaxError as e: + print(f"Escaped newline: SyntaxError") + +print("\n=== Format spec with special characters ===") diff --git a/tests/basics/string_tstring_interpolation1.py.exp b/tests/basics/string_tstring_interpolation1.py.exp new file mode 100644 index 00000000000..8a62f331826 --- /dev/null +++ b/tests/basics/string_tstring_interpolation1.py.exp @@ -0,0 +1,20 @@ + +=== Bracket/paren depth tracking === +Colon in key: expr=d['b:c'], value=2 +Slice colon: expr=items[1:3], value=[20, 30] +Function call: expr=pair(1, 2), value=(1, 2) +Default arg colon: expr=pair(3), value=(3, ':') +Nested brackets: expr=matrix[0][1], value=2 + +=== Inline template literal tests === +Empty nested: Template +With content: Template +Concatenation: Template +Mixed quotes: Template +Nested interp: Template + +=== Backslashes not allowed in expressions (PEP 498/750) === +Backslash in expression: SyntaxError +Escaped newline: SyntaxError + +=== Format spec with special characters === diff --git a/tests/basics/string_tstring_operations.py b/tests/basics/string_tstring_operations.py new file mode 100644 index 00000000000..3c3222025f8 --- /dev/null +++ b/tests/basics/string_tstring_operations.py @@ -0,0 +1,55 @@ +from string.templatelib import Template, Interpolation + +print("\n=== Binary operations ===") +t1 = t"template" + +try: + t1 + "string" +except TypeError as e: + print(f"Template+str: TypeError") + +try: + "string" + t1 +except TypeError as e: + print(f"str+Template: TypeError") + +try: + 42 + t1 +except TypeError: + print("int+Template: TypeError") + +for op in ["-", "*", "/", "%", "**", "&", "|", "^", "<<", ">>"]: + try: + eval(f"t1 {op} t1") + except TypeError: + print(f"{op}: unsupported") + +print("\n=== Template.__add__ with multiple interpolations ===") +try: + exprs = ["a", "b", "c", "d", "e"] + interps1 = [Interpolation(i, exprs[i % len(exprs)]) for i in range(20)] + strings1 = [""] * 21 + t1 = Template(*strings1, *interps1) + + interps2 = [Interpolation(i + 20, exprs[i % len(exprs)]) for i in range(20)] + strings2 = [""] * 21 + t2 = Template(*strings2, *interps2) + + result = t1 + t2 + print(f"Template.__add__: OK ({len(result.interpolations)} interpolations)") +except Exception as e: + print(f"Template.__add__: {type(e).__name__}") + +print("\n=== Template + non-string object ===") +try: + + class CustomObj: + pass + + t1 = t"hello" + result = t1 + CustomObj() + print("ERROR: Should have raised TypeError") +except TypeError as e: + print(f"Template + custom object: TypeError (correct)") + +print("\n=== 2-tuple constructor overflow ===") diff --git a/tests/basics/string_tstring_operations.py.exp b/tests/basics/string_tstring_operations.py.exp new file mode 100644 index 00000000000..a565db8fec9 --- /dev/null +++ b/tests/basics/string_tstring_operations.py.exp @@ -0,0 +1,23 @@ + +=== Binary operations === +Template+str: TypeError +str+Template: TypeError +int+Template: TypeError +-: unsupported +*: unsupported +/: unsupported +%: unsupported +**: unsupported +&: unsupported +|: unsupported +^: unsupported +<<: unsupported +>>: unsupported + +=== Template.__add__ with multiple interpolations === +Template.__add__: OK (40 interpolations) + +=== Template + non-string object === +Template + custom object: TypeError (correct) + +=== 2-tuple constructor overflow === diff --git a/tests/basics/string_tstring_parser1.py b/tests/basics/string_tstring_parser1.py new file mode 100644 index 00000000000..b8853ccd1e7 --- /dev/null +++ b/tests/basics/string_tstring_parser1.py @@ -0,0 +1,288 @@ +from string.templatelib import Template + +print("\n=== Empty expression tests ===") +try: + exec("t'{}'") + print("ERROR: Empty expression should have raised SyntaxError") +except SyntaxError as e: + print("Empty expr SyntaxError:", e) + +try: + exec("t'{ }'") + print("ERROR: Whitespace-only expression should have raised SyntaxError") +except SyntaxError as e: + print("Whitespace expr SyntaxError:", e) + +try: + exec("t'{\\n\\t \\n}'") + print("ERROR: Whitespace-only expr should fail") +except SyntaxError as e: + print(f"Whitespace expr: SyntaxError") + +try: + exec("t'{value'") + print("ERROR: Unterminated expr should fail") +except SyntaxError as e: + print(f"Unterminated expr: {type(e).__name__}") + +try: + exec("t'{ !r}'") + print("ERROR: Whitespace-conversion expr should fail") +except SyntaxError as e: + print(f"Whitespace + conversion: {type(e).__name__}") + +try: + long_expr = 'x' * 104 + code = f"t'{{{long_expr}}}'" + exec(f"{long_expr} = 'test'; result = {code}") + print("Long expression: OK") +except Exception as e: + print(f"Long expr error: {type(e).__name__}: {e}") + +try: + large_str = 'x' * 1099 + tmpl = Template(large_str) + result = str(tmpl) + print("Template.__str__ with large string: OK") +except Exception as e: + print(f"Template.__str__ error: {type(e).__name__}: {e}") + +try: + x = 'a' + result = t'{x}' + print(f"Basic t-string: OK, result={result}") +except Exception as e: + print(f"Basic t-string error: {type(e).__name__}: {e}") + +try: + code = 't"' + 'text{nested{x}}more' * 50 + '"' + x = 'test' + exec(f"x = 'test'; result = {code}") + print("Nested interpolations: OK") +except Exception as e: + print(f"Nested interpolations error: {type(e).__name__}: {e}") + +width = 5 +x = 42 +try: + tmpl = t'{x:{width}{{literal}}}' + result = str(tmpl) + print(f'Escaped braces result: {result}') +except ValueError as e: + print(f'Escaped braces ValueError: {e}') + +print("\n=== Expression parser tests ===") +try: + exec("t'{[{(x,y):[1,2,3]} for x,y in [(1,2),(3,4)]]}'") + print("Complex expr: OK") +except Exception as e: + print(f"Complex expr error: {type(e).__name__}") + +print("\n=== Lexer edge cases ===") +print("Lexer NULL case: Tested via heapalloc_fail_tstring.py") + +print("\n=== Parser allocation edge cases ===") +try: + deep_expr = "(" * 50 + "x" + ")" * 50 + exec(f"x = 1; result = t'{{{deep_expr}}}'") + print("Deep nesting: OK") +except Exception as e: + print(f"Deep nesting error: {type(e).__name__} - {e}") + +try: + long_expr = "(" * 100 + "x" + ")" * 100 + exec(f"x = 1; result = t'{{{long_expr}}}'") + print("Very long expression: OK") +except Exception as e: + print(f"Long expression error: {e}") + + +print("\n=== Additional parser regression tests ===") + +try: + fmt = '"QUOTED"' + value = 'raw' + tmpl = rt"{value:{fmt}}" + print(f"Raw nested spec: {tmpl.interpolations[0].format_spec}") +except Exception as e: + print(f"Raw nested spec error: {type(e).__name__}: {e}") + +try: + value = 'fmt' + tmpl = t'{value:"QUOTED"}' + print(f"Escaped quote spec: {tmpl.interpolations[0].format_spec}") +except Exception as e: + print(f"Escaped quote spec error: {type(e).__name__}: {e}") + +try: + value = 'fmt' + tmpl = t"{value:\\n}" + print(f"Escaped backslash spec: {tmpl.interpolations[0].format_spec}") +except Exception as e: + print(f"Escaped backslash spec error: {type(e).__name__}: {e}") + +try: + ns = {} + exec("value = 'fmt'\nresult = t\"{value:{'\\\\'}}\"", globals(), ns) + tmpl = ns['result'] + print(f"Nested quoted spec: {tmpl.interpolations[0].format_spec}") +except Exception as e: + print(f"Nested quoted spec error: {type(e).__name__}: {e}") + +try: + value = 'brace' + exec('tmpl = t"{value}}}}tail"') + print(f"Literal brace strings: {tmpl.strings}") +except Exception as e: + print(f"Literal brace error: {type(e).__name__}: {e}") + +try: + exec('t"{value"') + print("ERROR: Unterminated field should have raised") +except SyntaxError as e: + print(f"Unterminated field: {e}") + + +print("\n=== Complex expression stress test ===") +try: + vars_dict = {f"x{i}": i for i in range(60)} + exec_globals = globals().copy() + exec_globals.update(vars_dict) + expr_parts = [f"x{i}" for i in range(60)] + expr_str = " + ".join(expr_parts) + code = f't"{{{expr_str}}}"' + result = eval(code, exec_globals) + expected = sum(range(60)) + actual = result.interpolations[0].value + if actual == expected: + print(f"60 terms: PASS") + else: + print(f"60 terms: FAIL (expected={expected}, got={actual})") +except Exception as e: + print(f"60 terms: ERROR - {type(e).__name__}: {e}") + +try: + vars_dict = {f"x{i}": i for i in range(100)} + exec_globals = globals().copy() + exec_globals.update(vars_dict) + expr_parts = [f"x{i}" for i in range(100)] + expr_str = " + ".join(expr_parts) + code = f't"{{{expr_str}}}"' + result = eval(code, exec_globals) + expected = sum(range(100)) + actual = result.interpolations[0].value + if actual == expected: + print(f"100 terms: PASS") + else: + print(f"100 terms: FAIL (expected={expected}, got={actual})") +except Exception as e: + print(f"100 terms: ERROR - {type(e).__name__}: {e}") + +print("\n=== Stack expansion test (copy_parse_node) ===") +try: + depth = 20 + nested_expr = "[" * depth + "1" + "]" * depth + code = f't"{{{nested_expr}}}"' + result = eval(code) + if len(str(result.interpolations[0].value)) > 30: + print("Stack expansion: OK") + else: + print(f"Stack expansion: FAIL") +except Exception as e: + print(f"Stack expansion error: {type(e).__name__}") + +print("\n=== Format spec with triple-quoted strings ===") +try: + x = 42 + code = '''t'{x:{"""test"""}}' ''' + result = eval(code) + print(f"Triple-quote in format spec: OK") +except Exception as e: + print(f"Triple-quote error: {type(e).__name__}: {e}") + +try: + x = 42 + code = """t"{x:{'''test'''}}" """ + result = eval(code) + print(f"Triple single-quote in format spec: OK") +except Exception as e: + print(f"Triple single-quote error: {type(e).__name__}: {e}") + +print("\n=== Format spec parse error (exception handler) ===") +try: + x = 42 + code = "t\"{x:{1 + (}}\" " # Unclosed parenthesis + result = eval(code) + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print(f"Parse error handler: OK") +except Exception as e: + print(f"Unexpected error: {type(e).__name__}: {e}") + +try: + x = 42 + code = "t\"{x:{1 2 3}}\" " # Invalid syntax + result = eval(code) + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print(f"Parse error handler 2: OK") +except Exception as e: + print(f"Unexpected error 2: {type(e).__name__}: {e}") + +print("\n=== Large integer in t-string ===") +try: + # Use eval() to avoid compile-time overflow in longlong variant + large_int = eval("10**30") + tmpl = t"{large_int}" + if tmpl.interpolations[0].value == large_int: + print("Large integer: OK") + else: + print(f"Large integer: FAIL") +except OverflowError: + # longlong variant doesn't support arbitrary precision integers + print("Large integer: OK") +except Exception as e: + print(f"Large integer error: {type(e).__name__}: {e}") + +print("\n=== Bytes in expression ===") +try: + data = b"test" + tmpl = t"{data}" + if tmpl.interpolations[0].value == b"test": + print("Bytes in expression: OK") + else: + print(f"Bytes: FAIL") +except Exception as e: + print(f"Bytes error: {type(e).__name__}: {e}") + +print("\n=== Boolean constants ===") +try: + tmpl = t"{True} {False}" + if tmpl.values == (True, False): + print("Boolean constants: OK") + else: + print(f"Booleans: FAIL") +except Exception as e: + print(f"Boolean error: {type(e).__name__}: {e}") + +print("\n=== None constant ===") +try: + tmpl = t"{None}" + if tmpl.values[0] is None: + print("None constant: OK") + else: + print(f"None: FAIL") +except Exception as e: + print(f"None error: {type(e).__name__}: {e}") + +print("\n=== Deep nesting for rule stack expansion ===") +try: + depth = 80 + code = '(' * depth + '1' + ')' * depth + result = eval(code) + if result == 1: + print(f"Deep nesting (depth={depth}): OK") + else: + print(f"Deep nesting: FAIL (result={result})") +except Exception as e: + print(f"Deep nesting error: {type(e).__name__}: {e}") diff --git a/tests/basics/string_tstring_parser1.py.exp b/tests/basics/string_tstring_parser1.py.exp new file mode 100644 index 00000000000..cd55b258546 --- /dev/null +++ b/tests/basics/string_tstring_parser1.py.exp @@ -0,0 +1,60 @@ + +=== Empty expression tests === +Empty expr SyntaxError: malformed f-string +Whitespace expr SyntaxError: malformed f-string +Whitespace expr: SyntaxError +Unterminated expr: SyntaxError +Whitespace + conversion: SyntaxError +Long expression: OK +Template.__str__ with large string: OK +Basic t-string: OK, result=Template(strings=('', ''), interpolations=(Interpolation('a', 'x', None, ''),)) +Nested interpolations error: SyntaxError: invalid syntax +Escaped braces result: Template(strings=('', ''), interpolations=(Interpolation(42, 'x', None, '5{literal}'),)) + +=== Expression parser tests === +Complex expr: OK + +=== Lexer edge cases === +Lexer NULL case: Tested via heapalloc_fail_tstring.py + +=== Parser allocation edge cases === +Deep nesting: OK +Very long expression: OK + +=== Additional parser regression tests === +Raw nested spec: "QUOTED" +Escaped quote spec: "QUOTED" +Escaped backslash spec: \n +Nested quoted spec: \ +Literal brace error: SyntaxError: malformed f-string +Unterminated field: malformed f-string + +=== Complex expression stress test === +60 terms: PASS +100 terms: PASS + +=== Stack expansion test (copy_parse_node) === +Stack expansion: OK + +=== Format spec with triple-quoted strings === +Triple-quote in format spec: OK +Triple single-quote in format spec: OK + +=== Format spec parse error (exception handler) === +Parse error handler: OK +Parse error handler 2: OK + +=== Large integer in t-string === +Large integer: OK + +=== Bytes in expression === +Bytes in expression: OK + +=== Boolean constants === +Boolean constants: OK + +=== None constant === +None constant: OK + +=== Deep nesting for rule stack expansion === +Deep nesting (depth=80): OK diff --git a/tests/basics/string_tstring_whitespace.py b/tests/basics/string_tstring_whitespace.py new file mode 100644 index 00000000000..08dd7933c38 --- /dev/null +++ b/tests/basics/string_tstring_whitespace.py @@ -0,0 +1,25 @@ +print("# Empty expression (whitespace only)") +try: + exec('t"{ }"') + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print("Empty expr (space): SyntaxError (correct)") + +try: + exec('t"{\t}"') + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print("Empty expr (tab): SyntaxError (correct)") + +try: + exec('t"{\n}"') + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print("Empty expr (newline): SyntaxError (correct)") + +print("\n# Lexer escape sequence: invalid then valid") +try: + exec("t'\\U00200000\\x41'") + print("ERROR: Should have raised SyntaxError") +except SyntaxError as e: + print("Invalid escape sequence: SyntaxError (correct)") diff --git a/tests/basics/string_tstring_whitespace.py.exp b/tests/basics/string_tstring_whitespace.py.exp new file mode 100644 index 00000000000..f5868faf68f --- /dev/null +++ b/tests/basics/string_tstring_whitespace.py.exp @@ -0,0 +1,7 @@ +# Empty expression (whitespace only) +Empty expr (space): SyntaxError (correct) +Empty expr (tab): SyntaxError (correct) +Empty expr (newline): SyntaxError (correct) + +# Lexer escape sequence: invalid then valid +Invalid escape sequence: SyntaxError (correct) diff --git a/tests/basics/try_finally_break.py.exp b/tests/basics/try_finally_break.py.exp new file mode 100644 index 00000000000..d0b36985489 --- /dev/null +++ b/tests/basics/try_finally_break.py.exp @@ -0,0 +1,25 @@ +1 +2 +5 +a 1 +1 +iter 0 +1 +None +1 +2 +None +1 +2 +None +1 +3 +4 +7 +1 +2 +4 +7 +1 +4 +7 diff --git a/tests/basics/try_finally_break2.py.exp b/tests/basics/try_finally_break2.py.exp new file mode 100644 index 00000000000..bc5e931fdb4 --- /dev/null +++ b/tests/basics/try_finally_break2.py.exp @@ -0,0 +1,17 @@ +4 0 0 1 +4 0 0 2 +4 0 0 3 +4 0 0 4 +4 1 0 1 +4 1 0 2 +4 1 0 3 +4 1 0 4 +4 2 0 1 +4 2 0 2 +4 2 0 3 +4 2 0 4 +4 3 0 1 +4 3 0 2 +4 3 0 3 +4 3 0 4 +None diff --git a/tests/basics/try_finally_continue.py.exp b/tests/basics/try_finally_continue.py.exp new file mode 100644 index 00000000000..2901997b1aa --- /dev/null +++ b/tests/basics/try_finally_continue.py.exp @@ -0,0 +1,9 @@ +4 0 +continue +4 1 +continue +4 2 +continue +4 3 +continue +None diff --git a/tests/basics/try_finally_return2.py.exp b/tests/basics/try_finally_return2.py.exp new file mode 100644 index 00000000000..b4c364f81d9 --- /dev/null +++ b/tests/basics/try_finally_return2.py.exp @@ -0,0 +1,19 @@ +finally +0 +finally 1 +finally 2 +2 +finally 1 +finally 2 +1 +finally 1 +finally 2 +2 +finally +0 +finally +0 +finally +0 +finally +0 diff --git a/tests/basics/try_finally_return3.py.exp b/tests/basics/try_finally_return3.py.exp new file mode 100644 index 00000000000..5fef648ea67 --- /dev/null +++ b/tests/basics/try_finally_return3.py.exp @@ -0,0 +1,23 @@ +1 +42 +1 +2 +42 +1 +2 +3 +42 +2 +1 +42 +2 +1 +42 +1 +3 +2 +42 +1 +3 +2 +42 diff --git a/tests/basics/try_finally_return4.py.exp b/tests/basics/try_finally_return4.py.exp new file mode 100644 index 00000000000..f92572ff611 --- /dev/null +++ b/tests/basics/try_finally_return4.py.exp @@ -0,0 +1,38 @@ +1 +2 +3 +4 +5 +None +1 +2 +3 +5 +42 +1 +2 +5 +43 +1 +2 +5 +43 +1 +2 +5 +caught +1 +2 +5 +caught +1 +2 +3 +4 +5 +None +1 +2 +3 +5 +42 diff --git a/tests/basics/try_finally_return5.py.exp b/tests/basics/try_finally_return5.py.exp new file mode 100644 index 00000000000..dfa403cfd47 --- /dev/null +++ b/tests/basics/try_finally_return5.py.exp @@ -0,0 +1,3 @@ +4 0 +return +43 diff --git a/tests/basics/weakref_callback_exception.py b/tests/basics/weakref_callback_exception.py new file mode 100644 index 00000000000..df8e5129803 --- /dev/null +++ b/tests/basics/weakref_callback_exception.py @@ -0,0 +1,42 @@ +# Test weakref ref/finalize raising an exception within the callback. +# +# This test has different output to CPython due to the way that MicroPython +# prints the exception. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +import gc + + +class A: + def __str__(self): + return "" + + +def callback(*args): + raise ValueError("weakref callback", args) + + +def test(): + print("test ref with exception in the callback") + a = A() + r = weakref.ref(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("collect done") + + print("test finalize with exception in the callback") + a = A() + weakref.finalize(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("collect done") + + +test() diff --git a/tests/basics/weakref_callback_exception.py.exp b/tests/basics/weakref_callback_exception.py.exp new file mode 100644 index 00000000000..c2f7796310c --- /dev/null +++ b/tests/basics/weakref_callback_exception.py.exp @@ -0,0 +1,12 @@ +test ref with exception in the callback +Unhandled exception in weakref callback: +Traceback (most recent call last): + File "\.\+weakref_callback_exception.py", line 21, in callback +ValueError: ('weakref callback', (,)) +collect done +test finalize with exception in the callback +Unhandled exception in weakref callback: +Traceback (most recent call last): + File "\.\+weakref_callback_exception.py", line 21, in callback +ValueError: ('weakref callback', ()) +collect done diff --git a/tests/basics/weakref_callback_exception.py.native.exp b/tests/basics/weakref_callback_exception.py.native.exp new file mode 100644 index 00000000000..a06a35c3b7e --- /dev/null +++ b/tests/basics/weakref_callback_exception.py.native.exp @@ -0,0 +1,8 @@ +test ref with exception in the callback +Unhandled exception in weakref callback: +ValueError: ('weakref callback', (,)) +collect done +test finalize with exception in the callback +Unhandled exception in weakref callback: +ValueError: ('weakref callback', ()) +collect done diff --git a/tests/basics/weakref_finalize_basic.py b/tests/basics/weakref_finalize_basic.py new file mode 100644 index 00000000000..792cffacb13 --- /dev/null +++ b/tests/basics/weakref_finalize_basic.py @@ -0,0 +1,58 @@ +# Test weakref.finalize() functionality that doesn't require gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.finalize(value, lambda: None) + except TypeError: + print(value, "TypeError") + + +# Convert (obj, func, args, kwargs) so CPython and MicroPython have a chance to match. +def convert_4_tuple(values): + if values is None: + return None + return (type(values[0]).__name__, type(values[1]), values[2], values[3]) + + +class A: + def __str__(self): + return "" + + +print("test alive, peek, detach") +a = A() +f = weakref.finalize(a, lambda: None, 1, 2, kwarg=3) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("detach", convert_4_tuple(f.detach())) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("detach", convert_4_tuple(f.detach())) +print("call", f()) +a = None + +print("test alive, peek, call") +a = A() +f = weakref.finalize(a, lambda *args, **kwargs: (args, kwargs), 1, 2, kwarg=3) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("call", f()) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("call", f()) +print("detach", convert_4_tuple(f.detach())) + +print("test call which raises exception") +a = A() +f = weakref.finalize(a, lambda: 1 / 0) +try: + f() +except ZeroDivisionError as er: + print("call ZeroDivisionError") diff --git a/tests/basics/weakref_finalize_collect.py b/tests/basics/weakref_finalize_collect.py new file mode 100644 index 00000000000..f6e7c14843e --- /dev/null +++ b/tests/basics/weakref_finalize_collect.py @@ -0,0 +1,76 @@ +# Test weakref.finalize() functionality requiring gc.collect(). +# Should be kept in sync with tests/ports/webassembly/weakref_finalize_collect.py. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + + +class A: + def __str__(self): + return "" + + +def callback(*args, **kwargs): + print("callback({}, {})".format(args, kwargs)) + return 42 + + +def test(): + print("test basic use of finalize() with a simple callback") + a = A() + f = weakref.finalize(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) + + print("test that a callback is passed the correct values") + a = A() + f = weakref.finalize(a, callback, 1, 2, kwarg=3) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) + + print("test that calling the finalizer cancels the finalizer") + a = A() + f = weakref.finalize(a, callback) + print(f()) + print(a) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + print("test that calling detach cancels the finalizer") + a = A() + f = weakref.finalize(a, callback) + print(len(f.detach())) + print(a) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + print("test that finalize does not get collected before its ref does") + a = A() + weakref.finalize(a, callback) + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("free a") + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + +test() diff --git a/tests/basics/weakref_multiple_refs.py b/tests/basics/weakref_multiple_refs.py new file mode 100644 index 00000000000..400e03a17c7 --- /dev/null +++ b/tests/basics/weakref_multiple_refs.py @@ -0,0 +1,36 @@ +# Test weakref when multiple weak references are active. +# +# This test has different output to CPython due to the order that MicroPython +# executes weak reference callbacks. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + + +class A: + def __str__(self): + return "" + + +def test(): + global r1, r2 # needed for webassembly port to retain references to them + + print("test having multiple ref and finalize objects referencing the same thing") + a = A() + r1 = weakref.ref(a, lambda r: print("ref1", r())) + f1 = weakref.finalize(a, lambda: print("finalize1")) + r2 = weakref.ref(a, lambda r: print("ref2", r())) + f2 = weakref.finalize(a, lambda: print("finalize2")) + print(r1(), f1.alive, r2(), f2.alive) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + +test() diff --git a/tests/basics/weakref_multiple_refs.py.exp b/tests/basics/weakref_multiple_refs.py.exp new file mode 100644 index 00000000000..1f2d366f776 --- /dev/null +++ b/tests/basics/weakref_multiple_refs.py.exp @@ -0,0 +1,6 @@ +test having multiple ref and finalize objects referencing the same thing + True True +finalize2 +finalize1 +ref2 None +ref1 None diff --git a/tests/basics/weakref_ref_basic.py b/tests/basics/weakref_ref_basic.py new file mode 100644 index 00000000000..058045f6c33 --- /dev/null +++ b/tests/basics/weakref_ref_basic.py @@ -0,0 +1,14 @@ +# Test weakref.ref() functionality that doesn't require gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.ref(value) + except TypeError: + print(value, "TypeError") diff --git a/tests/basics/weakref_ref_collect.py b/tests/basics/weakref_ref_collect.py new file mode 100644 index 00000000000..0e8db977d77 --- /dev/null +++ b/tests/basics/weakref_ref_collect.py @@ -0,0 +1,69 @@ +# Test weakref.ref() functionality requiring gc.collect(). +# Should be kept in sync with tests/ports/webassembly/weakref_ref_collect.py. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.ref(value) + except TypeError: + print(value, "TypeError") + + +class A: + def __str__(self): + return "" + + +def callback(r): + print("callback", r()) + + +def test(): + print("test basic use of ref() with only one argument") + a = A() + r = weakref.ref(a) + print(r()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r()) + + print("test use of ref() with a callback") + a = A() + r = weakref.ref(a, callback) + print(r()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r()) + + print("test when weakref gets collected before the object it refs") + a = A() + r = weakref.ref(a, callback) + print(r()) + r = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + a = None + + print("test a double reference") + a = A() + r1 = weakref.ref(a, callback) + r2 = weakref.ref(a, callback) + print(r1(), r2()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r1(), r2()) + + +test() diff --git a/tests/cmdline/cmd_file_variable.py.exp b/tests/cmdline/cmd_file_variable.py.exp index 6807569f662..420f789ddf3 100644 --- a/tests/cmdline/cmd_file_variable.py.exp +++ b/tests/cmdline/cmd_file_variable.py.exp @@ -1 +1 @@ -__file__ = cmdline/cmd_file_variable.py +__file__ = \.\*cmdline/cmd_file_variable.py diff --git a/tests/cmdline/cmd_module_atexit.py.exp b/tests/cmdline/cmd_module_atexit.py.exp index 2a0f756b1e7..334bf53bac9 100644 --- a/tests/cmdline/cmd_module_atexit.py.exp +++ b/tests/cmdline/cmd_module_atexit.py.exp @@ -1,3 +1,3 @@ -['cmdline.cmd_module_atexit', 'cmdline/cmd_module_atexit.py'] +['cmdline.cmd_module_atexit', '\.\*cmdline/cmd_module_atexit.py'] start done diff --git a/tests/cmdline/cmd_module_atexit_exc.py.exp b/tests/cmdline/cmd_module_atexit_exc.py.exp index 6320d9d2d30..210748afdfe 100644 --- a/tests/cmdline/cmd_module_atexit_exc.py.exp +++ b/tests/cmdline/cmd_module_atexit_exc.py.exp @@ -1,3 +1,3 @@ -['cmdline.cmd_module_atexit_exc', 'cmdline/cmd_module_atexit_exc.py'] +['cmdline.cmd_module_atexit_exc', '\.\*cmdline/cmd_module_atexit_exc.py'] start done diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index db06de92371..8ac408c16a6 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -1,4 +1,4 @@ -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 63 bytes) +File \.\*cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 63 bytes) Raw bytecode (code_info_size=18, bytecode_size=45): 10 20 01 60 20 84 7d 64 60 88 07 64 60 69 20 62 64 20 32 00 16 05 32 01 16 05 81 2a 01 53 33 02 @@ -47,7 +47,7 @@ arg names: 42 IMPORT_STAR 43 LOAD_CONST_NONE 44 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 46\[68\] bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 46\[68\] bytes) Raw bytecode (code_info_size=8\[46\], bytecode_size=382): a8 12 9\[bf\] 03 05 60 60 26 22 24 64 22 24 25 25 24 26 23 63 22 22 25 23 23 2f 6c 25 65 25 25 69 68 @@ -411,7 +411,7 @@ arg names: 379 RETURN_VALUE 380 LOAD_CONST_NONE 381 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 59 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 59 bytes) Raw bytecode (code_info_size=8, bytecode_size=51): a8 10 0a 05 80 82 34 38 81 57 c0 57 c1 57 c2 57 c3 57 c4 57 c5 57 c6 57 c7 57 c8 c9 82 57 ca 57 @@ -470,7 +470,7 @@ arg names: 48 POP_TOP 49 LOAD_CONST_NONE 50 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 20 bytes) Raw bytecode (code_info_size=9, bytecode_size=11): a1 01 0b 05 06 80 88 40 00 82 2a 01 53 b0 21 00 01 c1 51 63 @@ -489,7 +489,7 @@ arg names: a 08 STORE_FAST 1 09 LOAD_CONST_NONE 10 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 21 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 21 bytes) Raw bytecode (code_info_size=8, bytecode_size=13): 88 40 0a 05 80 8f 23 23 51 67 59 81 67 59 81 5e 51 68 59 51 63 @@ -513,7 +513,7 @@ arg names: 10 POP_TOP 11 LOAD_CONST_NONE 12 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ 1\[56\] bytes) +File \.\*cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ 1\[56\] bytes) Raw bytecode (code_info_size=\[56\], bytecode_size=10): 00 \.\+ 11 0f 16 10 10 02 16 11 51 63 arg names: @@ -528,7 +528,7 @@ arg names: 06 STORE_NAME __qualname__ 08 LOAD_CONST_NONE 09 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 18 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 18 bytes) Raw bytecode (code_info_size=6, bytecode_size=12): 19 08 05 12 80 9c 12 13 12 14 b0 15 05 36 00 59 51 63 @@ -545,7 +545,7 @@ arg names: self 09 POP_TOP 10 LOAD_CONST_NONE 11 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) +File \.\*cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) Raw bytecode (code_info_size=9, bytecode_size=19): c3 40 0c 09 03 03 03 80 3b 53 b2 53 53 4b 0b c3 25 01 44 39 25 00 67 59 42 33 51 63 @@ -568,7 +568,7 @@ arg names: * * * 15 JUMP 4 17 LOAD_CONST_NONE 18 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 26 bytes) +File \.\*cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 26 bytes) Raw bytecode (code_info_size=8, bytecode_size=18): 4b 0c 0a 03 03 03 80 3c 2b 00 b2 5f 4b 0b c3 25 01 44 39 25 00 2f 14 42 33 63 @@ -588,7 +588,7 @@ arg names: * * * 13 STORE_COMP 20 15 JUMP 4 17 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) +File \.\*cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ 28 bytes) Raw bytecode (code_info_size=8, bytecode_size=20): 53 0c 0b 03 03 03 80 3d 2c 00 b2 5f 4b 0d c3 25 01 44 39 25 00 25 00 2f 19 42 31 63 @@ -609,7 +609,7 @@ arg names: * * * 15 STORE_COMP 25 17 JUMP 4 19 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ 20 bytes) Raw bytecode (code_info_size=8, bytecode_size=12): 19 0c 0c 03 80 6f 25 23 25 00 81 f2 c1 81 27 00 29 00 51 63 @@ -629,7 +629,7 @@ arg names: * 08 DELETE_DEREF 0 10 LOAD_CONST_NONE 11 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 13 bytes) +File \.\*cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 13 bytes) Raw bytecode (code_info_size=8, bytecode_size=5): 9a 01 0a 05 03 08 80 8b b1 25 00 f2 63 arg names: * b diff --git a/tests/cmdline/cmd_showbc_const.py.exp b/tests/cmdline/cmd_showbc_const.py.exp index 6cdc3e9c963..a8be765c822 100644 --- a/tests/cmdline/cmd_showbc_const.py.exp +++ b/tests/cmdline/cmd_showbc_const.py.exp @@ -1,4 +1,4 @@ -File cmdline/cmd_showbc_const.py, code block '' (descriptor: \.\+, bytecode @\.\+ 198 bytes) +File \.\*cmdline/cmd_showbc_const.py, code block '' (descriptor: \.\+, bytecode @\.\+ 198 bytes) Raw bytecode (code_info_size=40, bytecode_size=158): 2c 4c 01 60 2c 46 22 65 27 4a 83 0c 20 27 40 20 27 20 27 40 60 20 27 24 40 60 40 24 27 47 24 27 diff --git a/tests/cmdline/cmd_showbc_opt.py.exp b/tests/cmdline/cmd_showbc_opt.py.exp index 9e4e4fae10c..c3047275a9c 100644 --- a/tests/cmdline/cmd_showbc_opt.py.exp +++ b/tests/cmdline/cmd_showbc_opt.py.exp @@ -1,4 +1,4 @@ -File cmdline/cmd_showbc_opt.py, code block '' (descriptor: \.\+, bytecode @\.\+ 35 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block '' (descriptor: \.\+, bytecode @\.\+ 35 bytes) Raw bytecode (code_info_size=13, bytecode_size=22): 00 16 01 60 20 64 40 84 07 64 40 84 07 32 00 16 02 32 01 16 03 32 02 16 04 32 03 16 05 32 04 16 @@ -27,7 +27,7 @@ arg names: 18 STORE_NAME f4 20 LOAD_CONST_NONE 21 RETURN_VALUE -File cmdline/cmd_showbc_opt.py, code block 'f0' (descriptor: \.\+, bytecode @\.\+ 8 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block 'f0' (descriptor: \.\+, bytecode @\.\+ 8 bytes) Raw bytecode (code_info_size=6, bytecode_size=2): 08 08 02 60 40 22 80 63 arg names: @@ -39,7 +39,7 @@ arg names: bc=2 line=7 00 LOAD_CONST_SMALL_INT 0 01 RETURN_VALUE -File cmdline/cmd_showbc_opt.py, code block 'f1' (descriptor: \.\+, bytecode @\.\+ 22 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block 'f1' (descriptor: \.\+, bytecode @\.\+ 22 bytes) Raw bytecode (code_info_size=9, bytecode_size=13): 11 0e 03 08 80 0a 23 22 20 b0 44 42 51 63 12 07 82 34 01 59 51 63 @@ -61,7 +61,7 @@ arg names: x 10 POP_TOP 11 LOAD_CONST_NONE 12 RETURN_VALUE -File cmdline/cmd_showbc_opt.py, code block 'f2' (descriptor: \.\+, bytecode @\.\+ 10 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block 'f2' (descriptor: \.\+, bytecode @\.\+ 10 bytes) Raw bytecode (code_info_size=7, bytecode_size=3): 11 0a 04 08 80 11 23 12 09 65 arg names: x @@ -72,7 +72,7 @@ arg names: x bc=3 line=19 00 LOAD_GLOBAL Exception 02 RAISE_OBJ -File cmdline/cmd_showbc_opt.py, code block 'f3' (descriptor: \.\+, bytecode @\.\+ 24 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block 'f3' (descriptor: \.\+, bytecode @\.\+ 24 bytes) Raw bytecode (code_info_size=9, bytecode_size=15): 11 0e 05 08 80 16 22 22 23 42 42 42 43 b0 43 3b 12 07 82 34 01 59 51 63 @@ -94,7 +94,7 @@ arg names: x 12 POP_TOP 13 LOAD_CONST_NONE 14 RETURN_VALUE -File cmdline/cmd_showbc_opt.py, code block 'f4' (descriptor: \.\+, bytecode @\.\+ 24 bytes) +File \.\*cmdline/cmd_showbc_opt.py, code block 'f4' (descriptor: \.\+, bytecode @\.\+ 24 bytes) Raw bytecode (code_info_size=9, bytecode_size=15): 11 0e 06 08 80 1d 22 22 23 42 42 42 40 b0 43 3b 12 07 82 34 01 59 51 63 diff --git a/tests/cmdline/cmd_verbose.py.exp b/tests/cmdline/cmd_verbose.py.exp index ae833dbec8d..1982f313cb7 100644 --- a/tests/cmdline/cmd_verbose.py.exp +++ b/tests/cmdline/cmd_verbose.py.exp @@ -1,4 +1,4 @@ -File cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode @\.\+ 12 bytes) +File \.\*cmdline/cmd_verbose.py, code block '' (descriptor: \.\+, bytecode @\.\+ 12 bytes) Raw bytecode (code_info_size=4, bytecode_size=8): 08 04 01 40 11 02 81 34 01 59 51 63 arg names: diff --git a/tests/cpydiff/core_exception_construction.py b/tests/cpydiff/core_exception_construction.py new file mode 100644 index 00000000000..56a358f4c45 --- /dev/null +++ b/tests/cpydiff/core_exception_construction.py @@ -0,0 +1,28 @@ +""" +categories: Core,Exceptions +description: Throwing a derived exception class instance in its `__init__` without first calling ``super().__init__`` is a TypeError +cause: In MicroPython, an object is incompletely constructed if it does not call its superclass init function or return normally from its ``__init__``. This prevents its usage in some circumstances. +workaround: Call the superclass `__init__` method before raising the exception. +""" + + +class C(Exception): + def __init__(self): + raise self + + +class C1(Exception): + def __init__(self): + super().__init__() + raise self + + +try: + C() +except Exception as e: + print(type(e).__name__) + +try: + C1() +except Exception as e: + print(type(e).__name__) diff --git a/tests/extmod/marshal_fun_nested.py b/tests/extmod/marshal_fun_nested.py new file mode 100644 index 00000000000..fcf8f9a0fa4 --- /dev/null +++ b/tests/extmod/marshal_fun_nested.py @@ -0,0 +1,79 @@ +# Test the marshal module, with functions that have children. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + + +def f_with_child(): + def child(): + return a + + return child + + +def f_with_child_defargs(): + def child(a="default"): + return a + + return child + + +def f_with_child_closure(): + a = "closure 1" + + def child(): + return a + + a = "closure 2" + return child + + +def f_with_child_closure_defargs(): + a = "closure defargs 1" + + def child(b="defargs default"): + return (a, b) + + a = "closure defargs 1" + return child + + +def f_with_list_comprehension(a): + return [i + a for i in range(4)] + + +ftype = type(lambda: 0) + +# Test function with a child. +f = ftype(marshal.loads(marshal.dumps(f_with_child.__code__)), {"a": "global"}) +print(f()()) + +# Test function with a child that has default arguments. +f = ftype(marshal.loads(marshal.dumps(f_with_child_defargs.__code__)), {}) +print(f()()) +print(f()("non-default")) + +# Test function with a child that is a closure. +f = ftype(marshal.loads(marshal.dumps(f_with_child_closure.__code__)), {}) +print(f()()) + +# Test function with a child that is a closure and has default arguments. +f = ftype(marshal.loads(marshal.dumps(f_with_child_closure_defargs.__code__)), {}) +print(f()()) +print(f()("defargs non-default")) + +# Test function with a list comprehension (which will be an anonymous child). +f = ftype(marshal.loads(marshal.dumps(f_with_list_comprehension.__code__)), {}) +print(f(10)) + +# Test child within a module (the outer scope). +code = compile("def child(a): return a", "", "exec") +f = marshal.loads(marshal.dumps(code)) +ctx = {} +exec(f, ctx) +print(ctx["child"]("arg")) diff --git a/tests/extmod/marshal_micropython.py b/tests/extmod/marshal_micropython.py deleted file mode 100644 index 213b3bf3189..00000000000 --- a/tests/extmod/marshal_micropython.py +++ /dev/null @@ -1,21 +0,0 @@ -# Test the marshal module, MicroPython-specific functionality. - -try: - import marshal -except ImportError: - print("SKIP") - raise SystemExit - -import unittest - - -class Test(unittest.TestCase): - def test_function_with_children(self): - # Can't marshal a function with children (in this case the module has a child function f). - code = compile("def f(): pass", "", "exec") - with self.assertRaises(ValueError): - marshal.dumps(code) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/extmod/os_urandom.py b/tests/extmod/os_urandom.py new file mode 100644 index 00000000000..982d6d04b96 --- /dev/null +++ b/tests/extmod/os_urandom.py @@ -0,0 +1,14 @@ +# Test os.urandom(). + +try: + from os import urandom +except ImportError: + print("SKIP") + raise SystemExit + +for n in range(-2, 5, 1): + try: + r = urandom(n) + print(n, type(r), len(r)) + except ValueError: + print(n, "ValueError") diff --git a/tests/extmod/socket_badconstructor.py b/tests/extmod/socket_badconstructor.py index 4a9d2668c7f..1ea5d750b3e 100644 --- a/tests/extmod/socket_badconstructor.py +++ b/tests/extmod/socket_badconstructor.py @@ -16,6 +16,11 @@ except TypeError: print("TypeError") +try: + s = socket.socket(socket.AF_INET, 123456) +except OSError: + print("OSError") + try: s = socket.socket(socket.AF_INET, socket.SOCK_RAW, None) except TypeError: diff --git a/tests/extmod/vfs_blockdev_invalid.py b/tests/extmod/vfs_blockdev_invalid.py index 29d6bd6b2f9..955f8495b3f 100644 --- a/tests/extmod/vfs_blockdev_invalid.py +++ b/tests/extmod/vfs_blockdev_invalid.py @@ -46,12 +46,16 @@ def ioctl(self, op, arg): try: bdev = RAMBlockDevice(50) except MemoryError: - print("SKIP") + print("SKIP-TOO-LARGE") raise SystemExit -def test(vfs_class): - print(vfs_class) +ERROR_EIO = (OSError, "[Errno 5] EIO") +ERROR_EINVAL = (OSError, "[Errno 22] EINVAL") +ERROR_TYPE = (TypeError, "can't convert str to int") + + +def test(vfs_class, test_data): bdev.read_res = 0 # reset function results bdev.write_res = 0 @@ -61,17 +65,15 @@ def test(vfs_class): with fs.open("test", "w") as f: f.write("a" * 64) - for res in (0, -5, 5, 33, "invalid"): - # -5 is a legitimate negative failure (EIO), positive integer - # is not - + for res, error_open, error_read in test_data: # This variant will fail on open bdev.read_res = res try: with fs.open("test", "r") as f: - print("opened") + assert error_open is None except Exception as e: - print(type(e), e) + assert error_open is not None + assert (type(e), str(e)) == error_open # This variant should succeed on open, may fail on read # unless the filesystem cached the contents already @@ -79,11 +81,35 @@ def test(vfs_class): try: with fs.open("test", "r") as f: bdev.read_res = res - print("read 1", f.read(1)) - print("read rest", f.read()) + assert f.read(1) == "a" + assert f.read() == "a" * 63 + assert error_read is None except Exception as e: - print(type(e), e) + assert error_read is not None + assert (type(e), str(e)) == error_read -test(vfs.VfsLfs2) -test(vfs.VfsFat) +try: + test( + vfs.VfsLfs2, + ( + (0, None, None), + (-5, ERROR_EIO, None), + (5, ERROR_EINVAL, None), + (33, ERROR_EINVAL, None), + ("invalid", ERROR_TYPE, None), + ), + ) + test( + vfs.VfsFat, + ( + (0, None, None), + (-5, ERROR_EIO, ERROR_EIO), + (5, ERROR_EIO, ERROR_EIO), + (33, ERROR_EIO, ERROR_EIO), + ("invalid", ERROR_TYPE, ERROR_TYPE), + ), + ) + print("OK") +except MemoryError: + print("SKIP-TOO-LARGE") diff --git a/tests/extmod/vfs_blockdev_invalid.py.exp b/tests/extmod/vfs_blockdev_invalid.py.exp index 0ea0353501d..d86bac9de59 100644 --- a/tests/extmod/vfs_blockdev_invalid.py.exp +++ b/tests/extmod/vfs_blockdev_invalid.py.exp @@ -1,28 +1 @@ - -opened -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - [Errno 5] EIO -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - [Errno 22] EINVAL -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - [Errno 22] EINVAL -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - can't convert str to int -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - -opened -read 1 a -read rest aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - [Errno 5] EIO - [Errno 5] EIO - [Errno 5] EIO - [Errno 5] EIO - [Errno 5] EIO - [Errno 5] EIO - can't convert str to int - can't convert str to int +OK diff --git a/tests/extmod_hardware/machine_can2.py b/tests/extmod_hardware/machine_can2.py new file mode 100644 index 00000000000..0ecced82865 --- /dev/null +++ b/tests/extmod_hardware/machine_can2.py @@ -0,0 +1,44 @@ +# Test machine.CAN(1) and machine.CAN(2) using loopback +# +# Single device test, assumes support for loopback and no connections to the CAN pins +# +# This test is ported from tests/ports/stm32/pyb_can2.py + +try: + from machine import CAN + + CAN(2, 125_000) +except (ImportError, ValueError): + print("SKIP") + raise SystemExit + +import time + +# Setting up each CAN peripheral independently is deliberate here, to catch +# catch cases where initialising CAN2 breaks CAN1 + +can1 = CAN(1, 125_000, mode=CAN.MODE_LOOPBACK) +can1.set_filters([(0x100, 0x700, 0)]) + +can2 = CAN(2, 125_000, mode=CAN.MODE_LOOPBACK) +can2.set_filters([(0x000, 0x7F0, 0)]) + +# Drain any old messages in RX FIFOs +for can in (can1, can2): + while can.recv(): + pass + +for id, can in ((1, can1), (2, can2)): + print("testing", id) + # message1 should only receive on can1, message2 on can2 + can.send(0x123, b"message1", 0) + can.send(0x003, "message2", 0) + time.sleep_ms(10) + did_recv = False + while res := can.recv(): + did_recv = True + print(hex(res[0]), bytes(res[1]), res[2], res[3]) + if not did_recv: + print("no rx!") + +print("done") diff --git a/tests/extmod_hardware/machine_can2.py.exp b/tests/extmod_hardware/machine_can2.py.exp new file mode 100644 index 00000000000..bfb6a5088ba --- /dev/null +++ b/tests/extmod_hardware/machine_can2.py.exp @@ -0,0 +1,5 @@ +testing 1 +0x123 b'message1' 0 0 +testing 2 +0x3 b'message2' 0 0 +done diff --git a/tests/extmod_hardware/machine_can_timings.py b/tests/extmod_hardware/machine_can_timings.py new file mode 100644 index 00000000000..441059f5da5 --- /dev/null +++ b/tests/extmod_hardware/machine_can_timings.py @@ -0,0 +1,60 @@ +# Test machine.CAN timings results +# +# Single device test, assumes no connections to the CAN pins + +try: + from machine import CAN +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + +from target_wiring import can_args, can_kwargs + + +class TestTimings(unittest.TestCase): + def test_bitrate(self): + for bitrate in (125_000, 250_000, 500_000, 1_000_000): + can = CAN(*can_args, bitrate=bitrate, **can_kwargs) + print(can) + timings = can.get_timings() + print(timings) + # Actual bitrate may not be exactly equal to requested rate + self.assertAlmostEqual(timings[0], bitrate, delta=1_000) + can.deinit() + + def test_sample_point(self): + # Verify that tseg1 and tseg2 are set correctly from the sample_point argument + for sample_point in (66, 75, 95): + can = CAN(*can_args, bitrate=500_000, sample_point=sample_point, **can_kwargs) + _bitrate, _sjw, tseg1, tseg2, _fd, _port = can.get_timings() + print(f"sample_point={sample_point}, tseg1={tseg1}, tseg2={tseg2}") + self.assertAlmostEqual(sample_point / 100, tseg1 / (tseg1 + tseg2), delta=0.05) + can.deinit() + + def test_tseg_args(self): + # Verify that tseg1 and tseg2 are set correctly and sample_point is ignored if these are provided + for tseg1, tseg2 in ((5, 2), (16, 8), (16, 5), (15, 5)): + print(f"tseg1={tseg1} tseg2={tseg2}") + can = CAN( + *can_args, bitrate=250_000, tseg1=tseg1, tseg2=tseg2, sample_point=99, **can_kwargs + ) + bitrate, _sjw, ret_tseg1, ret_tseg2, _fd, _port = can.get_timings() + self.assertEqual(ret_tseg1, tseg1) + self.assertEqual(ret_tseg2, tseg2) + + def test_invalid_timing_args(self): + # Test various kwargs out of their allowed value ranges + with self.assertRaises(ValueError): + CAN(*can_args, bitrate=250_000, tseg1=55, **can_kwargs) + with self.assertRaises(ValueError): + CAN(*can_args, bitrate=500_000, tseg2=9, **can_kwargs) + with self.assertRaises(ValueError): + CAN(*can_args, bitrate=-1, **can_kwargs) + with self.assertRaises(ValueError): + CAN(*can_args, bitrate=500_000, sample_point=101, **can_kwargs) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod_hardware/machine_counter.py b/tests/extmod_hardware/machine_counter.py index 62ac1fed47c..a91d262854e 100644 --- a/tests/extmod_hardware/machine_counter.py +++ b/tests/extmod_hardware/machine_counter.py @@ -16,6 +16,11 @@ id = 0 out_pin = 4 in_pin = 5 +elif sys.platform == "mimxrt": + if "Teensy" in sys.implementation._machine: + id = 0 + out_pin = "D2" + in_pin = "D3" else: print("Please add support for this test on this platform.") raise SystemExit @@ -32,6 +37,18 @@ def toggle(times): out_pin(0) +class TestConnections(unittest.TestCase): + def setUp(self): + in_pin.init(Pin.IN) + + def test_connections(self): + # Test the hardware connections are correct. If this test fails, all tests will fail. + out_pin(1) + self.assertEqual(1, in_pin()) + out_pin(0) + self.assertEqual(0, in_pin()) + + class TestCounter(unittest.TestCase): def setUp(self): out_pin(0) @@ -43,13 +60,6 @@ def tearDown(self): def assertCounter(self, value): self.assertEqual(self.counter.value(), value) - def test_connections(self): - # Test the hardware connections are correct. If this test fails, all tests will fail. - out_pin(1) - self.assertEqual(1, in_pin()) - out_pin(0) - self.assertEqual(0, in_pin()) - def test_count_rising(self): self.assertCounter(0) toggle(100) @@ -73,6 +83,7 @@ def test_change_directions(self): toggle(25) self.assertCounter(75) + @unittest.skipIf(sys.platform == "mimxrt", "FALLING edge not supported") def test_count_falling(self): self.counter.init(in_pin, direction=Counter.UP, edge=Counter.FALLING) toggle(20) diff --git a/tests/extmod_hardware/machine_encoder.py b/tests/extmod_hardware/machine_encoder.py index c218c8bfb64..67d1b5f443e 100644 --- a/tests/extmod_hardware/machine_encoder.py +++ b/tests/extmod_hardware/machine_encoder.py @@ -28,6 +28,21 @@ in1_pin = Pin(in1_pin, mode=Pin.IN) +class TestConnections(unittest.TestCase): + def setUp(self): + in0_pin.init(Pin.IN) + in1_pin.init(Pin.IN) + + def test_connections(self): + # Test the hardware connections are correct. If this test fails, all tests will fail. + for ch, outp, inp in ((0, out0_pin, in0_pin), (1, out1_pin, in1_pin)): + print("Testing channel ", ch) + outp(1) + self.assertEqual(1, inp()) + outp(0) + self.assertEqual(0, inp()) + + class TestEncoder(unittest.TestCase): def setUp(self): out0_pin(PIN_INIT_VALUE) @@ -93,16 +108,6 @@ def assertPosition(self, value, value2=None, value4=None): self.assertEqual(self.enc4.value(), value4) pass - @unittest.skipIf(sys.platform == "mimxrt", "cannot read back the pin") - def test_connections(self): - # Test the hardware connections are correct. If this test fails, all tests will fail. - for ch, outp, inp in ((0, out0_pin, in0_pin), (1, out1_pin, in1_pin)): - print("Testing channel ", ch) - outp(1) - self.assertEqual(1, inp()) - outp(0) - self.assertEqual(0, inp()) - def test_basics(self): self.assertPosition(0) self.rotate(100) diff --git a/tests/extmod_hardware/machine_pwm.py b/tests/extmod_hardware/machine_pwm.py index e27da325486..7d4f82fd2fe 100644 --- a/tests/extmod_hardware/machine_pwm.py +++ b/tests/extmod_hardware/machine_pwm.py @@ -1,10 +1,7 @@ # Test machine.PWM, frequency and duty cycle (using machine.time_pulse_us). # # IMPORTANT: This test requires hardware connections: the PWM-output and pulse-input -# pins must be wired together (see the variable `pwm_pulse_pins`). - -import sys -import time +# pins must be wired together (see the variable `pwm_loopback_pins`). try: from machine import time_pulse_us, Pin, PWM @@ -12,44 +9,34 @@ print("SKIP") raise SystemExit -import unittest +import machine, sys, time, unittest +from target_wiring import pwm_loopback_pins pwm_freq_limit = 1000000 freq_margin_per_thousand = 0 duty_margin_per_thousand = 0 timing_margin_us = 5 -# Configure pins based on the target. +# Slow MCUs cannot capture short pulses using `time_pulse_us` so limit the maximum PWM +# frequency tested on such targets. +if hasattr(machine, "freq"): + f = machine.freq() + if isinstance(f, tuple): + f = f[0] + if f <= 48_000_000: + pwm_freq_limit = 2_000 + elif f <= 64_000_000: + pwm_freq_limit = 5_000 + +# Tune test parameters based on the target. if "esp32" in sys.platform: - pwm_pulse_pins = ((4, 5),) freq_margin_per_thousand = 2 duty_margin_per_thousand = 1 timing_margin_us = 20 elif "esp8266" in sys.platform: - pwm_pulse_pins = ((4, 5),) pwm_freq_limit = 1_000 duty_margin_per_thousand = 3 timing_margin_us = 50 -elif "mimxrt" in sys.platform: - if "Teensy" in sys.implementation._machine: - # Teensy 4.x - pwm_pulse_pins = ( - ("D0", "D1"), # FLEXPWM X and UART 1 - ("D2", "D3"), # FLEXPWM A/B - ("D11", "D12"), # QTMR and MOSI/MISO of SPI 0 - ) - else: - pwm_pulse_pins = (("D0", "D1"),) -elif "rp2" in sys.platform: - pwm_pulse_pins = (("GPIO0", "GPIO1"),) -elif "samd" in sys.platform: - pwm_pulse_pins = (("D0", "D1"),) - if "SAMD21" in sys.implementation._machine: - # MCU is too slow to capture short pulses. - pwm_freq_limit = 2_000 -else: - print("Please add support for this test on this platform.") - raise SystemExit # Test a specific frequency and duty cycle. @@ -66,8 +53,8 @@ def _test_freq_duty(self, pulse_in, pwm, freq, duty_u16): self.assertLessEqual(duty_error, duty_margin_per_thousand) # Calculate expected timing. - expected_total_us = 1_000_000 // freq - expected_high_us = expected_total_us * duty_u16 // 65535 + expected_total_us = (1_000_000 + freq // 2) // freq + expected_high_us = (expected_total_us * duty_u16 + 65535 // 2) // 65535 expected_low_us = expected_total_us - expected_high_us expected_us = (expected_low_us, expected_high_us) timeout = 2 * expected_total_us @@ -156,7 +143,7 @@ def test_freq_10000(self): # Generate test classes, one for each set of pins to test. -for pwm, pulse in pwm_pulse_pins: +for pwm, pulse in pwm_loopback_pins: cls_name = "Test_{}_{}".format(pwm, pulse) globals()[cls_name] = type( cls_name, (TestBase, unittest.TestCase), {"pwm_pin": pwm, "pulse_pin": pulse} diff --git a/tests/feature_check/tstring.py b/tests/feature_check/tstring.py new file mode 100644 index 00000000000..05322b2ae61 --- /dev/null +++ b/tests/feature_check/tstring.py @@ -0,0 +1,5 @@ +# check whether t-strings (PEP-750) are supported + +a = 1 +t = t"a={a}" +print("tstring") diff --git a/tests/feature_check/tstring.py.exp b/tests/feature_check/tstring.py.exp new file mode 100644 index 00000000000..ba42b0ec666 --- /dev/null +++ b/tests/feature_check/tstring.py.exp @@ -0,0 +1 @@ +tstring diff --git a/tests/float/complex1.py b/tests/float/complex1.py index 0a1d98b9af3..4decea2ac6b 100644 --- a/tests/float/complex1.py +++ b/tests/float/complex1.py @@ -20,7 +20,6 @@ print(complex("nanj")) print(complex("nan-infj")) print(complex(1, 2)) -print(complex(1j, 2j)) # unary ops print(bool(1j)) diff --git a/tests/float/complex1_micropython.py b/tests/float/complex1_micropython.py new file mode 100644 index 00000000000..6b92f593ef2 --- /dev/null +++ b/tests/float/complex1_micropython.py @@ -0,0 +1,6 @@ +# test basic complex number functionality + +# CPython 3.14 marks this constructor as deprecated, but it is still currently +# supported by MicroPython. + +print(complex(1j, 2j)) diff --git a/tests/float/complex1_micropython.py.exp b/tests/float/complex1_micropython.py.exp new file mode 100644 index 00000000000..1defdb822cd --- /dev/null +++ b/tests/float/complex1_micropython.py.exp @@ -0,0 +1 @@ +(-2+1j) diff --git a/tests/float/math_fun.py b/tests/float/math_fun.py index 789158c0e2b..05f8be08fa0 100644 --- a/tests/float/math_fun.py +++ b/tests/float/math_fun.py @@ -43,6 +43,9 @@ ans = "{:.5g}".format(function(value)) except ValueError as e: ans = str(e) + if ans.startswith("expected a "): + # CPython 3.14 changed messages to be more detailed; convert them back to simple ones + ans = "math domain error" print("{}({:.5g}) = {}".format(function_name, value, ans)) tuple_functions = [ diff --git a/tests/float/math_fun_special.py b/tests/float/math_fun_special.py index ecacedec552..fcf6175af73 100644 --- a/tests/float/math_fun_special.py +++ b/tests/float/math_fun_special.py @@ -51,6 +51,9 @@ ans = "{:.4g}".format(function(value)) except ValueError as e: ans = str(e) + if ans.startswith("expected a "): + # CPython 3.14 changed messages to be more detailed; convert them back to simple ones + ans = "math domain error" # a tiny error in REPR_C value for 1.5204998778 causes a wrong rounded value if is_REPR_C and function_name == "erfc" and ans == "1.521": ans = "1.52" diff --git a/tests/inlineasm/rv32/asm_ext_zba.py b/tests/inlineasm/rv32/asm_ext_zba.py new file mode 100644 index 00000000000..75f3573c864 --- /dev/null +++ b/tests/inlineasm/rv32/asm_ext_zba.py @@ -0,0 +1,18 @@ +@micropython.asm_rv32 +def test_sh1add(a0, a1): + sh1add(a0, a0, a1) + + +@micropython.asm_rv32 +def test_sh2add(a0, a1): + sh2add(a0, a0, a1) + + +@micropython.asm_rv32 +def test_sh3add(a0, a1): + sh3add(a0, a0, a1) + + +print(hex(test_sh1add(10, 20))) +print(hex(test_sh2add(10, 20))) +print(hex(test_sh3add(10, 20))) diff --git a/tests/inlineasm/rv32/asm_ext_zba.py.exp b/tests/inlineasm/rv32/asm_ext_zba.py.exp new file mode 100644 index 00000000000..5f56bd95642 --- /dev/null +++ b/tests/inlineasm/rv32/asm_ext_zba.py.exp @@ -0,0 +1,3 @@ +0x28 +0x3c +0x64 diff --git a/tests/inlineasm/xtensa/asmloadstore.py b/tests/inlineasm/xtensa/asmloadstore.py index b185e30520c..85f1f8a561b 100644 --- a/tests/inlineasm/xtensa/asmloadstore.py +++ b/tests/inlineasm/xtensa/asmloadstore.py @@ -1,8 +1,9 @@ import array -# On the 8266 the generated code gets put into the IRAM segment, which is only -# word-addressable. Therefore, to test byte and halfword load/store opcodes -# some memory must be reserved in the DRAM segment. +# On the ESP8266 the generated code gets put into the IRAM segment, which is +# only word-addressable. Therefore, to test byte and halfword load/store +# opcodes some memory must be reserved in the DRAM segment. This also happens +# to work on the ESP32 too. BYTE_DATA = array.array("B", (0x11, 0x22, 0x33, 0x44)) WORD_DATA = array.array("h", (100, 200, -100, -200)) @@ -29,8 +30,10 @@ def tl32r() -> int: @micropython.asm_xtensa def tl32i() -> uint: call0(ENTRY) + align(4) label(ENTRY) l32i(a2, a0, 0) + nop() print(hex(tl32i())) diff --git a/tests/inlineasm/xtensa/asmloadstore.py.exp b/tests/inlineasm/xtensa/asmloadstore.py.exp index e6672df6f81..ec453a1cad6 100644 --- a/tests/inlineasm/xtensa/asmloadstore.py.exp +++ b/tests/inlineasm/xtensa/asmloadstore.py.exp @@ -1,5 +1,5 @@ 0x4030201 -0xf8002022 +0xf0002022 0x22 200 -200 diff --git a/tests/micropython/builtin_tstring.py b/tests/micropython/builtin_tstring.py new file mode 100644 index 00000000000..8de0acfd98b --- /dev/null +++ b/tests/micropython/builtin_tstring.py @@ -0,0 +1,24 @@ +# This tests the built-in __template__ (which is MicroPython specific). + +# Test a varying number of arguments. +print(__template__(())) +print(__template__((), ())) +print(__template__((), None, None)) +print(__template__((), None, None, None)) +print(__template__((), None, None, None, None)) +print(__template__((), None, None, None, None, None)) + +# Test two strings and one interpolation. +print(__template__(("Hello ", "!"), (42, "x", None, ""))) + +# Test not enough arguments. +try: + print(__template__()) +except TypeError as er: + print(repr(er)) + +# Test two arguments with second not being a tuple/list. +try: + print(__template__((), None)) +except TypeError as er: + print(repr(er)) diff --git a/tests/micropython/builtin_tstring.py.exp b/tests/micropython/builtin_tstring.py.exp new file mode 100644 index 00000000000..b6b47306124 --- /dev/null +++ b/tests/micropython/builtin_tstring.py.exp @@ -0,0 +1,9 @@ +Template(strings=(), interpolations=()) +Template(strings=(), interpolations=()) +Template(strings=(), interpolations=()) +Template(strings=(), interpolations=()) +Template(strings=(), interpolations=(Interpolation(None, None, None, None),)) +Template(strings=(), interpolations=(Interpolation(None, None, None, None),)) +Template(strings=('Hello ', '!'), interpolations=(Interpolation(42, 'x', None, ''),)) +TypeError('function missing 1 required positional arguments',) +TypeError("object 'NoneType' isn't a tuple or list",) diff --git a/tests/micropython/heapalloc_fail_tstring.py b/tests/micropython/heapalloc_fail_tstring.py new file mode 100644 index 00000000000..94687d57bfb --- /dev/null +++ b/tests/micropython/heapalloc_fail_tstring.py @@ -0,0 +1,475 @@ +# Test template string (t-string) operations with heap allocation failure + +import micropython +from string.templatelib import Template, Interpolation + +i1 = Interpolation(1, "1") +i2 = Interpolation(2, "2") + +micropython.heap_lock() +try: + args = [] + for i in range(9): + args.append("x" * 100) + args.append(Interpolation(i, "x" + str(i))) + args.append("x" * 100) + Template(*args) + print("FAIL: Template creation") +except MemoryError: + print("OK: Template creation") +micropython.heap_unlock() + +# Multiple string concatenation +micropython.heap_lock() +try: + Template("first", "second", "third", "fourth") + print("FAIL: Multi string concat") +except MemoryError: + print("OK: Multi string concat") +micropython.heap_unlock() + +# Mixed constructor +micropython.heap_lock() +try: + Template("a", i1, "b", i2, "c", "d", "e") + print("FAIL: Mixed constructor") +except MemoryError: + print("OK: Mixed constructor") +micropython.heap_unlock() + +# Template.__str__() +t = t"Hello {42} world {99}" +micropython.heap_lock() +try: + str(t) + print("FAIL: Template.__str__()") +except MemoryError: + print("OK: Template.__str__()") +micropython.heap_unlock() + +# Template.values property +vals = list(range(10)) +t_many = t"{vals[0]}{vals[1]}{vals[2]}{vals[3]}{vals[4]}" +micropython.heap_lock() +try: + t_many.values + print("FAIL: Template.values") +except MemoryError: + print("OK: Template.values") +micropython.heap_unlock() + +n = 20 +args_large = [] +for i in range(n): + args_large.append("") + args_large.append(Interpolation(i, "x" + str(i))) +args_large.append("") +t_large = Template(*args_large) +micropython.heap_lock() +try: + t_large.values + print("FAIL: Large values array") +except MemoryError: + print("OK: Large values array") +micropython.heap_unlock() + +# Template concatenation +t1 = t"Hello" +t2 = t"World" +micropython.heap_lock() +try: + t1 + t2 + print("FAIL: Template concatenation") +except MemoryError: + print("OK: Template concatenation") +micropython.heap_unlock() + +# Template iterator +t_iter = t"a{1}b{2}c" +micropython.heap_lock() +try: + iter(t_iter) + print("FAIL: Template iterator") +except MemoryError: + print("OK: Template iterator") +micropython.heap_unlock() + +# Iterator next +t_iter2 = t"a{1}b{2}c{3}d" +it = iter(t_iter2) +micropython.heap_lock() +try: + list(it) + print("FAIL: Iterator next") +except MemoryError: + print("OK: Iterator next") +micropython.heap_unlock() + +# __template__ builtin +strings2 = ("test",) * 5 +interps2 = ((42, "x", None, ""),) * 4 +micropython.heap_lock() +try: + __template__(strings2, interps2) + print("FAIL: __template__ builtin") +except MemoryError: + print("OK: __template__ builtin") +micropython.heap_unlock() + +# Format spec interpolation +width = 10 +t_fmt = t"{42:{width}d}" +micropython.heap_lock() +try: + str(t_fmt) + print("FAIL: Format spec interpolation") +except MemoryError: + print("OK: Format spec interpolation") +micropython.heap_unlock() + +# Debug format +x = 42 +t_debug = t"{x=}" +micropython.heap_lock() +try: + str(t_debug) + print("FAIL: Debug format") +except MemoryError: + print("OK: Debug format") +micropython.heap_unlock() + +# Conversion with format +obj = "test" +t_conv = t"{obj!r:>10}" +micropython.heap_lock() +try: + str(t_conv) + print("FAIL: Conversion + format") +except MemoryError: + print("OK: Conversion + format") +micropython.heap_unlock() + +# String conversion (s) +t_s = t"{'test'!s}" +micropython.heap_lock() +try: + str(t_s) + print("FAIL: s conversion") +except MemoryError: + print("OK: s conversion") +micropython.heap_unlock() + +# ASCII conversion (a) +# t_a = t"{'test'!a}" +# micropython.heap_lock() +# try: +# str(t_a) +# print("FAIL: a conversion") +# except MemoryError: +# print("OK: a conversion") +# micropython.heap_unlock() + +# Complex format spec +fill = "*" +align = ">" +width = 10 +precision = 2 +t_complex = t"{3.14159:{fill}{align}{width}.{precision}f}" +micropython.heap_lock() +try: + str(t_complex) + print("FAIL: Complex format spec") +except MemoryError: + print("OK: Complex format spec") +micropython.heap_unlock() + +# Simple expression +x = 42 +t_simple = t"{x}" +micropython.heap_lock() +try: + str(t_simple) + print("FAIL: Simple expression") +except MemoryError: + print("OK: Simple expression") +micropython.heap_unlock() + +# Interpolation creation +micropython.heap_lock() +try: + Interpolation("value", "expr", "r", ".2f") + print("FAIL: Interpolation creation") +except MemoryError: + print("OK: Interpolation creation") +micropython.heap_unlock() + +# Template repr +t = t"test" +micropython.heap_lock() +try: + repr(t) + print("FAIL: Template repr") +except MemoryError: + print("OK: Template repr") +micropython.heap_unlock() + +# Interpolation repr +i = Interpolation(42, "x") +micropython.heap_lock() +try: + repr(i) + print("FAIL: Interpolation repr") +except MemoryError: + print("OK: Interpolation repr") +micropython.heap_unlock() + +def exhaust_heap(limit): + allocations = [] + try: + for _ in range(limit): + allocations.append("x" * 1024) + except MemoryError: + pass + return allocations + +def test_many_interpolations_heap(): + # Pre-create variables before exhausting heap + x1, x2, x3, x4, x5, x6, x7, x8, x9 = 1, 2, 3, 4, 5, 6, 7, 8, 9 + micropython.heap_lock() + try: + t = t"{x1}{x2}{x3}{x4}{x5}{x6}{x7}{x8}{x9}" + print("FAIL: Many interpolations heap test") + except MemoryError: + print("OK: Many interpolations heap test") + micropython.heap_unlock() + +def test_template_str_heap(): + t = t"x{1}x{2}x{3}x{4}x" + micropython.heap_lock() + try: + s = str(t) + print("FAIL: Template str heap test") + except MemoryError: + print("OK: Template str heap test") + micropython.heap_unlock() + +def test_template_iter_heap(): + t = t"a{1}b{2}c" + micropython.heap_lock() + try: + parts = list(iter(t)) + print("FAIL: Template iter heap test") + except MemoryError: + print("OK: Template iter heap test") + micropython.heap_unlock() + +def test_template_concat_heap(): + t1 = t"first" + t2 = t"second" + micropython.heap_lock() + try: + t3 = t1 + t2 + print("FAIL: Template concat heap test") + except MemoryError: + print("OK: Template concat heap test") + micropython.heap_unlock() + +def test_format_spec_heap(): + width = 10 + t = t"{42:{width}d}" + micropython.heap_lock() + try: + s = str(t) + print("FAIL: Format spec heap test") + except MemoryError: + print("OK: Format spec heap test") + micropython.heap_unlock() + +def test_debug_format_heap(): + value = 42 + t = t"{value}" + micropython.heap_lock() + try: + s = str(t) + print("FAIL: Debug format heap test") + except MemoryError: + print("OK: Debug format heap test") + micropython.heap_unlock() + +print("\n=== Heap allocation failure tests ===") +test_many_interpolations_heap() +test_template_str_heap() +test_template_iter_heap() +test_template_concat_heap() +test_format_spec_heap() +test_debug_format_heap() + +# Additional tests from coverage.c +print("\n=== Coverage.c heap tests ===") + +# Test creating interpolation with heap locked (from coverage.c) +micropython.heap_lock() +try: + i = Interpolation(42, "x", None, None) + print("FAIL: Interpolation with heap locked") +except MemoryError: + print("OK: Interpolation creation with heap locked") +micropython.heap_unlock() + +# Test parsing expression with heap locked (from coverage.c) +micropython.heap_lock() +try: + t = eval('t"{x + 1}"') + print("FAIL: Parse with heap locked") +except Exception as e: + if type(e).__name__ == "MemoryError": + print("OK: Parse with heap locked") + else: + print(f"Parse with heap locked: {type(e).__name__}") +micropython.heap_unlock() + +# Test lexer allocation failure for t-string expression parsing +print("\n=== Lexer/Parser allocation tests ===") + +# Wrap all tests in a try-catch to handle any unexpected errors +try: + # Test 1: Empty expression (tests tstring_expr_parser.c empty check) + try: + micropython.heap_lock() + try: + # This tests the empty expression path + exec("t'{}'") + print("FAIL: Empty expression with heap locked") + except Exception as e: + print(f"OK: Empty expression - {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 2: Whitespace-only expression + try: + micropython.heap_lock() + try: + exec("t'{ }'") + print("FAIL: Whitespace expression with heap locked") + except Exception as e: + print(f"OK: Whitespace expression - {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 3: Very complex expression to stress parser allocation + # Pre-create variables + x, y, z = 1, 2, 3 + try: + micropython.heap_lock() + try: + # Complex nested expression that requires many parse nodes + result = t'{[x*y for x in range(10) for y in range(10) if x+y > 5]}' + print("FAIL: Complex expression with heap locked") + except MemoryError: + print("OK: Complex expression parser allocation") + except Exception as e: + print(f"Complex expression: {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 4: Expression parser lexer failure (simulates NULL lexer) + # This happens when memory is exhausted during lexer creation + try: + micropython.heap_lock() + try: + # Try to create t-string with expression during low memory + long_expr = "x" * 100 # Use a moderately long expression + exec(f"{long_expr} = 42; result = t'{{{long_expr}}}'") + print("FAIL: Lexer creation with heap locked") + except MemoryError: + print("OK: Lexer creation failure") + except Exception as e: + print(f"Lexer creation: {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 5: Format spec and conversion parsing under memory pressure + val = 42 + try: + micropython.heap_lock() + try: + # This tests format spec node creation + result = t'{val!r:>10.2f}' + print("FAIL: Format spec with heap locked") + except MemoryError: + print("OK: Format spec allocation") + except Exception as e: + print(f"Format spec: {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 6: Debug format with memory pressure + x = 100 + try: + micropython.heap_lock() + try: + result = t'{x=:.2f}' + print("FAIL: Debug format with heap locked") + except MemoryError: + print("OK: Debug format allocation") + except Exception as e: + print(f"Debug format: {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 7: Multiple interpolations causing allocation failure + vals = list(range(20)) + try: + micropython.heap_lock() + try: + # Many interpolations to stress allocation + result = t'{vals[0]}{vals[1]}{vals[2]}{vals[3]}{vals[4]}{vals[5]}{vals[6]}{vals[7]}{vals[8]}{vals[9]}' + print("FAIL: Many interpolations with heap locked") + except MemoryError: + print("OK: Many interpolations allocation") + except Exception as e: + print(f"Many interpolations: {type(e).__name__}") + finally: + micropython.heap_unlock() + except: + pass + + # Test 8: Template string size limit (tests overflow check) + # Note: We can't directly test integer overflow, but we can test large templates + print("Template size limit: Tested via exec in coverage tests") + + # Test 9: Compile-time parsing under allocation failure + try: + micropython.heap_lock() + try: + compile("t'{x}'", "", "exec") + print("FAIL: Compile-time parsing under heap lock") + except MemoryError: + print("OK: Compile-time parsing under heap lock") + finally: + micropython.heap_unlock() + except: + pass + +except Exception as e: + # Catch any unexpected errors from the tests + print(f"\nTest error: {type(e).__name__}: {e}") + # Ensure heap is unlocked + try: + micropython.heap_unlock() + except: + pass + +print("\n=== Tests completed ===") diff --git a/tests/micropython/heapalloc_fail_tstring.py.exp b/tests/micropython/heapalloc_fail_tstring.py.exp new file mode 100644 index 00000000000..3bc90ef418a --- /dev/null +++ b/tests/micropython/heapalloc_fail_tstring.py.exp @@ -0,0 +1,42 @@ +OK: Template creation +OK: Multi string concat +OK: Mixed constructor +OK: Template.__str__() +OK: Template.values +OK: Large values array +OK: Template concatenation +OK: Template iterator +OK: Iterator next +OK: __template__ builtin +OK: Format spec interpolation +OK: Debug format +OK: Conversion + format +OK: s conversion +OK: Complex format spec +OK: Simple expression +OK: Interpolation creation +OK: Template repr +OK: Interpolation repr + +=== Heap allocation failure tests === +OK: Many interpolations heap test +OK: Template str heap test +OK: Template iter heap test +OK: Template concat heap test +OK: Format spec heap test +OK: Debug format heap test + +=== Coverage.c heap tests === +OK: Interpolation creation with heap locked +OK: Parse with heap locked + +=== Lexer/Parser allocation tests === +OK: Complex expression parser allocation +OK: Lexer creation failure +OK: Format spec allocation +OK: Debug format allocation +OK: Many interpolations allocation +Template size limit: Tested via exec in coverage tests +OK: Compile-time parsing under heap lock + +=== Tests completed === diff --git a/tests/micropython/incomplete_exc.py b/tests/micropython/incomplete_exc.py new file mode 100644 index 00000000000..2aec9a47b3a --- /dev/null +++ b/tests/micropython/incomplete_exc.py @@ -0,0 +1,40 @@ +# Test raising an incomplete exception. + + +class C(Exception): + def __init__(self): + raise self + + +class C1(C): + pass + + +class B: + pass + + +class C2(B, Exception): + def __init__(self): + raise self + + +class C3(Exception, B): + def __init__(self): + raise self + + +class D(Exception): + pass + + +class C4(D): + def __init__(self): + raise self + + +for cls in C, C1, C2, C3, C4: + try: + cls() + except TypeError as e: + print("TypeError") diff --git a/tests/micropython/incomplete_exc.py.exp b/tests/micropython/incomplete_exc.py.exp new file mode 100644 index 00000000000..f2e9c12f7f9 --- /dev/null +++ b/tests/micropython/incomplete_exc.py.exp @@ -0,0 +1,5 @@ +TypeError +TypeError +TypeError +TypeError +TypeError diff --git a/tests/micropython/native_marshal.py b/tests/micropython/native_marshal.py new file mode 100644 index 00000000000..09a27a374b8 --- /dev/null +++ b/tests/micropython/native_marshal.py @@ -0,0 +1,45 @@ +# Test the marshal module in combination with native/viper functions. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +import unittest + + +def f_native(): + @micropython.native + def g(): + pass + + return g + + +def f_viper(): + @micropython.viper + def g(): + pass + + return g + + +class Test(unittest.TestCase): + def test_native_function(self): + # Can't marshal a function with native code. + code = f_native.__code__ + with self.assertRaises(ValueError): + marshal.dumps(code) + + def test_viper_function(self): + # Can't marshal a function with viper code. + code = f_viper.__code__ + with self.assertRaises(ValueError): + marshal.dumps(code) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/micropython/schedule.py b/tests/micropython/schedule.py index f3dd3266126..e629edb3eb3 100644 --- a/tests/micropython/schedule.py +++ b/tests/micropython/schedule.py @@ -1,4 +1,6 @@ # test micropython.schedule() function +# this test should be manually kept in synch with +# tests/micrpython/schedule_sleep.py. try: import micropython diff --git a/tests/micropython/schedule_sleep.py b/tests/micropython/schedule_sleep.py new file mode 100644 index 00000000000..9aadde7b084 --- /dev/null +++ b/tests/micropython/schedule_sleep.py @@ -0,0 +1,72 @@ +# test micropython.schedule() function +# this is the same as tests/micropython/schedule.py but the busy loops are +# replaced with sleep/sleep_ms which allows the test to succeed when run under +# the native emitter. + +try: + import micropython + import time + + micropython.schedule +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + + +# Basic test of scheduling a function. + + +def callback(arg): + global done + print(arg) + done = True + + +done = False +micropython.schedule(callback, 1) +while not done: + time.sleep(0) + +# Test that callbacks can be scheduled from within a callback, but +# that they don't execute until the outer callback is finished. + + +def callback_inner(arg): + global done + print("inner") + done += 1 + + +def callback_outer(arg): + global done + micropython.schedule(callback_inner, 0) + # need a loop so that the VM can check for pending events + for i in range(2): + pass + print("outer") + done += 1 + + +done = 0 +micropython.schedule(callback_outer, 0) +while done != 2: + time.sleep(0) + +# Test that scheduling too many callbacks leads to an exception. To do this we +# must schedule from within a callback to guarantee that the scheduler is locked. + + +def callback(arg): + global done + try: + for i in range(100): + micropython.schedule(lambda x: x, None) + except RuntimeError: + print("RuntimeError") + done = True + + +done = False +micropython.schedule(callback, None) +while not done: + time.sleep_ms(0) diff --git a/tests/micropython/schedule_sleep.py.exp b/tests/micropython/schedule_sleep.py.exp new file mode 100644 index 00000000000..c4a3e1227e2 --- /dev/null +++ b/tests/micropython/schedule_sleep.py.exp @@ -0,0 +1,4 @@ +1 +outer +inner +RuntimeError diff --git a/tests/micropython/viper_preserve_invariants.py b/tests/micropython/viper_preserve_invariants.py new file mode 100644 index 00000000000..26f6003aba7 --- /dev/null +++ b/tests/micropython/viper_preserve_invariants.py @@ -0,0 +1,39 @@ +RESULTS = [] + +LOAD_TEMPLATE_REG = """ +BUFFER = bytearray([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) +OFFSET = 1 +@micropython.viper +def test_invariants_load{}_reg(buf: ptr{}, offset: int): + offset_copy1: int = offset + temporary1 = buf[offset] + offset_copy2: int = offset + temporary2 = buf[offset] + RESULTS.append(["LOAD{}", temporary1 == temporary2, offset == offset_copy1 == offset_copy2]) +test_invariants_load{}_reg(BUFFER, OFFSET) +""" + + +STORE_TEMPLATE_REG = """ +BUFFER = bytearray([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) +OFFSET = 1 +@micropython.viper +def test_invariants_store{}_reg(buf: ptr{}, offset: int): + offset_copy: int = offset + value: uint = {} + buf[offset] = value + temporary = buf[offset] + RESULTS.append(["STORE{}", temporary == value, offset == offset_copy]) +test_invariants_store{}_reg(BUFFER, OFFSET) +""" + +try: + for width, value in ((8, 0x11), (16, 0x1111), (32, 0x11111111)): + exec(LOAD_TEMPLATE_REG.format(width, width, width, width, width)) + exec(STORE_TEMPLATE_REG.format(width, width, value, width, width, width)) +except MemoryError: + print("SKIP-TOO-LARGE") + raise SystemExit + +for line in RESULTS: + print(" ".join([str(i) for i in line])) diff --git a/tests/micropython/viper_preserve_invariants.py.exp b/tests/micropython/viper_preserve_invariants.py.exp new file mode 100644 index 00000000000..21c66252430 --- /dev/null +++ b/tests/micropython/viper_preserve_invariants.py.exp @@ -0,0 +1,6 @@ +LOAD8 True True +STORE8 True True +LOAD16 True True +STORE16 True True +LOAD32 True True +STORE32 True True diff --git a/tests/multi_extmod/machine_can_01_rxtx_simple.py b/tests/multi_extmod/machine_can_01_rxtx_simple.py new file mode 100644 index 00000000000..fbc7692926d --- /dev/null +++ b/tests/multi_extmod/machine_can_01_rxtx_simple.py @@ -0,0 +1,35 @@ +from machine import CAN +import time + +ID_0 = 0x50 +ID_1 = 0x50333 +FLAGS_1 = CAN.FLAG_EXT_ID # ID_1 is an extended CAN ID + +can = CAN(1, 500_000) +can.set_filters(None) # receive all + + +def print_rx(can_id, data, flags, errors): + print(hex(can_id), bytes(data), hex(flags), hex(errors)) + + +def instance0(): + multitest.next() + + # receive from instance1 first + while not (rx := can.recv()): + time.sleep(0) + print_rx(*rx) + + # now send one + can.send(ID_0, b"1234") + + +def instance1(): + multitest.next() + + can.send(ID_1, b"ABCD", FLAGS_1) + + while not (rx := can.recv()): + time.sleep(0) + print_rx(*rx) diff --git a/tests/multi_extmod/machine_can_01_rxtx_simple.py.exp b/tests/multi_extmod/machine_can_01_rxtx_simple.py.exp new file mode 100644 index 00000000000..479b6809490 --- /dev/null +++ b/tests/multi_extmod/machine_can_01_rxtx_simple.py.exp @@ -0,0 +1,4 @@ +--- instance0 --- +0x50333 b'ABCD' 0x2 0x0 +--- instance1 --- +0x50 b'1234' 0x0 0x0 diff --git a/tests/multi_extmod/machine_can_02_rx_callback.py b/tests/multi_extmod/machine_can_02_rx_callback.py new file mode 100644 index 00000000000..80b13dcd622 --- /dev/null +++ b/tests/multi_extmod/machine_can_02_rx_callback.py @@ -0,0 +1,122 @@ +from machine import CAN +import time + +# Test the CAN.IRQ_RX irq handler, including overflow + +rx_overflow = False +rx_full = False +received = [] + +# CAN IDs +ID_SPAM = 0x345 # messages spammed into the receive FIFO +ID_ACK_OFLOW = 0x055 # message the receiver sends after it's seen an overflow +ID_AFTER = 0x100 # message the sender sends after the ACK + +can = CAN(1, 500_000) + + +# A very basic "soft" receiver handler that stores received messages into a global list +def receiver_irq_recv(can): + global rx_overflow, rx_full + + assert can.irq().flags() & can.IRQ_RX # the only enabled IRQ + + can_id, data, _flags, errors = can.recv() + + received.append((can_id, None)) + + # The FIFO is expected not to overflow by itself, wait until 40 messages + # have been received and then block the receive handler to induce an overflow + if len(received) == 40: + assert not rx_overflow # shouldn't have already happened, either + time.sleep_ms(500) + + if not rx_overflow and (errors & CAN.RECV_ERR_OVERRUN): + # expected this should happen on the very next message after + # the one where we slept for 500ms + print("irq_recv overrun", len(received)) + received.clear() # check we still get some messages, see rx_spam print line below + rx_overflow = True + + # also expect the FIFO to be FULL again immediately after overrunning and rx_overflow event + if rx_overflow and (errors & CAN.RECV_ERR_OVERRUN | CAN.RECV_ERR_FULL) == CAN.RECV_ERR_FULL: + rx_full = True + + +# Receiver +def instance0(): + can.irq(receiver_irq_recv, trigger=can.IRQ_RX, hard=False) + + can.set_filters(None) # receive all + + multitest.next() + + while not rx_overflow: + pass # Resume ASAP after FIFO0 overflows + + can.send(ID_ACK_OFLOW, b"overflow") + + # at least one ID_SPAM message should have been received + # *after* we overflowed and 'received' was clear in the irq handler + print("rx_spam", any(r[0] == ID_SPAM for r in received)) + + # wait until the "after" message is received + for n in range(100): + if any(r[0] == ID_AFTER for r in received): + break + time.sleep_ms(10) + + can.irq(None) # disable the IRQ + received.clear() + + # at some point while waiting for ID_AFTER the FIFO should have gotten + # full again + print("rx_full", rx_full) + + # now IRQ is disabled, no new messages should be received + time.sleep_ms(250) + print("len", len(received)) + + +received_ack = False + +# reusing the result buffer so sender_irq_recv can be 'hard' +sender_irq_result = [None, memoryview(bytearray(64)), None, None] + + +def sender_irq_recv(can): + global received_ack + + assert can.irq().flags() & can.IRQ_RX # the only enabled IRQ + + can_id, data, _flags, _errors = can.recv(sender_irq_result) + print("sender_irq_recv", can_id, len(data)) # should be ID_ACK_OFLOW and "overflow" payload + received_ack = True + + +# Sender +def instance1(): + can.irq(sender_irq_recv, CAN.IRQ_RX, hard=True) + + can.set_filters(None) + + multitest.next() + + # Spam out messages until the receiver tells us its RX FIFO is full. + # + # The RX FIFO on the receiver can vary from 3 deep (BXCAN) to 25 deep (STM32H7), + # so we keep sending to it until we see a CAN message on ID_ACK_OFLOW indicating + # the receiver's FIFO has overflowed + while not received_ack: + for i in range(255): + while can.send(ID_SPAM, bytes([i] * 8)) is None and not received_ack: + # Don't overflow the TX FIFO + time.sleep_ms(1) + if received_ack: + break + + # give the receiver some time to make space in the FIFO + time.sleep_ms(200) + + # send the final message, the receiver should get this one + can.send(ID_AFTER, b"aaaaa") diff --git a/tests/multi_extmod/machine_can_02_rx_callback.py.exp b/tests/multi_extmod/machine_can_02_rx_callback.py.exp new file mode 100644 index 00000000000..8e4e5dc6475 --- /dev/null +++ b/tests/multi_extmod/machine_can_02_rx_callback.py.exp @@ -0,0 +1,7 @@ +--- instance0 --- +irq_recv overrun 41 +rx_spam True +rx_full True +len 0 +--- instance1 --- +sender_irq_recv 85 8 diff --git a/tests/multi_extmod/machine_can_03_rx_filters.py b/tests/multi_extmod/machine_can_03_rx_filters.py new file mode 100644 index 00000000000..a56acbbe91e --- /dev/null +++ b/tests/multi_extmod/machine_can_03_rx_filters.py @@ -0,0 +1,103 @@ +from machine import CAN +import time + +# Test for filtering capabilities + +can = CAN(1, 500_000) + +# IDs and filter phases used for the 'single id' part of the test +SINGLE_EXT_ID = (0x1234_5678, 0x1FFF_FFFF, CAN.FLAG_EXT_ID) +SINGLE_STD_ID = (0x505, 0x7FF, 0) +SINGLE_ID_PHASES = [ + ("single ext id", [SINGLE_EXT_ID]), + ("single std id", [SINGLE_STD_ID]), + ("ext+std ids", [SINGLE_EXT_ID, SINGLE_STD_ID]), + ("std+ext ids", [SINGLE_STD_ID, SINGLE_EXT_ID]), # these two should be equivalent + ("accept none", []), + ("accept all", None), + ("accept none again", ()), +] + + +# Receiver +def receiver_irq_recv(can): + assert can.irq().flags() & can.IRQ_RX # the only enabled IRQ + can_id, data, _flags, _errors = can.recv() + print("recv", hex(can_id), data.hex()) + + +def instance0(): + can.irq(receiver_irq_recv, trigger=can.IRQ_RX, hard=False) + + multitest.next() + + # Configure to receive standard frames (in a range), and + # extended frames (in a range). + can.set_filters([(0x300, 0x300, 0), (0x3000, 0x3000, CAN.FLAG_EXT_ID)]) + multitest.broadcast("ready id ranges") + + # Run through the phases of filtering for individual IDs + for phase, filters in SINGLE_ID_PHASES: + multitest.wait("configure " + phase) + if filters and len(filters) > CAN.FILTERS_MAX: + # this check really exists to add test coverage for the FILTERS_MAX constant + print("Warning: Too many filters for hardware!") + can.set_filters(filters) + print("receiver configured " + phase) + multitest.broadcast("ready " + phase) + + multitest.wait("Sender done") + + +def send_messages(messages): + for can_id, payload, flags in messages: + r = can.send(can_id, payload, flags) + if r is None: + print("Failed to send:", hex(can_id), payload.hex()) + time.sleep_ms(5) # avoid flooding either our or the receiver's FIFO + + +# Sender +def instance1(): + multitest.next() + multitest.wait("ready id ranges") + + print("Sending ID ranges...") + for i in range(3): + send_messages( + [ + (0x345, bytes([i, 0xFF] * (i + 1)), 0), + (0x3700 + i, bytes([0xEE] * (i + 1)), CAN.FLAG_EXT_ID), + (0x123, b"abcdef", 0), # matches no filter, expect ACKed but not received + ] + ) + + # Now move on to single ID filtering + + single_id_messages = [ + (0x1234_5678, b"\x01\x02\x03\x04\x05", CAN.FLAG_EXT_ID), # matches ext id + (0x0234_5678, b"\x00\x00", CAN.FLAG_EXT_ID), # no match + (0x678, b"\x00\x01", 0), # no match + (0x505, b"\x06\x07\x08\x09\x0a\x0b", 0), # matches standard id + (0x345, b"\x00\x02", 0), # no match (in prev filter) + (0x1234_5679, b"\x00\x03", CAN.FLAG_EXT_ID), # no match + (0x3705, b"\x00\x04", CAN.FLAG_EXT_ID), # no match (in prev filter) + (0x1234_5678, b"\x01\x02\x03", CAN.FLAG_EXT_ID), # matches ext id + (0x505, b"\x04\x05\x06", 0), # matches standard id + (0x505, b"\x00\x05", CAN.FLAG_EXT_ID), # no match (is ext id) + (0x507, b"\x00\x06", 0), # no match + (0x1334_5678, b"\x00\x07", CAN.FLAG_EXT_ID), # no match + (0x1234_5670, b"\x00\x08", CAN.FLAG_EXT_ID), # no match + ] + + # Send the same list of messages for each phase of the test. + # The receiver will have configured different filters, and the .exp + # file is what selects which messages should be received or not. + for phase, _ in SINGLE_ID_PHASES: + multitest.broadcast("configure " + phase) + multitest.wait("ready " + phase) + print("Sending for " + phase + "...") + send_messages(single_id_messages) + + print("Sender done") + multitest.broadcast("Sender done") diff --git a/tests/multi_extmod/machine_can_03_rx_filters.py.exp b/tests/multi_extmod/machine_can_03_rx_filters.py.exp new file mode 100644 index 00000000000..d55c0c97d15 --- /dev/null +++ b/tests/multi_extmod/machine_can_03_rx_filters.py.exp @@ -0,0 +1,49 @@ +--- instance0 --- +recv 0x345 00ff +recv 0x3700 ee +recv 0x345 01ff01ff +recv 0x3701 eeee +recv 0x345 02ff02ff02ff +recv 0x3702 eeeeee +receiver configured single ext id +recv 0x12345678 0102030405 +recv 0x12345678 010203 +receiver configured single std id +recv 0x505 060708090a0b +recv 0x505 040506 +receiver configured ext+std ids +recv 0x12345678 0102030405 +recv 0x505 060708090a0b +recv 0x12345678 010203 +recv 0x505 040506 +receiver configured std+ext ids +recv 0x12345678 0102030405 +recv 0x505 060708090a0b +recv 0x12345678 010203 +recv 0x505 040506 +receiver configured accept none +receiver configured accept all +recv 0x12345678 0102030405 +recv 0x2345678 0000 +recv 0x678 0001 +recv 0x505 060708090a0b +recv 0x345 0002 +recv 0x12345679 0003 +recv 0x3705 0004 +recv 0x12345678 010203 +recv 0x505 040506 +recv 0x505 0005 +recv 0x507 0006 +recv 0x13345678 0007 +recv 0x12345670 0008 +receiver configured accept none again +--- instance1 --- +Sending ID ranges... +Sending for single ext id... +Sending for single std id... +Sending for ext+std ids... +Sending for std+ext ids... +Sending for accept none... +Sending for accept all... +Sending for accept none again... +Sender done diff --git a/tests/multi_extmod/machine_can_04_tx_order.py b/tests/multi_extmod/machine_can_04_tx_order.py new file mode 100644 index 00000000000..204bdafd59d --- /dev/null +++ b/tests/multi_extmod/machine_can_04_tx_order.py @@ -0,0 +1,170 @@ +from machine import CAN +import time +from random import seed, randrange + +import micropython + +micropython.alloc_emergency_exception_buf(256) +seed(0) + +# Testing that transmit order obeys the priority ordering + +ID_LOW = 0x500 +ID_HIGH = 0x200 + +NUM_MSGS = 255 + +MSG_LEN = 4 + +can = CAN(1, 500_000) + + +def check_sequence(items, label): + # The full range of NUM_MSGS values should have been received or sent, in + # order, without duplicates + if len(items) != NUM_MSGS: + print(label, "wrong count", len(items), "vs", NUM_MSGS) + if items == list(range(NUM_MSGS)): + print(label, "OK") + else: + print(label, "error:") + print(items) + + +# Receiver + +# lists of received messages, one list per ID +received_low = [] +received_high = [] + + +def irq_recv(can): + while can.irq().flags() & can.IRQ_RX: + can_id, data, flags, _errors = can.recv() + + if can_id == ID_LOW and len(data) == MSG_LEN: + received_low.append(data[0]) + elif can_id == ID_HIGH and len(data) == MSG_LEN: + received_high.append(data[0]) + else: + print("unexpected recv", can_id, data, flags) + + +def instance0(): + can.irq(irq_recv, trigger=can.IRQ_RX, hard=False) + can.set_filters(None) # receive all + + multitest.next() + + multitest.wait("sender done") + check_sequence(received_low, "Low prio received") + check_sequence(received_high, "High prio received") + + +# Sender + +## Messages pending to send +pending_low = list(range(NUM_MSGS)) +pending_high = list(range(NUM_MSGS)) + +# List of the messages currently queued to send +tx_queue = [None] * CAN.TX_QUEUE_LEN + +# Messages sent, recorded in order as [high_prio, val] +sent = [] +for _ in range(NUM_MSGS * 2): + sent.append([None, None]) +num_sent = 0 + + +def irq_send(can): + global num_sent + + while flags := can.irq().flags(): + assert flags & can.IRQ_TX # the only enabled IRQ + + idx = (flags >> can.IRQ_TX_IDX_SHIFT) & can.IRQ_TX_IDX_MASK + success = not (flags & can.IRQ_TX_FAILED) + + if not success: + return # We don't worry about failures here + + if not tx_queue[idx]: + print("bad done", idx, success) + return + + was_high, val = tx_queue[idx] + tx_queue[idx] = None + sent[num_sent][0] = was_high + sent[num_sent][1] = val + num_sent += 1 + + +def instance1(): + # note: this test can pass with hard=True, but in a debug build + # the completion IRQ may race ahead of setting tx_queue[idx], below + can.irq(irq_send, trigger=can.IRQ_TX, hard=False) + data = bytearray(MSG_LEN) + + multitest.next() + + while pending_low or pending_high: + if pending_high: + val = pending_high.pop(0) + data[0] = val + data[1] = 1 + while True: + idx = can.send(ID_HIGH, data) + if idx is None: + continue # keep trying until a queue spot opens up + old = tx_queue[idx] + tx_queue[idx] = (True, val) + if old: + print("error high priority queue race", idx, val, old) + break + + for _ in range(randrange(4)): + # Try and queue many low priority messages (expecting most will fail) + if pending_low: + val = pending_low[0] + data[0] = val + data[1] = 0 + idx = can.send(ID_LOW, data) + if idx is None: + # don't retry indefinitely for low priority messages + continue + + old = tx_queue[idx] + tx_queue[idx] = (False, val) + pending_low.pop(0) + if old is not None: + print("error low priority queue race", idx, val, old) + + print("waiting for tx queue to empty...") + while any(x is not None for x in tx_queue): + pass + + multitest.broadcast("sender done") + + # Check we sent the right number of messages + if num_sent != 2 * NUM_MSGS: + print("Sent %d expected %d" % (num_sent, 2 * NUM_MSGS)) + else: + print("Sent right number of messages") + + # Check the low and high priority messages all arrived in order + sent_low = [val for (prio, val) in sent[:num_sent] if prio == False] + sent_high = [val for (prio, val) in sent[:num_sent] if prio == True] + check_sequence(sent_low, "Low prio sent") + check_sequence(sent_high, "High prio sent") + + # check that high priority message queue items always stayed ahead of the low priority + high_val = -1 + for idx, (prio, val) in enumerate(sent): + if prio: + high_val = val + elif high_val <= val and val < NUM_MSGS - 1: + print( + "Low priority message %d overtook high priority %d at index %d" + % (val, high_val, idx) + ) diff --git a/tests/multi_extmod/machine_can_04_tx_order.py.exp b/tests/multi_extmod/machine_can_04_tx_order.py.exp new file mode 100644 index 00000000000..a2de6ee2706 --- /dev/null +++ b/tests/multi_extmod/machine_can_04_tx_order.py.exp @@ -0,0 +1,8 @@ +--- instance0 --- +Low prio received OK +High prio received OK +--- instance1 --- +waiting for tx queue to empty... +Sent right number of messages +Low prio sent OK +High prio sent OK diff --git a/tests/multi_extmod/machine_can_05_tx_prio_cancel.py b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py new file mode 100644 index 00000000000..64756a1a1af --- /dev/null +++ b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py @@ -0,0 +1,118 @@ +from machine import CAN +import time + +# Check that cancelling a low priority outgoing message and replacing it with a +# high priority message causes it to be transmitted successfully onto a busy bus + +recv = [] + +ITERS = 5 + +can = CAN(1, 500_000) + + +def irq_recv(can): + global recv_std_id + while can.irq().flags() & can.IRQ_RX: + can_id, data, flags, _errors = can.recv() + assert flags & CAN.FLAG_EXT_ID # test uses all extended IDs + if len(recv) < ITERS: + recv.append(can_id) + + +def instance0(): + can.irq(irq_recv, trigger=can.IRQ_RX, hard=False) + can.set_filters(None) # receive all + + multitest.next() + + # "Babble" medium priority messages onto the bus to prevent + # instance1() from sending anything lower priority than this + while len(recv) < ITERS: + for id in range(0x5000, 0x6000): + can.send(id, b"BABBLE", CAN.FLAG_EXT_ID) + if len(recv) >= ITERS: + break + + print("received", ITERS, "messages") + for can_id in recv: + print(hex(can_id)) # should be the high priority messages from instance1, only + + multitest.wait("sender done") + print("done") + + +last_idx = 0 +total_cancels = 0 +total_sent = 0 + + +def irq_send(can): + global total_cancels, total_sent + + while flags := can.irq().flags(): + assert flags & can.IRQ_TX # the only enabled IRQ + + idx = (flags >> can.IRQ_TX_IDX_SHIFT) & can.IRQ_TX_IDX_MASK + + if flags & can.IRQ_TX_FAILED: + # we should only see failed transmits due to cancels in buffer 'last_idx' + assert idx == last_idx + total_cancels += 1 + else: + # this includes the messages we explicitly send, plus queued low + # priority messages once the receiver stops 'babbling' on the bus + total_sent += 1 + + +def instance1(): + global last_idx + can.irq(irq_send, trigger=can.IRQ_TX, hard=True) + + multitest.next() + + for i in range(ITERS): + # Fill the transmit queue with low priority messages (all extended IDs) + last_idx = 0 + if i < 3: + # For the first 3 iterations, send unique message IDs + id_range = range(0x7000, 0x7FFF) + flags = CAN.FLAG_EXT_ID + else: + # For the last iterations, repeat the same ID but tell controller to ignore + # ordering (allows it to queue more than one despite hardware limitations) + id_range = [0x50000 + i] * CAN.TX_QUEUE_LEN + flags = CAN.FLAG_EXT_ID | CAN.FLAG_UNORDERED + + for id in id_range: + idx = can.send(id, b"LOWPRIO", flags) + if idx is None: + break # send queue is full, stop trying to send + last_idx = idx + + time.sleep_ms(50) # the send queue shouldn't empty as instance0 is "babbling" + + # try and cancel the last message we queued + res = can.cancel_send(last_idx) + print(i, "cancel result", res) + + # send a high priority message, that we expect to go out + idx = can.send(0x500 + i, b"HIPRIO", CAN.FLAG_EXT_ID) + print(i, "send result", idx is not None) + + # make sure this message is sent onto the bus + time.sleep_ms(1) + + multitest.broadcast("sender done") + + # let the entire transmit queue drain, now instance0 should have gone quiet + time.sleep_ms(50) + + print("total cancels", total_cancels) # should equal ITERS + + if total_sent == CAN.TX_QUEUE_LEN - 1 + ITERS: + # expect we send one message for each of ITERS, plus all low priority + # queued messages once instance0 stops babbling on the bus + print("total sent OK") + else: + print("total sent", total_sent, CAN.TX_QUEUE_LEN - 1 + ITERS) diff --git a/tests/multi_extmod/machine_can_05_tx_prio_cancel.py.exp b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py.exp new file mode 100644 index 00000000000..26bab097c44 --- /dev/null +++ b/tests/multi_extmod/machine_can_05_tx_prio_cancel.py.exp @@ -0,0 +1,21 @@ +--- instance0 --- +received 5 messages +0x500 +0x501 +0x502 +0x503 +0x504 +done +--- instance1 --- +0 cancel result True +0 send result True +1 cancel result True +1 send result True +2 cancel result True +2 send result True +3 cancel result True +3 send result True +4 cancel result True +4 send result True +total cancels 5 +total sent OK diff --git a/tests/multi_extmod/machine_can_06_remote_req.py b/tests/multi_extmod/machine_can_06_remote_req.py new file mode 100644 index 00000000000..4a1414c0946 --- /dev/null +++ b/tests/multi_extmod/machine_can_06_remote_req.py @@ -0,0 +1,70 @@ +from machine import CAN +import time + +# Test CAN remote transmission requests + +ID = 0x750 + +FILTER_ID = 0x101 +FILTER_MASK = 0x7FF + +can = CAN(1, 500_000) + + +def receiver_irq_recv(can): + assert can.irq().flags() & can.IRQ_RX # the only enabled IRQ + can_id, data, flags, _errors = can.recv() + is_rtr = flags == CAN.FLAG_RTR + print( + "recv", + hex(can_id), + is_rtr, + len(data) if is_rtr else bytes(data), + ) + if is_rtr: + # The 'data' response of a remote request should be all zeroes + assert bytes(data) == b"\x00" * len(data) + + +# Receiver +def instance0(): + can.irq(receiver_irq_recv, trigger=can.IRQ_RX, hard=False) + can.set_filters(None) # receive all + + multitest.next() + + multitest.wait("enable filter") + can.set_filters([(FILTER_ID, FILTER_MASK, 0)]) + multitest.broadcast("filter set") + + multitest.wait("done") + + +# Sender +def instance1(): + multitest.next() + + can.send(ID, b"abc", CAN.FLAG_RTR) # length==3 remote request + time.sleep_ms(5) + can.send(ID, b"abc", 0) # regular message using the same ID + time.sleep_ms(5) + can.send(ID, b"abcde", CAN.FLAG_RTR) # length==5 remote request + time.sleep_ms(5) + + multitest.broadcast("enable filter") + multitest.wait("filter set") + + # these two messages should be filtered out + can.send(ID, b"abc", CAN.FLAG_RTR) # length==3 remote request + time.sleep_ms(5) + can.send(ID, b"abc", 0) # regular message using the same ID + time.sleep_ms(5) + + # these messages should be filtered in + can.send(FILTER_ID, b"def", CAN.FLAG_RTR) # length==3 remote request + time.sleep_ms(5) + can.send(FILTER_ID, b"hij", 0) # regular message using the same ID + time.sleep_ms(5) + + multitest.broadcast("done") + print("done") diff --git a/tests/multi_extmod/machine_can_06_remote_req.py.exp b/tests/multi_extmod/machine_can_06_remote_req.py.exp new file mode 100644 index 00000000000..fe94508ba3d --- /dev/null +++ b/tests/multi_extmod/machine_can_06_remote_req.py.exp @@ -0,0 +1,8 @@ +--- instance0 --- +recv 0x750 True 3 +recv 0x750 False b'abc' +recv 0x750 True 5 +recv 0x101 True 3 +recv 0x101 False b'hij' +--- instance1 --- +done diff --git a/tests/multi_extmod/machine_can_07_error_states.py b/tests/multi_extmod/machine_can_07_error_states.py new file mode 100644 index 00000000000..e7eec513ee1 --- /dev/null +++ b/tests/multi_extmod/machine_can_07_error_states.py @@ -0,0 +1,205 @@ +from machine import CAN +from micropython import const +import time + +# Test that without a second CAN node on the network the controller will +# correctly go into the correct error states, and can then recover. +# +# Note this test depends on no other CAN node being active on the network apart +# from the two test instances. (Although it's OK for extra nodes to be in a +# "listen only" mode where they won't ACK messages.) + +rx_overflow = False +rx_full = False +received = [] + +# CAN IDs +_ID = const(0x100) + +# can.state() result list indexes, as constants +_IDX_TEC = const(0) +_IDX_REC = const(1) +_IDX_NUM_WARNING = const(2) +_IDX_NUM_PASSIVE = const(3) +_IDX_NUM_BUS_OFF = const(4) +_IDX_PEND_TX = const(5) + +can = CAN(1, 500_000) + + +def state_name(state): + for name in dir(CAN): + if name.startswith("STATE_") and state == getattr(CAN, name): + return name + return f"UNKNOWN-{state}" + + +def irq_recv(can): + while can.irq().flags() & can.IRQ_RX: + can_id, data, _flags, errors = can.recv() + print("recv", hex(can_id), data.hex()) + + +# Receiver +def instance0(): + can.irq(irq_recv, trigger=can.IRQ_RX, hard=False) + + can.set_filters(None) # receive all + + multitest.next() + + # Receive at least one CAN message before sender asks us to disable the controller + + multitest.wait("disable receiver") + can.deinit() + print("can deinit()") + multitest.broadcast("receiver disabled") + + # Wait for the sender to tell us to re-enable + + multitest.wait("enable receiver") + # note the irq is no longer active after deinit() + can.init(500_000) + print("can init()") + multitest.broadcast("receiver enabled") + + # Receive CAN messages until the sender asks us to switch to an invalid baud rate + + multitest.wait("switch baud") + can.init(125_000) + print("can switch baud") + multitest.broadcast("switched baud") + + time.sleep_ms(1) + + print("sending bad msg") + # trying to send this frame should introduce more bus errors + idx_bad = can.send(_ID, b"BADBAUD") + + multitest.wait("fix baud") + print("sending cancelling") + print("cancelled", can.cancel_send(idx_bad)) + print("re-init") + can.init(500_000) + multitest.broadcast("fixed baud") + + # Should be receiving CAN messages OK again + + print("done") + + +def irq_sender(can): + while flags := can.irq().flags(): + if flags & can.IRQ_STATE: + print("irq state", can.state()) + if flags & can.IRQ_TX: + print("irq sent", not (flags & can.IRQ_TX_FAILED)) + + +# Sender +def instance1(): + can.irq(irq_sender, CAN.IRQ_STATE | CAN.IRQ_TX, hard=False) + + can.set_filters(None) + + multitest.next() + + print("started", state_name(can.state())) # should be ERROR_ACTIVE + + # Send a single message to the receiver, to verify it's working + can.send(_ID, b"PAYLOAD") + active_counters = can.get_counters() + # print(active_counters) # DEBUG + + multitest.broadcast("disable receiver") + multitest.wait("receiver disabled") + + # Now the receiver shouldn't be ACKing our frames, queue will stay full + # ... we will get the ISR for ERROR_WARNING but it'll go from ERROR_WARNING to ERROR_PASSIVE + # very quickly as all messages are failing + can.send(_ID, b"MORE") + while can.state() in (CAN.STATE_ACTIVE, CAN.STATE_WARNING): + pass + + print(state_name(can.state())) # should be ERROR_PASSIVE now + passive_counters = can.get_counters() + # print(passive_counters) # DEBUG + print("tec increased", passive_counters[_IDX_TEC] > active_counters[_IDX_TEC]) + print("tec over thresh", passive_counters[_IDX_TEC] >= 128) + # we should have counted exactly one ERROR_WARNING and ERROR_PASSIVE transition + print( + "counted warning", + passive_counters[_IDX_NUM_WARNING] == active_counters[_IDX_NUM_WARNING] + 1, + ) + print( + "counted passive", + passive_counters[_IDX_NUM_PASSIVE] == active_counters[_IDX_NUM_PASSIVE] + 1, + ) + print("some pending tx", passive_counters[_IDX_PEND_TX] > 0) + + # Re-enable the receiver which should allow us to go from ERROR_PASSIVE to ERROR_WARNING + multitest.broadcast("enable receiver") + multitest.wait("receiver enabled") + + can.send(_ID, b"MORE") + while can.state() == CAN.STATE_PASSIVE: + pass + + print(state_name(can.state())) # should be ERROR_WARNING now + warning_counters = can.get_counters() + # print(warning_counters) # DEBUG + print("tec decreased", warning_counters[_IDX_TEC] < passive_counters[_IDX_TEC]) + print( + "tec below thresh", warning_counters[_IDX_TEC] < 128 + ) # and should be more than error passive threshold + # error warning count should stay the same, as we went "down" in severity not up + print( + "no new warning", warning_counters[_IDX_NUM_WARNING] == passive_counters[_IDX_NUM_WARNING] + ) + print( + "no new passive", warning_counters[_IDX_NUM_PASSIVE] == passive_counters[_IDX_NUM_PASSIVE] + ) + + # Tell the receiver to change to the wrong baud rate, which should create both RX and TX errorxs + multitest.broadcast("switch baud") + multitest.wait("switched baud") + + # queue another message. This will keep trying to send until we revert back to ERROR_PASSIVE + idx = can.send(_ID, b"YETMORE") + print("queued yetmore", idx is not None) + while can.state() != CAN.STATE_PASSIVE: + pass + + print(state_name(can.state())) # should be ERROR_PASSIVE again + passive_counters = can.get_counters() + # print(passive_counters) # DEBUG + # we can't say for sure which error counter will hit the ERROR_PASSIVE threshold first + print( + "one over thresh", passive_counters[_IDX_TEC] >= 128 or passive_counters[_IDX_REC] >= 128 + ) + print( + "no new warning", passive_counters[_IDX_NUM_WARNING] == warning_counters[_IDX_NUM_WARNING] + ) + print( + "counted passive", + passive_counters[_IDX_NUM_PASSIVE] == warning_counters[_IDX_NUM_PASSIVE] + 1, + ) + + # Note that we can't get all the way to the most severe BUS_OFF error state + # with this test setup, as Bus Off requires more than just "normal" frame + # transmit errors. + + # restarting the controller may cause it to leave its error state, or not, depending + # on the implementation - but it shouldn't cause any recovery issues. Also cancels all pending TX + # (note: have to do this before 'fix baud' or we create a race condition for pending tx) + can.restart() + + # tell the receiver to go back to a valid baud rate + multitest.broadcast("fix baud") + multitest.wait("fixed baud") + + idx_more = can.send(_ID, b"MOREMORE") + time.sleep_ms(50) # irq_sender should fire during this window + print("queued moremore", idx_more is not None) + + print("done") diff --git a/tests/multi_extmod/machine_can_07_error_states.py.exp b/tests/multi_extmod/machine_can_07_error_states.py.exp new file mode 100644 index 00000000000..158ae63bfbc --- /dev/null +++ b/tests/multi_extmod/machine_can_07_error_states.py.exp @@ -0,0 +1,37 @@ +--- instance0 --- +recv 0x100 5041594c4f4144 +can deinit() +can init() +can switch baud +sending bad msg +sending cancelling +cancelled True +re-init +done +--- instance1 --- +started STATE_ACTIVE +irq sent True +irq state 2 +irq state 3 +STATE_PASSIVE +tec increased True +tec over thresh True +counted warning True +counted passive True +some pending tx True +irq sent True +irq sent True +STATE_WARNING +tec decreased True +tec below thresh True +no new warning True +no new passive True +irq state 3 +queued yetmore True +STATE_PASSIVE +one over thresh True +no new warning True +counted passive True +irq sent True +queued moremore True +done diff --git a/tests/multi_extmod/machine_can_08_init_mode.py b/tests/multi_extmod/machine_can_08_init_mode.py new file mode 100644 index 00000000000..459e6180ab6 --- /dev/null +++ b/tests/multi_extmod/machine_can_08_init_mode.py @@ -0,0 +1,102 @@ +from machine import CAN +from micropython import const +import time + +# instance0 transitions through various modes, instance1 +# listens for various messages (or not) +# +# Note this test assumes no other CAN nodes are connected apart from the test +# instances (or if they are connected they must be in silent mode.) +# +# TODO: This test needs to eventually support the case where modes aren't supported +# on a controller, maybe by printing fake output if the mode switch fails? + +# MODE_NORMAL, MODE_SLEEP, MODE_LOOPBACK, MODE_SILENT, MODE_SILENT_LOOPBACK +can = CAN(1, 500_000, mode=CAN.MODE_NORMAL) + +# While instance0 is in Silent mode, instance1 sends a message with this ID +# that will be retried for 100ms (as instance0 won't ACK). So don't print every one. +_SILENT_RX_ID = const(0x53) +silent_rx_count = 0 + + +def irq_print(can): + global silent_rx_count + while flags := can.irq().flags(): + if flags & can.IRQ_RX: + can_id, data, _flags, errors = can.recv() + if can_id != _SILENT_RX_ID: + print("recv", hex(can_id), bytes(data)) + else: + silent_rx_count += 1 + if flags & can.IRQ_TX: # note: only enabled on instance1 to avoid race conditions + print("send", "failed" if flags & can.IRQ_TX_FAILED else "ok") + + +def reinit_with_mode(mode): + can.deinit() + can.init(bitrate=500_000, mode=mode) + can.irq(irq_print, trigger=can.IRQ_RX, hard=False) + can.set_filters(None) # receive all + + +def instance0(): + multitest.next() + multitest.wait("instance1 ready") + + reinit_with_mode(can.MODE_NORMAL) + print("Normal", "MODE_NORMAL" in str(can)) + can.send(0x50, b"Normal") + time.sleep_ms(100) + + # Skipping MODE_SLEEP as means different things on different hardware + + reinit_with_mode(can.MODE_LOOPBACK) + print("Loopback", "MODE_LOOPBACK" in str(can)) + + # This message should go out to the bus, but will also be received by instance0 itself + can.send(0x51, b"Loopback") + time.sleep_ms(100) + + reinit_with_mode(can.MODE_SILENT) + print("Silent", "MODE_SLIENT" in str(can)) + + # This message shouldn't go out onto the bus + idx = can.send(0x52, b"Silent") + multitest.broadcast("silent") + multitest.wait("silent done") + # we should have received the message from instance1 many times, as instance0 won't have ACKed it + print("silent_rx_count", silent_rx_count > 5) + can.cancel_send(idx) + + reinit_with_mode(can.MODE_SILENT_LOOPBACK) + print("Silent Loopback", "MODE_SILENT_LOOPBACK" in str(can)) + + # This message should be received by instance0 only + idx = can.send(0x54, b"SiLoop") + time.sleep_ms(50) + + reinit_with_mode(can.MODE_NORMAL) + print("Normal again", "MODE_NORMAL" in str(can)) + can.send(0x55, b"Normal2") # should be received by instance1 only, again + multitest.broadcast("normal done") + + +# Receiver +def instance1(): + can.irq(irq_print, trigger=can.IRQ_RX | can.IRQ_TX, hard=False) + can.set_filters(None) # receive all + multitest.next() + + multitest.broadcast("instance1 ready") + + # The IRQ does most of the work on this instance + + multitest.wait("silent") + # Sending this message back, it should fail to send as Silent mode won't ACK it + idx = can.send(0x53, b"Silent2") + time.sleep_ms(20) + can.cancel_send(idx) + multitest.broadcast("silent done") + + multitest.wait("normal done") diff --git a/tests/multi_extmod/machine_can_08_init_mode.py.exp b/tests/multi_extmod/machine_can_08_init_mode.py.exp new file mode 100644 index 00000000000..b9f0f11ae10 --- /dev/null +++ b/tests/multi_extmod/machine_can_08_init_mode.py.exp @@ -0,0 +1,14 @@ +--- instance0 --- +Normal True +Loopback True +recv 0x51 b'Loopback' +Silent False +silent_rx_count True +Silent Loopback True +recv 0x54 b'SiLoop' +Normal again True +--- instance1 --- +recv 0x50 b'Normal' +recv 0x51 b'Loopback' +send failed +recv 0x55 b'Normal2' diff --git a/tests/run-internalbench.py b/tests/run-internalbench.py index 99c6304afe9..715b0ffc9ed 100755 --- a/tests/run-internalbench.py +++ b/tests/run-internalbench.py @@ -8,16 +8,27 @@ from glob import glob from collections import defaultdict -run_tests_module = __import__("run-tests") -sys.path.append(run_tests_module.base_path("../tools")) -import pyboard +from test_utils import ( + base_path, + pyboard, + test_instance_description, + test_instance_epilog, + test_directory_description, + get_test_instance, +) if os.name == "nt": MICROPYTHON = os.getenv( "MICROPY_MICROPYTHON", "../ports/windows/build-standard/micropython.exe" ) + CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") else: MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/build-standard/micropython") + CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") + +MICROPYTHON_CMD = [MICROPYTHON, "-X", "emit=bytecode"] +CPYTHON3_CMD = [CPYTHON3, "-BS"] + injected_bench_code = b""" import time @@ -38,14 +49,14 @@ def run(test): """ -def execbench(pyb, filename, iters): +def execbench(test_instance, filename, iters): with open(filename, "rb") as f: pyfile = f.read() code = (injected_bench_code + pyfile).replace(b"20000000", str(iters).encode("utf-8")) - return pyb.exec(code).replace(b"\r\n", b"\n") + return test_instance.exec(code).replace(b"\r\n", b"\n") -def run_tests(pyb, test_dict, iters): +def run_tests(test_instance, test_dict, iters): test_count = 0 testcase_count = 0 @@ -54,19 +65,17 @@ def run_tests(pyb, test_dict, iters): baseline = None for test_file in tests: # run MicroPython - if pyb is None: + if isinstance(test_instance, list): # run on PC try: - output_mupy = subprocess.check_output( - [MICROPYTHON, "-X", "emit=bytecode", test_file[0]] - ) + output_mupy = subprocess.check_output(test_instance + [test_file[0]]) except subprocess.CalledProcessError: output_mupy = b"CRASH" else: # run on pyboard - pyb.enter_raw_repl() + test_instance.enter_raw_repl() try: - output_mupy = execbench(pyb, test_file[0], iters) + output_mupy = execbench(test_instance, test_file[0], iters) except pyboard.PyboardError: output_mupy = b"CRASH" @@ -97,10 +106,10 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, description=f"""Run and manage tests for MicroPython. -{run_tests_module.test_instance_description} -{run_tests_module.test_directory_description} +{test_instance_description} +{test_directory_description} """, - epilog=run_tests_module.test_instance_epilog, + epilog=f"""{test_instance_epilog}- cpython - use CPython to run the benchmarks instead\n""", ) cmd_parser.add_argument( "-t", "--test-instance", default="unix", help="the MicroPython instance to test" @@ -123,10 +132,15 @@ def main(): cmd_parser.add_argument("files", nargs="*", help="input test files") args = cmd_parser.parse_args() - # Note pyboard support is copied over from run-tests.py, not tests, and likely needs revamping - pyb = run_tests_module.get_test_instance( - args.test_instance, args.baudrate, args.user, args.password - ) + if args.test_instance == "cpython": + test_instance = CPYTHON3_CMD + else: + # Note pyboard support is copied over from run-tests.py, not tests, and likely needs revamping + test_instance = get_test_instance( + args.test_instance, args.baudrate, args.user, args.password + ) + if test_instance is None: + test_instance = MICROPYTHON_CMD if len(args.files) == 0: if args.test_dirs: @@ -150,7 +164,7 @@ def main(): continue test_dict[m.group(1)].append([t, None]) - if not run_tests(pyb, test_dict, args.iters): + if not run_tests(test_instance, test_dict, args.iters): sys.exit(1) diff --git a/tests/run-multitests.py b/tests/run-multitests.py index e5458ffe0d0..40aac16c16f 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -15,7 +15,13 @@ import subprocess import tempfile -run_tests_module = __import__("run-tests") +from test_utils import ( + base_path, + pyboard, + test_instance_epilog, + convert_device_shortcut_to_real_device, + create_test_report, +) test_dir = os.path.abspath(os.path.dirname(__file__)) @@ -24,9 +30,6 @@ # accidentally importing tests like micropython/const.py sys.path.pop(0) -sys.path.insert(0, test_dir + "/../tools") -import pyboard - if os.name == "nt": CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3.exe") MICROPYTHON = os.path.abspath( @@ -273,19 +276,22 @@ def stop(self): def readline(self): if self.finished: return None, None - if self.pyb.serial.inWaiting() == 0: - return None, None - out = self.pyb.read_until(1, (b"\r\n", b"\x04")) - if out.endswith(b"\x04"): - self.finished = True - out = out[:-1] - err = decode(self.pyb.read_until(1, b"\x04")) - err = err[:-1] - if not out and not err: + try: + if self.pyb.serial.inWaiting() == 0: return None, None - else: - err = None - return decode(out.rstrip()), err + out = self.pyb.read_until(1, (b"\r\n", b"\x04")) + if out.endswith(b"\x04"): + self.finished = True + out = out[:-1] + err = decode(self.pyb.read_until(1, b"\x04")) + err = err[:-1] + if not out and not err: + return None, None + else: + err = None + return decode(out.rstrip()), err + except OSError as e: + return None, "Failed to read from instance: {}".format(e) def write(self, data): self.pyb.serial.write(data) @@ -431,7 +437,10 @@ def run_test_on_instances(test_file, num_instances, instances): # Stop all instances for idx in range(num_instances): - instances[idx].stop() + try: + instances[idx].stop() + except OSError as e: + output[idx].append("Runner failed to stop instance: {}".format(e)) output_str = "" for idx, lines in enumerate(output): @@ -554,7 +563,7 @@ def main(): cmd_parser = argparse.ArgumentParser( description="Run network tests for MicroPython", epilog=( - run_tests_module.test_instance_epilog + test_instance_epilog + "Each instance arg can optionally have custom env provided, eg. ,ENV=VAR,ENV=VAR...\n" ), formatter_class=argparse.RawTextHelpFormatter, @@ -582,7 +591,7 @@ def main(): cmd_parser.add_argument( "-r", "--result-dir", - default=run_tests_module.base_path("results"), + default=base_path("results"), help="directory for test results", ) cmd_parser.add_argument("files", nargs="+", help="input test files") @@ -612,7 +621,7 @@ def main(): print("unsupported instance string: {}".format(cmd), file=sys.stderr) sys.exit(2) else: - device = run_tests_module.convert_device_shortcut_to_real_device(cmd) + device = convert_device_shortcut_to_real_device(cmd) instances_test.append(PyInstancePyboard(device)) for _ in range(max_instances - len(instances_test)): @@ -626,7 +635,7 @@ def main(): break test_results = run_tests(test_files, instances_truth, instances_test_permutation) - all_pass &= run_tests_module.create_test_report(cmd_args, test_results) + all_pass &= create_test_report(cmd_args, test_results) finally: for i in instances_truth: diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index 6d2a975b82a..103450f0231 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -9,7 +9,15 @@ import sys import argparse -run_tests_module = __import__("run-tests") +from test_utils import ( + base_path, + pyboard, + TEST_ENTER_RAW_REPL_TIMEOUT, + TEST_MAXIMUM_RAW_REPL_FAILURES, + test_instance_epilog, + get_test_instance, + create_test_report, +) # Paths for host executables CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") @@ -38,6 +46,7 @@ "xtensa", "xtensawin", "rv32imc", + "rv64imc", ) ARCH_MAPPINGS = {"armv7em": "armv7m"} @@ -105,11 +114,11 @@ def close(self): def run_script(self, script): try: - self.pyb.enter_raw_repl() + self.pyb.enter_raw_repl(timeout_overall=TEST_ENTER_RAW_REPL_TIMEOUT) output = self.pyb.exec_(script) output = output.replace(b"\r\n", b"\n") return output, None - except run_tests_module.pyboard.PyboardError as er: + except pyboard.PyboardError as er: return b"", er @@ -133,13 +142,7 @@ def detect_architecture(target): def run_tests(target_truth, target, args, resolved_arch): - global injected_import_hook_code - - prelude = "" - if args.begin: - prelude = args.begin.read() - injected_import_hook_code = injected_import_hook_code.replace("{import_prelude}", prelude) - + raw_repl_failure_count = 0 test_results = [] for test_file in args.files: # Find supported test @@ -190,6 +193,8 @@ def run_tests(target_truth, target, args, resolved_arch): elif error is not None: result = "FAIL" extra = " - " + str(error) + if str(error).startswith("could not enter raw repl"): + raw_repl_failure_count += 1 else: # Check result against truth try: @@ -219,13 +224,19 @@ def run_tests(target_truth, target, args, resolved_arch): # Print result print("{:4} {}{}".format(result, test_file, extra)) + if raw_repl_failure_count > TEST_MAXIMUM_RAW_REPL_FAILURES: + print("Too many raw REPL failures, aborting test run") + break + return test_results def main(): + global injected_import_hook_code + cmd_parser = argparse.ArgumentParser( description="Run dynamic-native-module tests under MicroPython", - epilog=run_tests_module.test_instance_epilog, + epilog=test_instance_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, ) cmd_parser.add_argument( @@ -240,24 +251,28 @@ def main(): cmd_parser.add_argument( "-b", "--begin", - type=argparse.FileType("rt"), + metavar="PROLOGUE", default=None, help="prologue python file to execute before module import", ) cmd_parser.add_argument( "-r", "--result-dir", - default=run_tests_module.base_path("results"), + default=base_path("results"), help="directory for test results", ) cmd_parser.add_argument("files", nargs="*", help="input test files") args = cmd_parser.parse_args() + prologue = "" + if args.begin: + with open(args.begin, "rt") as source: + prologue = source.read() + injected_import_hook_code = injected_import_hook_code.replace("{import_prelude}", prologue) + target_truth = TargetSubprocess([CPYTHON3]) - target = run_tests_module.get_test_instance( - args.test_instance, args.baudrate, args.user, args.password - ) + target = get_test_instance(args.test_instance, args.baudrate, args.user, args.password) if target is None: # Use the unix port of MicroPython. target = TargetSubprocess([MICROPYTHON]) @@ -272,7 +287,7 @@ def main(): target_platform, target_arch, error = detect_architecture(target) if error: print("Cannot run tests: {}".format(error)) - sys.exit(1) + sys.exit(2) target_arch = ARCH_MAPPINGS.get(target_arch, target_arch) if target_platform: @@ -281,7 +296,7 @@ def main(): os.makedirs(args.result_dir, exist_ok=True) test_results = run_tests(target_truth, target, args, target_arch) - res = run_tests_module.create_test_report(args, test_results) + res = create_test_report(args, test_results) target.close() target_truth.close() diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py index 039d11a3611..16182bc8a9f 100755 --- a/tests/run-perfbench.py +++ b/tests/run-perfbench.py @@ -10,9 +10,13 @@ import argparse from glob import glob -run_tests_module = __import__("run-tests") - -prepare_script_for_target = run_tests_module.prepare_script_for_target +from test_utils import ( + base_path, + pyboard, + get_test_instance, + prepare_script_for_target, + create_test_report, +) # Paths for host executables if os.name == "nt": @@ -49,7 +53,7 @@ def run_script_on_target(target, script): try: target.enter_raw_repl() output = target.exec_(script) - except run_tests_module.pyboard.PyboardError as er: + except pyboard.PyboardError as er: err = er else: # Run local executable @@ -277,7 +281,7 @@ def main(): cmd_parser.add_argument( "-r", "--result-dir", - default=run_tests_module.base_path("results"), + default=base_path("results"), help="directory for test results", ) cmd_parser.add_argument( @@ -298,9 +302,7 @@ def main(): M = int(args.M[0]) n_average = int(args.average) - target = run_tests_module.get_test_instance( - args.test_instance, args.baudrate, args.user, args.password - ) + target = get_test_instance(args.test_instance, args.baudrate, args.user, args.password) if target is None: # Use the unix port of MicroPython. target = [MICROPYTHON, "-X", "emit=" + args.emit] @@ -328,7 +330,7 @@ def main(): os.makedirs(args.result_dir, exist_ok=True) test_results = run_benchmarks(args, target, N, M, n_average, tests) - res = run_tests_module.create_test_report(args, test_results) + res = create_test_report(args, test_results) if hasattr(target, "exit_raw_repl"): target.exit_raw_repl() diff --git a/tests/run-tests.py b/tests/run-tests.py index d2849efa360..9ac8fdd0fbb 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -6,7 +6,6 @@ import sysconfig import platform import argparse -import inspect import json import re from glob import glob @@ -15,19 +14,30 @@ import threading import tempfile -# Maximum time to run a single test, in seconds. -TEST_TIMEOUT = float(os.environ.get("MICROPY_TEST_TIMEOUT", 30)) - -# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] -# are guaranteed to always work, this one should though. -BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None))) - -RV32_ARCH_FLAGS = {"zba": 1 << 0} - - -def base_path(*p): - return os.path.abspath(os.path.join(BASEPATH, *p)).replace("\\", "/") +from test_utils import ( + base_path, + pyboard, + TEST_TIMEOUT, + TEST_MAXIMUM_RAW_REPL_FAILURES, + MPYCROSS, + test_instance_description, + test_instance_epilog, + test_directory_description, + rm_f, + normalize_newlines, + set_injected_prologue, + get_results_filename, + convert_device_shortcut_to_real_device, + get_test_instance, + prepare_script_for_target, + create_test_report, + FLAKY_REASON_PREFIX, +) +RV32_ARCH_FLAGS = { + "zba": 1 << 0, + "zcmp": 1 << 1, +} # Tests require at least CPython 3.3. If your default python3 executable # is of lower version, you can point MICROPY_CPYTHON3 environment var @@ -37,88 +47,22 @@ def base_path(*p): MICROPYTHON = os.getenv( "MICROPY_MICROPYTHON", base_path("../ports/windows/build-standard/micropython.exe") ) - # mpy-cross is only needed if --via-mpy command-line arg is passed - MPYCROSS = os.getenv("MICROPY_MPYCROSS", base_path("../mpy-cross/build/mpy-cross.exe")) else: CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3") MICROPYTHON = os.getenv( "MICROPY_MICROPYTHON", base_path("../ports/unix/build-standard/micropython") ) - # mpy-cross is only needed if --via-mpy command-line arg is passed - MPYCROSS = os.getenv("MICROPY_MPYCROSS", base_path("../mpy-cross/build/mpy-cross")) # Use CPython options to not save .pyc files, to only access the core standard library # (not site packages which may clash with u-module names), and improve start up time. CPYTHON3_CMD = [CPYTHON3, "-BS"] -# File with the test results. -RESULTS_FILE = "_results.json" - # For diff'ing test output DIFF = os.getenv("MICROPY_DIFF", "diff -u") # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale os.environ["PYTHONIOENCODING"] = "utf-8" - -def normalize_newlines(data): - """Normalize newline variations to \\n. - - Only normalizes actual line endings, not literal \\r characters in strings. - Handles \\r\\r\\n and \\r\\n cases to ensure consistent comparison - across different platforms and terminals. - """ - if isinstance(data, bytes): - # Handle PTY double-newline issue first - data = data.replace(b"\r\r\n", b"\n") - # Then handle standard Windows line endings - data = data.replace(b"\r\n", b"\n") - # Don't convert standalone \r as it might be literal content - return data - - -# Code to allow a target MicroPython to import an .mpy from RAM -# Note: the module is named `__injected_test` but it needs to have `__name__` set to -# `__main__` so that the test sees itself as the main module, eg so unittest works. -injected_import_hook_code = """\ -import sys, os, io, vfs -class __File(io.IOBase): - def __init__(self): - module = sys.modules['__injected_test'] - module.__name__ = '__main__' - sys.modules['__main__'] = module - self.off = 0 - def ioctl(self, request, arg): - if request == 4: # MP_STREAM_CLOSE - return 0 - return -1 - def readinto(self, buf): - buf[:] = memoryview(__buf)[self.off:self.off + len(buf)] - self.off += len(buf) - return len(buf) -class __FS: - def mount(self, readonly, mkfs): - pass - def umount(self): - pass - def chdir(self, path): - pass - def getcwd(self): - return "" - def stat(self, path): - if path == '__injected_test.mpy': - return (0,0,0,0,0,0,0,0,0,0) - else: - raise OSError(2) # ENOENT - def open(self, path, mode): - self.stat(path) - return __File() -vfs.mount(__FS(), '/__vfstest') -os.chdir('/__vfstest') -{import_prologue} -__import__('__injected_test') -""" - # Platforms associated with the unix port, values of `sys.platform`. PC_PLATFORMS = ("darwin", "linux", "win32") @@ -161,10 +105,8 @@ def open(self, path, mode): "basics/exception_chain.py", # These require stack-allocated slice optimisation. "micropython/heapalloc_slice.py", - # These require running the scheduler. + # These require implicitly running the scheduler between bytecodes. "micropython/schedule.py", - "extmod/asyncio_event_queue.py", - "extmod/asyncio_iterator_event.py", # These require sys.exc_info(). "misc/sys_exc_info.py", # These require sys.settrace(). @@ -180,6 +122,9 @@ def open(self, path, mode): # Tests to skip on specific targets. # These are tests that are difficult to detect that they should not be run on the given target. platform_tests_to_skip = { + "esp8266": ( + "stress/list_sort.py", # watchdog kicks in because it takes too long + ), "minimal": ( "basics/class_inplace_op.py", # all special methods not supported "basics/subclass_native_init.py", # native subclassing corner cases not support @@ -213,6 +158,9 @@ def open(self, path, mode): "webassembly": ( "basics/string_format_modulo.py", # can't print nulls to stdout "basics/string_strip.py", # can't print nulls to stdout + "basics/weakref_callback_exception.py", # has different exception printing output + "basics/weakref_ref_collect.py", # requires custom test due to GC behaviour + "basics/weakref_finalize_collect.py", # requires custom test due to GC behaviour "extmod/asyncio_basic2.py", "extmod/asyncio_cancel_self.py", "extmod/asyncio_current_task.py", @@ -249,6 +197,23 @@ def open(self, path, mode): ), } +# Tests with known intermittent failures. These tests still run, but failures +# are reclassified as "ignored" instead of "fail" so they don't affect the CI +# exit code. Paths are relative to the tests/ directory (must match test_file +# format used by run_one_test, which normalises backslashes to forward slashes). +# +# Values are (reason, platforms) tuples where platforms is None (all platforms) +# or a tuple of sys.platform strings to restrict ignoring to those platforms. +flaky_tests_to_ignore = { + "thread/thread_gc1.py": ("GC race condition", None), + "thread/stress_schedule.py": ("intermittent crash under QEMU", None), + "thread/stress_recurse.py": ("stack overflow under emulation", None), + "thread/stress_heap.py": ("flaky on macOS", ("darwin",)), + "cmdline/repl_lock.py": ("REPL timing under QEMU", None), + "cmdline/repl_cont.py": ("REPL escaping on macOS", ("darwin",)), + "extmod/time_time_ns.py": ("CI runner clock precision", None), +} + # These tests don't test float explicitly but rather use it to perform the test. tests_requiring_float = ( "extmod/asyncio_basic.py", @@ -324,20 +289,18 @@ def open(self, path, mode): # Tests that require `import target_wiring` to work. tests_requiring_target_wiring = ( + "extmod/machine_spi_rate.py", "extmod/machine_uart_irq_txidle.py", "extmod/machine_uart_tx.py", + "extmod_hardware/machine_can_timings.py", "extmod_hardware/machine_encoder.py", + "extmod_hardware/machine_pwm.py", "extmod_hardware/machine_uart_irq_break.py", "extmod_hardware/machine_uart_irq_rx.py", "extmod_hardware/machine_uart_irq_rxidle.py", ) -def rm_f(fname): - if os.path.exists(fname): - os.remove(fname) - - # unescape wanted regex chars and escape unwanted ones def convert_regex_escapes(line): cs = [] @@ -362,38 +325,6 @@ def platform_to_port(platform): return platform_to_port_map.get(platform, platform) -def convert_device_shortcut_to_real_device(device): - if device.startswith("port:"): - return device.split(":", 1)[1] - elif device.startswith("a") and device[1:].isdigit(): - return "/dev/ttyACM" + device[1:] - elif device.startswith("u") and device[1:].isdigit(): - return "/dev/ttyUSB" + device[1:] - elif device.startswith("c") and device[1:].isdigit(): - return "COM" + device[1:] - else: - return device - - -def get_test_instance(test_instance, baudrate, user, password): - if test_instance == "unix": - return None - elif test_instance == "webassembly": - return PyboardNodeRunner() - else: - # Assume it's a device path. - port = convert_device_shortcut_to_real_device(test_instance) - - global pyboard - sys.path.append(base_path("../tools")) - import pyboard - - pyb = pyboard.Pyboard(port, baudrate, user, password) - pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target - pyb.enter_raw_repl() - return pyb - - def detect_inline_asm_arch(pyb, args): for arch in ("rv32", "thumb", "xtensa"): output = run_feature_check(pyb, args, "inlineasm_{}.py".format(arch)) @@ -500,90 +431,6 @@ def detect_target_wiring_script(pyb, args): pyb.target_wiring_script = tw_data -def prepare_script_for_target(args, *, script_text=None, force_plain=False): - if force_plain or (not args.via_mpy and args.emit == "bytecode"): - # A plain test to run as-is, no processing needed. - pass - elif args.via_mpy: - tempname = tempfile.mktemp(dir="") - mpy_filename = tempname + ".mpy" - - script_filename = tempname + ".py" - with open(script_filename, "wb") as f: - f.write(script_text) - - try: - subprocess.check_output( - [MPYCROSS] - + args.mpy_cross_flags.split() - + ["-o", mpy_filename, "-X", "emit=" + args.emit, script_filename], - stderr=subprocess.STDOUT, - ) - except subprocess.CalledProcessError as er: - return True, b"mpy-cross crash\n" + er.output - - with open(mpy_filename, "rb") as f: - script_text = b"__buf=" + bytes(repr(f.read()), "ascii") + b"\n" - - rm_f(mpy_filename) - rm_f(script_filename) - - script_text += bytes(injected_import_hook_code, "ascii") - else: - print("error: using emit={} must go via .mpy".format(args.emit)) - sys.exit(1) - - return False, script_text - - -def run_script_on_remote_target(pyb, args, test_file, is_special): - with open(test_file, "rb") as f: - script = f.read() - - # If the test is not a special test, prepend it with a print to indicate that it started. - # If the print does not execute this means that the test did not even start, eg it was - # too large for the target. - prepend_start_test = not is_special - if prepend_start_test: - if script.startswith(b"#"): - script = b"print('START TEST')" + script - else: - script = b"print('START TEST')\n" + script - - had_crash, script = prepare_script_for_target(args, script_text=script, force_plain=is_special) - - if had_crash: - return True, script - - try: - had_crash = False - pyb.enter_raw_repl() - if test_file.endswith(tests_requiring_target_wiring) and pyb.target_wiring_script: - pyb.exec_( - "import sys;sys.modules['target_wiring']=__build_class__(lambda:exec(" - + repr(pyb.target_wiring_script) - + "),'target_wiring')" - ) - output_mupy = pyb.exec_(script, timeout=TEST_TIMEOUT) - except pyboard.PyboardError as e: - had_crash = True - if not is_special and e.args[0] == "exception": - if prepend_start_test and e.args[1] == b"" and b"MemoryError" in e.args[2]: - output_mupy = b"SKIP-TOO-LARGE\n" - else: - output_mupy = e.args[1] + e.args[2] + b"CRASH" - else: - output_mupy = bytes(e.args[0], "ascii") + b"\nCRASH" - - if prepend_start_test: - if output_mupy.startswith(b"START TEST\r\n"): - output_mupy = output_mupy.removeprefix(b"START TEST\r\n") - else: - had_crash = True - - return had_crash, output_mupy - - tests_with_regex_output = [ base_path(file) for file in ( @@ -591,6 +438,7 @@ def run_script_on_remote_target(pyb, args, test_file, is_special): "micropython/meminfo.py", "basics/bytes_compare3.py", "basics/builtin_help.py", + "basics/weakref_callback_exception.py", "thread/thread_exc2.py", "circuitpython/traceback_test.py", "circuitpython/traceback_test_chained.py", @@ -719,8 +567,9 @@ def send_get(what): else: # run via pyboard interface + requires_target_wiring = test_file.endswith(tests_requiring_target_wiring) had_crash, output_mupy = pyb.run_script_on_remote_target( - args, test_file_abspath, is_special + args, test_file_abspath, is_special, requires_target_wiring ) # canonical form for all ports/platforms is to use \n for end-of-line @@ -810,53 +659,9 @@ def value(self): return self._value -class PyboardNodeRunner: - def __init__(self): - mjs = os.getenv("MICROPY_MICROPYTHON_MJS") - if mjs is None: - mjs = base_path("../ports/webassembly/build-standard/micropython.mjs") - else: - mjs = os.path.abspath(mjs) - self.micropython_mjs = mjs - - def close(self): - pass - - def run_script_on_remote_target(self, args, test_file, is_special): - cwd = os.path.dirname(test_file) - - # Create system command list. - cmdlist = ["node"] - if test_file.endswith(".py"): - # Run a Python script indirectly via "node micropython.mjs ". - cmdlist.append(self.micropython_mjs) - if args.heapsize is not None: - cmdlist.extend(["-X", "heapsize=" + args.heapsize]) - cmdlist.append(test_file) - else: - # Run a js/mjs script directly with Node, passing in the path to micropython.mjs. - cmdlist.append(test_file) - cmdlist.append(self.micropython_mjs) - - # Run the script. - try: - had_crash = False - output_mupy = subprocess.check_output( - cmdlist, stderr=subprocess.STDOUT, timeout=TEST_TIMEOUT, cwd=cwd - ) - except subprocess.CalledProcessError as er: - had_crash = True - output_mupy = er.output + b"CRASH" - except subprocess.TimeoutExpired as er: - had_crash = True - output_mupy = (er.output or b"") + b"TIMEOUT" - - # Return the results. - return had_crash, output_mupy - - def run_tests(pyb, tests, args, result_dir, num_threads=1): testcase_count = ThreadSafeCounter() + raw_repl_failure_count = ThreadSafeCounter() test_results = ThreadSafeCounter([]) skip_tests = set() @@ -870,6 +675,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_const = False skip_revops = False skip_fstring = False + skip_tstring = False skip_endian = False skip_inlineasm = False has_complex = True @@ -930,6 +736,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if output != b"a=1\n": skip_fstring = True + # Check if tstring feature is enabled, and skip such tests if it doesn't + output = run_feature_check(pyb, args, "tstring.py") + if output != b"tstring\n": + skip_tstring = True + if args.inlineasm_arch == "thumb": # Check if @micropython.asm_thumb supports Thumb2 instructions, and skip such tests if it doesn't output = run_feature_check(pyb, args, "inlineasm_thumb2.py") @@ -948,10 +759,15 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): skip_tests.add("inlineasm/thumb/asmfpsqrt.py") if args.inlineasm_arch == "rv32": - # Check if @micropython.asm_rv32 supports Zba instructions, and skip such tests if it doesn't - output = run_feature_check(pyb, args, "inlineasm_rv32_zba.py") - if output != b"rv32_zba\n": - skip_tests.add("inlineasm/rv32/asmzba.py") + # Discover extension-specific inlineasm tests and add them to the + # list of tests to run if applicable. + for extension in RV32_ARCH_FLAGS: + try: + output = run_feature_check(pyb, args, "inlineasm_rv32_{}.py".format(extension)) + if output.strip() != "rv32_{}".format(extension).encode(): + skip_tests.add("inlineasm/rv32/asm_ext_{}.py".format(extension)) + except FileNotFoundError: + pass # Check if emacs repl is supported, and skip such tests if it's not t = run_feature_check(pyb, args, "repl_emacs_check.py") @@ -975,8 +791,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # Some tests shouldn't be run on GitHub Actions if os.getenv("GITHUB_ACTIONS") == "true": - skip_tests.add("thread/stress_schedule.py") # has reliability issues - if os.getenv("RUNNER_OS") == "Windows" and os.getenv("CI_BUILD_CONFIGURATION") == "Debug": # fails with stack overflow on Debug builds skip_tests.add("misc/sys_settrace_features.py") @@ -1003,12 +817,16 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): if not args.unicode: skip_tests.add("extmod/json_loads.py") # tests loading a utf-8 character + # CIRCUITPY-CHANGE: asserts upstream's escaped Unicode repr, see py/objstrunicode.c + skip_tests.add("basics/string_tstring_basic1.py") + if skip_slice: skip_tests.update(tests_requiring_slice) if not has_complex: skip_tests.add("float/complex1.py") skip_tests.add("float/complex1_intbig.py") + skip_tests.add("float/complex1_micropython.py") skip_tests.add("float/complex_reverse_op.py") skip_tests.add("float/complex_special_methods.py") skip_tests.add("float/int_big_float.py") @@ -1054,9 +872,23 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): # Works but CPython uses '\' path separator skip_tests.add("import/import_file.py") + skip_tests = [os.path.realpath(base_path(skip_test)) for skip_test in skip_tests] + def run_one_test(test_file): - test_file = test_file.replace("\\", "/") test_file_abspath = os.path.abspath(test_file).replace("\\", "/") + # If test_file is one of our own tests always make it relative to our tests/ dir and + # otherwise use the absolute path, regardless of actual path passed, + # such that display and result output is always the same. + try: + test_file_relpath = os.path.relpath(test_file, start=base_path()) + if not test_file_relpath.startswith(".."): + test_file = test_file_relpath + else: + test_file = test_file_abspath + except ValueError: + # Path on different drive on Windows. + test_file = test_file_abspath + test_file = test_file.replace("\\", "/") if args.filters: # Default verdict is the opposite of the first action @@ -1083,9 +915,10 @@ def run_one_test(test_file): is_async = test_name.startswith(("async_", "asyncio_")) or test_name.endswith("_async") is_const = test_name.startswith("const") is_fstring = test_name.startswith("string_fstring") + is_tstring = test_name.startswith("string_tstring") or test_name.endswith("_tstring") is_inlineasm = test_name.startswith("asm") - skip_it = test_file in skip_tests + skip_it = os.path.realpath(test_file) in skip_tests skip_it |= skip_native and is_native skip_it |= skip_endian and is_endian skip_it |= skip_int_big and is_int_big @@ -1097,12 +930,17 @@ def run_one_test(test_file): skip_it |= skip_const and is_const skip_it |= skip_revops and "reverse_op" in test_name skip_it |= skip_fstring and is_fstring + skip_it |= skip_tstring and is_tstring skip_it |= skip_inlineasm and is_inlineasm if skip_it: print("skip ", test_file) test_results.append((test_file, "skip", "")) return + elif args.dry_run: + print("found", test_file) + test_results.append((test_file, "found", "")) + return # Run the test on the MicroPython target. output_mupy = run_micropython(pyb, args, test_file, test_file_abspath) @@ -1223,6 +1061,9 @@ def run_one_test(test_file): rm_f(filename_expected) rm_f(filename_mupy) else: + if output_mupy.startswith(b"could not enter raw repl"): + extra_info = "raw REPL failed" + raw_repl_failure_count.increment() print("FAIL ", test_file, extra_info) if output_expected is not None: with open(filename_expected, "wb") as f: @@ -1253,79 +1094,28 @@ def run_one_test(test_file): else: for test in tests: run_one_test(test) + if raw_repl_failure_count.value > TEST_MAXIMUM_RAW_REPL_FAILURES: + print("Too many raw REPL failures, aborting test run") + break except TestError as er: for line in er.args[0]: print(line) - sys.exit(1) + sys.exit(2) + + # Reclassify known-flaky test failures as ignored. + # Safe to mutate: thread pool has joined. + results = test_results.value + for i, r in enumerate(results): + if r[1] == "fail": + reason, platforms = flaky_tests_to_ignore.get(r[0], (None, None)) + if reason is not None: + if platforms is None or sys.platform in platforms: + results[i] = (r[0], "ignored", "{}: {}".format(FLAKY_REASON_PREFIX, reason)) # Return test results. return test_results.value, testcase_count.value -# Print a summary of the results and save them to a JSON file. -# Returns True if everything succeeded, False otherwise. -def create_test_report(args, test_results, testcase_count=None): - passed_tests = list(r for r in test_results if r[1] == "pass") - skipped_tests = list(r for r in test_results if r[1] == "skip" and r[2] != "too large") - skipped_tests_too_large = list( - r for r in test_results if r[1] == "skip" and r[2] == "too large" - ) - failed_tests = list(r for r in test_results if r[1] == "fail") - - num_tests_performed = len(passed_tests) + len(failed_tests) - - testcase_count_info = "" - if testcase_count is not None: - testcase_count_info = " ({} individual testcases)".format(testcase_count) - print("{} tests performed{}".format(num_tests_performed, testcase_count_info)) - - print("{} tests passed".format(len(passed_tests))) - - if len(skipped_tests) > 0: - print( - "{} tests skipped: {}".format( - len(skipped_tests), " ".join(test[0] for test in skipped_tests) - ) - ) - - if len(skipped_tests_too_large) > 0: - print( - "{} tests skipped because they are too large: {}".format( - len(skipped_tests_too_large), " ".join(test[0] for test in skipped_tests_too_large) - ) - ) - - if len(failed_tests) > 0: - print( - "{} tests failed: {}".format( - len(failed_tests), " ".join(test[0] for test in failed_tests) - ) - ) - - # Serialize regex added by append_filter. - def to_json(obj): - if isinstance(obj, re.Pattern): - return obj.pattern - return obj - - with open(os.path.join(args.result_dir, RESULTS_FILE), "w") as f: - json.dump( - { - # The arguments passed on the command-line. - "args": vars(args), - # A list of all results of the form [(test, result, reason), ...]. - "results": list(test for test in test_results), - # A list of failed tests. This is deprecated, use the "results" above instead. - "failed_tests": [test[0] for test in failed_tests], - }, - f, - default=to_json, - ) - - # Return True only if all tests succeeded. - return len(failed_tests) == 0 - - class append_filter(argparse.Action): def __init__(self, option_strings, dest, **kwargs): super().__init__(option_strings, dest, default=[], **kwargs) @@ -1340,39 +1130,7 @@ def __call__(self, parser, args, value, option): args.filters.append((option, re.compile(value))) -test_instance_description = """\ -By default the tests are run against the unix port of MicroPython. To run it -against something else, use the -t option. See below for details. -""" -test_instance_epilog = """\ -The -t option accepts the following for the test instance: -- unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON - environment variable (which defaults to the standard variant of either the unix - or windows ports, depending on the host platform) -- webassembly - use the webassembly port of MicroPython, specified by the - MICROPY_MICROPYTHON_MJS environment variable (which defaults to the standard - variant of the webassembly port) -- port: - connect to and use the given serial port device -- a - connect to and use /dev/ttyACM -- u - connect to and use /dev/ttyUSB -- c - connect to and use COM -- exec: - execute a command and attach to its stdin/stdout -- execpty: - execute a command and attach to the printed /dev/pts/ device -- ... - connect to the given IPv4 address -- anything else specifies a serial port -""" - -test_directory_description = """\ -Tests are discovered by scanning test directories for .py files or using the -specified test files. If test files nor directories are specified, the script -expects to be ran in the tests directory (where this file is located) and the -builtin tests suitable for the target platform are ran. -""" - - def main(): - global injected_import_hook_code - cmd_parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=f"""Run and manage tests for MicroPython. @@ -1427,6 +1185,11 @@ def main(): dest="filters", help="include test by regex on path/name.py", ) + cmd_parser.add_argument( + "--dry-run", + action="store_true", + help="Show tests which would run (though might still be skipped at runtime)", + ) cmd_parser.add_argument( "--emit", default="bytecode", help="MicroPython emitter to use (bytecode or native)" ) @@ -1479,7 +1242,7 @@ def main(): if args.begin: with open(args.begin, "rt") as source: prologue = source.read() - injected_import_hook_code = injected_import_hook_code.replace("{import_prologue}", prologue) + set_injected_prologue(prologue) if args.print_failures: for out in glob(os.path.join(args.result_dir, "*.out")): @@ -1502,7 +1265,7 @@ def main(): os.path.join(args.result_dir, "*.out") ): os.remove(f) - rm_f(os.path.join(args.result_dir, RESULTS_FILE)) + rm_f(get_results_filename(args)) sys.exit(0) @@ -1518,7 +1281,7 @@ def main(): ) if args.run_failures: - results_file = os.path.join(args.result_dir, RESULTS_FILE) + results_file = get_results_filename(args) if os.path.exists(results_file): with open(results_file, "r") as f: tests = list(test[0] for test in json.load(f)["results"] if test[1] == "fail") @@ -1529,7 +1292,19 @@ def main(): if args.platform == "webassembly": test_extensions += ("*.js", "*.mjs") - if args.test_dirs is None: + all_test_dirs = [] + main_tests_dir_in_args = None + if args.test_dirs is not None: + # Run tests from given directories though if user explicitly passes this directory as argument + # still do the normal test discovery to be consistent with running from within this directory. + main_tests_dir = os.path.realpath(base_path()) + for test_dir in args.test_dirs: + if os.path.realpath(test_dir) == main_tests_dir: + main_tests_dir_in_args = test_dir + else: + all_test_dirs.append(test_dir) + + if args.test_dirs is None or main_tests_dir_in_args is not None: test_dirs = ( "basics", "micropython", @@ -1553,13 +1328,18 @@ def main(): test_dirs += ("import",) if args.build != "minimal": test_dirs += ("cmdline", "io") - else: - # run tests from these directories - test_dirs = args.test_dirs + + all_test_dirs.extend( + test_dir + if main_tests_dir_in_args is None + else os.path.join(main_tests_dir_in_args, test_dir) + for test_dir in test_dirs + ) + tests = sorted( test_file for test_files in ( - glob(os.path.join(dir, ext)) for dir in test_dirs for ext in test_extensions + glob(os.path.join(dir, ext)) for dir in all_test_dirs for ext in test_extensions ) for test_file in test_files ) diff --git a/tests/serial_test.py b/tests/serial_test.py index 3b5940d91a9..eebea402fa4 100755 --- a/tests/serial_test.py +++ b/tests/serial_test.py @@ -13,7 +13,7 @@ import sys import time -run_tests_module = __import__("run-tests") +from test_utils import test_instance_epilog, convert_device_shortcut_to_real_device echo_test_script = """ import sys @@ -307,7 +307,7 @@ def main(): cmd_parser = argparse.ArgumentParser( description="Test performance and reliability of serial port communication.", - epilog=run_tests_module.test_instance_epilog, + epilog=test_instance_epilog, formatter_class=argparse.RawTextHelpFormatter, ) cmd_parser.add_argument( @@ -321,7 +321,7 @@ def main(): ) args = cmd_parser.parse_args() - dev_repl = run_tests_module.convert_device_shortcut_to_real_device(args.test_instance) + dev_repl = convert_device_shortcut_to_real_device(args.test_instance) test_passed = True try: diff --git a/tests/target_wiring/NUCLEO_WB55.py b/tests/target_wiring/NUCLEO_WB55.py index ad7c120d377..e40d35e225b 100644 --- a/tests/target_wiring/NUCLEO_WB55.py +++ b/tests/target_wiring/NUCLEO_WB55.py @@ -6,3 +6,7 @@ # LPUART(1) is on PA2/PA3. uart_loopback_args = ("LP1",) uart_loopback_kwargs = {} + +spi_standalone_args_list = [(1,), (2,)] + +pwm_loopback_pins = [("D1", "D0")] diff --git a/tests/target_wiring/PYBx.py b/tests/target_wiring/PYBx.py index 10ce520ef0a..b36e342e944 100644 --- a/tests/target_wiring/PYBx.py +++ b/tests/target_wiring/PYBx.py @@ -6,3 +6,11 @@ # UART("XA") is on X1/X2 (usually UART(4) on PA0/PA1). uart_loopback_args = ("XA",) uart_loopback_kwargs = {} + +spi_standalone_args_list = [(1,), (2,)] + +# CAN args assume no connection for single device tests +can_args = (1,) +can_kwargs = {} + +pwm_loopback_pins = [("X1", "X2")] diff --git a/tests/target_wiring/README.md b/tests/target_wiring/README.md new file mode 100644 index 00000000000..c3a7038bb91 --- /dev/null +++ b/tests/target_wiring/README.md @@ -0,0 +1,81 @@ +# Target wiring + +Some tests require hardware configuration and/or external connections, for example +bridging a pair of GPIO pins. Each board that such tests run on needs to be configured +individually. That is achieved by providing a target wiring configuration script that +defines the necessary hardware parameters for each test. + +## Selecting the target wiring + +There are three ways to provide the target wiring configuration: + +1. Specify it explicitly when running the test: `./run-tests.py --target-wiring