[wasm] Compute CoreCLR P/Invoke struct sizes with crossgen2's type system

docs/workflow/wasm-documentation.md

@@ -51,7 +51,7 @@ For debugging instructions including VS Code and Chrome DevTools setup, see the
 
 ### Running coreclr callhelpers generator
 
-After building the runtime, use the `generate-coreclr-helpers` script for your platform (`.cmd` or `.sh`) in `src/tasks/WasmAppBuilder` to [re]generate the call helpers in `src/coreclr/vm/wasm`.
+After building the runtime, use the `generate-coreclr-helpers` script for your platform (`.cmd` or `.sh`) in `src/coreclr/vm/wasm` to [re]generate the call helpers in `src/coreclr/vm/wasm`.
 
 ## Features and Configuration
 

eng/Subsets.props

@@ -759,6 +759,17 @@
               Non-VMR builds normally use the crossgen2 built for the target host SDK from another build leg, but builds without one can opt in.
             -->
             <ProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono' and ('$(TargetsMobile)' != 'true' or '$(TargetOS)' == 'browser') and '$(TargetsLinuxBionic)' != 'true' and '$(BuildHostTools)' == 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Crossgen2.Host.sfxproj" Category="packs" />
+
+            <!--
+              The wasm-tools workload manifest declares the host crossgen2 pack, so the workload
+              testing legs need it in their local package feed. Those legs do not set
+              $(BuildHostTools), so the project above does not produce one for them. The Host variant
+              pins the RID to the build host, which is exactly the pack the workload resolves.
+
+              Opt-in only. The official build already publishes this pack from the host platform
+              legs, and building it here as well would produce a second package with the same id.
+            -->
+            <ProjectToBuild Condition="'$(BuildCrossgen2HostPackForWorkloadTesting)' == 'true' and '$(RuntimeFlavor)' != 'Mono' and '$(BuildHostTools)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Crossgen2.Host.sfxproj" Category="packs" />
           </ItemGroup>
           <ItemGroup>
             <SharedFrameworkProjectToBuild Condition="'$(_BuildHostPack)' == 'true'" Include="$(InstallerProjectRoot)pkg\archives\dotnet-nethost.proj" />

eng/pipelines/common/templates/browser-wasm-build-tests.yml

@@ -92,9 +92,10 @@ jobs:
               TargetFolder: '$(Build.SourcesDirectory)/artifacts'
               CleanTargetFolder: false
 
-          # Download the CoreCLR runtime pack. The wasm-tools workload manifest now includes
-          # the CoreCLR browser-wasm runtime pack, so installing the workload for testing
-          # requires the pack to be present in the local package feed.
+          # Download the CoreCLR runtime pack and the host crossgen2 pack. The wasm-tools workload
+          # manifest includes both, so installing the workload for testing requires them to be
+          # present in the local package feed. Only pipelines that build the CoreCLR browser-wasm
+          # runtime (e.g. runtime.yml) can stage them here.
           - ${{ if eq(parameters.includeCoreClrRuntimePack, true) }}:
             - task: DownloadPipelineArtifact@2
               displayName: Download built nugets for CoreCLR runtime
@@ -104,10 +105,12 @@ jobs:
                 targetPath: '$(Build.SourcesDirectory)/artifacts/BuildArtifacts_browser_wasm_$(_hostedOs)_Release_CoreCLR'
 
             - task: CopyFiles@2
-              displayName: Copy CoreCLR runtime pack
+              displayName: Copy CoreCLR runtime pack and host crossgen2 pack
               inputs:
                 SourceFolder: '$(Build.SourcesDirectory)/artifacts/BuildArtifacts_browser_wasm_$(_hostedOs)_Release_CoreCLR'
-                Contents: packages/$(_BuildConfig)/Shipping/Microsoft.NETCore.App.Runtime.browser-wasm.*
+                Contents: |
+                  packages/$(_BuildConfig)/Shipping/Microsoft.NETCore.App.Runtime.browser-wasm.*
+                  packages/$(_BuildConfig)/Shipping/Microsoft.NETCore.App.Crossgen2.*
                 TargetFolder: '$(Build.SourcesDirectory)/artifacts'
                 CleanTargetFolder: false
 

eng/pipelines/extra-platforms/runtime-extra-platforms-wasm.yml

@@ -209,7 +209,7 @@ jobs:
         - browser_wasm_win
       jobParameters:
         nameSuffix: CoreCLR
-        buildArgs: -s clr+libs+packs -c Release -rc $(_BuildConfig) /p:TestAssemblies=false /p:InstallWorkloadForTesting=false
+        buildArgs: -s clr+libs+packs -c Release -rc $(_BuildConfig) /p:TestAssemblies=false /p:InstallWorkloadForTesting=false /p:BuildCrossgen2HostPackForWorkloadTesting=true
         timeoutInMinutes: 120
         postBuildSteps:
           - template: /eng/pipelines/common/wasm-post-build-steps.yml

eng/pipelines/runtime.yml

@@ -130,7 +130,7 @@ extends:
           - browser_wasm_win
           jobParameters:
             nameSuffix: CoreCLR
-            buildArgs: -s clr+libs+libs.tests+packs -c Release -rc $(_BuildConfig) /p:TestAssemblies=false /p:TestWasmBuildTests=true /p:ArchiveTests=true /p:InstallWorkloadForTesting=false
+            buildArgs: -s clr+libs+libs.tests+packs -c Release -rc $(_BuildConfig) /p:TestAssemblies=false /p:TestWasmBuildTests=true /p:ArchiveTests=true /p:InstallWorkloadForTesting=false /p:BuildCrossgen2HostPackForWorkloadTesting=true
             timeoutInMinutes: 120
             postBuildSteps:
               - template: /eng/pipelines/common/wasm-post-build-steps.yml

eng/wasm/WasmPInvokeModules.props

@@ -1,12 +1,12 @@
 <!--
   The framework P/Invoke modules that CoreCLR resolves without dynamic loading on WebAssembly.
 
-  ManagedToNativeGenerator emits a direct-call entry for every [DllImport] whose module name is on
-  this list and nothing for the rest, so two builds have to agree on it:
+  The portable call-helpers generator emits a direct-call entry for every [DllImport] whose module
+  name is on this list and nothing for the rest, so two builds have to agree on it:
 
-    * src/tasks/WasmAppBuilder/WasmAppBuilder.csproj generates the checked-in call tables under
-      src/coreclr/vm/wasm from it (see generate-coreclr-helpers.md). They are compiled into
-      coreclr_gen_static, which the product's corerun links.
+    * src/coreclr/vm/wasm/generate-coreclr-helpers.proj imports this list to regenerate the
+      checked-in call tables under src/coreclr/vm/wasm (see generate-coreclr-helpers.md). They are
+      compiled into coreclr_gen_static, which the product's corerun links.
     * src/tests/Common/CLRTest.WasmCorerun.targets leaves coreclr_gen_static out when it links a
       test-specific corerun, and regenerates equivalent tables in its place. Those have to cover
       the same modules: one missing here surfaces as a DllNotFoundException at run time, inside

src/coreclr/tools/Common/JitInterface/WasmLowering.cs

@@ -389,7 +389,9 @@ public static WasmValueType LowerType(TypeDesc type)
         /// <summary>
         /// Maps a WasmValueType to its single-character signature encoding.
         /// </summary>
-        private static char WasmValueTypeToSigChar(WasmValueType vt) => vt switch
+        // internal rather than private so the call-helper generator can encode a single type with the
+        // same table the signature builder below uses (see ILCompiler.PortableCallHelpers.InteropSignature).
+        internal static char WasmValueTypeToSigChar(WasmValueType vt) => vt switch
         {
             WasmValueType.I32 => 'i',
             WasmValueType.I64 => 'l',
@@ -409,6 +411,22 @@ public static WasmValueType LowerType(TypeDesc type)
             _ => throw new InvalidOperationException($"Unknown signature char: {c}")
         };
 
+        internal static string DescribeSigChar(char c) => c switch
+        {
+            'v' => "a void result",
+            'i' => "a 32-bit integer",
+            'l' => "a 64-bit integer",
+            'f' => "a 32-bit float",
+            'd' => "a 64-bit float",
+            'V' => "a 128-bit vector",
+            'S' or 'A' => "a struct passed by reference",
+            'T' => "the 'this' argument",
+            'p' => "the portable entry point argument",
+            'a' => "the async continuation argument",
+            'e' => "an empty struct",
+            _ => $"an unrecognized element '{c}'"
+        };
+
         private static int ParseStructSize(string sig, ref int pos)
         {
             Debug.Assert(sig[pos] is 'S' or 'A');

src/coreclr/tools/aot/ILCompiler.ReadyToRun.Tests/WasmArgumentLayoutTests.cs

@@ -8,9 +8,14 @@
 using System.IO;
 using System.Linq;
 
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Emit;
+
 using crossgen2::ILCompiler;
 using crossgen2::ILCompiler.DependencyAnalysis.ReadyToRun;
 using crossgen2::ILCompiler.DependencyAnalysis.Wasm;
+using crossgen2::ILCompiler.PortableCallHelpers;
 using crossgen2::Internal.CallingConvention;
 using crossgen2::Internal.JitInterface;
 
@@ -546,26 +551,220 @@ private static MethodSignature MakeProbeSignature(ReadyToRunCompilerContext cont
             context.GetWellKnownType(WellKnownType.Int32).MakeByRefType());
 
 
+    /// <summary>
+    /// The generator encodes a type in parameter position with a single token. These are the three
+    /// shapes that encoding exists to tell apart: a multi-field struct, which goes by reference and
+    /// carries its size; a single-field wrapper, which is passed as the field it wraps; and a
+    /// primitive.
+    /// </summary>
+    [Theory]
+    [InlineData("Guid", "S16")]
+    [InlineData("DateTime", "l")]
+    [InlineData("Int32", "i")]
+    public void PortableCallHelpersGeneratorEncodesTypesTheWayTheCompilerLowersThem(string typeName, string expected)
+    {
+        ReadyToRunCompilerContext context = CreateWasmContext();
+
+        Assert.Equal(expected, InteropSignature.GetAbiToken(GetSystemType(context, typeName)));
+    }
+
+    /// <summary>
+    /// A struct that holds a reference lays out through the auto-layout path, which asks the
+    /// compilation group whether the base offset needs aligning. Generation is not a compilation, so
+    /// it has to configure a group itself for that question to have an answer at all.
+    /// </summary>
+    [Fact]
+    public void PortableCallHelpersGeneratorComputesLayoutOfStructsHoldingReferences()
+    {
+        ReadyToRunCompilerContext context = CreateWasmContext();
+        var type = GetSystemType(context, "RuntimeTypeHandle");
+
+        // If this stops holding, the test no longer covers the auto-layout path it was written for.
+        Assert.True(type.ContainsGCPointers, $"{type} was chosen because it holds a reference");
+
+        // One field the size of the whole struct: lowered to that field, a reference, passed as i32.
+        Assert.Equal("i", InteropSignature.GetAbiToken(type));
+    }
+
+    /// <summary>
+    /// The thunk a method gets is keyed by its lowered signature, so the generator has to encode a
+    /// method exactly as the compiler lowers it. Anything else and the interpreter calls through a
+    /// thunk built for a different shape.
+    /// </summary>
+    [Fact]
+    public void PortableCallHelpersGeneratorEncodesMethodsLikeTheCompiler()
+    {
+        ReadyToRunCompilerContext context = CreateWasmContext();
+        var method = (EcmaMethod)GetSystemType(context, "DateTime").GetMethod("AddTicks"u8, null);
+
+        string expected = WasmLowering.GetSignature(method.Signature, WasmLowering.LoweringFlags.None).SignatureString;
+        _output.WriteLine($"{method} lowers to '{expected}'");
+
+        Assert.Equal(expected, InteropSignature.GetMethodSignature(method));
+    }
+
+    /// <summary>
+    /// A type has to get the same token at the interop boundary as it does inside a lowered method
+    /// signature, because the runtime looks a thunk up by the signature the compiler produced. The
+    /// two encoders are separate code, so this pins them together for each shape the ABI treats
+    /// differently: multi-segment types passed by value across several slots, structs passed by
+    /// reference, single-field wrappers, and primitives.
+    /// </summary>
+    [Theory]
+    [InlineData("Int128")]
+    [InlineData("UInt128")]
+    [InlineData("Guid")]
+    [InlineData("DateTime")]
+    [InlineData("Int32")]
+    [InlineData("Double")]
+    public void PortableCallHelpersGeneratorEncodesTypesTheSameWayInAndOutOfASignature(string typeName)
+    {
+        ReadyToRunCompilerContext context = CreateWasmContext();
+        TypeDesc type = GetSystemType(context, typeName);
+
+        string signature = WasmLowering.GetSignature(
+            MakeStaticVoidSignature(context, type),
+            WasmLowering.LoweringFlags.None).SignatureString;
+        _output.WriteLine($"{typeName} lowers to '{signature}' in a signature");
+
+        // 'v' return, then the single parameter, then the 'p' entrypoint suffix.
+        List<string> tokens = InteropSignature.ParseSignatureTokens(signature);
+        Assert.Equal(tokens[1], InteropSignature.GetAbiToken(type));
+    }
+
+    private const string CoreLibSimpleName = "System.Private.CoreLib";
+
+    /// <summary>
+    /// An exported callback resolves its MethodDesc at run time through
+    /// LookupUnmanagedCallersOnlyMethodByName, which matches on the declaring type and the method name
+    /// alone. Overloads are indistinguishable to it, so generation has to reject a name it could not
+    /// resolve rather than emit a wrapper that calls whichever one the walk reaches first.
+    /// </summary>
+    [Theory]
+    // Two exported overloads: the lookup cannot tell them apart.
+    [InlineData("[UnmanagedCallersOnly(EntryPoint = \"cb_one\")]", "Handle",
+                "[UnmanagedCallersOnly(EntryPoint = \"cb_two\")]", "Handle", true)]
+    // The twin does not have to be exported to be returned by the walk, which only tests the attribute.
+    [InlineData("[UnmanagedCallersOnly(EntryPoint = \"cb_one\")]", "Handle",
+                "[UnmanagedCallersOnly]", "Handle", true)]
+    // Distinct names resolve unambiguously.
+    [InlineData("[UnmanagedCallersOnly(EntryPoint = \"cb_one\")]", "HandleOne",
+                "[UnmanagedCallersOnly(EntryPoint = \"cb_two\")]", "HandleTwo", false)]
+    // Nothing is exported, so neither wrapper reaches the name lookup: the runtime hands both their
+    // MethodDesc through the arity-aware g_ReverseThunks key instead.
+    [InlineData("[UnmanagedCallersOnly]", "Handle", "[UnmanagedCallersOnly]", "Handle", false)]
+    public void PortableCallHelpersGeneratorRejectsAnExportItCouldNotResolveByName(
+        string firstAttribute, string firstName, string secondAttribute, string secondName, bool expectRejected)
+    {
+        string source = $$"""
+            using System.Runtime.InteropServices;
+
+            public static class Exports
+            {
+                {{firstAttribute}}
+                public static int {{firstName}}(int a) => a;
+
+                {{secondAttribute}}
+                public static int {{secondName}}(int a, int b) => a + b;
+            }
+            """;
+
+        string workingDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+        Directory.CreateDirectory(workingDirectory);
+
+        try
+        {
+            string inputAssembly = CompileCallbackAssembly(source, Path.Combine(workingDirectory, "Callbacks.dll"));
+            string outputDirectory = Path.Combine(workingDirectory, "generated");
+
+            var options = new PortableCallHelpersGeneratorOptions
+            {
+                OutputDirectory = outputDirectory,
+                TargetOS = "browser",
+                PInvokeModules = new[] { "libSystem.Native" },
+            };
+
+            var log = new StringWriter();
+            int exitCode = PortableCallHelpersGenerator.Run(
+                CreateWasmContext(inputAssembly), options, new Logger(log, isVerbose: false));
+
+            if (expectRejected)
+            {
+                Assert.Equal(1, exitCode);
+                Assert.Contains($"declares more than one [UnmanagedCallersOnly] method named '{firstName}'", log.ToString());
+            }
+            else
+            {
+                Assert.Equal(0, exitCode);
+                Assert.DoesNotContain("declares more than one", log.ToString());
+            }
+        }
+        finally
+        {
+            // The type system maps an input assembly with FileShare.Read and never releases it - the
+            // context is not disposable - so on Windows the compiled input cannot be deleted while
+            // this process lives. Cleaning up is best effort rather than a second way to fail.
+            try
+            {
+                Directory.Delete(workingDirectory, recursive: true);
+            }
+            catch (Exception e) when (e is IOException or UnauthorizedAccessException)
+            {
+            }
+        }
+    }
+
+    /// <summary>
+    /// Builds an input assembly for the generator to scan. It references the same CoreLib the context
+    /// reads, so the attributes it applies are the ones the type system will resolve.
+    /// </summary>
+    private static string CompileCallbackAssembly(string source, string outputPath)
+    {
+        CSharpCompilation compilation = CSharpCompilation.Create(
+            Path.GetFileNameWithoutExtension(outputPath),
+            new[] { CSharpSyntaxTree.ParseText(source) },
+            new[] { MetadataReference.CreateFromFile(TestPaths.SystemPrivateCoreLibPath) },
+            new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary));
+
+        EmitResult result = compilation.Emit(outputPath);
+        Assert.True(result.Success,
+            string.Join(Environment.NewLine, result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)));
+
+        return outputPath;
+    }
+
+    private static EcmaType GetSystemType(ReadyToRunCompilerContext context, string typeName)
+    {
+        return (EcmaType)context.SystemModule.GetType("System"u8, System.Text.Encoding.UTF8.GetBytes(typeName));
+    }
+
     /// <summary>
     /// Configures a type system context the way crossgen2 does for
-    /// <c>--targetarch wasm --targetos browser</c>.
+    /// <c>--targetarch wasm --targetos browser</c>. Extra input assemblies stand in for the rest of an
+    /// app closure, which a real build always supplies alongside CoreLib.
     /// </summary>
-    private ReadyToRunCompilerContext CreateWasmContext()
+    private ReadyToRunCompilerContext CreateWasmContext(params string[] extraInputAssemblyPaths)
     {
         string coreLibPath = TestPaths.SystemPrivateCoreLibPath;
         Assert.True(File.Exists(coreLibPath), $"System.Private.CoreLib.dll not found at '{coreLibPath}'");
 
         InstructionSetSupport instructionSetSupport = new(default, default, TargetArchitecture.Wasm32);
         TargetDetails target = new(TargetArchitecture.Wasm32, TargetOS.Browser, TargetAbi.NativeAot, instructionSetSupport.GetVectorTSimdVector());
 
+        Dictionary<string, string> inputFilePaths = new(StringComparer.OrdinalIgnoreCase) { { CoreLibSimpleName, coreLibPath } };
+        foreach (string path in extraInputAssemblyPaths)
+        {
+            inputFilePaths.Add(Path.GetFileNameWithoutExtension(path), path);
+        }
+
         // Wasm cannot generate code at runtime, matching what crossgen2's Program computes for this target.
         ReadyToRunCompilerContext context = new(target, SharedGenericsMode.CanonicalReferenceTypes, bubbleIncludesCoreModule: true, targetAllowsRuntimeCodeGeneration: false, instructionSetSupport, oldTypeSystemContext: null)
         {
-            InputFilePaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) { { "System.Private.CoreLib", coreLibPath } },
+            InputFilePaths = inputFilePaths,
             ReferenceFilePaths = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase),
         };
 
-        EcmaModule coreLib = (EcmaModule)context.GetModuleForSimpleName("System.Private.CoreLib");
+        EcmaModule coreLib = (EcmaModule)context.GetModuleForSimpleName(CoreLibSimpleName);
         context.SetSystemModule(coreLib);
 
         // The R2R field layout algorithm reaches into the compilation group to decide whether base