diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/CHANGELOG.md b/Frends.MicrosoftSQL.ExecuteQueryToFile/CHANGELOG.md index 2fe47cc..c257477 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/CHANGELOG.md +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [2.4.0] - 2026-09-07 +### Added +- Added JSON output support to the ExecuteQueryToFile method, allowing SQL query results to be streamed directly to a JSON file. + ## [2.3.0] - 2026-01-30 ### Fixed - Fixed an issue that was causing problems with Frends processes' cleanup and assembly unloading. diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/GlobalSuppressions.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/GlobalSuppressions.cs index 82dc8de..8bca355 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/GlobalSuppressions.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/GlobalSuppressions.cs @@ -1,9 +1,10 @@ using System.Diagnostics.CodeAnalysis; -[assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1600:Elements should be documented", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] -[assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1633:File should have header", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] -[assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1633:File should have header", Justification = "Following Frends documentation guidelines")] -[assembly: SuppressMessage("StyleCop.CSharp.NamingRules", "SA1309:Field names should not begin with underscore", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] +[assembly: SuppressMessage("StyleCop.CSharp.SpecialRules", "SA0001::XmlCommentAnalysisDisabled", Justification = "Following Frends documentation guidelines")] +[assembly: SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1101:PrefixLocalCallsWithThis", Justification = "Following Frends documentation guidelines")] +[assembly: SuppressMessage("StyleCop.CSharp.OrderingRules", "SA1200:UsingDirectivesMustBePlacedWithinNamespace", Justification = "Following Frends documentation guidelines")] +[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:BracesMustNotBeOmitted", Justification = "Following Frends documentation guidelines")] +[assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1600:ElementsMustBeDocumented", Justification = "Documentation checked by custom analyzers")] +[assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1633:FileMustHaveHeader", Justification = "Following Frends documentation guidelines")] [assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1629:Documentation text should end with a period", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] -[assembly: SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1101:Prefix local calls with this", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] -[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:Braces should not be omitted", Justification = "Following latest .Net6 langVersion", Scope = "member", Target = "~M:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests.Helper.CreateTestTable(System.String,System.String,System.String)")] +[assembly: SuppressMessage("StyleCop.CSharp.NamingRules", "SA1309:Field names should not begin with underscore", Justification = "Following Frends documentation guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Tests")] \ No newline at end of file diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/JsonUnitTests.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/JsonUnitTests.cs new file mode 100644 index 0000000..31aa443 --- /dev/null +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/JsonUnitTests.cs @@ -0,0 +1,503 @@ +using System; +using System.Data; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Frends.MicrosoftSQL.ExecuteQueryToFile.Definitions; +using Frends.MicrosoftSQL.ExecuteQueryToFile.Enums; +using Microsoft.Data.SqlClient; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Tests; + +[TestFixture] +public class JsonUnitTests +{ + private static readonly string _connString = Helper.GetConnectionString(); + private static readonly string _tableName = "TestTable"; + private static readonly string _destination = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "../../../TestData/test.json"); + + private Options _options; + + [SetUp] + public void Init() + { + Helper.ExecuteNonQuery(_connString, $"IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{_tableName}') DROP TABLE {_tableName}"); + + _options = new Options + { + TimeoutSeconds = 30, + ReturnFormat = ReturnFormat.JSON, + JsonOptions = new JsonOptions + { + JsonOutputMode = JsonOutputMode.Indented, + DateFormat = "yyyy-MM-dd", + DateTimeFormat = "yyyy-MM-ddTHH:mm:ss.fff", + TimeFormat = @"hh\:mm\:ss\.fff", + HandleNullAsEmpty = false, + FileBufferSize = 65536, + }, + }; + + Helper.CreateTestTable(_connString, _tableName); + + var parameters = new Microsoft.Data.SqlClient.SqlParameter[] + { + new Microsoft.Data.SqlClient.SqlParameter("@Hash", SqlDbType.VarBinary) + { + Value = File.ReadAllBytes(Path.Combine(Path.GetDirectoryName(_destination), "Test_image.png")), + }, + new Microsoft.Data.SqlClient.SqlParameter("@TestText", SqlDbType.VarBinary) + { + Value = File.ReadAllBytes(Path.Combine(Path.GetDirectoryName(_destination), "Test_text.txt")), + }, + }; + + Helper.ExecuteNonQuery(_connString, $"Insert into {_tableName} (Id, LastName, FirstName, Salary, Image, TestText) values (1,'Meikalainen','Matti',1523.25, {parameters[0].ParameterName}, {parameters[1].ParameterName});", parameters); + } + + [TearDown] + public void CleanUp() + { + using var connection = new SqlConnection(_connString); + connection.Open(); + var cmd = connection.CreateCommand(); + cmd.CommandText = $"IF EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{_tableName}') BEGIN DROP TABLE IF EXISTS {_tableName}; END"; + cmd.ExecuteNonQuery(); + + File.Delete(_destination); + } + + [Test] + public async Task ExecuteQueryToFile_Json_Indented_WritesValidJsonArray() + { + var input = new Input + { + Query = $"SELECT Id, LastName, FirstName, Salary FROM {_tableName}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + var result = await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var content = await File.ReadAllTextAsync(_destination); + var array = JArray.Parse(content); // throws if invalid JSON + + Assert.AreEqual(1, result.EntriesWritten); + Assert.AreEqual(1, array.Count); + Assert.AreEqual(1, array[0]["Id"].Value()); + Assert.AreEqual("Meikalainen", array[0]["LastName"].Value()); + Assert.AreEqual("Matti", array[0]["FirstName"].Value()); + Assert.AreEqual(1523.25m, array[0]["Salary"].Value()); + } + + [Test] + public async Task ExecuteQueryToFile_Json_JsonLines_EachRowOnSeparateLine() + { + _options.JsonOptions.JsonOutputMode = JsonOutputMode.JsonLines; + + var input = new Input + { + Query = $"SELECT Id, LastName, FirstName, Salary FROM {_tableName}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var lines = (await File.ReadAllLinesAsync(_destination)) + .Where(l => !string.IsNullOrWhiteSpace(l)) + .ToArray(); + + Assert.AreEqual(1, lines.Length); + + // Each line must be a valid, self-contained JSON object. + var row = JObject.Parse(lines[0]); + Assert.AreEqual(1, row["Id"].Value()); + Assert.AreEqual("Meikalainen", row["LastName"].Value()); + } + + [Test] + public async Task ExecuteQueryToFile_Json_DataTypes_NumbersBooleansDatesWrittenCorrectly() + { + var table = "DataTypeTest"; + var createSql = $@" + IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{table}') + BEGIN + CREATE TABLE {table} ( + IntCol int, + BigIntCol bigint, + DecimalCol decimal(18,4), + BitCol bit, + DateCol date, + DateTimeCol datetime2, + GuidCol uniqueidentifier + ); + END"; + + Helper.CreateTestTable(_connString, table, createSql); + + var guid = Guid.NewGuid(); + var insertSql = $@" + INSERT INTO {table} VALUES ( + 42, 9999999999, 123.4567, 1, + '2024-06-15', '2024-06-15T13:45:00.123', + '{guid}' + )"; + + Helper.ExecuteNonQuery(_connString, insertSql); + + var input = new Input + { + Query = $"SELECT * FROM {table}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + try + { + await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var content = await File.ReadAllTextAsync(_destination); + using var stringReader = new StringReader(content); + using var jsonReader = new JsonTextReader(stringReader) { DateParseHandling = DateParseHandling.None }; + var array = JArray.Load(jsonReader); + var row = array[0]; + + // Numbers written as numbers, not strings. + Assert.AreEqual(JTokenType.Integer, row["IntCol"].Type); + Assert.AreEqual(42, row["IntCol"].Value()); + Assert.AreEqual(JTokenType.Integer, row["BigIntCol"].Type); + Assert.AreEqual(JTokenType.Float, row["DecimalCol"].Type); + Assert.AreEqual(123.4567m, row["DecimalCol"].Value()); + + // Bit written as boolean. + Assert.AreEqual(JTokenType.Boolean, row["BitCol"].Type); + Assert.IsTrue(row["BitCol"].Value()); + + // Dates written as strings in configured format. + Assert.AreEqual(JTokenType.String, row["DateCol"].Type); + Assert.AreEqual("2024-06-15", row["DateCol"].Value()); + Assert.AreEqual(JTokenType.String, row["DateTimeCol"].Type); + Assert.AreEqual("2024-06-15T13:45:00.123", row["DateTimeCol"].Value()); + + // GUID written as string. + Assert.AreEqual(JTokenType.String, row["GuidCol"].Type); + Assert.AreEqual(guid.ToString(), row["GuidCol"].Value()); + } + finally + { + Helper.ExecuteNonQuery(_connString, $"DROP TABLE {table}"); + } + } + + [Test] + public async Task ExecuteQueryToFile_Json_NullValues_WrittenAsJsonNull() + { + var input = new Input + { + Query = $"SELECT Id, LastName, TestNull FROM {_tableName}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var content = await File.ReadAllTextAsync(_destination); + var row = JArray.Parse(content)[0]; + + Assert.AreEqual(JTokenType.Null, row["TestNull"].Type); + } + + [Test] + public async Task ExecuteQueryToFile_Json_HandleNullAsEmpty_NullStringWrittenAsEmptyString() + { + _options.JsonOptions.HandleNullAsEmpty = true; + + var input = new Input + { + Query = $"SELECT Id, TestNull FROM {_tableName}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var content = await File.ReadAllTextAsync(_destination); + var row = JArray.Parse(content)[0]; + + // String null → empty string. + Assert.AreEqual(JTokenType.String, row["TestNull"].Type); + Assert.AreEqual(string.Empty, row["TestNull"].Value()); + } + + [Test] + public async Task ExecuteQueryToFile_Json_CustomDateFormat_AppliedCorrectly() + { + _options.JsonOptions.DateFormat = "dd.MM.yyyy"; + _options.JsonOptions.DateTimeFormat = "dd.MM.yyyy HH:mm"; + + var table = "DateFormatTest"; + var createSql = $@" + IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{table}') + BEGIN + CREATE TABLE {table} (DateCol date, DateTimeCol datetime2); + END"; + + Helper.CreateTestTable(_connString, table, createSql); + + Helper.ExecuteNonQuery(_connString, $"INSERT INTO {table} VALUES ('2024-06-15', '2024-06-15T13:45:00')"); + + var input = new Input + { + Query = $"SELECT * FROM {table}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + try + { + await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + var content = await File.ReadAllTextAsync(_destination); + var row = JArray.Parse(content)[0]; + + Assert.AreEqual("15.06.2024", row["DateCol"].Value()); + Assert.AreEqual("15.06.2024 13:45", row["DateTimeCol"].Value()); + } + finally + { + Helper.ExecuteNonQuery(_connString, $"DROP TABLE {table}"); + } + } + + [Test] + public async Task ExecuteQueryToFile_Json_LargeRowCount_StreamsWithoutLoadingAllRowsIntoMemory() + { + const int RowCount = 100_000; + const long MaxAllowedMemoryGrowthBytes = 200 * 1024 * 1024; // 200 MB + + var table = "LargeRowTest"; + var createSql = $@" + IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{table}') + BEGIN + CREATE TABLE {table} ( + Id int, + Name nvarchar(100), + Salary decimal(18,2), + IsActive bit, + CreatedAt datetime2 + ); + END"; + + Helper.CreateTestTable(_connString, table, createSql); + + var insertSql = $@" + WITH Numbers AS (SELECT TOP {RowCount} ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS N FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c) + INSERT INTO {table} + SELECT N, 'Name_' + CAST(N AS nvarchar), CAST(N AS decimal(18,2)) * 1.5, CAST(N % 2 AS bit), GETDATE() + FROM Numbers"; + + Helper.ExecuteNonQuery(_connString, insertSql); + + var input = new Input + { + Query = $"SELECT * FROM {table}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + try + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryBefore = Process.GetCurrentProcess().WorkingSet64; + + var result = await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryAfter = Process.GetCurrentProcess().WorkingSet64; + + Assert.AreEqual(RowCount, result.EntriesWritten); + Assert.IsTrue(File.Exists(_destination)); + + var memoryGrowth = memoryAfter - memoryBefore; + Assert.Less( + memoryGrowth, + MaxAllowedMemoryGrowthBytes, + $"Memory grew by {memoryGrowth / 1024 / 1024} MB, expected less than {MaxAllowedMemoryGrowthBytes / 1024 / 1024} MB"); + } + finally + { + Helper.ExecuteNonQuery(_connString, $"DROP TABLE {table}"); + } + } + + [Test] + public async Task ExecuteQueryToFile_Json_LargeBinaryColumn_StreamsWithoutLoadingIntoMemory() + { + const int RowCount = 10; + const int BinaryColumnSizeMb = 10; + const long MaxAllowedMemoryGrowthBytes = 200 * 1024 * 1024; // 200 MB + + var table = "LargeBinaryTest"; + var createSql = $@" + IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='{table}') + BEGIN + CREATE TABLE {table} (Id int, Data varbinary(max)); + END"; + + Helper.CreateTestTable(_connString, table, createSql); + + // Insert rows with ~10 MB binary each = ~100 MB total. + for (int i = 1; i <= RowCount; i++) + { + var param = new Microsoft.Data.SqlClient.SqlParameter("@Data", SqlDbType.VarBinary) + { + Value = new byte[BinaryColumnSizeMb * 1024 * 1024], + }; + Helper.ExecuteNonQuery(_connString, $"INSERT INTO {table} VALUES ({i}, @Data)", new[] { param }); + } + + var input = new Input + { + Query = $"SELECT * FROM {table}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + try + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryBefore = Process.GetCurrentProcess().WorkingSet64; + + var result = await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryAfter = Process.GetCurrentProcess().WorkingSet64; + + Assert.AreEqual(RowCount, result.EntriesWritten); + Assert.IsTrue(File.Exists(_destination)); + + var memoryGrowth = memoryAfter - memoryBefore; + Assert.Less( + memoryGrowth, + MaxAllowedMemoryGrowthBytes, + $"Memory grew by {memoryGrowth / 1024 / 1024} MB, expected less than {MaxAllowedMemoryGrowthBytes / 1024 / 1024} MB"); + } + finally + { + Helper.ExecuteNonQuery(_connString, $"DROP TABLE {table}"); + } + } + + [Test] + [Ignore("Requires significant disk space and time. Run manually to verify multi-GB streaming.")] + public async Task ExecuteQueryToFile_Json_MultiGigabyte_StreamsWithoutLoadingAllRowsIntoMemory() + { + const int RowCount = 5_000_000; + const int BatchSize = 100_000; + const long MaxAllowedMemoryGrowthBytes = 200 * 1024 * 1024; // 200 MB + + var createSql = $@" + IF NOT EXISTS (SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='MultiGigabyteTest') + BEGIN + CREATE TABLE MultiGigabyteTest ( + Id int, + Name nvarchar(200), + Description nvarchar(500), + Salary decimal(18,2), + IsActive bit, + CreatedAt datetime2 + ); + END"; + + var table = "MultiGigabyteTest"; + Helper.CreateTestTable(_connString, table, createSql); + + // Insert in batches to avoid timeout + int inserted = 0; + while (inserted < RowCount) + { + int batchEnd = Math.Min(inserted + BatchSize, RowCount); + + var insertSql = $@" + WITH Numbers AS ( + SELECT TOP {batchEnd - inserted} ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) + {inserted} AS N + FROM sys.objects a CROSS JOIN sys.objects b CROSS JOIN sys.objects c + ) + INSERT INTO {table} + SELECT + N, + REPLICATE('Name_' + CAST(N AS nvarchar), 5), + REPLICATE('Description_' + CAST(N AS nvarchar), 10), + CAST(N AS decimal(18,2)) * 1.5, + CAST(N % 2 AS bit), + GETDATE() + FROM Numbers"; + + Helper.ExecuteNonQuery(_connString, insertSql); + inserted = batchEnd; + } + + var input = new Input + { + Query = $"SELECT * FROM {table}", + QueryParameters = Array.Empty(), + ConnectionString = _connString, + OutputFilePath = _destination, + }; + + try + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryBefore = Process.GetCurrentProcess().WorkingSet64; + + var result = await MicrosoftSQL.ExecuteQueryToFile(input, _options, default); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var memoryAfter = Process.GetCurrentProcess().WorkingSet64; + + Assert.AreEqual(RowCount, result.EntriesWritten); + Assert.IsTrue(File.Exists(_destination)); + + var fileSizeGb = new FileInfo(_destination).Length / 1024.0 / 1024.0 / 1024.0; + TestContext.WriteLine($"File size: {fileSizeGb:F2} GB"); + TestContext.WriteLine($"Memory before: {memoryBefore / 1024 / 1024} MB"); + TestContext.WriteLine($"Memory after: {memoryAfter / 1024 / 1024} MB"); + TestContext.WriteLine($"Memory growth: {(memoryAfter - memoryBefore) / 1024 / 1024} MB"); + + var memoryGrowth = memoryAfter - memoryBefore; + Assert.Less( + memoryGrowth, + MaxAllowedMemoryGrowthBytes, + $"Memory grew by {memoryGrowth / 1024 / 1024} MB, expected less than {MaxAllowedMemoryGrowthBytes / 1024 / 1024} MB"); + } + finally + { + Helper.ExecuteNonQuery(_connString, $"DROP TABLE {table}"); + } + } +} diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/UnitTests.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/UnitTests.cs index a1f5e4e..2f0c2d1 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/UnitTests.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.Tests/UnitTests.cs @@ -7,7 +7,6 @@ namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Tests; using Frends.MicrosoftSQL.ExecuteQueryToFile.Definitions; using Frends.MicrosoftSQL.ExecuteQueryToFile.Enums; using Microsoft.Data.SqlClient; -using Newtonsoft.Json.Linq; using NUnit.Framework; /// diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.sln b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.sln index df720a7..6ec359a 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.sln +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32112.339 +# Visual Studio Version 18 +VisualStudioVersion = 18.10.12113.136 insiders MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Frends.MicrosoftSQL.ExecuteQueryToFile", "Frends.MicrosoftSQL.ExecuteQueryToFile\Frends.MicrosoftSQL.ExecuteQueryToFile.csproj", "{35C305C0-8108-4A98-BB1D-AFE5C926239E}" EndProject @@ -9,7 +9,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Frends.MicrosoftSQL.Execute EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{78F7F22E-6E20-4BCE-8362-0C558568B729}" ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig CHANGELOG.md = CHANGELOG.md ..\.github\workflows\ExecuteQueryToFile_build_and_test_on_main.yml = ..\.github\workflows\ExecuteQueryToFile_build_and_test_on_main.yml ..\.github\workflows\ExecuteQueryToFile_build_and_test_on_push.yml = ..\.github\workflows\ExecuteQueryToFile_build_and_test_on_push.yml diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/ColumnSchema.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/ColumnSchema.cs new file mode 100644 index 0000000..ab8eaf6 --- /dev/null +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/ColumnSchema.cs @@ -0,0 +1,46 @@ +using System; + +namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Definitions; + +/// +/// Represents schema metadata for a single database column, including ordinal position, column name, .NET type, and +/// database type name. +/// +/// Instances are immutable and expose read-only properties initialized in the constructor. +public sealed class ColumnSchema +{ + /// + /// Initializes a new instance of the class. + /// + /// The zero-based ordinal position of the column in the result set. + /// The name of the column. + /// The .NET type of the column. + /// The database type name of the column. + public ColumnSchema(int index, string name, Type dotnetType, string dbTypeName) + { + Index = index; + Name = name; + DotnetType = dotnetType; + DbTypeName = dbTypeName; + } + + /// + /// Gets the zero-based ordinal position of the column in the result set. + /// + public int Index { get; } + + /// + /// Gets the name of the column. + /// + public string Name { get; } + + /// + /// Gets the .NET type of the column. + /// + public Type DotnetType { get; } + + /// + /// Gets the database type name of the column. + /// + public string DbTypeName { get; } +} diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonFileWriter .cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonFileWriter .cs new file mode 100644 index 0000000..71b00ee --- /dev/null +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonFileWriter .cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Frends.MicrosoftSQL.ExecuteQueryToFile.Enums; +using Microsoft.Data.SqlClient; +using Newtonsoft.Json; + +namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Definitions; + +internal class JsonFileWriter : IAsyncDisposable +{ + internal JsonFileWriter(SqlCommand sqlCommand, Input input, JsonOptions options) + { + SqlCommand = sqlCommand; + Input = input; + Options = options; + } + + private SqlCommand SqlCommand { get; } + + private Input Input { get; } + + private JsonOptions Options { get; } + + public async ValueTask DisposeAsync() + { + if (SqlCommand != null) + await SqlCommand.DisposeAsync(); + } + + public async Task SaveQueryToJson(CancellationToken cancellationToken) + { + await using var fileStream = new FileStream( + Input.OutputFilePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: Options.FileBufferSize, + useAsync: true); + + await using var streamWriter = new StreamWriter(fileStream); + + using var jsonWriter = new JsonTextWriter(streamWriter) + { + Formatting = Options.JsonOutputMode == JsonOutputMode.Indented + ? Formatting.Indented + : Formatting.None, + + DateFormatHandling = DateFormatHandling.IsoDateFormat, + }; + + using var reader = await SqlCommand + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false); + + int count = Options.JsonOutputMode == JsonOutputMode.Indented + ? await WriteIndented(reader, jsonWriter, cancellationToken).ConfigureAwait(false) + : await WriteJsonLines(reader, streamWriter, cancellationToken).ConfigureAwait(false); + + await streamWriter.FlushAsync().ConfigureAwait(false); + + return new Result(count, Input.OutputFilePath, Path.GetFileName(Input.OutputFilePath)); + } + + private async Task WriteIndented( + DbDataReader reader, + JsonTextWriter jsonWriter, + CancellationToken cancellationToken) + { + var schema = BuildSchema(reader); + int count = 0; + + await jsonWriter.WriteStartArrayAsync(cancellationToken).ConfigureAwait(false); + + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + await jsonWriter.WriteStartObjectAsync(cancellationToken).ConfigureAwait(false); + + foreach (var col in schema) + { + cancellationToken.ThrowIfCancellationRequested(); + + await jsonWriter.WritePropertyNameAsync(col.Name, cancellationToken).ConfigureAwait(false); + await WriteValue(jsonWriter, reader, col, cancellationToken).ConfigureAwait(false); + } + + await jsonWriter.WriteEndObjectAsync(cancellationToken).ConfigureAwait(false); + count++; + } + + await jsonWriter.WriteEndArrayAsync(cancellationToken).ConfigureAwait(false); + return count; + } + + private async Task WriteJsonLines( + DbDataReader reader, + StreamWriter streamWriter, + CancellationToken cancellationToken) + { + var schema = BuildSchema(reader); + int count = 0; + + using var jsonWriter = new JsonTextWriter(streamWriter) + { + Formatting = Formatting.None, + CloseOutput = false, + }; + + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + await jsonWriter.WriteStartObjectAsync(cancellationToken).ConfigureAwait(false); + + foreach (var col in schema) + { + cancellationToken.ThrowIfCancellationRequested(); + await jsonWriter.WritePropertyNameAsync(col.Name, cancellationToken).ConfigureAwait(false); + await WriteValue(jsonWriter, reader, col, cancellationToken).ConfigureAwait(false); + } + + await jsonWriter.WriteEndObjectAsync(cancellationToken).ConfigureAwait(false); + await jsonWriter.WriteRawAsync("\n", cancellationToken).ConfigureAwait(false); + count++; + } + + return count; + } + + private async Task WriteValue( + JsonTextWriter jsonWriter, + DbDataReader reader, + ColumnSchema col, + CancellationToken cancellationToken) + { + if (reader.IsDBNull(col.Index)) + { + if (Options.HandleNullAsEmpty && col.DotnetType == typeof(string)) + await jsonWriter.WriteValueAsync(string.Empty, cancellationToken).ConfigureAwait(false); + else + await jsonWriter.WriteNullAsync(cancellationToken).ConfigureAwait(false); + return; + } + + if (col.DotnetType == typeof(byte[])) + { + await WriteBinaryAsBase64Async(jsonWriter, reader, col.Index, cancellationToken).ConfigureAwait(false); + return; + } + + var value = MapValue(reader, col); + await jsonWriter.WriteValueAsync(value, cancellationToken).ConfigureAwait(false); + } + + private object MapValue(DbDataReader reader, ColumnSchema col) + { + var value = reader.GetValue(col.Index); + var type = col.DotnetType; + + if (type == typeof(DateTime)) + { + var fmt = col.DbTypeName == "date" ? Options.DateFormat : Options.DateTimeFormat; + return ((DateTime)value).ToString(fmt, CultureInfo.InvariantCulture); + } + + if (type == typeof(DateTimeOffset)) + return ((DateTimeOffset)value).ToString(Options.DateTimeOffsetFormat, CultureInfo.InvariantCulture); + + if (type == typeof(TimeSpan)) + return ((TimeSpan)value).ToString(Options.TimeFormat, CultureInfo.InvariantCulture); + + if (type == typeof(Guid)) + return ((Guid)value).ToString(); + + return value; + } + + /// + /// Streams a binary column to JSON as a Base64-encoded string using fixed-size chunks, + /// avoiding loading the entire value into memory. + /// + private static async Task WriteBinaryAsBase64Async( + JsonTextWriter jsonWriter, + DbDataReader reader, + int columnIndex, + CancellationToken cancellationToken) + { + const int InputChunkSize = 3 * 1024; + + await using var binaryStream = reader.GetStream(columnIndex); + + await jsonWriter.WriteRawValueAsync("\"", cancellationToken).ConfigureAwait(false); + + var inputBuffer = new byte[InputChunkSize]; + + var outputBuffer = new char[((InputChunkSize / 3) + 1) * 4]; + + int bytesRead; + while ((bytesRead = await ReadChunkAsync(binaryStream, inputBuffer, cancellationToken).ConfigureAwait(false)) > 0) + { + int charsWritten = Convert.ToBase64CharArray( + inputBuffer, 0, bytesRead, outputBuffer, 0); + + await jsonWriter.WriteRawAsync( + new string(outputBuffer, 0, charsWritten), cancellationToken) + .ConfigureAwait(false); + } + + await jsonWriter.WriteRawAsync("\"", cancellationToken).ConfigureAwait(false); + } + + private static List BuildSchema(DbDataReader reader) + { + var schema = new List(reader.FieldCount); + + for (int i = 0; i < reader.FieldCount; i++) + { + schema.Add(new ColumnSchema( + index: i, + name: reader.GetName(i), + dotnetType: reader.GetFieldType(i), + dbTypeName: reader.GetDataTypeName(i)?.ToLowerInvariant() ?? string.Empty)); + } + + return schema; + } + + private static async Task ReadChunkAsync(Stream stream, byte[] buffer, CancellationToken cancellationToken) + { + int total = 0; + while (total < buffer.Length) + { + int read = await stream + .ReadAsync(buffer, total, buffer.Length - total, cancellationToken) + .ConfigureAwait(false); + if (read == 0) + break; + total += read; + } + + return total; + } +} diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonOptions.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonOptions.cs new file mode 100644 index 0000000..aa2d042 --- /dev/null +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/JsonOptions.cs @@ -0,0 +1,65 @@ +using System.ComponentModel; +using Frends.MicrosoftSQL.ExecuteQueryToFile.Enums; + +namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Definitions; + +/// +/// Options for writing SQL query results to a JSON file. +/// +public class JsonOptions +{ + /// + /// How the JSON output is structured. + /// + /// JsonOutputMode.Indented + [DefaultValue(JsonOutputMode.Indented)] + public JsonOutputMode JsonOutputMode { get; set; } = JsonOutputMode.Indented; + + /// + /// Date format to use for formatting DATE columns, use .NET formatting tokens. + /// Note that formatting is done using invariant culture. + /// + /// yyyy-MM-dd + [DefaultValue("\"yyyy-MM-dd\"")] + public string DateFormat { get; set; } = "yyyy-MM-dd"; + + /// + /// Date format to use for formatting DATETIME columns, use .NET formatting tokens. + /// Note that formatting is done using invariant culture. + /// + /// yyyy-MM-dd HH:mm:ss + [DefaultValue("\"yyyy-MM-dd HH:mm:ss\"")] + public string DateTimeFormat { get; set; } = "yyyy-MM-dd HH:mm:ss"; + + /// + /// Format for SQL time columns (mapped to TimeSpan in .NET). + /// Default: ISO 8601 time (HH:mm:ss.fff). + /// + /// hh\:mm\:ss\.fff + [DefaultValue(@"hh\:mm\:ss\.fff")] + public string TimeFormat { get; set; } = @"hh\:mm\:ss\.fff"; + + /// + /// Format for SQL datetimeoffset columns (mapped to DateTimeOffset in .NET). + /// Default: ISO 8601 round-trip format preserving the UTC offset. + /// + /// 0 + [DefaultValue(0)] + public string DateTimeOffsetFormat { get; set; } = "O"; + + /// + /// When true, SQL NULL values for string columns are written as empty strings instead of JSON null. + /// Non-string types such as numbers and booleans are always written as JSON null regardless of this setting. + /// + /// false + [DefaultValue(false)] + public bool HandleNullAsEmpty { get; set; } = false; + + /// + /// Size in bytes of the file write buffer. Larger values reduce I/O syscalls for big exports; + /// smaller values reduce memory usage. Default: 65536 (64 KB). + /// + /// 65536 + [DefaultValue(65536)] + public int FileBufferSize { get; set; } = 65536; +} diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/Options.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/Options.cs index 91ba4f7..6465304 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/Options.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Definitions/Options.cs @@ -28,4 +28,10 @@ public class Options /// [UIHint(nameof(ReturnFormat), "", ReturnFormat.CSV)] public CsvOptions CsvOptions { get; set; } + + /// + /// Json options. + /// + [UIHint(nameof(ReturnFormat), "", ReturnFormat.JSON)] + public JsonOptions JsonOptions { get; set; } = new JsonOptions(); } \ No newline at end of file diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/JsonOutputMode.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/JsonOutputMode.cs new file mode 100644 index 0000000..a18ff02 --- /dev/null +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/JsonOutputMode.cs @@ -0,0 +1,17 @@ +namespace Frends.MicrosoftSQL.ExecuteQueryToFile.Enums; + +/// +/// Specifies the format used when writing JSON output. +/// +public enum JsonOutputMode +{ + /// + /// Writes the result as a single indented JSON array containing all rows. + /// + Indented, + + /// + /// Writes the result as JSON Lines (newline-delimited JSON), with each row on a separate line. + /// + JsonLines, +} diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/ReturnFormat.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/ReturnFormat.cs index 909d941..a000e14 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/ReturnFormat.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Enums/ReturnFormat.cs @@ -7,6 +7,7 @@ public enum ReturnFormat { CSV, + JSON, } #pragma warning restore CS1591 // Self-explanatory diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.cs index 2c51983..80e6a0b 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.cs @@ -65,6 +65,13 @@ public static async Task ExecuteQueryToFile( break; } + + case ReturnFormat.JSON: + { + await using var jsonWriter = new JsonFileWriter(command, input, options.JsonOptions); + result = await jsonWriter.SaveQueryToJson(cancellationToken).ConfigureAwait(false); + break; + } } return result; diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.csproj b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.csproj index 19115b6..11aa035 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.csproj +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile.csproj @@ -3,7 +3,7 @@ net6.0 Latest - 2.3.0 + 2.4.0 Frends Frends Frends @@ -17,7 +17,6 @@ - PreserveNewest @@ -30,12 +29,12 @@ - - - - - - + + + + + + diff --git a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/GlobalSuppressions.cs b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/GlobalSuppressions.cs index 6e9785b..9baa66b 100644 --- a/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/GlobalSuppressions.cs +++ b/Frends.MicrosoftSQL.ExecuteQueryToFile/Frends.MicrosoftSQL.ExecuteQueryToFile/GlobalSuppressions.cs @@ -9,4 +9,5 @@ [assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1633:File should have header", Justification = "Following Frends guidelines")] [assembly: SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1602:Enumeration items should be documented", Justification = "Following Frends guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile.Enums")] [assembly: SuppressMessage("StyleCop.CSharp.OrderingRules", "SA1200:Using directives should be placed correctly", Justification = "Following Frends guidelines")] -[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:Braces should not be omitted", Justification = "Following Frends guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile")] \ No newline at end of file +[assembly: SuppressMessage("StyleCop.CSharp.LayoutRules", "SA1503:Braces should not be omitted", Justification = "Following Frends guidelines", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile")] +[assembly: SuppressMessage("StyleCop.CSharp.OrderingRules", "SA1204:StaticElementsMustAppearBeforeInstanceElements", Justification = "Static helper methods are placed after the instance methods they support for readability.", Scope = "namespaceanddescendants", Target = "~N:Frends.MicrosoftSQL.ExecuteQueryToFile")] \ No newline at end of file