Resolve comments

Author: SicongLiu2000

language-extensions/dotnet-core-CSharp/README.md

@@ -7,7 +7,7 @@ For more information about SQL Server Language Extensions, refer to this [docume
 
 The dotnet-core-CSharp-extension version in this repository is compatible with SQL Server 2019 CU3 onwards. It integrates .NET core in SQL Server and works with .NET 6.0 in **Windows only**.
 
-Currently, the extension supports the following data types: SQL_C_SLONG, SQL_C_ULONG, SQL_C_SSHORT, SQL_C_USHORT, SQL_C_SBIGINT, SQL_C_UBIGINT, SQL_C_STINYINT, SQL_C_UTINYINT, SQL_C_BIT, SQL_C_FLOAT, SQL_C_DOUBLE, SQL_C_CHAR, SQL_C_WCHAR. It supports the following SQL data types: int, bigint, smallint, tinyint, real, float, bit, varchar(n), nvarchar(n).
+Currently, the extension supports the following data types: SQL_C_SLONG, SQL_C_ULONG, SQL_C_SSHORT, SQL_C_USHORT, SQL_C_SBIGINT, SQL_C_UBIGINT, SQL_C_STINYINT, SQL_C_UTINYINT, SQL_C_BIT, SQL_C_FLOAT, SQL_C_DOUBLE, SQL_C_CHAR, and SQL_C_WCHAR. It supports the following SQL data types: int, bigint, smallint, tinyint, real, float, bit, char(n), varchar(n), nchar(n), and nvarchar(n).
 
 To use this dotnet-core-CSharp-lang-extension.zip package, follow [this tutorial](./sample/regex/README.md). For any fixes or enhancements, you are welcome to modify, rebuild and use the binaries using the following instructions.
 

language-extensions/dotnet-core-CSharp/src/managed/CSharpInputDataSet.cs

@@ -130,24 +130,21 @@ private unsafe void AddColumn(
                     int[] strLens = new int[rowsNumber];
                     Interop.Copy((int*)colMap, strLens, 0, (int)rowsNumber);
 
-                    // Use byte-based splitting to properly handle multi-byte UTF-8 characters
+                    // SQL Server sends UTF-8 encoded strings with byte lengths in strLenOrNullMap.
+                    // We decode each segment directly from the byte buffer to properly handle
+                    // multi-byte UTF-8 characters where byte count != character count.
                     //
                     CSharpDataFrame.Columns.Add(new StringDataFrameColumn(_columns[columnNumber].Name, DataSetUtils.UTF8ByteSplitToArray((byte*)colData, strLens)));
                     break;
                 case SqlDataType.DotNetWChar:
-                    int[] wcharStrLens = new int[rowsNumber];
-                    Interop.Copy((int*)colMap, wcharStrLens, 0, (int)rowsNumber);
-    
-                    // For NCHAR/WCHAR, the strLenOrNullMap contains byte lengths, but we need character counts for UTF16PtrToStr
-                    // Each Unicode character is 2 bytes (sizeof(wchar_t)), so divide by 2 to get character count
-                    //
-                    int[] wcharCharLens = new int[rowsNumber];
-                    for (int i = 0; i < (int)rowsNumber; i++)
-                    {
-                        wcharCharLens[i] = wcharStrLens[i] > 0 ? wcharStrLens[i] / sizeof(char) : wcharStrLens[i];
-                    }
+                    int[] wcharByteLens = new int[rowsNumber];
+                    Interop.Copy((int*)colMap, wcharByteLens, 0, (int)rowsNumber);
 
-                    CSharpDataFrame.Columns.Add(new StringDataFrameColumn(_columns[columnNumber].Name, DataSetUtils.StringSplitToArray(Interop.UTF16PtrToStr((char*)colData), wcharCharLens)));
+                    // SQL Server sends UTF-16 (nvarchar/nchar) encoded strings with byte lengths in strLenOrNullMap.
+                    // We decode each segment directly from the byte buffer to properly handle
+                    // multi-string buffers where strings are concatenated without null terminators.
+                    //
+                    CSharpDataFrame.Columns.Add(new StringDataFrameColumn(_columns[columnNumber].Name, DataSetUtils.UTF16ByteSplitToArray((byte*)colData, wcharByteLens)));
                     break;
                 default:
                     throw new NotImplementedException("Column type for " + _columns[columnNumber].DataType.ToString() + " has not been implemented yet");

language-extensions/dotnet-core-CSharp/src/managed/CSharpOutputDataSet.cs

@@ -42,41 +42,25 @@ public class CSharpOutputDataSet: CSharpDataSet
         /// This method extracts metadata and actual data for each column supplied
         /// by extracting data and information from every DataFrameColumn.
         /// </summary>
-        /// <param name="CSharpDataFrame">The DataFrame containing the output data.</param>
-        /// <param name="inputColumns">
-        /// Optional dictionary of input column metadata (not used - C# extension always outputs ANSI strings
-        /// like Python/R extensions).
-        /// </param>
-        public unsafe void ExtractColumns(DataFrame CSharpDataFrame, Dictionary<ushort, CSharpColumn> inputColumns = null)
+        public unsafe void ExtractColumns(DataFrame dataFrame)
         {
             Logging.Trace("CSharpOutputDataSet::ExtractColumns");
             _strLenOrNullMapPtrs = new int*[ColumnsNumber];
             _dataPtrs = new void*[ColumnsNumber];
             for(ushort columnNumber = 0; columnNumber < ColumnsNumber; ++columnNumber)
             {
-                DataFrameColumn column = CSharpDataFrame.Columns[columnNumber];
-
-                // Determine the SQL data type for this column
-                // Prefer the incoming input column metadata when available so nvarchar metadata is preserved.
-                // Default to ANSI mapping to keep existing behavior when no metadata is supplied.
-                //
-                SqlDataType dataType = DataTypeMap[column.DataType];
-                ulong columnSize = (ulong)DataTypeSize[dataType];
-                if(inputColumns != null && inputColumns.ContainsKey(columnNumber))
-                {
-                    dataType = inputColumns[columnNumber].DataType;
-                    columnSize = inputColumns[columnNumber].Size;
-                }
+                DataFrameColumn column = dataFrame.Columns[columnNumber];
 
                 // Add column metadata to a CSharpColumn dictionary
                 //
+                SqlDataType dataType = DataTypeMap[column.DataType];
                 _columns[columnNumber] = new CSharpColumn
                 {
                     Name = column.Name,
                     DataType = dataType,
                     Nullable = (short)(column.NullCount > 0 ? 1 : 0),
                     DecimalDigits = 0,
-                    Size = columnSize,
+                    Size = (ulong)DataTypeSize[dataType],
                     Id = columnNumber
                 };
 
@@ -185,8 +169,9 @@ DataFrameColumn column
                     break;
                 case SqlDataType.DotNetChar:
                     // Modify the size of the string column to be the max size of bytes.
+                    // Handle all-null columns by checking if any positive values exist.
                     //
-                    int maxStrLen = colMap.Max();
+                    int maxStrLen = colMap.Length > 0 ? colMap.Where(x => x > 0).DefaultIfEmpty(0).Max() : 0;
                     if(maxStrLen > 0)
                     {
                         _columns[columnNumber].Size = (ulong)maxStrLen;
@@ -196,11 +181,13 @@ DataFrameColumn column
                     break;
                 case SqlDataType.DotNetWChar:
                     // Preserve nvarchar metadata by emitting UTF-16 data and byte counts.
+                    // Handle all-null columns by checking if any positive values exist.
+                    // Column size is reported in characters (byte length / 2 for UTF-16).
                     //
-                    int maxUnicodeStrLen = colMap.Max();
-                    if(maxUnicodeStrLen > 0)
+                    int maxUnicodeByteLen = colMap.Length > 0 ? colMap.Where(x => x > 0).DefaultIfEmpty(0).Max() : 0;
+                    if(maxUnicodeByteLen > 0)
                     {
-                        _columns[columnNumber].Size = (ulong)maxUnicodeStrLen;
+                        _columns[columnNumber].Size = (ulong)(maxUnicodeByteLen / sizeof(char));
                     }
 
                     SetDataPtrs<char>(columnNumber, GetUnicodeStringArray(column));
@@ -252,8 +239,15 @@ private T[] GetArray<T>(DataFrameColumn column) where T : unmanaged
         /// This method gets the array from a DataFrameColumn Column for string types by
         /// building a long string from the column and returning the underlying bytes as an array.
         /// </summary>
+        /// <param name="column">The DataFrameColumn containing string data.</param>
+        /// <returns>A byte array containing all non-null string values as UTF-8 encoded bytes.</returns>
         private byte[] GetStringArray(DataFrameColumn column)
         {
+            if (column == null)
+            {
+                return Array.Empty<byte>();
+            }
+
             StringBuilder builder = new StringBuilder();
             int totalBytes = 0;
             for(int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
@@ -282,8 +276,15 @@ private byte[] GetStringArray(DataFrameColumn column)
         /// <summary>
         /// This method builds a contiguous UTF-16 buffer for string types (nvarchar/nchar).
         /// </summary>
+        /// <param name="column">The DataFrameColumn containing string data.</param>
+        /// <returns>A char array containing all non-null string values concatenated.</returns>
         private char[] GetUnicodeStringArray(DataFrameColumn column)
         {
+            if (column == null)
+            {
+                return Array.Empty<char>();
+            }
+
             StringBuilder builder = new StringBuilder();
             int totalBytes = 0;
             for(int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
@@ -339,11 +340,11 @@ private int[] GetStrLenNullMap(ushort columnNumber, DataFrameColumn column)
                             Logging.Trace($"GetStrLenNullMap: Row {rowNumber}, Value='{column[rowNumber]}', ByteLen={colMap[rowNumber]}");
                             break;
                         default:
-                            if(!DataTypeSize.ContainsKey(dataType))
+                            if(!DataTypeSize.TryGetValue(dataType, out short size))
                             {
-                                throw new NotImplementedException("Parameter type for " + dataType.ToString() + " has not been implemented yet");
+                                throw new NotImplementedException("Parameter type for " + dataType + " has not been implemented yet");
                             }
-                            colMap[rowNumber] = DataTypeSize[dataType];
+                            colMap[rowNumber] = size;
                             break;
                     }
                 }

language-extensions/dotnet-core-CSharp/src/managed/CSharpParamContainer.cs

@@ -136,7 +136,10 @@ public unsafe void AddParam(
                     _params[paramNumber].Value = Interop.UTF8PtrToStr((char*)paramValue, (ulong)strLenOrNullMap);
                     break;
                 case SqlDataType.DotNetWChar:
-                    // For NCHAR/WCHAR, strLenOrNullMap contains byte length, divide by 2 to get character count
+                    // For NCHAR/NVARCHAR, strLenOrNullMap contains byte length.
+                    // In C#, sizeof(char) is always 2 bytes (UTF-16), regardless of platform.
+                    // Note: C++ wchar_t is 2 bytes on Windows but 4 bytes on Linux - this extension only supports Windows.
+                    // The cast to (char*) is correct here because C# char is UTF-16 (same as Windows wchar_t).
                     //
                     _params[paramNumber].Value = Interop.UTF16PtrToStr((char*)paramValue, strLenOrNullMap / sizeof(char));
                     break;
@@ -216,8 +219,13 @@ public unsafe void ReplaceParam(
                     ReplaceStringParam((string)param.Value, paramValue);
                     break;
                 case SqlDataType.DotNetWChar:
-                    // For NCHAR/WCHAR, strLenOrNullMap is in bytes
-                    // param.Size is in characters, so convert to bytes for comparison
+                    // For NCHAR/NVARCHAR, strLenOrNullMap must be in bytes (UTF-16: 2 bytes per character).
+                    // In C#, sizeof(char) is always 2 bytes (UTF-16), regardless of platform.
+                    // Note: C++ wchar_t is 2 bytes on Windows but 4 bytes on Linux - this extension only supports Windows.
+                    // param.Size is the declared parameter size in characters (from SQL Server's NCHAR(n)/NVARCHAR(n)),
+                    // so we multiply by sizeof(char) to convert to bytes.
+                    // Note: *strLenOrNullMap was pre-set to param.Size on line 176, but that value is in
+                    // characters, not bytes, so we recalculate the max byte length here.
                     //
                     int wcharByteLen = param.Value.Length * sizeof(char);
                     int maxByteLen = (int)param.Size * sizeof(char);
@@ -248,13 +256,19 @@ public void HandleCleanup()
 
         /// <summary>
         /// This method replaces parameter value for numeric data types.
+        /// Uses proper memory pinning to ensure the value remains valid after method returns.
         /// </summary>
         private unsafe void ReplaceNumericParam<T>(
             T    value,
             void **paramValue) where T : unmanaged
         {
-            _handleList.Add(GCHandle.Alloc(value));
-            *paramValue = &value;
+            // Box the value into a single-element array to create a heap-allocated copy, then pin it.
+            // This ensures the pointer remains valid after the method returns.
+            //
+            T[] valueArray = new T[1] { value };
+            GCHandle handle = GCHandle.Alloc(valueArray, GCHandleType.Pinned);
+            _handleList.Add(handle);
+            *paramValue = (void*)handle.AddrOfPinnedObject();
         }
 
         /// <summary>
@@ -268,20 +282,19 @@ private unsafe void ReplaceStringParam(
         {
             if(string.IsNullOrEmpty(value))
             {
-                _handleList.Add(GCHandle.Alloc(value));
-                fixed(void* strPtr = value)
-                {
-                    *paramValue = strPtr;
-                }
+                // For empty/null strings, allocate a single null byte
+                //
+                byte[] emptyBytes = new byte[1];
+                GCHandle handle = GCHandle.Alloc(emptyBytes, GCHandleType.Pinned);
+                _handleList.Add(handle);
+                *paramValue = (void*)handle.AddrOfPinnedObject();
             }
             else
             {
                 byte[] strBytes = Encoding.UTF8.GetBytes(value);
-                _handleList.Add(GCHandle.Alloc(strBytes));
-                fixed(void* strPtr = strBytes)
-                {
-                    *paramValue = strPtr;
-                }
+                GCHandle handle = GCHandle.Alloc(strBytes, GCHandleType.Pinned);
+                _handleList.Add(handle);
+                *paramValue = (void*)handle.AddrOfPinnedObject();
             }
         }
 
@@ -296,20 +309,19 @@ private unsafe void ReplaceUnicodeStringParam(
         {
             if(string.IsNullOrEmpty(value))
             {
-                _handleList.Add(GCHandle.Alloc(value));
-                fixed(void* strPtr = value)
-                {
-                    *paramValue = strPtr;
-                }
+                // For empty/null strings, allocate a single null wchar
+                //
+                byte[] emptyBytes = new byte[2];
+                GCHandle handle = GCHandle.Alloc(emptyBytes, GCHandleType.Pinned);
+                _handleList.Add(handle);
+                *paramValue = (void*)handle.AddrOfPinnedObject();
             }
             else
             {
                 byte[] strBytes = Encoding.Unicode.GetBytes(value);
-                _handleList.Add(GCHandle.Alloc(strBytes));
-                fixed(void* strPtr = strBytes)
-                {
-                    *paramValue = strPtr;
-                }
+                GCHandle handle = GCHandle.Alloc(strBytes, GCHandleType.Pinned);
+                _handleList.Add(handle);
+                *paramValue = (void*)handle.AddrOfPinnedObject();
             }
         }
     }

language-extensions/dotnet-core-CSharp/src/managed/CSharpSession.cs

@@ -192,9 +192,7 @@ public void Execute(
             if(_outputDataSet.CSharpDataFrame != null)
             {
                 _outputDataSet.ColumnsNumber = (ushort)_outputDataSet.CSharpDataFrame.Columns.Count;
-                // Pass input column metadata to preserve data types (e.g., nvarchar vs varchar).
-                //
-                _outputDataSet.ExtractColumns(_outputDataSet.CSharpDataFrame, _inputDataSet.Columns);
+                _outputDataSet.ExtractColumns(_outputDataSet.CSharpDataFrame);
                 *outputSchemaColumnsNumber = _outputDataSet.ColumnsNumber;
             }
             else