Add MCCP4 support - New protocol created using zstd with substantial performance improvements.

MUSHclient.vcxproj

@@ -255,6 +255,10 @@
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
+    <ClCompile Include="mccp4.cpp">
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
     <ClCompile Include="MiniWindow.cpp">
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
@@ -894,6 +898,46 @@
       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
       </PrecompiledHeader>
     </ClCompile>
+    <ClCompile Include="zstd\common\debug.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\common\entropy_common.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\common\error_private.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\common\fse_decompress.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\common\xxhash.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\common\zstd_common.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\decompress\huf_decompress.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\decompress\zstd_ddict.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\decompress\zstd_decompress.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
+    <ClCompile Include="zstd\decompress\zstd_decompress_block.c">
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"></PrecompiledHeader>
+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"></PrecompiledHeader>
+    </ClCompile>
     <ClCompile Include="xml\xml_load_world.cpp">
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(PreprocessorDefinitions)</PreprocessorDefinitions>
@@ -1443,6 +1487,7 @@
     <ClInclude Include="luacom\tUtil.h" />
     <ClInclude Include="stdafx.h" />
     <ClInclude Include="version.h" />
+    <ClInclude Include="zstd\zstd.h" />
   </ItemGroup>
   <ItemGroup>
     <None Include="res\cursor1.cur" />

dialogs/world_prefs/configuration.cpp

@@ -531,6 +531,8 @@ void CMUSHclientDoc:: LoadPrefsP15 (CPrefsP15 &page15)
        page15.m_strCompressionRatio += " MCCP v 1";
     else if (m_iMCCP_type == 2)
        page15.m_strCompressionRatio += " MCCP v 2";
+    else if (m_iMCCP_type == 4)
+       page15.m_strCompressionRatio += " MCCP v 4 (Zstandard)";
     page15.m_strCompressedIn = CFormat ("%I64d", m_nTotalCompressed);
     page15.m_strCompressedOut = CFormat ("%I64d", m_nTotalUncompressed);
     }

doc.cpp

@@ -1422,23 +1422,52 @@ int count = m_pSocket->Receive (buff, sizeof (buff) - 1);
   m_iInputPacketCount++;       // count packets
   m_nBytesIn += count;    // count bytes in
 
-  if (m_bCompress)   // if we are compressing, put data into zlib buffer
+  if (m_bCompress)   // if we are compressing, handle based on compression type
     {
-    if ((COMPRESS_BUFFER_LENGTH - m_zCompress.avail_in) < (uInt) count)
+    if (m_iMCCP_type == 4 && m_MCCP4_active_encoding == "zstd")  // MCCP4 with Zstandard
       {
-      OnConnectionDisconnect ();    // close the world
-      TMessageBox ("Insufficient memory in buffer to decompress text", MB_ICONEXCLAMATION);
-      return;
+      // Process Zstandard compressed data directly
+      int result = ProcessZstdCompressed((const unsigned char*)buff, count);
+      if (result < 0)
+        {
+        // Error processing Zstandard data - already handled in ProcessZstdCompressed
+        return;
+        }
+      // Stats already updated in ProcessZstdCompressed
+
+      // If not all bytes were consumed and compression is now off,
+      // we need to process the remaining bytes as uncompressed data
+      if (result > 0 && result < count && !m_bCompress)
+        {
+        // Move remaining bytes to the beginning of the buffer
+        memmove(buff, buff + result, count - result);
+        count = count - result;
+        // Fall through to process remaining bytes as uncompressed
+        }
+      else
+        {
+        // All bytes consumed or still compressing - we're done
+        return;
+        }
+      }
+    else  // MCCP1/MCCP2/MCCP4-deflate (zlib)
+      {
+      if ((COMPRESS_BUFFER_LENGTH - m_zCompress.avail_in) < (uInt) count)
+        {
+        OnConnectionDisconnect ();    // close the world
+        TMessageBox ("Insufficient memory in buffer to decompress text", MB_ICONEXCLAMATION);
+        return;
+        }
+      // shuffle existing input to-be-processed to start of buffer
+      if (m_zCompress.avail_in)
+        memmove (m_CompressInput, m_zCompress.next_in, m_zCompress.avail_in);
+      m_zCompress.next_in = m_CompressInput;
+      // add new stuff
+      memcpy (&m_zCompress.next_in [m_zCompress.avail_in], buff, count);
+      m_zCompress.avail_in += count;
+      // stats - count compressed bytes
+      m_nTotalCompressed += count;
       }
-    // shuffle existing input to-be-processed to start of buffer
-    if (m_zCompress.avail_in)
-      memmove (m_CompressInput, m_zCompress.next_in, m_zCompress.avail_in);
-    m_zCompress.next_in = m_CompressInput;
-    // add new stuff
-    memcpy (&m_zCompress.next_in [m_zCompress.avail_in], buff, count);
-    m_zCompress.avail_in += count;
-    // stats - count compressed bytes
-    m_nTotalCompressed += count;
     }
 
   // now display the message, unless we are getting compressed data
@@ -1460,8 +1489,12 @@ int count = m_pSocket->Receive (buff, sizeof (buff) - 1);
       break;    // still not compressed - exit
 
     // is compressed now
+
+    // For MCCP4 (Zstandard), decompression is already handled above
+    if (m_iMCCP_type == 4)
+      break;  // Zstandard processing is complete
 
-    // give up if nothing to display
+    // give up if nothing to display (zlib only)
     if (m_zCompress.avail_in <= 0)
        break;    // no data to process
 
@@ -1492,6 +1525,12 @@ int count = m_pSocket->Receive (buff, sizeof (buff) - 1);
 
          if (m_CompressOutput == NULL)
            {
+            // Stop timing before early return
+            if (App.m_iCounterFrequency)
+              {
+              QueryPerformanceCounter (&finish);
+              m_iCompressionTimeTaken += finish.QuadPart - start.QuadPart;
+              }
             OnConnectionDisconnect ();    // close the world
             free (m_CompressInput);       // may as well get rid of compression input as well
             m_CompressInput = NULL;
@@ -1502,14 +1541,15 @@ int count = m_pSocket->Receive (buff, sizeof (buff) - 1);
 
       } while (iCompressResult == Z_BUF_ERROR);
 
+    // End timing here - after decompression but before error handling
     if (App.m_iCounterFrequency)
       {
       QueryPerformanceCounter (&finish);
       m_iCompressionTimeTaken += finish.QuadPart - start.QuadPart;
       }
 
-    // error?
-    if (iCompressResult < 0)
+      // error?
+      if (iCompressResult < 0)
       {
       char * pMsg = m_zCompress.msg;    // closing may clear the message
 
@@ -1546,6 +1586,9 @@ int count = m_pSocket->Receive (buff, sizeof (buff) - 1);
     if (iCompressResult == Z_STREAM_END)
       {    // we can stop decompressing
       m_bCompress = false;
+      // Clean up compression resources based on type
+      if (m_iMCCP_type == 4)
+        CleanupZstd();  // Clean up Zstandard resources
       // put remaining stuff back into buff
       memcpy (buff, m_zCompress.next_in, m_zCompress.avail_in);
       count = m_zCompress.avail_in;
@@ -2042,6 +2085,28 @@ CString strLine (lpszText, size);
             // data as it it was not compressed
             if (!bWasCompressing && m_bCompress)
               {
+              // MCCP4 with zstd doesn't use m_CompressInput - process remaining data immediately
+              if (m_iMCCP_type == 4 && m_MCCP4_active_encoding == "zstd")
+                {
+                // Skip the SE byte and process any remaining compressed data immediately
+                p++;    // skip SE
+                size--; // one less byte
+                if (size > 0)  // if there's remaining data, it's compressed
+                  {
+                  // Process the remaining compressed data immediately
+                  int result = ProcessZstdCompressed((const unsigned char*)p, size);
+                  if (result < 0)
+                    return;  // Error already handled
+                  // If not all bytes were consumed, there might be uncompressed data
+                  if (result > 0 && result < size && !m_bCompress)
+                    {
+                    // Process remaining uncompressed bytes
+                    DisplayMsg((const char*)(p + result), size - result, 0);
+                    }
+                  }
+                return;
+                }
+
               p++;    // skip SE  (normally done at end of loop)
               size--; // one less of these
               if (size)  // copy compressed data to compression buffer
@@ -2064,6 +2129,28 @@ CString strLine (lpszText, size);
             // data as it it was not compressed
             if (!bWasCompressing && m_bCompress)
               {
+              // MCCP4 with zstd doesn't use m_CompressInput - process remaining data immediately
+              if (m_iMCCP_type == 4 && m_MCCP4_active_encoding == "zstd")
+                {
+                // Skip the SE byte and process any remaining compressed data immediately
+                p++;    // skip SE
+                size--; // one less byte
+                if (size > 0)  // if there's remaining data, it's compressed
+                  {
+                  // Process the remaining compressed data immediately
+                  int result = ProcessZstdCompressed((const unsigned char*)p, size);
+                  if (result < 0)
+                    return;  // Error already handled
+                  // If not all bytes were consumed, there might be uncompressed data
+                  if (result > 0 && result < size && !m_bCompress)
+                    {
+                    // Process remaining uncompressed bytes
+                    DisplayMsg((const char*)(p + result), size - result, 0);
+                    }
+                  }
+                return;
+                }
+
               p++;    // skip SE  (normally done at end of loop)
               size--; // one less of these
               if (size)  // copy compressed data to compression buffer
@@ -6788,6 +6875,12 @@ void CMUSHclientDoc::ConnectionEstablished (void)
 	m_bEnableAutoSay = FALSE;		// auto-say off at start of session
   App.m_bUpdateActivity = TRUE;   // new activity!
   m_bCompress = FALSE;        // not compressing yet
+  // Clean up any existing compression
+  if (m_iMCCP_type == 4)
+    CleanupZstd();
+  m_iMCCP_type = 0;
+  m_bSupports_MCCP_2 = false;
+  m_bSupports_MCCP_4 = false;
   m_nTotalLinesSent = 0;    // no lines sent yet
   m_nTotalLinesReceived = 0;  // no lines received yet
   m_iTriggersMatchedThisSessionCount = 0;   

doc.h

@@ -206,7 +206,13 @@ enum {
 #define TELOPT_TERMINAL_TYPE 24  // want to know terminal type
 #define TELOPT_COMPRESS 85   // telet negotiation code for starting compression v1
 #define TELOPT_COMPRESS2 86  // telet negotiation code for starting compression v2
+#define TELOPT_COMPRESS4 88  // telet negotiation code for starting compression v4 (Zstandard)
 #define TELOPT_MUD_SPECIFIC 102  // telet negotiation code MUD-specific negotiations
+
+// MCCP4 subnegotiation constants
+#define MCCP4_ACCEPT_ENCODING 1
+#define MCCP4_BEGIN_ENCODING  2  // Server is actually sending 0x02 for this
+#define MCCP4_WONT            3  // Refuse to do MCCP4 compression.
 #define SUPPORT_VERSIONS 0   // MCCP support-versions query
 #define VERSION_IS 1         // MCCP version number
 #define VERSION_NONE 0   // see web page below for details
@@ -1088,8 +1094,21 @@ class CMUSHclientDoc : public CDocument
   long m_nCompressionOutputBufferSize; // size of decompression output buffer
 
 
-  int m_iMCCP_type;   // MCCP protocol type in use: 0 = none, 1 = v1, 2 = v2
+  int m_iMCCP_type;   // MCCP protocol type in use: 0 = none, 1 = v1, 2 = v2, 4 = v4 (Zstandard)
   bool m_bSupports_MCCP_2;    // if true we have agreed to support MCCP v 2
+  bool m_bSupports_MCCP_4;    // if true we have agreed to support MCCP v 4 (Zstandard)
+
+  // MCCP4 protocol negotiation state
+  string m_MCCP4_accepted_encodings;  // client's advertised supported encodings
+  string m_MCCP4_active_encoding;     // currently active encoding (zstd/deflate)
+  bool m_MCCP4_negotiation_active;    // true if we're in MCCP4 negotiation
+
+  // MCCP4 (Zstandard) stuff
+  void *m_zstd_dstream;         // ZSTD_DStream* decompression context
+  unsigned char *m_zstd_inbuf;  // input buffer
+  size_t m_zstd_incap;          // capacity of input buffer
+  unsigned char *m_zstd_outbuf; // output buffer
+  size_t m_zstd_outcap;         // capacity of output buffer
 
   // end MCCP stuff
 
@@ -1449,10 +1468,17 @@ class CMUSHclientDoc : public CDocument
   void Handle_IAC_GA ();
 
   void Handle_TELOPT_COMPRESS2 ();
+  void Handle_TELOPT_COMPRESS4 ();
   void Handle_TELOPT_MUD_SPECIFIC ();
   void Handle_TELOPT_MXP ();
   void Handle_TELOPT_TERMINAL_TYPE ();
   void Handle_TELOPT_CHARSET ();
+
+  // MCCP4 (Zstandard) support functions
+  bool InitZstd();
+  void CleanupZstd();
+  int ProcessZstdCompressed(const unsigned char* input, unsigned int inputSize);
+  void Send_MCCP4_Accept_Encoding();
 
   void Send_IAC_DO (const unsigned char c);
   void Send_IAC_DONT (const unsigned char c);

doc_construct.cpp

@@ -97,6 +97,12 @@ int i;
   m_bShowingMapperStatus = false;
   m_iMCCP_type = 0;
   m_bSupports_MCCP_2 = false;
+  m_bSupports_MCCP_4 = false;
+  m_zstd_dstream = NULL;
+  m_zstd_inbuf = NULL;
+  m_zstd_incap = 0;
+  m_zstd_outbuf = NULL;
+  m_zstd_outcap = 0;
   m_bNoEcho = false;
   m_bDebugIncomingPackets = false;
   m_iInputPacketCount = 0;