Update OpenSSL (to 3.6.0) and libssh2 (to 1.11.1)

.github/lib-versions.env

@@ -1,2 +1,2 @@
-OPENSSL_BRANCH=OpenSSL_1_1_1w
-LIBSSH2_VERSION=1.9.0
+OPENSSL_BRANCH=openssl-3.6.0
+LIBSSH2_VERSION=1.11.1

.github/workflows/build-libs.yml

@@ -5,8 +5,9 @@ on:
   workflow_dispatch:
 
 env:
+  JOM_DOWNLOAD_URL: http://download.qt.io/official_releases/jom/jom.zip
   OPENSSL_REPO: https://github.com/openssl/openssl.git
-  LIBSSH2_REPO: https://github.com/libssh2/libssh2.git
+  LIBSSH2_URL: https://github.com/libssh2/libssh2
 
 jobs:
   build-libs:
@@ -32,6 +33,15 @@ jobs:
       - name: Install NASM
         uses: ilammy/setup-nasm@v1
 
+      - name: Install jom
+        shell: cmd
+        run: |
+          curl.exe -LO "%JOM_DOWNLOAD_URL%"
+          set "JOM_DIR=%HOMEDRIVE%%HOMEPATH%\jom"
+          md "%JOM_DIR%" 2>NUL || type NUL
+          tar.exe xz -C "%JOM_DIR%" -f jom.zip
+          echo %JOM_DIR%>>%GITHUB_PATH%
+
       - name: Setup MSBuild
         uses: microsoft/setup-msbuild@v2
         with:
@@ -75,11 +85,15 @@ jobs:
 
             $opensslPrefix = "$env:GITHUB_WORKSPACE\deps\openssl\$arch"
 
-            cmd /c "`"$vcvars`" $vc && cd openssl-$arch && perl Configure $targetOpenSSL no-shared no-tests --prefix=$opensslPrefix && nmake && nmake install" `
+            cmd /c "`"$vcvars`" $vc && cd openssl-$arch && perl Configure $targetOpenSSL no-shared no-tests --prefix=$opensslPrefix /FS && jom /J 8 && jom install" `
               2>&1 | Tee-Object "logs\openssl-$arch.log"
 
             # --- libssh2 ---
-            git clone --depth=1 --branch "libssh2-$($env:LIBSSH2_VERSION)" $env:LIBSSH2_REPO "libssh2-$arch"
+            curl.exe -LO "$($env:LIBSSH2_URL)/releases/download/libssh2-$($env:LIBSSH2_VERSION)/libssh2-$($env:LIBSSH2_VERSION).zip"
+            Set-PSDebug -Off
+            Expand-Archive ".\libssh2-$($env:LIBSSH2_VERSION).zip" -DestinationPath .
+            Rename-Item "libssh2-$($env:LIBSSH2_VERSION)" "libssh2-$arch"
+            Set-PSDebug -Trace 1
             Push-Location "libssh2-$arch"
 
             $libssh2Prefix = "$env:GITHUB_WORKSPACE\deps\libssh2\$arch"

ShellDLL/Auth.cpp

@@ -351,6 +351,10 @@ bool CAuthentication::CanRetry(IEasySFTPAuthentication* pAuth)
 
 	if (!pAuthSession->lpPageantKeyList)
 		return false;
+
+	if (AssignAgentFlags(pAuthSession))
+		return true;
+
 	pAuthSession->dwKeyIndex++;
 	if (pAuthSession->dwKeyIndex >= pAuthSession->dwKeyCount)
 	{
@@ -363,6 +367,33 @@ bool CAuthentication::CanRetry(IEasySFTPAuthentication* pAuth)
 	pAuthSession->lpCurrentKey += dw + 4;
 	dw = ConvertEndian(*reinterpret_cast<DWORD*>(pAuthSession->lpCurrentKey));
 	pAuthSession->lpCurrentKey += dw + 4;
+	pAuthSession->nPrevFlags = -1;
+	return AssignAgentFlags(pAuthSession);
+}
+
+bool CAuthentication::AssignAgentFlags(CAuthSession* pAuthSession)
+{
+	if (pAuthSession->nPrevFlags == 0)
+		return false;
+	// get key type data (in the head of blob data)
+	DWORD dwKeyTypeLen = ConvertEndian(*((DWORD*)(pAuthSession->lpCurrentKey + 4)));
+	LPCSTR lpszKeyType = (LPCSTR)(pAuthSession->lpCurrentKey + 8);
+
+	if ((dwKeyTypeLen == 7 && memcmp(lpszKeyType, "ssh-rsa", dwKeyTypeLen) == 0) ||
+		(dwKeyTypeLen == 28 && memcmp(lpszKeyType, "ssh-rsa-cert-v01@openssh.com", dwKeyTypeLen) == 0))
+	{
+		// both rsa-sha2-512 and rsa-sha2-256 are supported, so use rsa-sha2-512
+		if (pAuthSession->nPrevFlags < 0)
+			pAuthSession->nPrevFlags = SSH_AGENT_RSA_SHA2_512;
+		else if (pAuthSession->nPrevFlags == SSH_AGENT_RSA_SHA2_512)
+			pAuthSession->nPrevFlags = SSH_AGENT_RSA_SHA2_256;
+		else
+			pAuthSession->nPrevFlags = 0;
+	}
+	else
+	{
+		pAuthSession->nPrevFlags = 0;
+	}
 	return true;
 }
 
@@ -403,47 +434,49 @@ AuthReturnType CAuthentication::SSHAuthenticateWithAgent(IEasySFTPAuthentication
 		pAuthSession->dwKeyCount = ConvertEndian(*((DWORD*)lpKeyList));
 		pAuthSession->dwKeyIndex = 0;
 		pAuthSession->lpCurrentKey = lpKeyList + 4;
+		pAuthSession->nPrevFlags = -1;
 		if (FAILED(pAuth->put_AuthSession(reinterpret_cast<__int3264>(pAuthSession))))
 		{
 			delete pAuthSession;
 			return AuthReturnType::Error;
 		}
+		AssignAgentFlags(pAuthSession);
 	}
 	LPBYTE p = pAuthSession->lpCurrentKey;
 
-	LPCSTR lpszKeyType;
 	LPCBYTE pBlob;
 	size_t nBlobLen;
 
-	// get key type data (in the head of blob data)
-	DWORD dwKeyTypeLen = ConvertEndian(*((DWORD*)(p + 4)));
-	lpszKeyType = (LPCSTR)(p + 8);
-
 	nBlobLen = (size_t)ConvertEndian(*((DWORD*)p));
 	pBlob = (p + 4);
 	p += nBlobLen + 4;
 
-	// get the comment of key
 	{
+		DWORD dwKeyTypeLen = ConvertEndian(*((DWORD*)(pBlob)));
+		LPCSTR lpszKeyType = (LPCSTR)(pBlob + 4);
+
+		// get the comment of key
 		DWORD dwCommentLen = ConvertEndian(*((DWORD*)p));
 		CMyStringW str;
 		str.SetUTF8String((LPCBYTE)(p + 4), static_cast<size_t>(dwCommentLen));
 		p += dwCommentLen + 4;
-		CMyStringW strType(lpszKeyType), strDebug;
+		CMyStringW strType, strDebug;
+		strType.SetString(lpszKeyType, dwKeyTypeLen);
 		strDebug.Format(L"trying key '%s' (type: %s)", str.operator LPCWSTR(), strType.operator LPCWSTR());
 		theApp.Log(EasySFTPLogLevel::Debug, strDebug, S_OK);
 	}
 
 	void* abstract = pAuthSession;
 	auto ret = libssh2_userauth_publickey(pSession, lpszUser, pBlob, nBlobLen,
-		[](LIBSSH2_SESSION*, LPBYTE* sig, size_t* sig_len, LPCBYTE data, size_t data_len, void** abstract) -> int
+		[](LIBSSH2_SESSION* session, LPBYTE* sig, size_t* sig_len, LPCBYTE data, size_t data_len, void** abstract) -> int
 		{
 			*sig = NULL;
 			*sig_len = 0;
 			CAuthSession* pAuthSession = static_cast<CAuthSession*>(*abstract);
 			LPBYTE lpCurrentKey = pAuthSession->lpCurrentKey;
+
 			size_t nSignedLen;
-			auto buff = pAuthSession->pAgent->SignSSH2Key(lpCurrentKey, data, data_len, &nSignedLen);
+			auto buff = pAuthSession->pAgent->SignSSH2Key(lpCurrentKey, pAuthSession->nPrevFlags, data, data_len, &nSignedLen);
 			LPBYTE pSignedData = static_cast<LPBYTE>(buff);
 			if (nSignedLen < 4 || !buff)
 			{
@@ -505,7 +538,12 @@ AuthReturnType CAuthentication::SSHAuthenticateWithAgent(IEasySFTPAuthentication
 		return AuthReturnType::Again;
 
 	if (ret != 0)
+	{
+		// trying any flags only needed when error is 'unverified'
+		if (ret != LIBSSH2_ERROR_PUBLICKEY_UNVERIFIED)
+			pAuthSession->nPrevFlags = 0;
 		return AuthReturnType::Error;
+	}
 
 	delete pAuthSession;
 	pAuth->put_AuthSession(reinterpret_cast<__int3264>(nullptr));

ShellDLL/Auth.h

@@ -37,6 +37,7 @@ struct CAuthSession
 	LPBYTE lpCurrentKey;
 	DWORD dwKeyCount;
 	DWORD dwKeyIndex;
+	int nPrevFlags;
 
 	~CAuthSession();
 };
@@ -69,6 +70,7 @@ class CAuthentication : public CDispatchImplT<IEasySFTPAuthentication2>
 	static bool CanRetry(IEasySFTPAuthentication* pAuth);
 
 private:
+	static bool AssignAgentFlags(CAuthSession* pAuthSession);
 	static AuthReturnType SSHAuthenticateWithAgent(IEasySFTPAuthentication* pAuth, CMyStringW& strUserName, LIBSSH2_SESSION* pSession, LPCSTR lpszService, CSSHAgent* (* CreateAgent)());
 
 public:

ShellDLL/SSHAgent.cpp

@@ -62,7 +62,7 @@ int CSSHAgent::GetKeyList2(LPBYTE* ppKeyList)
 	return static_cast<int>(nResponseLen) - 5;
 }
 
-void* CSSHAgent::SignSSH2Key(LPCBYTE pszPubKey, LPCBYTE pszData, size_t nDataLen, size_t* pnOutLen)
+void* CSSHAgent::SignSSH2Key(LPCBYTE pszPubKey, int flags, LPCBYTE pszData, size_t nDataLen, size_t* pnOutLen)
 {
 	void* ret;
 
@@ -85,7 +85,7 @@ void* CSSHAgent::SignSSH2Key(LPCBYTE pszPubKey, LPCBYTE pszData, size_t nDataLen
 	// sign data (length + data)
 	request.AppendToBufferWithLenCE(pszData, nDataLen);
 	// flags
-	request.AppendToBufferCE(static_cast<DWORD>(0));
+	request.AppendToBufferCE(static_cast<DWORD>(flags));
 
 	retval = Query(request, nReqLen, (void**)&pResponse, &nResponseLen);
 	if (!retval)

ShellDLL/SSHAgent.h

@@ -1,5 +1,9 @@
 #pragma once
 
+/* Signature request methods */
+#define SSH_AGENT_RSA_SHA2_256 2
+#define SSH_AGENT_RSA_SHA2_512 4
+
 class __declspec(novtable) CSSHAgent
 {
 public:
@@ -8,6 +12,6 @@ class __declspec(novtable) CSSHAgent
 
 public:
 	int GetKeyList2(LPBYTE* ppKeyList);
-	void* SignSSH2Key(LPCBYTE pszPubKey, LPCBYTE pszData, size_t nDataLen, size_t* pnOutLen);
+	void* SignSSH2Key(LPCBYTE pszPubKey, int flags, LPCBYTE pszData, size_t nDataLen, size_t* pnOutLen);
 	void FreeKeyList(LPBYTE pKeyList);
 };

ShellDLL/ShellDLL.cpp

@@ -53,6 +53,13 @@ ITypeInfo* GetTypeInfo(const GUID& guid)
 
 ////////////////////////////////////////////////////////////////////////////////
 
+// workaround (related: https://github.com/openssl/openssl/issues/27701 )
+#if !defined(WIN64)
+extern "C" unsigned _int64 _dtoul3_legacy(double v) { return (unsigned _int64)llround(v); }
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
 #if !defined(NTDDI_WIN7) || (NTDDI_VERSION < NTDDI_WIN7)
 #define INITGUID
 #include <guiddef.h>
@@ -1232,9 +1239,6 @@ bool CMainDLL::InitInstance()
 
 	// for SSL library
 	SSL_library_init();
-	ERR_load_BIO_strings();
-	ERR_load_CRYPTO_strings();
-	ERR_load_SSL_strings();
 
 	::srand((unsigned int) (time(NULL) * GetTickCount()));
 
@@ -1661,13 +1665,9 @@ int CMainDLL::ExitInstance()
 	::DeleteCriticalSection(&m_csHosts);
 	::DeleteCriticalSection(&m_csRootRefs);
 
-	::ERR_remove_state(0);
 	//::ENGINE_cleanup();
 	//::CONF_modules_unload();
 	//::RAND_cleanup();
-	ERR_free_strings();
-	EVP_cleanup();
-	CRYPTO_cleanup_all_ex_data();
 	libssh2_exit();
 
 	m_TimerThread.Finalize();

ShellDLL/stdafx.h

@@ -74,6 +74,8 @@ typedef IDataObjectAsyncCapability IAsyncOperation;
 #include <openssl/rand.h>
 
 // libssh2
+// woraround definition (related: https://github.com/libssh2/libssh2/issues/1578 )
+#define LIBSSH2_API
 #include <libssh2.h>
 #include <libssh2_sftp.h>