Thanks for all comments. I made it work by changing all arguments and return values in DLL public functions to variants.
jonscott8:
Here are the function declarations in the DLL, both the original (which didn't work) and the new (which does):
Original:
Public Function GetRegistrySettings(keyname As String, encryptkey As String) As Variant
'this function returns an array
Dim c_sValue As String
Dim c_sArray(3, 1) As String
c_sArray(0, 0) = "Server"
c_sArray(0, 1) = QueryValue(&H80000002, keyname, "Server"

c_sArray(1, 0) = "Database"
c_sArray(1, 1) = QueryValue(&H80000002, keyname, "Database"

c_sArray(2, 0) = "Login"
c_sArray(2, 1) = QueryValue(&H80000002, keyname, "Login"

c_sArray(2, 1) = Encrypt(c_sArray(2, 1), encryptkey)
c_sArray(3, 0) = "Password"
c_sArray(3, 1) = QueryValue(&H80000002, keyname, "Password"

c_sArray(3, 1) = Encrypt(c_sArray(3, 1), encryptkey)
GetRegistrySettings = c_sArray
End Function
Public Function CheckKey(keyname As String) As Long
'determine if specified key exists
Dim lResult As Long
CheckKey = RegOpenKeyEx(HKEY_LOCAL_MACHINE, keyname, 0, &H8, lResult)
End Function
Public Function GetRegistrySettings(keyname As Variant, encryptkey As Variant) As Variant
Dim c_sValue As Variant
Dim c_sArray(3, 1) As Variant
Dim sKeyname As String
Dim sEncKey As String
'convert variant arguments to strings for internal use
sKeyname = CStr(keyname)
sEncKey = CStr(encryptkey)
c_sArray(0, 0) = "Server"
c_sArray(0, 1) = QueryValue(&H80000002, sKeyname, "Server"

c_sArray(1, 0) = "Database"
c_sArray(1, 1) = QueryValue(&H80000002, sKeyname, "Database"

c_sArray(2, 0) = "Login"
c_sArray(2, 1) = QueryValue(&H80000002, sKeyname, "Login"

c_sArray(2, 1) = Encrypt(c_sArray(2, 1), sEncKey)
c_sArray(3, 0) = "Password"
c_sArray(3, 1) = QueryValue(&H80000002, sKeyname, "Password"

c_sArray(3, 1) = Encrypt(c_sArray(3, 1), sEncKey)
GetRegistrySettings = c_sArray
End Function
Public Function CheckKey(keyname As Variant) As Variant
'determine if specified key exists
Dim lResult As Long
Dim sKeyname As String
sKeyname = CStr(keyname)
CheckKey = RegOpenKeyEx(HKEY_LOCAL_MACHINE, sKeyname, 0, &H8, lResult)
End Function