Allow server public key to be anywhere from 512 bits to 2048 bits.

(Vista Beta 2 uses a 2048 bit key)


git-svn-id: svn://svn.code.sf.net/p/rdesktop/code/trunk/rdesktop@1236 423420c4-83ab-492f-b58f-81f9feb106b5
This commit is contained in:
Matt Chapman 2006-06-14 08:26:00 +00:00
parent 997e354a24
commit 69692c7014
2 changed files with 41 additions and 31 deletions

View File

@ -62,6 +62,7 @@ enum MCS_PDU_TYPE
/* RDP secure transport constants */ /* RDP secure transport constants */
#define SEC_RANDOM_SIZE 32 #define SEC_RANDOM_SIZE 32
#define SEC_MODULUS_SIZE 64 #define SEC_MODULUS_SIZE 64
#define SEC_MAX_MODULUS_SIZE 256
#define SEC_PADDING_SIZE 8 #define SEC_PADDING_SIZE 8
#define SEC_EXPONENT_SIZE 4 #define SEC_EXPONENT_SIZE 4

View File

@ -46,13 +46,14 @@ static int rc4_key_len;
static RC4_KEY rc4_decrypt_key; static RC4_KEY rc4_decrypt_key;
static RC4_KEY rc4_encrypt_key; static RC4_KEY rc4_encrypt_key;
static RSA *server_public_key; static RSA *server_public_key;
static uint32 server_public_key_len;
static uint8 sec_sign_key[16]; static uint8 sec_sign_key[16];
static uint8 sec_decrypt_key[16]; static uint8 sec_decrypt_key[16];
static uint8 sec_encrypt_key[16]; static uint8 sec_encrypt_key[16];
static uint8 sec_decrypt_update_key[16]; static uint8 sec_decrypt_update_key[16];
static uint8 sec_encrypt_update_key[16]; static uint8 sec_encrypt_update_key[16];
static uint8 sec_crypted_random[SEC_MODULUS_SIZE]; static uint8 sec_crypted_random[SEC_MAX_MODULUS_SIZE];
uint16 g_server_rdp_version = 0; uint16 g_server_rdp_version = 0;
@ -297,14 +298,14 @@ reverse(uint8 * p, int len)
/* Perform an RSA public key encryption operation */ /* Perform an RSA public key encryption operation */
static void static void
sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint8 * modulus, uint8 * exponent) sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * modulus, uint8 * exponent)
{ {
BN_CTX *ctx; BN_CTX *ctx;
BIGNUM mod, exp, x, y; BIGNUM mod, exp, x, y;
uint8 inr[SEC_MODULUS_SIZE]; uint8 inr[SEC_MAX_MODULUS_SIZE];
int outlen; int outlen;
reverse(modulus, SEC_MODULUS_SIZE); reverse(modulus, modulus_size);
reverse(exponent, SEC_EXPONENT_SIZE); reverse(exponent, SEC_EXPONENT_SIZE);
memcpy(inr, in, len); memcpy(inr, in, len);
reverse(inr, len); reverse(inr, len);
@ -315,14 +316,14 @@ sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint8 * modulus, uint8 * expon
BN_init(&x); BN_init(&x);
BN_init(&y); BN_init(&y);
BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod); BN_bin2bn(modulus, modulus_size, &mod);
BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp); BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
BN_bin2bn(inr, len, &x); BN_bin2bn(inr, len, &x);
BN_mod_exp(&y, &x, &exp, &mod, ctx); BN_mod_exp(&y, &x, &exp, &mod, ctx);
outlen = BN_bn2bin(&y, out); outlen = BN_bn2bin(&y, out);
reverse(out, outlen); reverse(out, outlen);
if (outlen < SEC_MODULUS_SIZE) if (outlen < modulus_size)
memset(out + outlen, 0, SEC_MODULUS_SIZE - outlen); memset(out + outlen, 0, modulus_size - outlen);
BN_free(&y); BN_free(&y);
BN_clear_free(&x); BN_clear_free(&x);
@ -388,14 +389,14 @@ sec_send(STREAM s, uint32 flags)
static void static void
sec_establish_key(void) sec_establish_key(void)
{ {
uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE; uint32 length = server_public_key_len + SEC_PADDING_SIZE;
uint32 flags = SEC_CLIENT_RANDOM; uint32 flags = SEC_CLIENT_RANDOM;
STREAM s; STREAM s;
s = sec_init(flags, 76); s = sec_init(flags, length+4);
out_uint32_le(s, length); out_uint32_le(s, length);
out_uint8p(s, sec_crypted_random, SEC_MODULUS_SIZE); out_uint8p(s, sec_crypted_random, server_public_key_len);
out_uint8s(s, SEC_PADDING_SIZE); out_uint8s(s, SEC_PADDING_SIZE);
s_mark_end(s); s_mark_end(s);
@ -507,16 +508,18 @@ sec_parse_public_key(STREAM s, uint8 ** modulus, uint8 ** exponent)
} }
in_uint32_le(s, modulus_len); in_uint32_le(s, modulus_len);
if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE) modulus_len -= SEC_PADDING_SIZE;
if ((modulus_len < 64) || (modulus_len > SEC_MAX_MODULUS_SIZE))
{ {
error("modulus len 0x%x\n", modulus_len); error("Bad server public key size (%u bits)\n", modulus_len*8);
return False; return False;
} }
in_uint8s(s, 8); /* modulus_bits, unknown */ in_uint8s(s, 8); /* modulus_bits, unknown */
in_uint8p(s, *exponent, SEC_EXPONENT_SIZE); in_uint8p(s, *exponent, SEC_EXPONENT_SIZE);
in_uint8p(s, *modulus, SEC_MODULUS_SIZE); in_uint8p(s, *modulus, modulus_len);
in_uint8s(s, SEC_PADDING_SIZE); in_uint8s(s, SEC_PADDING_SIZE);
server_public_key_len = modulus_len;
return s_check(s); return s_check(s);
} }
@ -544,9 +547,15 @@ sec_parse_x509_key(X509 * cert)
} }
server_public_key = RSAPublicKey_dup((RSA *) epk->pkey.ptr); server_public_key = RSAPublicKey_dup((RSA *) epk->pkey.ptr);
EVP_PKEY_free(epk); EVP_PKEY_free(epk);
server_public_key_len = RSA_size(server_public_key);
if ((server_public_key_len < 64) || (server_public_key_len > SEC_MAX_MODULUS_SIZE))
{
error("Bad server public key size (%u bits)\n", server_public_key_len*8);
return False;
}
return True; return True;
} }
@ -719,7 +728,6 @@ sec_process_crypt_info(STREAM s)
uint8 *server_random, *modulus, *exponent; uint8 *server_random, *modulus, *exponent;
uint8 client_random[SEC_RANDOM_SIZE]; uint8 client_random[SEC_RANDOM_SIZE];
uint32 rc4_key_size; uint32 rc4_key_size;
uint8 inr[SEC_MODULUS_SIZE];
if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random, &modulus, &exponent)) if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random, &modulus, &exponent))
{ {
@ -728,29 +736,30 @@ sec_process_crypt_info(STREAM s)
} }
DEBUG(("Generating client random\n")); DEBUG(("Generating client random\n"));
/* Generate a client random, and hence determine encryption keys */
/* This is what the MS client do: */
memset(inr, 0, SEC_RANDOM_SIZE);
/* *ARIGL!* Plaintext attack, anyone?
I tried doing:
generate_random(inr);
..but that generates connection errors now and then (yes,
"now and then". Something like 0 to 3 attempts needed before a
successful connection. Nice. Not!
*/
generate_random(client_random); generate_random(client_random);
if (NULL != server_public_key) if (NULL != server_public_key)
{ /* Which means we should use { /* Which means we should use
RDP5-style encryption */ RDP5-style encryption */
uint8 inr[SEC_MAX_MODULUS_SIZE];
uint32 padding_len = server_public_key_len - SEC_RANDOM_SIZE;
memcpy(inr + SEC_RANDOM_SIZE, client_random, SEC_RANDOM_SIZE); /* This is what the MS client do: */
reverse(inr + SEC_RANDOM_SIZE, SEC_RANDOM_SIZE); memset(inr, 0, padding_len);
/* *ARIGL!* Plaintext attack, anyone?
I tried doing:
generate_random(inr);
..but that generates connection errors now and then (yes,
"now and then". Something like 0 to 3 attempts needed before a
successful connection. Nice. Not!
*/
memcpy(inr + padding_len, client_random, SEC_RANDOM_SIZE);
reverse(inr + padding_len, SEC_RANDOM_SIZE);
RSA_public_encrypt(SEC_MODULUS_SIZE, RSA_public_encrypt(server_public_key_len,
inr, sec_crypted_random, server_public_key, RSA_NO_PADDING); inr, sec_crypted_random, server_public_key, RSA_NO_PADDING);
reverse(sec_crypted_random, SEC_MODULUS_SIZE); reverse(sec_crypted_random, server_public_key_len);
RSA_free(server_public_key); RSA_free(server_public_key);
server_public_key = NULL; server_public_key = NULL;
@ -758,7 +767,7 @@ sec_process_crypt_info(STREAM s)
else else
{ /* RDP4-style encryption */ { /* RDP4-style encryption */
sec_rsa_encrypt(sec_crypted_random, sec_rsa_encrypt(sec_crypted_random,
client_random, SEC_RANDOM_SIZE, modulus, exponent); client_random, SEC_RANDOM_SIZE, server_public_key_len, modulus, exponent);
} }
sec_generate_keys(client_random, server_random, rc4_key_size); sec_generate_keys(client_random, server_random, rc4_key_size);
} }