From fdb2da450a9ce0569ff85ed9c7aebea93b069dad Mon Sep 17 00:00:00 2001 From: Pierre Ossman Date: Fri, 20 Sep 2019 08:36:10 +0200 Subject: [PATCH] Fix initial security handshake for old systems The conversion from OpenSSL to GnuTLS (or GMP in this case) got the endianness mixed up in the initial security handshake. It got it wrong in two places though, which cancelled each other out when X.509 certificates are used. But servers using the older system, like Windows XP, would fail the handshake as different endianness was then used when reading the RSA key and when using it. --- ssl.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ssl.c b/ssl.c index 41cd843..930c7f9 100644 --- a/ssl.c +++ b/ssl.c @@ -90,9 +90,8 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * mpz_init(exp); mpz_init(mod); - mpz_import(mod, modulus_size, 1, sizeof(modulus[0]), 0, 0, modulus); - // TODO: Need exponent size - mpz_import(exp, 3, 1, sizeof(exponent[0]), 0, 0, exponent); + mpz_import(mod, modulus_size, -1, sizeof(modulus[0]), 0, 0, modulus); + mpz_import(exp, SEC_EXPONENT_SIZE, -1, sizeof(exponent[0]), 0, 0, exponent); mpz_import(x, len, -1, sizeof(in[0]), 0, 0, in); @@ -308,9 +307,15 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, { size_t outlen; - // TODO: Check size before exporing - mpz_export(modulus, &outlen, 1, sizeof(uint8), 0, 0, rkey->n); - mpz_export(exponent, &outlen, 1, sizeof(uint8), 0, 0, rkey->e); + outlen = (mpz_sizeinbase(modulus, 2) + 7) / 8; + if (outlen > max_mod_len) + return 1; + outlen = (mpz_sizeinbase(exponent, 2) + 7) / 8; + if (outlen > max_exp_len) + return 1; + + mpz_export(modulus, &outlen, -1, sizeof(uint8), 0, 0, rkey->n); + mpz_export(exponent, &outlen, -1, sizeof(uint8), 0, 0, rkey->e); /* * Note that gnutls_x509_crt_get_pk_rsa_raw() exports modulus with additional