static int get_client_hello(SSL *s)
{
    int i, n;
    unsigned long len;
    unsigned char *p;
    STACK_OF(SSL_CIPHER) *cs;   /* a stack of SSL_CIPHERS */
    STACK_OF(SSL_CIPHER) *cl;   /* the ones we want to use */
    STACK_OF(SSL_CIPHER) *prio, *allow;
    int z;

    /*
     * This is a bit of a hack to check for the correct packet type the first
     * time round.
     */
    if (s->state == SSL2_ST_GET_CLIENT_HELLO_A) {
        s->first_packet = 1;
        s->state = SSL2_ST_GET_CLIENT_HELLO_B;
    }

    p = (unsigned char *)s->init_buf->data;
    if (s->state == SSL2_ST_GET_CLIENT_HELLO_B) {
        i = ssl2_read(s, (char *)&(p[s->init_num]), 9 - s->init_num);
        if (i < (9 - s->init_num))
            return (ssl2_part_read(s, SSL_F_GET_CLIENT_HELLO, i));
        s->init_num = 9;

        if (*(p++) != SSL2_MT_CLIENT_HELLO) {
            if (p[-1] != SSL2_MT_ERROR) {
                ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
                SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_READ_WRONG_PACKET_TYPE);
            } else
                SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_PEER_ERROR);
            return (-1);
        }
        n2s(p, i);
        if (i < s->version)
            s->version = i;
        n2s(p, i);
        s->s2->tmp.cipher_spec_length = i;
        n2s(p, i);
        s->s2->tmp.session_id_length = i;
        if ((i < 0) || (i > SSL_MAX_SSL_SESSION_ID_LENGTH)) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_LENGTH_MISMATCH);
            return -1;
        }
        n2s(p, i);
        s->s2->challenge_length = i;
        if ((i < SSL2_MIN_CHALLENGE_LENGTH) ||
            (i > SSL2_MAX_CHALLENGE_LENGTH)) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_INVALID_CHALLENGE_LENGTH);
            return (-1);
        }
        s->state = SSL2_ST_GET_CLIENT_HELLO_C;
    }

    /* SSL2_ST_GET_CLIENT_HELLO_C */
    p = (unsigned char *)s->init_buf->data;
    len =
        9 + (unsigned long)s->s2->tmp.cipher_spec_length +
        (unsigned long)s->s2->challenge_length +
        (unsigned long)s->s2->tmp.session_id_length;
    if (len > SSL2_MAX_RECORD_LENGTH_3_BYTE_HEADER) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_MESSAGE_TOO_LONG);
        return -1;
    }
    n = (int)len - s->init_num;
    i = ssl2_read(s, (char *)&(p[s->init_num]), n);
    if (i != n)
        return (ssl2_part_read(s, SSL_F_GET_CLIENT_HELLO, i));
    if (s->msg_callback) {
        /* CLIENT-HELLO */
        s->msg_callback(0, s->version, 0, p, (size_t)len, s,
                        s->msg_callback_arg);
    }
    p += 9;

    /*
     * get session-id before cipher stuff so we can get out session structure
     * if it is cached
     */
    /* session-id */
    if ((s->s2->tmp.session_id_length != 0) &&
        (s->s2->tmp.session_id_length != SSL2_SSL_SESSION_ID_LENGTH)) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_BAD_SSL_SESSION_ID_LENGTH);
        return (-1);
    }

    if (s->s2->tmp.session_id_length == 0) {
        if (!ssl_get_new_session(s, 1)) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            return (-1);
        }
    } else {
        i = ssl_get_prev_session(s, &(p[s->s2->tmp.cipher_spec_length]),
                                 s->s2->tmp.session_id_length, NULL);
        if (i == 1) {           /* previous session */
            s->hit = 1;
        } else if (i == -1) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            return (-1);
        } else {
            if (s->cert == NULL) {
                ssl2_return_error(s, SSL2_PE_NO_CERTIFICATE);
                SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_NO_CERTIFICATE_SET);
                return (-1);
            }

            if (!ssl_get_new_session(s, 1)) {
                ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
                return (-1);
            }
        }
    }

    if (!s->hit) {
        cs = ssl_bytes_to_cipher_list(s, p, s->s2->tmp.cipher_spec_length,
                                      &s->session->ciphers);
        if (cs == NULL)
            goto mem_err;

        cl = SSL_get_ciphers(s);

        if (s->options & SSL_OP_CIPHER_SERVER_PREFERENCE) {
            prio = sk_SSL_CIPHER_dup(cl);
            if (prio == NULL)
                goto mem_err;
            allow = cs;
        } else {
             prio = cs;
             allow = cl;
         }

        /* Generate list of SSLv2 ciphers shared between client and server */
         for (z = 0; z < sk_SSL_CIPHER_num(prio); z++) {
            const SSL_CIPHER *cp = sk_SSL_CIPHER_value(prio, z);
            if ((cp->algorithm_ssl & SSL_SSLV2) == 0 ||
                sk_SSL_CIPHER_find(allow, cp) < 0) {
                 (void)sk_SSL_CIPHER_delete(prio, z);
                 z--;
             }
        }
        /*
             sk_SSL_CIPHER_free(s->session->ciphers);
             s->session->ciphers = prio;
         }

        /* Make sure we have at least one cipher in common */
        if (sk_SSL_CIPHER_num(s->session->ciphers) == 0) {
            ssl2_return_error(s, SSL2_PE_NO_CIPHER);
            SSLerr(SSL_F_GET_CLIENT_HELLO, SSL_R_NO_CIPHER_MATCH);
            return -1;
        }
         /*
          * s->session->ciphers should now have a list of ciphers that are on
          * both the client and server. This list is ordered by the order the
    if (s->s2->challenge_length > sizeof s->s2->challenge) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_HELLO, ERR_R_INTERNAL_ERROR);
        return -1;
    }
    memcpy(s->s2->challenge, p, (unsigned int)s->s2->challenge_length);
    return (1);
 mem_err:
    SSLerr(SSL_F_GET_CLIENT_HELLO, ERR_R_MALLOC_FAILURE);
    return (0);
}


Better SSLv2 cipher-suite enforcement

Based on patch by: Nimrod Aviram <nimrod.aviram@gmail.com>

CVE-2015-3197

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>

static int get_client_master_key(SSL *s)
{
    int is_export, i, n, keya;
    unsigned int num_encrypted_key_bytes, key_length;
    unsigned long len;
    unsigned char *p;
    const SSL_CIPHER *cp;
    const EVP_CIPHER *c;
    const EVP_MD *md;
    unsigned char rand_premaster_secret[SSL_MAX_MASTER_KEY_LENGTH];
    unsigned char decrypt_good;
    size_t j;

    p = (unsigned char *)s->init_buf->data;
    if (s->state == SSL2_ST_GET_CLIENT_MASTER_KEY_A) {
        i = ssl2_read(s, (char *)&(p[s->init_num]), 10 - s->init_num);

        if (i < (10 - s->init_num))
            return (ssl2_part_read(s, SSL_F_GET_CLIENT_MASTER_KEY, i));
        s->init_num = 10;

        if (*(p++) != SSL2_MT_CLIENT_MASTER_KEY) {
            if (p[-1] != SSL2_MT_ERROR) {
                ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
                SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,
                       SSL_R_READ_WRONG_PACKET_TYPE);
            } else
                SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_PEER_ERROR);
            return (-1);
         }

         cp = ssl2_get_cipher_by_char(p);
        if (cp == NULL) {
             ssl2_return_error(s, SSL2_PE_NO_CIPHER);
             SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_NO_CIPHER_MATCH);
             return (-1);
        }
        s->session->cipher = cp;

        p += 3;
        n2s(p, i);
        s->s2->tmp.clear = i;
        n2s(p, i);
        s->s2->tmp.enc = i;
        n2s(p, i);
        if (i > SSL_MAX_KEY_ARG_LENGTH) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_KEY_ARG_TOO_LONG);
            return -1;
        }
        s->session->key_arg_length = i;
        s->state = SSL2_ST_GET_CLIENT_MASTER_KEY_B;
    }

    /* SSL2_ST_GET_CLIENT_MASTER_KEY_B */
    p = (unsigned char *)s->init_buf->data;
    if (s->init_buf->length < SSL2_MAX_RECORD_LENGTH_3_BYTE_HEADER) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, ERR_R_INTERNAL_ERROR);
        return -1;
    }
    keya = s->session->key_arg_length;
    len =
        10 + (unsigned long)s->s2->tmp.clear + (unsigned long)s->s2->tmp.enc +
        (unsigned long)keya;
    if (len > SSL2_MAX_RECORD_LENGTH_3_BYTE_HEADER) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_MESSAGE_TOO_LONG);
        return -1;
    }
    n = (int)len - s->init_num;
    i = ssl2_read(s, (char *)&(p[s->init_num]), n);
    if (i != n)
        return (ssl2_part_read(s, SSL_F_GET_CLIENT_MASTER_KEY, i));
    if (s->msg_callback) {
        /* CLIENT-MASTER-KEY */
        s->msg_callback(0, s->version, 0, p, (size_t)len, s,
                        s->msg_callback_arg);
    }
    p += 10;

    memcpy(s->session->key_arg, &(p[s->s2->tmp.clear + s->s2->tmp.enc]),
           (unsigned int)keya);

    if (s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey == NULL) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_NO_PRIVATEKEY);
        return (-1);
    }

    is_export = SSL_C_IS_EXPORT(s->session->cipher);

    if (!ssl_cipher_get_evp(s->session, &c, &md, NULL, NULL, NULL)) {
        ssl2_return_error(s, SSL2_PE_NO_CIPHER);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,
               SSL_R_PROBLEMS_MAPPING_CIPHER_FUNCTIONS);
        return (0);
    }

    /*
     * The format of the CLIENT-MASTER-KEY message is
     * 1 byte message type
     * 3 bytes cipher
     * 2-byte clear key length (stored in s->s2->tmp.clear)
     * 2-byte encrypted key length (stored in s->s2->tmp.enc)
     * 2-byte key args length (IV etc)
     * clear key
     * encrypted key
     * key args
     *
     * If the cipher is an export cipher, then the encrypted key bytes
     * are a fixed portion of the total key (5 or 8 bytes). The size of
     * this portion is in |num_encrypted_key_bytes|. If the cipher is not an
     * export cipher, then the entire key material is encrypted (i.e., clear
     * key length must be zero).
     */
    key_length = (unsigned int)EVP_CIPHER_key_length(c);
    if (key_length > SSL_MAX_MASTER_KEY_LENGTH) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, ERR_R_INTERNAL_ERROR);
        return -1;
    }

    if (s->session->cipher->algorithm2 & SSL2_CF_8_BYTE_ENC) {
        is_export = 1;
        num_encrypted_key_bytes = 8;
    } else if (is_export) {
        num_encrypted_key_bytes = 5;
    } else {
        num_encrypted_key_bytes = key_length;
    }

    if (s->s2->tmp.clear + num_encrypted_key_bytes != key_length) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,SSL_R_BAD_LENGTH);
        return -1;
    }
    /*
     * The encrypted blob must decrypt to the encrypted portion of the key.
     * Decryption can't be expanding, so if we don't have enough encrypted
     * bytes to fit the key in the buffer, stop now.
     */
    if (s->s2->tmp.enc < num_encrypted_key_bytes) {
        ssl2_return_error(s,SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,SSL_R_LENGTH_TOO_SHORT);
        return -1;
    }

    /*
     * We must not leak whether a decryption failure occurs because of
     * Bleichenbacher's attack on PKCS #1 v1.5 RSA padding (see RFC 2246,
     * section 7.4.7.1). The code follows that advice of the TLS RFC and
     * generates a random premaster secret for the case that the decrypt
     * fails. See https://tools.ietf.org/html/rfc5246#section-7.4.7.1
     */

    /*
     * should be RAND_bytes, but we cannot work around a failure.
     */
    if (RAND_pseudo_bytes(rand_premaster_secret,
                          (int)num_encrypted_key_bytes) <= 0)
        return 0;

    i = ssl_rsa_private_decrypt(s->cert, s->s2->tmp.enc,
                                &(p[s->s2->tmp.clear]),
                                &(p[s->s2->tmp.clear]),
                                (s->s2->ssl2_rollback) ? RSA_SSLV23_PADDING :
                                RSA_PKCS1_PADDING);
    ERR_clear_error();
    /*
     * If a bad decrypt, continue with protocol but with a random master
     * secret (Bleichenbacher attack)
     */
    decrypt_good = constant_time_eq_int_8(i, (int)num_encrypted_key_bytes);
    for (j = 0; j < num_encrypted_key_bytes; j++) {
        p[s->s2->tmp.clear + j] =
                constant_time_select_8(decrypt_good, p[s->s2->tmp.clear + j],
                                       rand_premaster_secret[j]);
    }

    s->session->master_key_length = (int)key_length;
    memcpy(s->session->master_key, p, key_length);
    OPENSSL_cleanse(p, key_length);

    return 1;
}

ssl/s2_srvr.c in OpenSSL 1.0.1 before 1.0.1r and 1.0.2 before 1.0.2f does not prevent use of disabled ciphers, which makes it easier for man-in-the-middle attackers to defeat cryptographic protection mechanisms by performing computations on SSLv2 traffic, related to the get_client_master_key and get_client_hello functions.


static int get_client_master_key(SSL *s)
{
    int is_export, i, n, keya;
    unsigned int num_encrypted_key_bytes, key_length;
    unsigned long len;
    unsigned char *p;
    const SSL_CIPHER *cp;
    const EVP_CIPHER *c;
    const EVP_MD *md;
    unsigned char rand_premaster_secret[SSL_MAX_MASTER_KEY_LENGTH];
    unsigned char decrypt_good;
    size_t j;

    p = (unsigned char *)s->init_buf->data;
    if (s->state == SSL2_ST_GET_CLIENT_MASTER_KEY_A) {
        i = ssl2_read(s, (char *)&(p[s->init_num]), 10 - s->init_num);

        if (i < (10 - s->init_num))
            return (ssl2_part_read(s, SSL_F_GET_CLIENT_MASTER_KEY, i));
        s->init_num = 10;

        if (*(p++) != SSL2_MT_CLIENT_MASTER_KEY) {
            if (p[-1] != SSL2_MT_ERROR) {
                ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
                SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,
                       SSL_R_READ_WRONG_PACKET_TYPE);
            } else
                SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_PEER_ERROR);
            return (-1);
         }

         cp = ssl2_get_cipher_by_char(p);
        if (cp == NULL || sk_SSL_CIPHER_find(s->session->ciphers, cp) < 0) {
             ssl2_return_error(s, SSL2_PE_NO_CIPHER);
             SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_NO_CIPHER_MATCH);
             return (-1);
        }
        s->session->cipher = cp;

        p += 3;
        n2s(p, i);
        s->s2->tmp.clear = i;
        n2s(p, i);
        s->s2->tmp.enc = i;
        n2s(p, i);
        if (i > SSL_MAX_KEY_ARG_LENGTH) {
            ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
            SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_KEY_ARG_TOO_LONG);
            return -1;
        }
        s->session->key_arg_length = i;
        s->state = SSL2_ST_GET_CLIENT_MASTER_KEY_B;
    }

    /* SSL2_ST_GET_CLIENT_MASTER_KEY_B */
    p = (unsigned char *)s->init_buf->data;
    if (s->init_buf->length < SSL2_MAX_RECORD_LENGTH_3_BYTE_HEADER) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, ERR_R_INTERNAL_ERROR);
        return -1;
    }
    keya = s->session->key_arg_length;
    len =
        10 + (unsigned long)s->s2->tmp.clear + (unsigned long)s->s2->tmp.enc +
        (unsigned long)keya;
    if (len > SSL2_MAX_RECORD_LENGTH_3_BYTE_HEADER) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_MESSAGE_TOO_LONG);
        return -1;
    }
    n = (int)len - s->init_num;
    i = ssl2_read(s, (char *)&(p[s->init_num]), n);
    if (i != n)
        return (ssl2_part_read(s, SSL_F_GET_CLIENT_MASTER_KEY, i));
    if (s->msg_callback) {
        /* CLIENT-MASTER-KEY */
        s->msg_callback(0, s->version, 0, p, (size_t)len, s,
                        s->msg_callback_arg);
    }
    p += 10;

    memcpy(s->session->key_arg, &(p[s->s2->tmp.clear + s->s2->tmp.enc]),
           (unsigned int)keya);

    if (s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey == NULL) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, SSL_R_NO_PRIVATEKEY);
        return (-1);
    }

    is_export = SSL_C_IS_EXPORT(s->session->cipher);

    if (!ssl_cipher_get_evp(s->session, &c, &md, NULL, NULL, NULL)) {
        ssl2_return_error(s, SSL2_PE_NO_CIPHER);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,
               SSL_R_PROBLEMS_MAPPING_CIPHER_FUNCTIONS);
        return (0);
    }

    /*
     * The format of the CLIENT-MASTER-KEY message is
     * 1 byte message type
     * 3 bytes cipher
     * 2-byte clear key length (stored in s->s2->tmp.clear)
     * 2-byte encrypted key length (stored in s->s2->tmp.enc)
     * 2-byte key args length (IV etc)
     * clear key
     * encrypted key
     * key args
     *
     * If the cipher is an export cipher, then the encrypted key bytes
     * are a fixed portion of the total key (5 or 8 bytes). The size of
     * this portion is in |num_encrypted_key_bytes|. If the cipher is not an
     * export cipher, then the entire key material is encrypted (i.e., clear
     * key length must be zero).
     */
    key_length = (unsigned int)EVP_CIPHER_key_length(c);
    if (key_length > SSL_MAX_MASTER_KEY_LENGTH) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY, ERR_R_INTERNAL_ERROR);
        return -1;
    }

    if (s->session->cipher->algorithm2 & SSL2_CF_8_BYTE_ENC) {
        is_export = 1;
        num_encrypted_key_bytes = 8;
    } else if (is_export) {
        num_encrypted_key_bytes = 5;
    } else {
        num_encrypted_key_bytes = key_length;
    }

    if (s->s2->tmp.clear + num_encrypted_key_bytes != key_length) {
        ssl2_return_error(s, SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,SSL_R_BAD_LENGTH);
        return -1;
    }
    /*
     * The encrypted blob must decrypt to the encrypted portion of the key.
     * Decryption can't be expanding, so if we don't have enough encrypted
     * bytes to fit the key in the buffer, stop now.
     */
    if (s->s2->tmp.enc < num_encrypted_key_bytes) {
        ssl2_return_error(s,SSL2_PE_UNDEFINED_ERROR);
        SSLerr(SSL_F_GET_CLIENT_MASTER_KEY,SSL_R_LENGTH_TOO_SHORT);
        return -1;
    }

    /*
     * We must not leak whether a decryption failure occurs because of
     * Bleichenbacher's attack on PKCS #1 v1.5 RSA padding (see RFC 2246,
     * section 7.4.7.1). The code follows that advice of the TLS RFC and
     * generates a random premaster secret for the case that the decrypt
     * fails. See https://tools.ietf.org/html/rfc5246#section-7.4.7.1
     */

    /*
     * should be RAND_bytes, but we cannot work around a failure.
     */
    if (RAND_pseudo_bytes(rand_premaster_secret,
                          (int)num_encrypted_key_bytes) <= 0)
        return 0;

    i = ssl_rsa_private_decrypt(s->cert, s->s2->tmp.enc,
                                &(p[s->s2->tmp.clear]),
                                &(p[s->s2->tmp.clear]),
                                (s->s2->ssl2_rollback) ? RSA_SSLV23_PADDING :
                                RSA_PKCS1_PADDING);
    ERR_clear_error();
    /*
     * If a bad decrypt, continue with protocol but with a random master
     * secret (Bleichenbacher attack)
     */
    decrypt_good = constant_time_eq_int_8(i, (int)num_encrypted_key_bytes);
    for (j = 0; j < num_encrypted_key_bytes; j++) {
        p[s->s2->tmp.clear + j] =
                constant_time_select_8(decrypt_good, p[s->s2->tmp.clear + j],
                                       rand_premaster_secret[j]);
    }

    s->session->master_key_length = (int)key_length;
    memcpy(s->session->master_key, p, key_length);
    OPENSSL_cleanse(p, key_length);

    return 1;
}


RSA key generation: ensure BN_mod_inverse and BN_mod_exp_mont both get called with BN_FLG_CONSTTIME flag set.

CVE-2018-0737

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>

static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
                              BN_GENCB *cb)
{
    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
    int bitsp, bitsq, ok = -1, n = 0;
    BN_CTX *ctx = NULL;
    unsigned long error = 0;

    /*
     * When generating ridiculously small keys, we can get stuck
     * continually regenerating the same prime values.
     */
    if (bits < 16) {
        ok = 0;             /* we set our own err */
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
        goto err;
    }

    ctx = BN_CTX_new();
    if (ctx == NULL)
        goto err;
    BN_CTX_start(ctx);
    r0 = BN_CTX_get(ctx);
    r1 = BN_CTX_get(ctx);
    r2 = BN_CTX_get(ctx);
    r3 = BN_CTX_get(ctx);
    if (r3 == NULL)
        goto err;

    bitsp = (bits + 1) / 2;
    bitsq = bits - bitsp;

    /* We need the RSA components non-NULL */
    if (!rsa->n && ((rsa->n = BN_new()) == NULL))
        goto err;
    if (!rsa->d && ((rsa->d = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->e && ((rsa->e = BN_new()) == NULL))
        goto err;
    if (!rsa->p && ((rsa->p = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->q && ((rsa->q = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->dmp1 && ((rsa->dmp1 = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->dmq1 && ((rsa->dmq1 = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->iqmp && ((rsa->iqmp = BN_secure_new()) == NULL))
        goto err;

     if (BN_copy(rsa->e, e_value) == NULL)
         goto err;

     BN_set_flags(r2, BN_FLG_CONSTTIME);
     /* generate p and q */
     for (;;) {
        if (!BN_sub(r2, rsa->p, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 0))
        goto err;
    for (;;) {
        do {
            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                goto err;
        } while (BN_cmp(rsa->p, rsa->q) == 0);
        if (!BN_sub(r2, rsa->q, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 1))
        goto err;
    if (BN_cmp(rsa->p, rsa->q) < 0) {
        tmp = rsa->p;
        rsa->p = rsa->q;
        rsa->q = tmp;
    }

    /* calculate n */
    if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
        goto err;

    /* calculate d */
    if (!BN_sub(r1, rsa->p, BN_value_one()))
        goto err;               /* p-1 */
    if (!BN_sub(r2, rsa->q, BN_value_one()))
        goto err;               /* q-1 */
    if (!BN_mul(r0, r1, r2, ctx))
        goto err;               /* (p-1)(q-1) */
    {
        BIGNUM *pr0 = BN_new();

        if (pr0 == NULL)
            goto err;
        BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);
        if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) {
            BN_free(pr0);
            goto err;               /* d */
        }
        /* We MUST free pr0 before any further use of r0 */
        BN_free(pr0);
    }

    {
        BIGNUM *d = BN_new();

        if (d == NULL)
            goto err;
        BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);

        if (   /* calculate d mod (p-1) */
               !BN_mod(rsa->dmp1, d, r1, ctx)
               /* calculate d mod (q-1) */
            || !BN_mod(rsa->dmq1, d, r2, ctx)) {
            BN_free(d);
            goto err;
        }
        /* We MUST free d before any further use of rsa->d */
        BN_free(d);
    }

    {
        BIGNUM *p = BN_new();

        if (p == NULL)
            goto err;
        BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);

        /* calculate inverse of q mod p */
        if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) {
            BN_free(p);
            goto err;
        }
        /* We MUST free p before any further use of rsa->p */
        BN_free(p);
    }

    ok = 1;
 err:
    if (ok == -1) {
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);
        ok = 0;
    }
    if (ctx != NULL)
        BN_CTX_end(ctx);
    BN_CTX_free(ctx);

    return ok;
}

The OpenSSL RSA Key generation algorithm has been shown to be vulnerable to a cache timing side channel attack. An attacker with sufficient access to mount cache timing attacks during the RSA key generation process could recover the private key. Fixed in OpenSSL 1.1.0i-dev (Affected 1.1.0-1.1.0h). Fixed in OpenSSL 1.0.2p-dev (Affected 1.0.2b-1.0.2o).


static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
                              BN_GENCB *cb)
{
    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
    int bitsp, bitsq, ok = -1, n = 0;
    BN_CTX *ctx = NULL;
    unsigned long error = 0;

    /*
     * When generating ridiculously small keys, we can get stuck
     * continually regenerating the same prime values.
     */
    if (bits < 16) {
        ok = 0;             /* we set our own err */
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
        goto err;
    }

    ctx = BN_CTX_new();
    if (ctx == NULL)
        goto err;
    BN_CTX_start(ctx);
    r0 = BN_CTX_get(ctx);
    r1 = BN_CTX_get(ctx);
    r2 = BN_CTX_get(ctx);
    r3 = BN_CTX_get(ctx);
    if (r3 == NULL)
        goto err;

    bitsp = (bits + 1) / 2;
    bitsq = bits - bitsp;

    /* We need the RSA components non-NULL */
    if (!rsa->n && ((rsa->n = BN_new()) == NULL))
        goto err;
    if (!rsa->d && ((rsa->d = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->e && ((rsa->e = BN_new()) == NULL))
        goto err;
    if (!rsa->p && ((rsa->p = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->q && ((rsa->q = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->dmp1 && ((rsa->dmp1 = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->dmq1 && ((rsa->dmq1 = BN_secure_new()) == NULL))
        goto err;
    if (!rsa->iqmp && ((rsa->iqmp = BN_secure_new()) == NULL))
        goto err;

     if (BN_copy(rsa->e, e_value) == NULL)
         goto err;

    BN_set_flags(rsa->p, BN_FLG_CONSTTIME);
    BN_set_flags(rsa->q, BN_FLG_CONSTTIME);
     BN_set_flags(r2, BN_FLG_CONSTTIME);
     /* generate p and q */
     for (;;) {
        if (!BN_sub(r2, rsa->p, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 0))
        goto err;
    for (;;) {
        do {
            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                goto err;
        } while (BN_cmp(rsa->p, rsa->q) == 0);
        if (!BN_sub(r2, rsa->q, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 1))
        goto err;
    if (BN_cmp(rsa->p, rsa->q) < 0) {
        tmp = rsa->p;
        rsa->p = rsa->q;
        rsa->q = tmp;
    }

    /* calculate n */
    if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
        goto err;

    /* calculate d */
    if (!BN_sub(r1, rsa->p, BN_value_one()))
        goto err;               /* p-1 */
    if (!BN_sub(r2, rsa->q, BN_value_one()))
        goto err;               /* q-1 */
    if (!BN_mul(r0, r1, r2, ctx))
        goto err;               /* (p-1)(q-1) */
    {
        BIGNUM *pr0 = BN_new();

        if (pr0 == NULL)
            goto err;
        BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);
        if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx)) {
            BN_free(pr0);
            goto err;               /* d */
        }
        /* We MUST free pr0 before any further use of r0 */
        BN_free(pr0);
    }

    {
        BIGNUM *d = BN_new();

        if (d == NULL)
            goto err;
        BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);

        if (   /* calculate d mod (p-1) */
               !BN_mod(rsa->dmp1, d, r1, ctx)
               /* calculate d mod (q-1) */
            || !BN_mod(rsa->dmq1, d, r2, ctx)) {
            BN_free(d);
            goto err;
        }
        /* We MUST free d before any further use of rsa->d */
        BN_free(d);
    }

    {
        BIGNUM *p = BN_new();

        if (p == NULL)
            goto err;
        BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);

        /* calculate inverse of q mod p */
        if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx)) {
            BN_free(p);
            goto err;
        }
        /* We MUST free p before any further use of rsa->p */
        BN_free(p);
    }

    ok = 1;
 err:
    if (ok == -1) {
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);
        ok = 0;
    }
    if (ctx != NULL)
        BN_CTX_end(ctx);
    BN_CTX_free(ctx);

    return ok;
}


RSA key generation: ensure BN_mod_inverse and BN_mod_exp_mont both get called with BN_FLG_CONSTTIME flag set.

CVE-2018-0737

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(cherry picked from commit 6939eab03a6e23d2bd2c3f5e34fe1d48e542e787)

static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
                              BN_GENCB *cb)
{
    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
    BIGNUM local_r0, local_d, local_p;
    BIGNUM *pr0, *d, *p;
    int bitsp, bitsq, ok = -1, n = 0;
    BN_CTX *ctx = NULL;
    unsigned long error = 0;

    /*
     * When generating ridiculously small keys, we can get stuck
     * continually regenerating the same prime values.
     */
    if (bits < 16) {
        ok = 0;             /* we set our own err */
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
        goto err;
    }

    ctx = BN_CTX_new();
    if (ctx == NULL)
        goto err;
    BN_CTX_start(ctx);
    r0 = BN_CTX_get(ctx);
    r1 = BN_CTX_get(ctx);
    r2 = BN_CTX_get(ctx);
    r3 = BN_CTX_get(ctx);
    if (r3 == NULL)
        goto err;

    bitsp = (bits + 1) / 2;
    bitsq = bits - bitsp;

    /* We need the RSA components non-NULL */
    if (!rsa->n && ((rsa->n = BN_new()) == NULL))
        goto err;
    if (!rsa->d && ((rsa->d = BN_new()) == NULL))
        goto err;
    if (!rsa->e && ((rsa->e = BN_new()) == NULL))
        goto err;
    if (!rsa->p && ((rsa->p = BN_new()) == NULL))
        goto err;
    if (!rsa->q && ((rsa->q = BN_new()) == NULL))
        goto err;
    if (!rsa->dmp1 && ((rsa->dmp1 = BN_new()) == NULL))
        goto err;
    if (!rsa->dmq1 && ((rsa->dmq1 = BN_new()) == NULL))
        goto err;
    if (!rsa->iqmp && ((rsa->iqmp = BN_new()) == NULL))
        goto err;

     if (BN_copy(rsa->e, e_value) == NULL)
         goto err;

     BN_set_flags(r2, BN_FLG_CONSTTIME);
     /* generate p and q */
     for (;;) {
        if (!BN_sub(r2, rsa->p, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 0))
        goto err;
    for (;;) {
        do {
            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                goto err;
        } while (BN_cmp(rsa->p, rsa->q) == 0);
        if (!BN_sub(r2, rsa->q, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 1))
        goto err;
    if (BN_cmp(rsa->p, rsa->q) < 0) {
        tmp = rsa->p;
        rsa->p = rsa->q;
        rsa->q = tmp;
    }

    /* calculate n */
    if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
        goto err;

    /* calculate d */
    if (!BN_sub(r1, rsa->p, BN_value_one()))
        goto err;               /* p-1 */
    if (!BN_sub(r2, rsa->q, BN_value_one()))
        goto err;               /* q-1 */
    if (!BN_mul(r0, r1, r2, ctx))
        goto err;               /* (p-1)(q-1) */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        pr0 = &local_r0;
        BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);
    } else
        pr0 = r0;
    if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx))
        goto err;               /* d */

    /* set up d for correct BN_FLG_CONSTTIME flag */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        d = &local_d;
        BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
    } else
        d = rsa->d;

    /* calculate d mod (p-1) */
    if (!BN_mod(rsa->dmp1, d, r1, ctx))
        goto err;

    /* calculate d mod (q-1) */
    if (!BN_mod(rsa->dmq1, d, r2, ctx))
        goto err;

    /* calculate inverse of q mod p */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        p = &local_p;
        BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);
    } else
        p = rsa->p;
    if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx))
        goto err;

    ok = 1;
 err:
    if (ok == -1) {
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);
        ok = 0;
    }
    if (ctx != NULL) {
        BN_CTX_end(ctx);
        BN_CTX_free(ctx);
    }

    return ok;
}


static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
                              BN_GENCB *cb)
{
    BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
    BIGNUM local_r0, local_d, local_p;
    BIGNUM *pr0, *d, *p;
    int bitsp, bitsq, ok = -1, n = 0;
    BN_CTX *ctx = NULL;
    unsigned long error = 0;

    /*
     * When generating ridiculously small keys, we can get stuck
     * continually regenerating the same prime values.
     */
    if (bits < 16) {
        ok = 0;             /* we set our own err */
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
        goto err;
    }

    ctx = BN_CTX_new();
    if (ctx == NULL)
        goto err;
    BN_CTX_start(ctx);
    r0 = BN_CTX_get(ctx);
    r1 = BN_CTX_get(ctx);
    r2 = BN_CTX_get(ctx);
    r3 = BN_CTX_get(ctx);
    if (r3 == NULL)
        goto err;

    bitsp = (bits + 1) / 2;
    bitsq = bits - bitsp;

    /* We need the RSA components non-NULL */
    if (!rsa->n && ((rsa->n = BN_new()) == NULL))
        goto err;
    if (!rsa->d && ((rsa->d = BN_new()) == NULL))
        goto err;
    if (!rsa->e && ((rsa->e = BN_new()) == NULL))
        goto err;
    if (!rsa->p && ((rsa->p = BN_new()) == NULL))
        goto err;
    if (!rsa->q && ((rsa->q = BN_new()) == NULL))
        goto err;
    if (!rsa->dmp1 && ((rsa->dmp1 = BN_new()) == NULL))
        goto err;
    if (!rsa->dmq1 && ((rsa->dmq1 = BN_new()) == NULL))
        goto err;
    if (!rsa->iqmp && ((rsa->iqmp = BN_new()) == NULL))
        goto err;

     if (BN_copy(rsa->e, e_value) == NULL)
         goto err;

    BN_set_flags(rsa->p, BN_FLG_CONSTTIME);
    BN_set_flags(rsa->q, BN_FLG_CONSTTIME);
     BN_set_flags(r2, BN_FLG_CONSTTIME);
     /* generate p and q */
     for (;;) {
        if (!BN_sub(r2, rsa->p, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 0))
        goto err;
    for (;;) {
        do {
            if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                goto err;
        } while (BN_cmp(rsa->p, rsa->q) == 0);
        if (!BN_sub(r2, rsa->q, BN_value_one()))
            goto err;
        ERR_set_mark();
        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
            /* GCD == 1 since inverse exists */
            break;
        }
        error = ERR_peek_last_error();
        if (ERR_GET_LIB(error) == ERR_LIB_BN
            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
            /* GCD != 1 */
            ERR_pop_to_mark();
        } else {
            goto err;
        }
        if (!BN_GENCB_call(cb, 2, n++))
            goto err;
    }
    if (!BN_GENCB_call(cb, 3, 1))
        goto err;
    if (BN_cmp(rsa->p, rsa->q) < 0) {
        tmp = rsa->p;
        rsa->p = rsa->q;
        rsa->q = tmp;
    }

    /* calculate n */
    if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
        goto err;

    /* calculate d */
    if (!BN_sub(r1, rsa->p, BN_value_one()))
        goto err;               /* p-1 */
    if (!BN_sub(r2, rsa->q, BN_value_one()))
        goto err;               /* q-1 */
    if (!BN_mul(r0, r1, r2, ctx))
        goto err;               /* (p-1)(q-1) */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        pr0 = &local_r0;
        BN_with_flags(pr0, r0, BN_FLG_CONSTTIME);
    } else
        pr0 = r0;
    if (!BN_mod_inverse(rsa->d, rsa->e, pr0, ctx))
        goto err;               /* d */

    /* set up d for correct BN_FLG_CONSTTIME flag */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        d = &local_d;
        BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
    } else
        d = rsa->d;

    /* calculate d mod (p-1) */
    if (!BN_mod(rsa->dmp1, d, r1, ctx))
        goto err;

    /* calculate d mod (q-1) */
    if (!BN_mod(rsa->dmq1, d, r2, ctx))
        goto err;

    /* calculate inverse of q mod p */
    if (!(rsa->flags & RSA_FLAG_NO_CONSTTIME)) {
        p = &local_p;
        BN_with_flags(p, rsa->p, BN_FLG_CONSTTIME);
    } else
        p = rsa->p;
    if (!BN_mod_inverse(rsa->iqmp, rsa->q, p, ctx))
        goto err;

    ok = 1;
 err:
    if (ok == -1) {
        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, ERR_LIB_BN);
        ok = 0;
    }
    if (ctx != NULL) {
        BN_CTX_end(ctx);
        BN_CTX_free(ctx);
    }

    return ok;
}


Timing vulnerability in ECDSA signature generation (CVE-2018-0735)

Preallocate an extra limb for some of the big numbers to avoid a reallocation
that can potentially provide a side channel.

Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/7486)

(cherry picked from commit 99540ec79491f59ed8b46b4edf130e17dc907f52)

static int ec_mul_consttime(const EC_GROUP *group, EC_POINT *r,
                            const BIGNUM *scalar, const EC_POINT *point,
                            BN_CTX *ctx)
{
    int i, cardinality_bits, group_top, kbit, pbit, Z_is_one;
    EC_POINT *s = NULL;
    BIGNUM *k = NULL;
    BIGNUM *lambda = NULL;
    BIGNUM *cardinality = NULL;
    BN_CTX *new_ctx = NULL;
    int ret = 0;

    if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new()) == NULL)
        return 0;

    BN_CTX_start(ctx);

    s = EC_POINT_new(group);
    if (s == NULL)
        goto err;

    if (point == NULL) {
        if (!EC_POINT_copy(s, group->generator))
            goto err;
    } else {
        if (!EC_POINT_copy(s, point))
            goto err;
    }

    EC_POINT_BN_set_flags(s, BN_FLG_CONSTTIME);

    cardinality = BN_CTX_get(ctx);
    lambda = BN_CTX_get(ctx);
    k = BN_CTX_get(ctx);
    if (k == NULL || !BN_mul(cardinality, group->order, group->cofactor, ctx))
        goto err;

    /*
     * Group cardinalities are often on a word boundary.
     * So when we pad the scalar, some timing diff might
     * pop if it needs to be expanded due to carries.
     * So expand ahead of time.
      */
     cardinality_bits = BN_num_bits(cardinality);
     group_top = bn_get_top(cardinality);
    if ((bn_wexpand(k, group_top + 1) == NULL)
        || (bn_wexpand(lambda, group_top + 1) == NULL))
         goto err;

     if (!BN_copy(k, scalar))
        goto err;

    BN_set_flags(k, BN_FLG_CONSTTIME);

    if ((BN_num_bits(k) > cardinality_bits) || (BN_is_negative(k))) {
        /*-
         * this is an unusual input, and we don't guarantee
         * constant-timeness
         */
        if (!BN_nnmod(k, k, cardinality, ctx))
            goto err;
    }

    if (!BN_add(lambda, k, cardinality))
        goto err;
    BN_set_flags(lambda, BN_FLG_CONSTTIME);
    if (!BN_add(k, lambda, cardinality))
        goto err;
    /*
     * lambda := scalar + cardinality
      * k := scalar + 2*cardinality
      */
     kbit = BN_is_bit_set(lambda, cardinality_bits);
    BN_consttime_swap(kbit, k, lambda, group_top + 1);

     group_top = bn_get_top(group->field);
     if ((bn_wexpand(s->X, group_top) == NULL)
        || (bn_wexpand(s->Y, group_top) == NULL)
        || (bn_wexpand(s->Z, group_top) == NULL)
        || (bn_wexpand(r->X, group_top) == NULL)
        || (bn_wexpand(r->Y, group_top) == NULL)
        || (bn_wexpand(r->Z, group_top) == NULL))
        goto err;

    /*-
     * Apply coordinate blinding for EC_POINT.
     *
     * The underlying EC_METHOD can optionally implement this function:
     * ec_point_blind_coordinates() returns 0 in case of errors or 1 on
     * success or if coordinate blinding is not implemented for this
     * group.
     */
    if (!ec_point_blind_coordinates(group, s, ctx))
        goto err;

    /* top bit is a 1, in a fixed pos */
    if (!EC_POINT_copy(r, s))
        goto err;

    EC_POINT_BN_set_flags(r, BN_FLG_CONSTTIME);

    if (!EC_POINT_dbl(group, s, s, ctx))
        goto err;

    pbit = 0;

#define EC_POINT_CSWAP(c, a, b, w, t) do {         \
        BN_consttime_swap(c, (a)->X, (b)->X, w);   \
        BN_consttime_swap(c, (a)->Y, (b)->Y, w);   \
        BN_consttime_swap(c, (a)->Z, (b)->Z, w);   \
        t = ((a)->Z_is_one ^ (b)->Z_is_one) & (c); \
        (a)->Z_is_one ^= (t);                      \
        (b)->Z_is_one ^= (t);                      \
} while(0)

    /*-
     * The ladder step, with branches, is
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * Swapping R, S conditionally on k[i] leaves you with state
     *
     * k[i] == 0: T, U = R, S
     * k[i] == 1: T, U = S, R
     *
     * Then perform the ECC ops.
     *
     * U = add(T, U)
     * T = dbl(T)
     *
     * Which leaves you with state
     *
     * k[i] == 0: U = add(R, S), T = dbl(R)
     * k[i] == 1: U = add(S, R), T = dbl(S)
     *
     * Swapping T, U conditionally on k[i] leaves you with state
     *
     * k[i] == 0: R, S = T, U
     * k[i] == 1: R, S = U, T
     *
     * Which leaves you with state
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * So we get the same logic, but instead of a branch it's a
     * conditional swap, followed by ECC ops, then another conditional swap.
     *
     * Optimization: The end of iteration i and start of i-1 looks like
     *
     * ...
     * CSWAP(k[i], R, S)
     * ECC
     * CSWAP(k[i], R, S)
     * (next iteration)
     * CSWAP(k[i-1], R, S)
     * ECC
     * CSWAP(k[i-1], R, S)
     * ...
     *
     * So instead of two contiguous swaps, you can merge the condition
     * bits and do a single swap.
     *
     * k[i]   k[i-1]    Outcome
     * 0      0         No Swap
     * 0      1         Swap
     * 1      0         Swap
     * 1      1         No Swap
     *
     * This is XOR. pbit tracks the previous bit of k.
     */

    for (i = cardinality_bits - 1; i >= 0; i--) {
        kbit = BN_is_bit_set(k, i) ^ pbit;
        EC_POINT_CSWAP(kbit, r, s, group_top, Z_is_one);
        if (!EC_POINT_add(group, s, r, s, ctx))
            goto err;
        if (!EC_POINT_dbl(group, r, r, ctx))
            goto err;
        /*
         * pbit logic merges this cswap with that of the
         * next iteration
         */
        pbit ^= kbit;
    }
    /* one final cswap to move the right value into r */
    EC_POINT_CSWAP(pbit, r, s, group_top, Z_is_one);
#undef EC_POINT_CSWAP

    ret = 1;

 err:
    EC_POINT_free(s);
    BN_CTX_end(ctx);
    BN_CTX_free(new_ctx);

    return ret;
}

The OpenSSL ECDSA signature algorithm has been shown to be vulnerable to a timing side channel attack. An attacker could use variations in the signing algorithm to recover the private key. Fixed in OpenSSL 1.1.0j (Affected 1.1.0-1.1.0i). Fixed in OpenSSL 1.1.1a (Affected 1.1.1).


static int ec_mul_consttime(const EC_GROUP *group, EC_POINT *r,
                            const BIGNUM *scalar, const EC_POINT *point,
                            BN_CTX *ctx)
{
    int i, cardinality_bits, group_top, kbit, pbit, Z_is_one;
    EC_POINT *s = NULL;
    BIGNUM *k = NULL;
    BIGNUM *lambda = NULL;
    BIGNUM *cardinality = NULL;
    BN_CTX *new_ctx = NULL;
    int ret = 0;

    if (ctx == NULL && (ctx = new_ctx = BN_CTX_secure_new()) == NULL)
        return 0;

    BN_CTX_start(ctx);

    s = EC_POINT_new(group);
    if (s == NULL)
        goto err;

    if (point == NULL) {
        if (!EC_POINT_copy(s, group->generator))
            goto err;
    } else {
        if (!EC_POINT_copy(s, point))
            goto err;
    }

    EC_POINT_BN_set_flags(s, BN_FLG_CONSTTIME);

    cardinality = BN_CTX_get(ctx);
    lambda = BN_CTX_get(ctx);
    k = BN_CTX_get(ctx);
    if (k == NULL || !BN_mul(cardinality, group->order, group->cofactor, ctx))
        goto err;

    /*
     * Group cardinalities are often on a word boundary.
     * So when we pad the scalar, some timing diff might
     * pop if it needs to be expanded due to carries.
     * So expand ahead of time.
      */
     cardinality_bits = BN_num_bits(cardinality);
     group_top = bn_get_top(cardinality);
    if ((bn_wexpand(k, group_top + 2) == NULL)
        || (bn_wexpand(lambda, group_top + 2) == NULL)) {
         goto err;

     if (!BN_copy(k, scalar))
        goto err;

    BN_set_flags(k, BN_FLG_CONSTTIME);

    if ((BN_num_bits(k) > cardinality_bits) || (BN_is_negative(k))) {
        /*-
         * this is an unusual input, and we don't guarantee
         * constant-timeness
         */
        if (!BN_nnmod(k, k, cardinality, ctx))
            goto err;
    }

    if (!BN_add(lambda, k, cardinality))
        goto err;
    BN_set_flags(lambda, BN_FLG_CONSTTIME);
    if (!BN_add(k, lambda, cardinality))
        goto err;
    /*
     * lambda := scalar + cardinality
      * k := scalar + 2*cardinality
      */
     kbit = BN_is_bit_set(lambda, cardinality_bits);
    BN_consttime_swap(kbit, k, lambda, group_top + 2);

     group_top = bn_get_top(group->field);
     if ((bn_wexpand(s->X, group_top) == NULL)
        || (bn_wexpand(s->Y, group_top) == NULL)
        || (bn_wexpand(s->Z, group_top) == NULL)
        || (bn_wexpand(r->X, group_top) == NULL)
        || (bn_wexpand(r->Y, group_top) == NULL)
        || (bn_wexpand(r->Z, group_top) == NULL))
        goto err;

    /*-
     * Apply coordinate blinding for EC_POINT.
     *
     * The underlying EC_METHOD can optionally implement this function:
     * ec_point_blind_coordinates() returns 0 in case of errors or 1 on
     * success or if coordinate blinding is not implemented for this
     * group.
     */
    if (!ec_point_blind_coordinates(group, s, ctx))
        goto err;

    /* top bit is a 1, in a fixed pos */
    if (!EC_POINT_copy(r, s))
        goto err;

    EC_POINT_BN_set_flags(r, BN_FLG_CONSTTIME);

    if (!EC_POINT_dbl(group, s, s, ctx))
        goto err;

    pbit = 0;

#define EC_POINT_CSWAP(c, a, b, w, t) do {         \
        BN_consttime_swap(c, (a)->X, (b)->X, w);   \
        BN_consttime_swap(c, (a)->Y, (b)->Y, w);   \
        BN_consttime_swap(c, (a)->Z, (b)->Z, w);   \
        t = ((a)->Z_is_one ^ (b)->Z_is_one) & (c); \
        (a)->Z_is_one ^= (t);                      \
        (b)->Z_is_one ^= (t);                      \
} while(0)

    /*-
     * The ladder step, with branches, is
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * Swapping R, S conditionally on k[i] leaves you with state
     *
     * k[i] == 0: T, U = R, S
     * k[i] == 1: T, U = S, R
     *
     * Then perform the ECC ops.
     *
     * U = add(T, U)
     * T = dbl(T)
     *
     * Which leaves you with state
     *
     * k[i] == 0: U = add(R, S), T = dbl(R)
     * k[i] == 1: U = add(S, R), T = dbl(S)
     *
     * Swapping T, U conditionally on k[i] leaves you with state
     *
     * k[i] == 0: R, S = T, U
     * k[i] == 1: R, S = U, T
     *
     * Which leaves you with state
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * So we get the same logic, but instead of a branch it's a
     * conditional swap, followed by ECC ops, then another conditional swap.
     *
     * Optimization: The end of iteration i and start of i-1 looks like
     *
     * ...
     * CSWAP(k[i], R, S)
     * ECC
     * CSWAP(k[i], R, S)
     * (next iteration)
     * CSWAP(k[i-1], R, S)
     * ECC
     * CSWAP(k[i-1], R, S)
     * ...
     *
     * So instead of two contiguous swaps, you can merge the condition
     * bits and do a single swap.
     *
     * k[i]   k[i-1]    Outcome
     * 0      0         No Swap
     * 0      1         Swap
     * 1      0         Swap
     * 1      1         No Swap
     *
     * This is XOR. pbit tracks the previous bit of k.
     */

    for (i = cardinality_bits - 1; i >= 0; i--) {
        kbit = BN_is_bit_set(k, i) ^ pbit;
        EC_POINT_CSWAP(kbit, r, s, group_top, Z_is_one);
        if (!EC_POINT_add(group, s, r, s, ctx))
            goto err;
        if (!EC_POINT_dbl(group, r, r, ctx))
            goto err;
        /*
         * pbit logic merges this cswap with that of the
         * next iteration
         */
        pbit ^= kbit;
    }
    /* one final cswap to move the right value into r */
    EC_POINT_CSWAP(pbit, r, s, group_top, Z_is_one);
#undef EC_POINT_CSWAP

    ret = 1;

 err:
    EC_POINT_free(s);
    BN_CTX_end(ctx);
    BN_CTX_free(new_ctx);

    return ret;
}


int ec_scalar_mul_ladder(const EC_GROUP *group, EC_POINT *r,
                         const BIGNUM *scalar, const EC_POINT *point,
                         BN_CTX *ctx)
{
    int i, cardinality_bits, group_top, kbit, pbit, Z_is_one;
    EC_POINT *p = NULL;
    EC_POINT *s = NULL;
    BIGNUM *k = NULL;
    BIGNUM *lambda = NULL;
    BIGNUM *cardinality = NULL;
    int ret = 0;

    /* early exit if the input point is the point at infinity */
    if (point != NULL && EC_POINT_is_at_infinity(group, point))
        return EC_POINT_set_to_infinity(group, r);

    if (BN_is_zero(group->order)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_UNKNOWN_ORDER);
        return 0;
    }
    if (BN_is_zero(group->cofactor)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_UNKNOWN_COFACTOR);
        return 0;
    }

    BN_CTX_start(ctx);

    if (((p = EC_POINT_new(group)) == NULL)
        || ((s = EC_POINT_new(group)) == NULL)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    if (point == NULL) {
        if (!EC_POINT_copy(p, group->generator)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_EC_LIB);
            goto err;
        }
    } else {
        if (!EC_POINT_copy(p, point)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_EC_LIB);
            goto err;
        }
    }

    EC_POINT_BN_set_flags(p, BN_FLG_CONSTTIME);
    EC_POINT_BN_set_flags(r, BN_FLG_CONSTTIME);
    EC_POINT_BN_set_flags(s, BN_FLG_CONSTTIME);

    cardinality = BN_CTX_get(ctx);
    lambda = BN_CTX_get(ctx);
    k = BN_CTX_get(ctx);
    if (k == NULL) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    if (!BN_mul(cardinality, group->order, group->cofactor, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    /*
     * Group cardinalities are often on a word boundary.
     * So when we pad the scalar, some timing diff might
     * pop if it needs to be expanded due to carries.
     * So expand ahead of time.
      */
     cardinality_bits = BN_num_bits(cardinality);
     group_top = bn_get_top(cardinality);
    if ((bn_wexpand(k, group_top + 1) == NULL)
        || (bn_wexpand(lambda, group_top + 1) == NULL)) {
         ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
         goto err;
     }

    if (!BN_copy(k, scalar)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    BN_set_flags(k, BN_FLG_CONSTTIME);

    if ((BN_num_bits(k) > cardinality_bits) || (BN_is_negative(k))) {
        /*-
         * this is an unusual input, and we don't guarantee
         * constant-timeness
         */
        if (!BN_nnmod(k, k, cardinality, ctx)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
            goto err;
        }
    }

    if (!BN_add(lambda, k, cardinality)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }
    BN_set_flags(lambda, BN_FLG_CONSTTIME);
    if (!BN_add(k, lambda, cardinality)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }
    /*
     * lambda := scalar + cardinality
      * k := scalar + 2*cardinality
      */
     kbit = BN_is_bit_set(lambda, cardinality_bits);
    BN_consttime_swap(kbit, k, lambda, group_top + 1);

     group_top = bn_get_top(group->field);
     if ((bn_wexpand(s->X, group_top) == NULL)
        || (bn_wexpand(s->Y, group_top) == NULL)
        || (bn_wexpand(s->Z, group_top) == NULL)
        || (bn_wexpand(r->X, group_top) == NULL)
        || (bn_wexpand(r->Y, group_top) == NULL)
        || (bn_wexpand(r->Z, group_top) == NULL)
        || (bn_wexpand(p->X, group_top) == NULL)
        || (bn_wexpand(p->Y, group_top) == NULL)
        || (bn_wexpand(p->Z, group_top) == NULL)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    /*-
     * Apply coordinate blinding for EC_POINT.
     *
     * The underlying EC_METHOD can optionally implement this function:
     * ec_point_blind_coordinates() returns 0 in case of errors or 1 on
     * success or if coordinate blinding is not implemented for this
     * group.
     */
    if (!ec_point_blind_coordinates(group, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_POINT_COORDINATES_BLIND_FAILURE);
        goto err;
    }

    /* Initialize the Montgomery ladder */
    if (!ec_point_ladder_pre(group, r, s, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_PRE_FAILURE);
        goto err;
    }

    /* top bit is a 1, in a fixed pos */
    pbit = 1;

#define EC_POINT_CSWAP(c, a, b, w, t) do {         \
        BN_consttime_swap(c, (a)->X, (b)->X, w);   \
        BN_consttime_swap(c, (a)->Y, (b)->Y, w);   \
        BN_consttime_swap(c, (a)->Z, (b)->Z, w);   \
        t = ((a)->Z_is_one ^ (b)->Z_is_one) & (c); \
        (a)->Z_is_one ^= (t);                      \
        (b)->Z_is_one ^= (t);                      \
} while(0)

    /*-
     * The ladder step, with branches, is
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * Swapping R, S conditionally on k[i] leaves you with state
     *
     * k[i] == 0: T, U = R, S
     * k[i] == 1: T, U = S, R
     *
     * Then perform the ECC ops.
     *
     * U = add(T, U)
     * T = dbl(T)
     *
     * Which leaves you with state
     *
     * k[i] == 0: U = add(R, S), T = dbl(R)
     * k[i] == 1: U = add(S, R), T = dbl(S)
     *
     * Swapping T, U conditionally on k[i] leaves you with state
     *
     * k[i] == 0: R, S = T, U
     * k[i] == 1: R, S = U, T
     *
     * Which leaves you with state
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * So we get the same logic, but instead of a branch it's a
     * conditional swap, followed by ECC ops, then another conditional swap.
     *
     * Optimization: The end of iteration i and start of i-1 looks like
     *
     * ...
     * CSWAP(k[i], R, S)
     * ECC
     * CSWAP(k[i], R, S)
     * (next iteration)
     * CSWAP(k[i-1], R, S)
     * ECC
     * CSWAP(k[i-1], R, S)
     * ...
     *
     * So instead of two contiguous swaps, you can merge the condition
     * bits and do a single swap.
     *
     * k[i]   k[i-1]    Outcome
     * 0      0         No Swap
     * 0      1         Swap
     * 1      0         Swap
     * 1      1         No Swap
     *
     * This is XOR. pbit tracks the previous bit of k.
     */

    for (i = cardinality_bits - 1; i >= 0; i--) {
        kbit = BN_is_bit_set(k, i) ^ pbit;
        EC_POINT_CSWAP(kbit, r, s, group_top, Z_is_one);

        /* Perform a single step of the Montgomery ladder */
        if (!ec_point_ladder_step(group, r, s, p, ctx)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_STEP_FAILURE);
            goto err;
        }
        /*
         * pbit logic merges this cswap with that of the
         * next iteration
         */
        pbit ^= kbit;
    }
    /* one final cswap to move the right value into r */
    EC_POINT_CSWAP(pbit, r, s, group_top, Z_is_one);
#undef EC_POINT_CSWAP

    /* Finalize ladder (and recover full point coordinates) */
    if (!ec_point_ladder_post(group, r, s, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_POST_FAILURE);
        goto err;
    }

    ret = 1;

 err:
    EC_POINT_free(p);
    EC_POINT_free(s);
    BN_CTX_end(ctx);

    return ret;
}


int ec_scalar_mul_ladder(const EC_GROUP *group, EC_POINT *r,
                         const BIGNUM *scalar, const EC_POINT *point,
                         BN_CTX *ctx)
{
    int i, cardinality_bits, group_top, kbit, pbit, Z_is_one;
    EC_POINT *p = NULL;
    EC_POINT *s = NULL;
    BIGNUM *k = NULL;
    BIGNUM *lambda = NULL;
    BIGNUM *cardinality = NULL;
    int ret = 0;

    /* early exit if the input point is the point at infinity */
    if (point != NULL && EC_POINT_is_at_infinity(group, point))
        return EC_POINT_set_to_infinity(group, r);

    if (BN_is_zero(group->order)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_UNKNOWN_ORDER);
        return 0;
    }
    if (BN_is_zero(group->cofactor)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_UNKNOWN_COFACTOR);
        return 0;
    }

    BN_CTX_start(ctx);

    if (((p = EC_POINT_new(group)) == NULL)
        || ((s = EC_POINT_new(group)) == NULL)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    if (point == NULL) {
        if (!EC_POINT_copy(p, group->generator)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_EC_LIB);
            goto err;
        }
    } else {
        if (!EC_POINT_copy(p, point)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_EC_LIB);
            goto err;
        }
    }

    EC_POINT_BN_set_flags(p, BN_FLG_CONSTTIME);
    EC_POINT_BN_set_flags(r, BN_FLG_CONSTTIME);
    EC_POINT_BN_set_flags(s, BN_FLG_CONSTTIME);

    cardinality = BN_CTX_get(ctx);
    lambda = BN_CTX_get(ctx);
    k = BN_CTX_get(ctx);
    if (k == NULL) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_MALLOC_FAILURE);
        goto err;
    }

    if (!BN_mul(cardinality, group->order, group->cofactor, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    /*
     * Group cardinalities are often on a word boundary.
     * So when we pad the scalar, some timing diff might
     * pop if it needs to be expanded due to carries.
     * So expand ahead of time.
      */
     cardinality_bits = BN_num_bits(cardinality);
     group_top = bn_get_top(cardinality);
    if ((bn_wexpand(k, group_top + 2) == NULL)
        || (bn_wexpand(lambda, group_top + 2) == NULL)) {
         ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
         goto err;
     }

    if (!BN_copy(k, scalar)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    BN_set_flags(k, BN_FLG_CONSTTIME);

    if ((BN_num_bits(k) > cardinality_bits) || (BN_is_negative(k))) {
        /*-
         * this is an unusual input, and we don't guarantee
         * constant-timeness
         */
        if (!BN_nnmod(k, k, cardinality, ctx)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
            goto err;
        }
    }

    if (!BN_add(lambda, k, cardinality)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }
    BN_set_flags(lambda, BN_FLG_CONSTTIME);
    if (!BN_add(k, lambda, cardinality)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }
    /*
     * lambda := scalar + cardinality
      * k := scalar + 2*cardinality
      */
     kbit = BN_is_bit_set(lambda, cardinality_bits);
    BN_consttime_swap(kbit, k, lambda, group_top + 2);

     group_top = bn_get_top(group->field);
     if ((bn_wexpand(s->X, group_top) == NULL)
        || (bn_wexpand(s->Y, group_top) == NULL)
        || (bn_wexpand(s->Z, group_top) == NULL)
        || (bn_wexpand(r->X, group_top) == NULL)
        || (bn_wexpand(r->Y, group_top) == NULL)
        || (bn_wexpand(r->Z, group_top) == NULL)
        || (bn_wexpand(p->X, group_top) == NULL)
        || (bn_wexpand(p->Y, group_top) == NULL)
        || (bn_wexpand(p->Z, group_top) == NULL)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_BN_LIB);
        goto err;
    }

    /*-
     * Apply coordinate blinding for EC_POINT.
     *
     * The underlying EC_METHOD can optionally implement this function:
     * ec_point_blind_coordinates() returns 0 in case of errors or 1 on
     * success or if coordinate blinding is not implemented for this
     * group.
     */
    if (!ec_point_blind_coordinates(group, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_POINT_COORDINATES_BLIND_FAILURE);
        goto err;
    }

    /* Initialize the Montgomery ladder */
    if (!ec_point_ladder_pre(group, r, s, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_PRE_FAILURE);
        goto err;
    }

    /* top bit is a 1, in a fixed pos */
    pbit = 1;

#define EC_POINT_CSWAP(c, a, b, w, t) do {         \
        BN_consttime_swap(c, (a)->X, (b)->X, w);   \
        BN_consttime_swap(c, (a)->Y, (b)->Y, w);   \
        BN_consttime_swap(c, (a)->Z, (b)->Z, w);   \
        t = ((a)->Z_is_one ^ (b)->Z_is_one) & (c); \
        (a)->Z_is_one ^= (t);                      \
        (b)->Z_is_one ^= (t);                      \
} while(0)

    /*-
     * The ladder step, with branches, is
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * Swapping R, S conditionally on k[i] leaves you with state
     *
     * k[i] == 0: T, U = R, S
     * k[i] == 1: T, U = S, R
     *
     * Then perform the ECC ops.
     *
     * U = add(T, U)
     * T = dbl(T)
     *
     * Which leaves you with state
     *
     * k[i] == 0: U = add(R, S), T = dbl(R)
     * k[i] == 1: U = add(S, R), T = dbl(S)
     *
     * Swapping T, U conditionally on k[i] leaves you with state
     *
     * k[i] == 0: R, S = T, U
     * k[i] == 1: R, S = U, T
     *
     * Which leaves you with state
     *
     * k[i] == 0: S = add(R, S), R = dbl(R)
     * k[i] == 1: R = add(S, R), S = dbl(S)
     *
     * So we get the same logic, but instead of a branch it's a
     * conditional swap, followed by ECC ops, then another conditional swap.
     *
     * Optimization: The end of iteration i and start of i-1 looks like
     *
     * ...
     * CSWAP(k[i], R, S)
     * ECC
     * CSWAP(k[i], R, S)
     * (next iteration)
     * CSWAP(k[i-1], R, S)
     * ECC
     * CSWAP(k[i-1], R, S)
     * ...
     *
     * So instead of two contiguous swaps, you can merge the condition
     * bits and do a single swap.
     *
     * k[i]   k[i-1]    Outcome
     * 0      0         No Swap
     * 0      1         Swap
     * 1      0         Swap
     * 1      1         No Swap
     *
     * This is XOR. pbit tracks the previous bit of k.
     */

    for (i = cardinality_bits - 1; i >= 0; i--) {
        kbit = BN_is_bit_set(k, i) ^ pbit;
        EC_POINT_CSWAP(kbit, r, s, group_top, Z_is_one);

        /* Perform a single step of the Montgomery ladder */
        if (!ec_point_ladder_step(group, r, s, p, ctx)) {
            ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_STEP_FAILURE);
            goto err;
        }
        /*
         * pbit logic merges this cswap with that of the
         * next iteration
         */
        pbit ^= kbit;
    }
    /* one final cswap to move the right value into r */
    EC_POINT_CSWAP(pbit, r, s, group_top, Z_is_one);
#undef EC_POINT_CSWAP

    /* Finalize ladder (and recover full point coordinates) */
    if (!ec_point_ladder_post(group, r, s, p, ctx)) {
        ECerr(EC_F_EC_SCALAR_MUL_LADDER, EC_R_LADDER_POST_FAILURE);
        goto err;
    }

    ret = 1;

 err:
    EC_POINT_free(p);
    EC_POINT_free(s);
    BN_CTX_end(ctx);

    return ret;
}


Merge DSA reallocation timing fix CVE-2018-0734.

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7513)

static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp,
                          BIGNUM **rp)
{
    BN_CTX *ctx;
    BIGNUM k, kq, *K, *kinv = NULL, *r = NULL;
    BIGNUM l, m;
    int ret = 0;
    int q_bits;

    if (!dsa->p || !dsa->q || !dsa->g) {
        DSAerr(DSA_F_DSA_SIGN_SETUP, DSA_R_MISSING_PARAMETERS);
        return 0;
    }

    BN_init(&k);
    BN_init(&kq);
    BN_init(&l);
    BN_init(&m);

    if (ctx_in == NULL) {
        if ((ctx = BN_CTX_new()) == NULL)
            goto err;
    } else
        ctx = ctx_in;

    if ((r = BN_new()) == NULL)
         goto err;

     /* Preallocate space */
    q_bits = BN_num_bits(dsa->q);
     if (!BN_set_bit(&k, q_bits)
         || !BN_set_bit(&l, q_bits)
         || !BN_set_bit(&m, q_bits))
        goto err;

    /* Get random k */
    do
        if (!BN_rand_range(&k, dsa->q))
            goto err;
    while (BN_is_zero(&k));

    if ((dsa->flags & DSA_FLAG_NO_EXP_CONSTTIME) == 0) {
        BN_set_flags(&k, BN_FLG_CONSTTIME);
    }

    if (dsa->flags & DSA_FLAG_CACHE_MONT_P) {
        if (!BN_MONT_CTX_set_locked(&dsa->method_mont_p,
                                    CRYPTO_LOCK_DSA, dsa->p, ctx))
            goto err;
    }

    /* Compute r = (g^k mod p) mod q */

    if ((dsa->flags & DSA_FLAG_NO_EXP_CONSTTIME) == 0) {
        /*
         * We do not want timing information to leak the length of k, so we
         * compute G^k using an equivalent scalar of fixed bit-length.
         *
         * We unconditionally perform both of these additions to prevent a
         * small timing information leakage.  We then choose the sum that is
         * one bit longer than the modulus.
         *
         * TODO: revisit the BN_copy aiming for a memory access agnostic
         * conditional copy.
         */
        if (!BN_add(&l, &k, dsa->q)
            || !BN_add(&m, &l, dsa->q)
            || !BN_copy(&kq, BN_num_bits(&l) > q_bits ? &l : &m))
            goto err;

        BN_set_flags(&kq, BN_FLG_CONSTTIME);

        K = &kq;
    } else {
        K = &k;
    }

    DSA_BN_MOD_EXP(goto err, dsa, r, dsa->g, K, dsa->p, ctx,
                   dsa->method_mont_p);
    if (!BN_mod(r, r, dsa->q, ctx))
        goto err;

    /* Compute  part of 's = inv(k) (m + xr) mod q' */
    if ((kinv = BN_mod_inverse(NULL, &k, dsa->q, ctx)) == NULL)
        goto err;

    if (*kinvp != NULL)
        BN_clear_free(*kinvp);
    *kinvp = kinv;
    kinv = NULL;
    if (*rp != NULL)
        BN_clear_free(*rp);
    *rp = r;
    ret = 1;
 err:
    if (!ret) {
        DSAerr(DSA_F_DSA_SIGN_SETUP, ERR_R_BN_LIB);
        if (r != NULL)
            BN_clear_free(r);
    }
    if (ctx_in == NULL)
        BN_CTX_free(ctx);
    BN_clear_free(&k);
    BN_clear_free(&kq);
    BN_clear_free(&l);
    BN_clear_free(&m);
    return ret;
}

The OpenSSL DSA signature algorithm has been shown to be vulnerable to a timing side channel attack. An attacker could use variations in the signing algorithm to recover the private key. Fixed in OpenSSL 1.1.1a (Affected 1.1.1). Fixed in OpenSSL 1.1.0j (Affected 1.1.0-1.1.0i). Fixed in OpenSSL 1.0.2q (Affected 1.0.2-1.0.2p).


static int dsa_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp,
                          BIGNUM **rp)
{
    BN_CTX *ctx;
    BIGNUM k, kq, *K, *kinv = NULL, *r = NULL;
    BIGNUM l, m;
    int ret = 0;
    int q_bits;

    if (!dsa->p || !dsa->q || !dsa->g) {
        DSAerr(DSA_F_DSA_SIGN_SETUP, DSA_R_MISSING_PARAMETERS);
        return 0;
    }

    BN_init(&k);
    BN_init(&kq);
    BN_init(&l);
    BN_init(&m);

    if (ctx_in == NULL) {
        if ((ctx = BN_CTX_new()) == NULL)
            goto err;
    } else
        ctx = ctx_in;

    if ((r = BN_new()) == NULL)
         goto err;

     /* Preallocate space */
    q_bits = BN_num_bits(dsa->q) + sizeof(dsa->q->d[0]) * 16;
     if (!BN_set_bit(&k, q_bits)
         || !BN_set_bit(&l, q_bits)
         || !BN_set_bit(&m, q_bits))
        goto err;

    /* Get random k */
    do
        if (!BN_rand_range(&k, dsa->q))
            goto err;
    while (BN_is_zero(&k));

    if ((dsa->flags & DSA_FLAG_NO_EXP_CONSTTIME) == 0) {
        BN_set_flags(&k, BN_FLG_CONSTTIME);
    }

    if (dsa->flags & DSA_FLAG_CACHE_MONT_P) {
        if (!BN_MONT_CTX_set_locked(&dsa->method_mont_p,
                                    CRYPTO_LOCK_DSA, dsa->p, ctx))
            goto err;
    }

    /* Compute r = (g^k mod p) mod q */

    if ((dsa->flags & DSA_FLAG_NO_EXP_CONSTTIME) == 0) {
        /*
         * We do not want timing information to leak the length of k, so we
         * compute G^k using an equivalent scalar of fixed bit-length.
         *
         * We unconditionally perform both of these additions to prevent a
         * small timing information leakage.  We then choose the sum that is
         * one bit longer than the modulus.
         *
         * TODO: revisit the BN_copy aiming for a memory access agnostic
         * conditional copy.
         */
        if (!BN_add(&l, &k, dsa->q)
            || !BN_add(&m, &l, dsa->q)
            || !BN_copy(&kq, BN_num_bits(&l) > q_bits ? &l : &m))
            goto err;

        BN_set_flags(&kq, BN_FLG_CONSTTIME);

        K = &kq;
    } else {
        K = &k;
    }

    DSA_BN_MOD_EXP(goto err, dsa, r, dsa->g, K, dsa->p, ctx,
                   dsa->method_mont_p);
    if (!BN_mod(r, r, dsa->q, ctx))
        goto err;

    /* Compute  part of 's = inv(k) (m + xr) mod q' */
    if ((kinv = BN_mod_inverse(NULL, &k, dsa->q, ctx)) == NULL)
        goto err;

    if (*kinvp != NULL)
        BN_clear_free(*kinvp);
    *kinvp = kinv;
    kinv = NULL;
    if (*rp != NULL)
        BN_clear_free(*rp);
    *rp = r;
    ret = 1;
 err:
    if (!ret) {
        DSAerr(DSA_F_DSA_SIGN_SETUP, ERR_R_BN_LIB);
        if (r != NULL)
            BN_clear_free(r);
    }
    if (ctx_in == NULL)
        BN_CTX_free(ctx);
    BN_clear_free(&k);
    BN_clear_free(&kq);
    BN_clear_free(&l);
    BN_clear_free(&m);
    return ret;
}


Reject excessively large primes in DH key generation.

CVE-2018-0732

Signed-off-by: Guido Vranken <guidovranken@gmail.com>

(cherry picked from commit 91f7361f47b082ae61ffe1a7b17bb2adf213c7fe)

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6457)

static int generate_key(DH *dh)
{
     int ok = 0;
     int generate_new_key = 0;
     unsigned l;
    BN_CTX *ctx;
     BN_MONT_CTX *mont = NULL;
     BIGNUM *pub_key = NULL, *priv_key = NULL;

     ctx = BN_CTX_new();
     if (ctx == NULL)
         goto err;
        generate_new_key = 1;
    } else

During key agreement in a TLS handshake using a DH(E) based ciphersuite a malicious server can send a very large prime value to the client. This will cause the client to spend an unreasonably long period of time generating a key for this prime resulting in a hang until the client has finished. This could be exploited in a Denial Of Service attack. Fixed in OpenSSL 1.1.0i-dev (Affected 1.1.0-1.1.0h). Fixed in OpenSSL 1.0.2p-dev (Affected 1.0.2-1.0.2o).


static int generate_key(DH *dh)
{
     int ok = 0;
     int generate_new_key = 0;
     unsigned l;
    BN_CTX *ctx = NULL;
     BN_MONT_CTX *mont = NULL;
     BIGNUM *pub_key = NULL, *priv_key = NULL;

    if (BN_num_bits(dh->p) > OPENSSL_DH_MAX_MODULUS_BITS) {
        DHerr(DH_F_GENERATE_KEY, DH_R_MODULUS_TOO_LARGE);
        return 0;
    }

     ctx = BN_CTX_new();
     if (ctx == NULL)
         goto err;
        generate_new_key = 1;
    } else


bdfReadCharacters(FontFilePtr file, FontPtr pFont, bdfFileState *pState,
		  int bit, int byte, int glyph, int scan)
{
    unsigned char *line;
    register CharInfoPtr ci;
    int         i,
                ndx,
                nchars,
                nignored;
    unsigned int char_row, char_col;
    int         numEncodedGlyphs = 0;
    CharInfoPtr *bdfEncoding[256];
    BitmapFontPtr  bitmapFont;
    BitmapExtraPtr bitmapExtra;
    CARD32     *bitmapsSizes;
    unsigned char        lineBuf[BDFLINELEN];
    int         nencoding;

    bitmapFont = (BitmapFontPtr) pFont->fontPrivate;
    bitmapExtra = (BitmapExtraPtr) bitmapFont->bitmapExtra;

    if (bitmapExtra) {
	bitmapsSizes = bitmapExtra->bitmapsSizes;
	for (i = 0; i < GLYPHPADOPTIONS; i++)
	    bitmapsSizes[i] = 0;
    } else
	bitmapsSizes = NULL;

    bzero(bdfEncoding, sizeof(bdfEncoding));
    bitmapFont->metrics = NULL;
    ndx = 0;

    line = bdfGetLine(file, lineBuf, BDFLINELEN);

    if ((!line) || (sscanf((char *) line, "CHARS %d", &nchars) != 1)) {
	bdfError("bad 'CHARS' in bdf file\n");
	return (FALSE);
    }
    if (nchars < 1) {
	bdfError("invalid number of CHARS in BDF file\n");
	return (FALSE);
    }
    if (nchars > INT32_MAX / sizeof(CharInfoRec)) {
	bdfError("Couldn't allocate pCI (%d*%d)\n", nchars,
		 (int) sizeof(CharInfoRec));
	goto BAILOUT;
    }
    ci = calloc(nchars, sizeof(CharInfoRec));
    if (!ci) {
	bdfError("Couldn't allocate pCI (%d*%d)\n", nchars,
		 (int) sizeof(CharInfoRec));
	goto BAILOUT;
    }
    bitmapFont->metrics = ci;

    if (bitmapExtra) {
	bitmapExtra->glyphNames = malloc(nchars * sizeof(Atom));
	if (!bitmapExtra->glyphNames) {
	    bdfError("Couldn't allocate glyphNames (%d*%d)\n",
		     nchars, (int) sizeof(Atom));
	    goto BAILOUT;
	}
    }
    if (bitmapExtra) {
	bitmapExtra->sWidths = malloc(nchars * sizeof(int));
	if (!bitmapExtra->sWidths) {
	    bdfError("Couldn't allocate sWidth (%d *%d)\n",
		     nchars, (int) sizeof(int));
	    return FALSE;
	}
    }
    line = bdfGetLine(file, lineBuf, BDFLINELEN);
    pFont->info.firstRow = 256;
    pFont->info.lastRow = 0;
    pFont->info.firstCol = 256;
    pFont->info.lastCol = 0;
    nignored = 0;
    for (ndx = 0; (ndx < nchars) && (line) && (bdfIsPrefix(line, "STARTCHAR"));) {
	int         t;
	int         wx;		/* x component of width */
	int         wy;		/* y component of width */
	int         bw;		/* bounding-box width */
	int         bh;		/* bounding-box height */
	int         bl;		/* bounding-box left */
	int         bb;		/* bounding-box bottom */
	int         enc,
	            enc2;	/* encoding */
	unsigned char *p;	/* temp pointer into line */
 	char        charName[100];
 	int         ignore;

	if (sscanf((char *) line, "STARTCHAR %s", charName) != 1) {
 	    bdfError("bad character name in BDF file\n");
 	    goto BAILOUT;	/* bottom of function, free and return error */
 	}
	if (bitmapExtra)
	    bitmapExtra->glyphNames[ndx] = bdfForceMakeAtom(charName, NULL);

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if (!line || (t = sscanf((char *) line, "ENCODING %d %d", &enc, &enc2)) < 1) {
	    bdfError("bad 'ENCODING' in BDF file\n");
	    goto BAILOUT;
	}
	if (enc < -1 || (t == 2 && enc2 < -1)) {
	    bdfError("bad ENCODING value");
	    goto BAILOUT;
	}
	if (t == 2 && enc == -1)
	    enc = enc2;
	ignore = 0;
	if (enc == -1) {
	    if (!bitmapExtra) {
		nignored++;
		ignore = 1;
	    }
	} else if (enc > MAXENCODING) {
	    bdfError("char '%s' has encoding too large (%d)\n",
		     charName, enc);
	} else {
	    char_row = (enc >> 8) & 0xFF;
	    char_col = enc & 0xFF;
	    if (char_row < pFont->info.firstRow)
		pFont->info.firstRow = char_row;
	    if (char_row > pFont->info.lastRow)
		pFont->info.lastRow = char_row;
	    if (char_col < pFont->info.firstCol)
		pFont->info.firstCol = char_col;
	    if (char_col > pFont->info.lastCol)
		pFont->info.lastCol = char_col;
	    if (bdfEncoding[char_row] == (CharInfoPtr *) NULL) {
		bdfEncoding[char_row] = malloc(256 * sizeof(CharInfoPtr));
		if (!bdfEncoding[char_row]) {
		    bdfError("Couldn't allocate row %d of encoding (%d*%d)\n",
			     char_row, INDICES, (int) sizeof(CharInfoPtr));
		    goto BAILOUT;
		}
		for (i = 0; i < 256; i++)
		    bdfEncoding[char_row][i] = (CharInfoPtr) NULL;
	    }
	    if (bdfEncoding[char_row] != NULL) {
		bdfEncoding[char_row][char_col] = ci;
		numEncodedGlyphs++;
	    }
	}

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "SWIDTH %d %d", &wx, &wy) != 2)) {
	    bdfError("bad 'SWIDTH'\n");
	    goto BAILOUT;
	}
	if (wy != 0) {
	    bdfError("SWIDTH y value must be zero\n");
	    goto BAILOUT;
	}
	if (bitmapExtra)
	    bitmapExtra->sWidths[ndx] = wx;

/* 5/31/89 (ef) -- we should be able to ditch the character and recover */
/*		from all of these.					*/

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "DWIDTH %d %d", &wx, &wy) != 2)) {
	    bdfError("bad 'DWIDTH'\n");
	    goto BAILOUT;
	}
	if (wy != 0) {
	    bdfError("DWIDTH y value must be zero\n");
	    goto BAILOUT;
	}
	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "BBX %d %d %d %d", &bw, &bh, &bl, &bb) != 4)) {
	    bdfError("bad 'BBX'\n");
	    goto BAILOUT;
	}
	if ((bh < 0) || (bw < 0)) {
	    bdfError("character '%s' has a negative sized bitmap, %dx%d\n",
		     charName, bw, bh);
	    goto BAILOUT;
	}
	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((line) && (bdfIsPrefix(line, "ATTRIBUTES"))) {
	    for (p = line + strlen("ATTRIBUTES ");
		    (*p == ' ') || (*p == '\t');
		    p++)
		 /* empty for loop */ ;
	    ci->metrics.attributes = (bdfHexByte(p) << 8) + bdfHexByte(p + 2);
	    line = bdfGetLine(file, lineBuf, BDFLINELEN);
	} else
	    ci->metrics.attributes = 0;

	if (!line || !bdfIsPrefix(line, "BITMAP")) {
	    bdfError("missing 'BITMAP'\n");
	    goto BAILOUT;
	}
	/* collect data for generated properties */
	if ((strlen(charName) == 1)) {
	    if ((charName[0] >= '0') && (charName[0] <= '9')) {
		pState->digitWidths += wx;
		pState->digitCount++;
	    } else if (charName[0] == 'x') {
		pState->exHeight = (bh + bb) <= 0 ? bh : bh + bb;
	    }
	}
	if (!ignore) {
	    ci->metrics.leftSideBearing = bl;
	    ci->metrics.rightSideBearing = bl + bw;
	    ci->metrics.ascent = bh + bb;
	    ci->metrics.descent = -bb;
	    ci->metrics.characterWidth = wx;
	    ci->bits = NULL;
	    bdfReadBitmap(ci, file, bit, byte, glyph, scan, bitmapsSizes);
	    ci++;
	    ndx++;
	} else
	    bdfSkipBitmap(file, bh);

	line = bdfGetLine(file, lineBuf, BDFLINELEN);	/* get STARTCHAR or
							 * ENDFONT */
    }

    if (ndx + nignored != nchars) {
	bdfError("%d too few characters\n", nchars - (ndx + nignored));
	goto BAILOUT;
    }
    nchars = ndx;
    bitmapFont->num_chars = nchars;
    if ((line) && (bdfIsPrefix(line, "STARTCHAR"))) {
	bdfError("more characters than specified\n");
	goto BAILOUT;
    }
    if ((!line) || (!bdfIsPrefix(line, "ENDFONT"))) {
	bdfError("missing 'ENDFONT'\n");
	goto BAILOUT;
    }
    if (numEncodedGlyphs == 0)
	bdfWarning("No characters with valid encodings\n");

    nencoding = (pFont->info.lastRow - pFont->info.firstRow + 1) *
	(pFont->info.lastCol - pFont->info.firstCol + 1);
    bitmapFont->encoding = calloc(NUM_SEGMENTS(nencoding),sizeof(CharInfoPtr*));
    if (!bitmapFont->encoding) {
	bdfError("Couldn't allocate ppCI (%d,%d)\n",
                 NUM_SEGMENTS(nencoding),
                 (int) sizeof(CharInfoPtr*));
	goto BAILOUT;
    }
    pFont->info.allExist = TRUE;
    i = 0;
    for (char_row = pFont->info.firstRow;
	    char_row <= pFont->info.lastRow;
	    char_row++) {
	if (bdfEncoding[char_row] == (CharInfoPtr *) NULL) {
	    pFont->info.allExist = FALSE;
            i += pFont->info.lastCol - pFont->info.firstCol + 1;
	} else {
	    for (char_col = pFont->info.firstCol;
		    char_col <= pFont->info.lastCol;
		    char_col++) {
		if (!bdfEncoding[char_row][char_col])
		    pFont->info.allExist = FALSE;
                else {
                    if (!bitmapFont->encoding[SEGMENT_MAJOR(i)]) {
                        bitmapFont->encoding[SEGMENT_MAJOR(i)]=
                            calloc(BITMAP_FONT_SEGMENT_SIZE,
                                   sizeof(CharInfoPtr));
                        if (!bitmapFont->encoding[SEGMENT_MAJOR(i)])
                            goto BAILOUT;
                    }
                    ACCESSENCODINGL(bitmapFont->encoding,i) =
                        bdfEncoding[char_row][char_col];
                }
                i++;
            }
	}
    }
    for (i = 0; i < 256; i++)
	if (bdfEncoding[i])
	    free(bdfEncoding[i]);
    return (TRUE);
BAILOUT:
    for (i = 0; i < 256; i++)
	if (bdfEncoding[i])
	    free(bdfEncoding[i]);
    /* bdfFreeFontBits will clean up the rest */
    return (FALSE);
}

Stack-based buffer overflow in the bdfReadCharacters function in bitmap/bdfread.c in X.Org libXfont 1.1 through 1.4.6 allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a long string in a character name in a BDF font file.


bdfReadCharacters(FontFilePtr file, FontPtr pFont, bdfFileState *pState,
		  int bit, int byte, int glyph, int scan)
{
    unsigned char *line;
    register CharInfoPtr ci;
    int         i,
                ndx,
                nchars,
                nignored;
    unsigned int char_row, char_col;
    int         numEncodedGlyphs = 0;
    CharInfoPtr *bdfEncoding[256];
    BitmapFontPtr  bitmapFont;
    BitmapExtraPtr bitmapExtra;
    CARD32     *bitmapsSizes;
    unsigned char        lineBuf[BDFLINELEN];
    int         nencoding;

    bitmapFont = (BitmapFontPtr) pFont->fontPrivate;
    bitmapExtra = (BitmapExtraPtr) bitmapFont->bitmapExtra;

    if (bitmapExtra) {
	bitmapsSizes = bitmapExtra->bitmapsSizes;
	for (i = 0; i < GLYPHPADOPTIONS; i++)
	    bitmapsSizes[i] = 0;
    } else
	bitmapsSizes = NULL;

    bzero(bdfEncoding, sizeof(bdfEncoding));
    bitmapFont->metrics = NULL;
    ndx = 0;

    line = bdfGetLine(file, lineBuf, BDFLINELEN);

    if ((!line) || (sscanf((char *) line, "CHARS %d", &nchars) != 1)) {
	bdfError("bad 'CHARS' in bdf file\n");
	return (FALSE);
    }
    if (nchars < 1) {
	bdfError("invalid number of CHARS in BDF file\n");
	return (FALSE);
    }
    if (nchars > INT32_MAX / sizeof(CharInfoRec)) {
	bdfError("Couldn't allocate pCI (%d*%d)\n", nchars,
		 (int) sizeof(CharInfoRec));
	goto BAILOUT;
    }
    ci = calloc(nchars, sizeof(CharInfoRec));
    if (!ci) {
	bdfError("Couldn't allocate pCI (%d*%d)\n", nchars,
		 (int) sizeof(CharInfoRec));
	goto BAILOUT;
    }
    bitmapFont->metrics = ci;

    if (bitmapExtra) {
	bitmapExtra->glyphNames = malloc(nchars * sizeof(Atom));
	if (!bitmapExtra->glyphNames) {
	    bdfError("Couldn't allocate glyphNames (%d*%d)\n",
		     nchars, (int) sizeof(Atom));
	    goto BAILOUT;
	}
    }
    if (bitmapExtra) {
	bitmapExtra->sWidths = malloc(nchars * sizeof(int));
	if (!bitmapExtra->sWidths) {
	    bdfError("Couldn't allocate sWidth (%d *%d)\n",
		     nchars, (int) sizeof(int));
	    return FALSE;
	}
    }
    line = bdfGetLine(file, lineBuf, BDFLINELEN);
    pFont->info.firstRow = 256;
    pFont->info.lastRow = 0;
    pFont->info.firstCol = 256;
    pFont->info.lastCol = 0;
    nignored = 0;
    for (ndx = 0; (ndx < nchars) && (line) && (bdfIsPrefix(line, "STARTCHAR"));) {
	int         t;
	int         wx;		/* x component of width */
	int         wy;		/* y component of width */
	int         bw;		/* bounding-box width */
	int         bh;		/* bounding-box height */
	int         bl;		/* bounding-box left */
	int         bb;		/* bounding-box bottom */
	int         enc,
	            enc2;	/* encoding */
	unsigned char *p;	/* temp pointer into line */
 	char        charName[100];
 	int         ignore;

	if (sscanf((char *) line, "STARTCHAR %99s", charName) != 1) {
 	    bdfError("bad character name in BDF file\n");
 	    goto BAILOUT;	/* bottom of function, free and return error */
 	}
	if (bitmapExtra)
	    bitmapExtra->glyphNames[ndx] = bdfForceMakeAtom(charName, NULL);

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if (!line || (t = sscanf((char *) line, "ENCODING %d %d", &enc, &enc2)) < 1) {
	    bdfError("bad 'ENCODING' in BDF file\n");
	    goto BAILOUT;
	}
	if (enc < -1 || (t == 2 && enc2 < -1)) {
	    bdfError("bad ENCODING value");
	    goto BAILOUT;
	}
	if (t == 2 && enc == -1)
	    enc = enc2;
	ignore = 0;
	if (enc == -1) {
	    if (!bitmapExtra) {
		nignored++;
		ignore = 1;
	    }
	} else if (enc > MAXENCODING) {
	    bdfError("char '%s' has encoding too large (%d)\n",
		     charName, enc);
	} else {
	    char_row = (enc >> 8) & 0xFF;
	    char_col = enc & 0xFF;
	    if (char_row < pFont->info.firstRow)
		pFont->info.firstRow = char_row;
	    if (char_row > pFont->info.lastRow)
		pFont->info.lastRow = char_row;
	    if (char_col < pFont->info.firstCol)
		pFont->info.firstCol = char_col;
	    if (char_col > pFont->info.lastCol)
		pFont->info.lastCol = char_col;
	    if (bdfEncoding[char_row] == (CharInfoPtr *) NULL) {
		bdfEncoding[char_row] = malloc(256 * sizeof(CharInfoPtr));
		if (!bdfEncoding[char_row]) {
		    bdfError("Couldn't allocate row %d of encoding (%d*%d)\n",
			     char_row, INDICES, (int) sizeof(CharInfoPtr));
		    goto BAILOUT;
		}
		for (i = 0; i < 256; i++)
		    bdfEncoding[char_row][i] = (CharInfoPtr) NULL;
	    }
	    if (bdfEncoding[char_row] != NULL) {
		bdfEncoding[char_row][char_col] = ci;
		numEncodedGlyphs++;
	    }
	}

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "SWIDTH %d %d", &wx, &wy) != 2)) {
	    bdfError("bad 'SWIDTH'\n");
	    goto BAILOUT;
	}
	if (wy != 0) {
	    bdfError("SWIDTH y value must be zero\n");
	    goto BAILOUT;
	}
	if (bitmapExtra)
	    bitmapExtra->sWidths[ndx] = wx;

/* 5/31/89 (ef) -- we should be able to ditch the character and recover */
/*		from all of these.					*/

	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "DWIDTH %d %d", &wx, &wy) != 2)) {
	    bdfError("bad 'DWIDTH'\n");
	    goto BAILOUT;
	}
	if (wy != 0) {
	    bdfError("DWIDTH y value must be zero\n");
	    goto BAILOUT;
	}
	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((!line) || (sscanf((char *) line, "BBX %d %d %d %d", &bw, &bh, &bl, &bb) != 4)) {
	    bdfError("bad 'BBX'\n");
	    goto BAILOUT;
	}
	if ((bh < 0) || (bw < 0)) {
	    bdfError("character '%s' has a negative sized bitmap, %dx%d\n",
		     charName, bw, bh);
	    goto BAILOUT;
	}
	line = bdfGetLine(file, lineBuf, BDFLINELEN);
	if ((line) && (bdfIsPrefix(line, "ATTRIBUTES"))) {
	    for (p = line + strlen("ATTRIBUTES ");
		    (*p == ' ') || (*p == '\t');
		    p++)
		 /* empty for loop */ ;
	    ci->metrics.attributes = (bdfHexByte(p) << 8) + bdfHexByte(p + 2);
	    line = bdfGetLine(file, lineBuf, BDFLINELEN);
	} else
	    ci->metrics.attributes = 0;

	if (!line || !bdfIsPrefix(line, "BITMAP")) {
	    bdfError("missing 'BITMAP'\n");
	    goto BAILOUT;
	}
	/* collect data for generated properties */
	if ((strlen(charName) == 1)) {
	    if ((charName[0] >= '0') && (charName[0] <= '9')) {
		pState->digitWidths += wx;
		pState->digitCount++;
	    } else if (charName[0] == 'x') {
		pState->exHeight = (bh + bb) <= 0 ? bh : bh + bb;
	    }
	}
	if (!ignore) {
	    ci->metrics.leftSideBearing = bl;
	    ci->metrics.rightSideBearing = bl + bw;
	    ci->metrics.ascent = bh + bb;
	    ci->metrics.descent = -bb;
	    ci->metrics.characterWidth = wx;
	    ci->bits = NULL;
	    bdfReadBitmap(ci, file, bit, byte, glyph, scan, bitmapsSizes);
	    ci++;
	    ndx++;
	} else
	    bdfSkipBitmap(file, bh);

	line = bdfGetLine(file, lineBuf, BDFLINELEN);	/* get STARTCHAR or
							 * ENDFONT */
    }

    if (ndx + nignored != nchars) {
	bdfError("%d too few characters\n", nchars - (ndx + nignored));
	goto BAILOUT;
    }
    nchars = ndx;
    bitmapFont->num_chars = nchars;
    if ((line) && (bdfIsPrefix(line, "STARTCHAR"))) {
	bdfError("more characters than specified\n");
	goto BAILOUT;
    }
    if ((!line) || (!bdfIsPrefix(line, "ENDFONT"))) {
	bdfError("missing 'ENDFONT'\n");
	goto BAILOUT;
    }
    if (numEncodedGlyphs == 0)
	bdfWarning("No characters with valid encodings\n");

    nencoding = (pFont->info.lastRow - pFont->info.firstRow + 1) *
	(pFont->info.lastCol - pFont->info.firstCol + 1);
    bitmapFont->encoding = calloc(NUM_SEGMENTS(nencoding),sizeof(CharInfoPtr*));
    if (!bitmapFont->encoding) {
	bdfError("Couldn't allocate ppCI (%d,%d)\n",
                 NUM_SEGMENTS(nencoding),
                 (int) sizeof(CharInfoPtr*));
	goto BAILOUT;
    }
    pFont->info.allExist = TRUE;
    i = 0;
    for (char_row = pFont->info.firstRow;
	    char_row <= pFont->info.lastRow;
	    char_row++) {
	if (bdfEncoding[char_row] == (CharInfoPtr *) NULL) {
	    pFont->info.allExist = FALSE;
            i += pFont->info.lastCol - pFont->info.firstCol + 1;
	} else {
	    for (char_col = pFont->info.firstCol;
		    char_col <= pFont->info.lastCol;
		    char_col++) {
		if (!bdfEncoding[char_row][char_col])
		    pFont->info.allExist = FALSE;
                else {
                    if (!bitmapFont->encoding[SEGMENT_MAJOR(i)]) {
                        bitmapFont->encoding[SEGMENT_MAJOR(i)]=
                            calloc(BITMAP_FONT_SEGMENT_SIZE,
                                   sizeof(CharInfoPtr));
                        if (!bitmapFont->encoding[SEGMENT_MAJOR(i)])
                            goto BAILOUT;
                    }
                    ACCESSENCODINGL(bitmapFont->encoding,i) =
                        bdfEncoding[char_row][char_col];
                }
                i++;
            }
	}
    }
    for (i = 0; i < 256; i++)
	if (bdfEncoding[i])
	    free(bdfEncoding[i]);
    return (TRUE);
BAILOUT:
    for (i = 0; i < 256; i++)
	if (bdfEncoding[i])
	    free(bdfEncoding[i]);
    /* bdfFreeFontBits will clean up the rest */
    return (FALSE);
}


load(ImlibImage * im, ImlibProgressFunction progress, char progress_granularity,
     char immediate_load)
{
   static const int    intoffset[] = { 0, 4, 2, 1 };
   static const int    intjump[] = { 8, 8, 4, 2 };
   int                 rc;
   DATA32             *ptr;
   GifFileType        *gif;
   GifRowType         *rows;
   GifRecordType       rec;
   ColorMapObject     *cmap;
   int                 i, j, done, bg, r, g, b, w = 0, h = 0;
   float               per = 0.0, per_inc;
   int                 last_per = 0, last_y = 0;
   int                 transp;
   int                 fd;

   done = 0;
   rows = NULL;
   transp = -1;

   /* if immediate_load is 1, then dont delay image laoding as below, or */
   /* already data in this image - dont load it again */
   if (im->data)
      return 0;

   fd = open(im->real_file, O_RDONLY);
   if (fd < 0)
      return 0;

#if GIFLIB_MAJOR >= 5
   gif = DGifOpenFileHandle(fd, NULL);
#else
   gif = DGifOpenFileHandle(fd);
#endif
   if (!gif)
     {
        close(fd);
        return 0;
     }

   rc = 0;                      /* Failure */

   do
     {
        if (DGifGetRecordType(gif, &rec) == GIF_ERROR)
          {
             /* PrintGifError(); */
             rec = TERMINATE_RECORD_TYPE;
          }
        if ((rec == IMAGE_DESC_RECORD_TYPE) && (!done))
          {
             if (DGifGetImageDesc(gif) == GIF_ERROR)
               {
                  /* PrintGifError(); */
                  rec = TERMINATE_RECORD_TYPE;
                  break;
               }
             w = gif->Image.Width;
             h = gif->Image.Height;
             if (!IMAGE_DIMENSIONS_OK(w, h))
                goto quit2;

             rows = calloc(h, sizeof(GifRowType *));
             if (!rows)
                goto quit2;

             for (i = 0; i < h; i++)
               {
                  rows[i] = malloc(w * sizeof(GifPixelType));
                  if (!rows[i])
                     goto quit;
               }

             if (gif->Image.Interlace)
               {
                  for (i = 0; i < 4; i++)
                    {
                       for (j = intoffset[i]; j < h; j += intjump[i])
                         {
                            if (DGifGetLine(gif, rows[i], w) == GIF_ERROR)
                               {
                                  break;
                               }
                         }
                    }
               }
             else
               {
                  for (i = 0; i < h; i++)
                    {
                       if (DGifGetLine(gif, rows[i], w) == GIF_ERROR)
                          {
                             break;
                          }
                    }
               }
             done = 1;
          }
        else if (rec == EXTENSION_RECORD_TYPE)
          {
             int                 ext_code;
             GifByteType        *ext;

             ext = NULL;
             DGifGetExtension(gif, &ext_code, &ext);
             while (ext)
               {
                  if ((ext_code == 0xf9) && (ext[1] & 1) && (transp < 0))
                    {
                       transp = (int)ext[4];
                    }
                  ext = NULL;
                  DGifGetExtensionNext(gif, &ext);
               }
          }
     }
   while (rec != TERMINATE_RECORD_TYPE);

   if (transp >= 0)
     {
        SET_FLAG(im->flags, F_HAS_ALPHA);
     }
   else
      {
         UNSET_FLAG(im->flags, F_HAS_ALPHA);
      }

    /* set the format string member to the lower-case full extension */
    /* name for the format - so example names would be: */
      im->format = strdup("gif");

   if (im->loader || immediate_load || progress)
     {
        bg = gif->SBackGroundColor;
        cmap = (gif->Image.ColorMap ? gif->Image.ColorMap : gif->SColorMap);
        im->data = (DATA32 *) malloc(sizeof(DATA32) * w * h);
        if (!im->data)
           goto quit;

        if (!cmap)
          {
             /* No colormap? Now what?? Let's clear the image (and not segv) */
             memset(im->data, 0, sizeof(DATA32) * w * h);
             rc = 1;
             goto finish;
          }

        ptr = im->data;
        per_inc = 100.0 / (((float)w) * h);
        for (i = 0; i < h; i++)
          {
             for (j = 0; j < w; j++)
               {
                  if (rows[i][j] == transp)
                    {
                       r = cmap->Colors[bg].Red;
                       g = cmap->Colors[bg].Green;
                       b = cmap->Colors[bg].Blue;
                       *ptr++ = 0x00ffffff & ((r << 16) | (g << 8) | b);
                    }
                  else
                    {
                       r = cmap->Colors[rows[i][j]].Red;
                       g = cmap->Colors[rows[i][j]].Green;
                       b = cmap->Colors[rows[i][j]].Blue;
                       *ptr++ = (0xff << 24) | (r << 16) | (g << 8) | b;
                    }
                  per += per_inc;
                  if (progress && (((int)per) != last_per)
                      && (((int)per) % progress_granularity == 0))
                    {
                       last_per = (int)per;
                       if (!(progress(im, (int)per, 0, last_y, w, i)))
                         {
                            rc = 2;
                            goto quit;
                         }
                       last_y = i;
                    }
               }
          }

      finish:
        if (progress)
           progress(im, 100, 0, last_y, w, h);
     }

   rc = 1;                      /* Success */

 quit:
   for (i = 0; i < h; i++)
      free(rows[i]);
   free(rows);

 quit2:
#if GIFLIB_MAJOR > 5 || (GIFLIB_MAJOR == 5 && GIFLIB_MINOR >= 1)
   DGifCloseFile(gif, NULL);
#else
   DGifCloseFile(gif);
#endif

   return rc;
}

imlib2 before 1.4.7 allows remote attackers to cause a denial of service (segmentation fault) via a crafted GIF file.