//============================================================================ // CAST-128 //============================================================================ static int ssh_cast5_ctr(EVP_CIPHER_CTX *ctx, unsigned char *dest, const unsigned char *src, unsigned int len) { struct ssh_cast5_ctr_ctx *c; unsigned int n = 0; unsigned char buf[CAST_BLOCK]; int i, j; CAST_LONG tmp[(CAST_BLOCK + 3) / 4]; unsigned char *p; if (len == 0) return (1); if ((c = EVP_CIPHER_CTX_get_app_data(ctx)) == NULL) return (0); while ((len--) > 0) { if (n == 0) { for (i = j = 0, p = c->cast5_counter; i < CAST_BLOCK; i += 4, j++) { tmp[j] = ((CAST_LONG)*p++) << 24; tmp[j] |= ((CAST_LONG)*p++) << 16; tmp[j] |= ((CAST_LONG)*p++) << 8; tmp[j] |= ((CAST_LONG)*p++); } CAST_encrypt(tmp, &c->cast5_ctx); for (i = j = 0, p = buf; i < CAST_BLOCK; i += 4, j++) { *p++ = (unsigned char)(tmp[j] >> 24); *p++ = (unsigned char)(tmp[j] >> 16); *p++ = (unsigned char)(tmp[j] >> 8); *p++ = (unsigned char)tmp[j]; } ssh_ctr_inc(c->cast5_counter, CAST_BLOCK); } *(dest++) = *(src++) ^ buf[n]; n = (n + 1) % CAST_BLOCK; } return (1); }
//============================================================================ // AES //============================================================================ static int ssh_aes_ctr(EVP_CIPHER_CTX *ctx, unsigned char *dest, const unsigned char *src, unsigned int len) { struct ssh_aes_ctr_ctx *c; unsigned int n = 0; unsigned char buf[AES_BLOCK_SIZE]; if (len == 0) return (1); if ((c = EVP_CIPHER_CTX_get_app_data(ctx)) == NULL) return (0); while ((len--) > 0) { if (n == 0) { AES_encrypt(c->aes_counter, buf, &c->aes_ctx); ssh_ctr_inc(c->aes_counter, AES_BLOCK_SIZE); } *(dest++) = *(src++) ^ buf[n]; n = (n + 1) % AES_BLOCK_SIZE; } return (1); }
//============================================================================ // Triple-DES //============================================================================ static int ssh_des3_ctr(EVP_CIPHER_CTX *ctx, unsigned char *dest, const unsigned char *src, unsigned int len) { struct ssh_des3_ctr_ctx *c; unsigned int n = 0; unsigned char buf[DES_BLOCK_SIZE]; if (len == 0) return (1); if ((c = EVP_CIPHER_CTX_get_app_data(ctx)) == NULL) return (0); while ((len--) > 0) { if (n == 0) { memcpy(buf, (unsigned char *)(c->des3_counter), DES_BLOCK_SIZE); DES_encrypt3((DES_LONG *)buf, &c->des3_ctx[0], &c->des3_ctx[1], &c->des3_ctx[2]); ssh_ctr_inc(c->des3_counter, DES_BLOCK_SIZE); } *(dest++) = *(src++) ^ buf[n]; n = (n + 1) % DES_BLOCK_SIZE; } return (1); }
static int ssh_aes_ctr(EVP_CIPHER_CTX *ctx, u_char *dest, const u_char *src, LIBCRYPTO_EVP_INL_TYPE len) { typedef union { #ifdef CIPHER_INT128_OK __uint128_t *u128; #endif uint64_t *u64; uint32_t *u32; uint8_t *u8; const uint8_t *cu8; uintptr_t u; } ptrs_t; ptrs_t destp, srcp, bufp; uintptr_t align; struct ssh_aes_ctr_ctx *c; struct kq *q, *oldq; int ridx; u_char *buf; if (len == 0) return 1; if ((c = EVP_CIPHER_CTX_get_app_data(ctx)) == NULL) return 0; q = &c->q[c->qidx]; ridx = c->ridx; /* src already padded to block multiple */ srcp.cu8 = src; destp.u8 = dest; while (len > 0) { buf = q->keys[ridx]; bufp.u8 = buf; /* figure out the alignment on the fly */ #ifdef CIPHER_UNALIGNED_OK align = 0; #else align = destp.u | srcp.u | bufp.u; #endif #ifdef CIPHER_INT128_OK if ((align & 0xf) == 0) { destp.u128[0] = srcp.u128[0] ^ bufp.u128[0]; } else #endif if ((align & 0x7) == 0) { destp.u64[0] = srcp.u64[0] ^ bufp.u64[0]; destp.u64[1] = srcp.u64[1] ^ bufp.u64[1]; } else if ((align & 0x3) == 0) { destp.u32[0] = srcp.u32[0] ^ bufp.u32[0]; destp.u32[1] = srcp.u32[1] ^ bufp.u32[1]; destp.u32[2] = srcp.u32[2] ^ bufp.u32[2]; destp.u32[3] = srcp.u32[3] ^ bufp.u32[3]; } else { size_t i; for (i = 0; i < AES_BLOCK_SIZE; ++i) dest[i] = src[i] ^ buf[i]; } destp.u += AES_BLOCK_SIZE; srcp.u += AES_BLOCK_SIZE; len -= AES_BLOCK_SIZE; ssh_ctr_inc(ctx->iv, AES_BLOCK_SIZE); /* Increment read index, switch queues on rollover */ if ((ridx = (ridx + 1) % KQLEN) == 0) { oldq = q; /* Mark next queue draining, may need to wait */ c->qidx = (c->qidx + 1) % NUMKQ; q = &c->q[c->qidx]; pthread_mutex_lock(&q->lock); while (q->qstate != KQFULL) { STATS_WAIT(c->stats); pthread_cond_wait(&q->cond, &q->lock); } q->qstate = KQDRAINING; pthread_mutex_unlock(&q->lock); /* Mark consumed queue empty and signal producers */ pthread_mutex_lock(&oldq->lock); oldq->qstate = KQEMPTY; STATS_DRAIN(c->stats); pthread_cond_broadcast(&oldq->cond); pthread_mutex_unlock(&oldq->lock); } } c->ridx = ridx; return 1; }
/* * The life of a pregen thread: * Find empty keystream queues and fill them using their counter. * When done, update counter for the next fill. */ static void * thread_loop(void *x) { AES_KEY key; STATS_STRUCT(stats); struct ssh_aes_ctr_ctx *c = x; struct kq *q; int i; int qidx; /* Threads stats on cancellation */ STATS_INIT(stats); #ifdef CIPHER_THREAD_STATS pthread_cleanup_push(thread_loop_stats, &stats); #endif /* Thread local copy of AES key */ memcpy(&key, &c->aes_ctx, sizeof(key)); /* * Handle the special case of startup, one thread must fill * the first KQ then mark it as draining. Lock held throughout. */ if (pthread_equal(pthread_self(), c->tid[0])) { q = &c->q[0]; pthread_mutex_lock(&q->lock); if (q->qstate == KQINIT) { for (i = 0; i < KQLEN; i++) { AES_encrypt(q->ctr, q->keys[i], &key); ssh_ctr_inc(q->ctr, AES_BLOCK_SIZE); } ssh_ctr_add(q->ctr, KQLEN * (NUMKQ - 1), AES_BLOCK_SIZE); q->qstate = KQDRAINING; STATS_FILL(stats); pthread_cond_broadcast(&q->cond); } pthread_mutex_unlock(&q->lock); } else STATS_SKIP(stats); /* * Normal case is to find empty queues and fill them, skipping over * queues already filled by other threads and stopping to wait for * a draining queue to become empty. * * Multiple threads may be waiting on a draining queue and awoken * when empty. The first thread to wake will mark it as filling, * others will move on to fill, skip, or wait on the next queue. */ for (qidx = 1;; qidx = (qidx + 1) % NUMKQ) { /* Check if I was cancelled, also checked in cond_wait */ pthread_testcancel(); /* Lock queue and block if its draining */ q = &c->q[qidx]; pthread_mutex_lock(&q->lock); pthread_cleanup_push(thread_loop_cleanup, &q->lock); while (q->qstate == KQDRAINING || q->qstate == KQINIT) { STATS_WAIT(stats); pthread_cond_wait(&q->cond, &q->lock); } pthread_cleanup_pop(0); /* If filling or full, somebody else got it, skip */ if (q->qstate != KQEMPTY) { pthread_mutex_unlock(&q->lock); STATS_SKIP(stats); continue; } /* * Empty, let's fill it. * Queue lock is relinquished while we do this so others * can see that it's being filled. */ q->qstate = KQFILLING; pthread_mutex_unlock(&q->lock); for (i = 0; i < KQLEN; i++) { AES_encrypt(q->ctr, q->keys[i], &key); ssh_ctr_inc(q->ctr, AES_BLOCK_SIZE); } /* Re-lock, mark full and signal consumer */ pthread_mutex_lock(&q->lock); ssh_ctr_add(q->ctr, KQLEN * (NUMKQ - 1), AES_BLOCK_SIZE); q->qstate = KQFULL; STATS_FILL(stats); pthread_cond_signal(&q->cond); pthread_mutex_unlock(&q->lock); } #ifdef CIPHER_THREAD_STATS /* Stats */ pthread_cleanup_pop(1); #endif return NULL; }
static int ssh_aes_ctr(EVP_CIPHER_CTX *ctx, u_char *dest, const u_char *src, LIBCRYPTO_EVP_INL_TYPE len) { struct ssh_aes_ctr_ctx *c; struct kq *q, *oldq; int ridx; u_char *buf; if (len == 0) return (1); if ((c = EVP_CIPHER_CTX_get_app_data(ctx)) == NULL) return (0); q = &c->q[c->qidx]; ridx = c->ridx; /* src already padded to block multiple */ while (len > 0) { buf = q->keys[ridx]; #ifdef CIPHER_BYTE_XOR dest[0] = src[0] ^ buf[0]; dest[1] = src[1] ^ buf[1]; dest[2] = src[2] ^ buf[2]; dest[3] = src[3] ^ buf[3]; dest[4] = src[4] ^ buf[4]; dest[5] = src[5] ^ buf[5]; dest[6] = src[6] ^ buf[6]; dest[7] = src[7] ^ buf[7]; dest[8] = src[8] ^ buf[8]; dest[9] = src[9] ^ buf[9]; dest[10] = src[10] ^ buf[10]; dest[11] = src[11] ^ buf[11]; dest[12] = src[12] ^ buf[12]; dest[13] = src[13] ^ buf[13]; dest[14] = src[14] ^ buf[14]; dest[15] = src[15] ^ buf[15]; #else *(uint64_t *)dest = *(uint64_t *)src ^ *(uint64_t *)buf; *(uint64_t *)(dest + 8) = *(uint64_t *)(src + 8) ^ *(uint64_t *)(buf + 8); #endif dest += 16; src += 16; len -= 16; ssh_ctr_inc(ctx->iv, AES_BLOCK_SIZE); /* Increment read index, switch queues on rollover */ if ((ridx = (ridx + 1) % KQLEN) == 0) { oldq = q; /* Mark next queue draining, may need to wait */ c->qidx = (c->qidx + 1) % NUMKQ; q = &c->q[c->qidx]; pthread_mutex_lock(&q->lock); while (q->qstate != KQFULL) { STATS_WAIT(c->stats); pthread_cond_wait(&q->cond, &q->lock); } q->qstate = KQDRAINING; pthread_mutex_unlock(&q->lock); /* Mark consumed queue empty and signal producers */ pthread_mutex_lock(&oldq->lock); oldq->qstate = KQEMPTY; STATS_DRAIN(c->stats); pthread_cond_broadcast(&oldq->cond); pthread_mutex_unlock(&oldq->lock); } } c->ridx = ridx; return (1); }