void *bucket_split(Bucket *bucket, Bucket *buckets, uint32_t bucket_index,
Heap *heap, uint32_t block_size) {
uint32_t first_block_index = bucket_get_first_block_index(bucket);
Block *first_block_header = block_get_header(heap, first_block_index);
uint32_t first_block_size = block_get_size(first_block_header);
// Calculate the remaining size of the first block.
uint32_t rem_size = first_block_size - block_size;
// Create a new header for the new block we have created.
uint32_t alloc_index = first_block_index + block_size_to_bytes(rem_size);
Block *alloc_header = block_get_header(heap, alloc_index);
uint32_t next_block_index = block_get_next_index(first_block_header);
Block *next_block_header = block_get_header(heap, next_block_index);
// If we are simply replacing the block, we don't need to update the previous
// and next indexes of memory, since they will not change: no blocks are
// being created or merged.
if (rem_size != 0) {
block_set_size(alloc_header, block_size);
block_set_prev_index(alloc_header, first_block_index);
block_set_next_index(alloc_header, next_block_index);
}
block_set_allocated(alloc_header, true);
// If there is no left over space in the block, there is no need to split it.
// Simply remove this block from the bucket and return an address to it.
if (rem_size == 0) {
bucket_remove(bucket, heap, first_block_index);
return &heap->memory[block_index_skip_header(alloc_index)];
}
// Update the header of the remaining part of the block.
block_set_next_index(first_block_header, alloc_index);
block_set_size(first_block_header, rem_size);
// Update the header of the next block.
next_block_header = block_get_header(heap, next_block_index);
block_set_prev_index(next_block_header, alloc_index);
// If the remaining space from the block is now too small for its bucket, it
// is removed from this bucket and reinserted into the bucket structure.
uint32_t first_block_bucket_index = block_get_bucket_index(
first_block_header);
if (first_block_bucket_index != bucket_index) {
bucket_remove(bucket, heap, first_block_index);
bucket_insert(&buckets[first_block_bucket_index], heap, first_block_index);
}
return &heap->memory[block_index_skip_header(alloc_index)];
}
void smap_put(smap *map, char *key, int value) {
if (!map) {
return;
}
size_t hash = hash_string(key) % map->num_buckets;
bucket buck = map->buckets[hash];
for (size_t i = 0; i < buck.num_pairs; i += 1) {
if (!strcmp(key, buck.pairs[i].key)) {
buck.pairs[i].val = value;
return;
}
}
if ((map->num_pairs + 1) < map->num_buckets * LOAD_FACTOR) {
bucket_insert(map->buckets + hash, key, value);
map->num_pairs += 1;
} else {
expand(map);
smap_put(map, key, value);
}
}
static ngx_int_t ngx_estreaming_handler(ngx_http_request_t * r) {
size_t root;
ngx_int_t rc;
ngx_uint_t level;
ngx_str_t path;
ngx_open_file_info_t of;
ngx_http_core_loc_conf_t *clcf;
video_buffer *destination;
if (!(r->method & (NGX_HTTP_GET | NGX_HTTP_HEAD)))
return NGX_HTTP_NOT_ALLOWED;
if (r->uri.data[r->uri.len - 1] == '/')
return NGX_DECLINED;
rc = ngx_http_discard_request_body(r);
if (rc != NGX_OK)
return rc;
mp4_split_options_t * options = mp4_split_options_init(r);
if (!ngx_http_map_uri_to_path(r, &path, &root, 1)) {
mp4_split_options_exit(r, options);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if (r->args.len && !mp4_split_options_set(r, options, (const char *) r->args.data, r->args.len)) {
mp4_split_options_exit(r, options);
return NGX_DECLINED;
}
if (!options) return NGX_DECLINED;
ngx_log_t * nlog = r->connection->log;
struct bucket_t * bucket = bucket_init(r);
int result = 0;
u_int m3u8 = 0, len_ = 0;
int64_t duration = 0;
if (ngx_memcmp(r->exten.data, "mp4", r->exten.len) == 0) {
return ngx_http_mp4_handler(r);
} else if (ngx_memcmp(r->exten.data, "m3u8", r->exten.len) == 0) {
m3u8 = 1;
} else if (ngx_memcmp(r->exten.data, "len", r->exten.len) == 0) {// this is for length request
len_ = 1;
} else if (ngx_memcmp(r->exten.data, "ts", r->exten.len) == 0) {
// don't do anything
} else {
return NGX_HTTP_UNSUPPORTED_MEDIA_TYPE;
}
// change file name to mp4
// in order to lookup file in filesystem
char *ext = strrchr((const char *) path.data, '.');
strcpy(ext, ".mp4");
path.len = ((u_char *) ext - path.data) + 4;
path.data[path.len] = '\0';
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
ngx_memzero(&of, sizeof (ngx_open_file_info_t));
of.read_ahead = clcf->read_ahead;
of.directio = NGX_MAX_OFF_T_VALUE;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
of.errors = clcf->open_file_cache_errors;
of.events = clcf->open_file_cache_events;
if (ngx_open_cached_file(clcf->open_file_cache, &path, &of, r->pool) != NGX_OK) {
mp4_split_options_exit(r, options);
switch (of.err) {
case 0:
return NGX_HTTP_INTERNAL_SERVER_ERROR;
case NGX_ENOENT:
case NGX_ENOTDIR:
case NGX_ENAMETOOLONG:
level = NGX_LOG_ERR;
rc = NGX_HTTP_NOT_FOUND;
break;
case NGX_EACCES:
level = NGX_LOG_ERR;
rc = NGX_HTTP_FORBIDDEN;
break;
default:
level = NGX_LOG_CRIT;
rc = NGX_HTTP_INTERNAL_SERVER_ERROR;
break;
}
if (rc != NGX_HTTP_NOT_FOUND || clcf->log_not_found) {
ngx_log_error(level, nlog, of.err,
ngx_open_file_n " \"%s\" failed", path.data);
}
return rc;
}
if (!of.is_file) {
mp4_split_options_exit(r, options);
if (ngx_close_file(of.fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, nlog, ngx_errno,
ngx_close_file_n " \"%s\" failed", path.data);
}
return NGX_DECLINED;
}
/* move atom to beginning of file if it's in the last*/
hls_conf_t *mlcf;
mlcf = ngx_http_get_module_loc_conf(r, ngx_http_estreaming_module);
if (mlcf->mp4_enhance == 1) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0,
"examine mp4 filename: \"%V\"", &path);
if (ngx_http_enable_fast_start(&path, of.fd, r) != NGX_OK) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
}
ngx_file_t *file = ngx_pcalloc(r->pool, sizeof (ngx_file_t));
if (file == NULL) {
mp4_split_options_exit(r, options);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
file->fd = of.fd;
file->name = path;
file->log = nlog;
mp4_context_t *mp4_context = mp4_open(r, file, of.size, MP4_OPEN_MOOV);
if (!mp4_context) {
mp4_split_options_exit(r, options);
ngx_log_error(NGX_LOG_ALERT, nlog, ngx_errno, "mp4_open failed");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
mp4_context->root = root;
if (m3u8 || len_) {
int ret, video_width = 0;
// only request for master playlist need to use ffmpeg api to open video
// this takes time and cpu, we should avoid it as much as possible
if ((m3u8) && (options->adbr || options->org)) goto no_ffmpeg;
AVFormatContext *fmt_ctx = NULL;
unsigned int i;
av_register_all();
if ((ret = avformat_open_input(&fmt_ctx, (const char*) path.data, NULL, NULL)) < 0) {
mp4_split_options_exit(r, options);
mp4_close(mp4_context);
if (fmt_ctx) avformat_close_input(&fmt_ctx);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if ((ret = avformat_find_stream_info(fmt_ctx, NULL)) < 0) {
if (fmt_ctx) avformat_close_input(&fmt_ctx);
mp4_close(mp4_context);
mp4_split_options_exit(r, options);
av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if (len_) {
duration = fmt_ctx->duration;
u_char *buffer = (u_char *) ngx_palloc(mp4_context->r->pool, 10 * sizeof (char));
u_char * p = buffer;
if (buffer == NULL) {
mp4_split_options_exit(r, options);
mp4_close(mp4_context);
if (fmt_ctx) avformat_close_input(&fmt_ctx);
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
// this is stolen from ffmpeg source code
if (duration != AV_NOPTS_VALUE) {
duration = duration + 5000;
int secs;
secs = duration / AV_TIME_BASE;
p = ngx_sprintf(p, "%02d\n", secs);
} else {
p = ngx_sprintf(p, "N/A\n");
}
bucket_insert(bucket, buffer, p - buffer);
ngx_pfree(mp4_context->r->pool, buffer);
if (fmt_ctx) avformat_close_input(&fmt_ctx);
r->allow_ranges = 0;
result = 1;
goto response;
} else {
for (i = 0; i < fmt_ctx->nb_streams; i++) {
AVStream *stream;
AVCodecContext *codec_ctx;
stream = fmt_ctx->streams[i];
codec_ctx = stream->codec;
if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
av_log(NULL, AV_LOG_ERROR, "source video w:%d", codec_ctx->width);
if (video_width == 0) {
video_width = codec_ctx->width;
} else if ((video_width != 0) && (video_width < codec_ctx->width)) {
// has 2 video streams
video_width = codec_ctx->width;
} else
break;
}
}
avformat_close_input(&fmt_ctx);
}
// finish getting video width
no_ffmpeg:
if ((result = mp4_create_m3u8(mp4_context, bucket, options, video_width))) {
char action[50];
sprintf(action, "ios_playlist&segments=%d", result);
view_count(mp4_context, (char *) path.data, options ? options->hash : NULL, action);
}
r->allow_ranges = 0;
} else {
result = output_ts(mp4_context, bucket, options);
if (!options || !result) {
mp4_close(mp4_context);
ngx_log_error(NGX_LOG_ALERT, nlog, ngx_errno, "output_ts failed");
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
if (options->adbr) {
destination = ngx_pcalloc(r->pool, sizeof (video_buffer));
destination->data = NULL;
destination->len = 0;
destination->pool = r->pool;
if (ngx_estreaming_adaptive_bitrate(r, bucket->first,
destination, options) == NGX_OK) {
ngx_buf_t *b = ngx_pcalloc(r->pool, sizeof (ngx_buf_t));
if (b == NULL) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
}
bucket->first->buf = b;
bucket->first->next = NULL;
b->pos = destination->data;
b->last = destination->data + (destination->len * sizeof (unsigned char));
b->memory = 1;
b->last_buf = 1;
bucket->content_length = destination->len;
}
ngx_pfree(r->pool, destination);
}
char action[50] = "ios_view";
view_count(mp4_context, (char *) path.data, options->hash, action);
r->allow_ranges = 1;
}
response:
mp4_close(mp4_context);
mp4_split_options_exit(r, options);
result = result == 0 ? 415 : 200;
r->root_tested = !r->error_page;
if (result && bucket) {
nlog->action = "sending mp4 to client";
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, nlog, 0, "content_length: %d", bucket->content_length);
if (bucket->content_length == 0) return NGX_HTTP_UNSUPPORTED_MEDIA_TYPE;
r->headers_out.status = NGX_HTTP_OK;
r->headers_out.content_length_n = bucket->content_length;
r->headers_out.last_modified_time = of.mtime;
if (m3u8) {
r->headers_out.content_type.len = sizeof ("application/vnd.apple.mpegurl") - 1;
r->headers_out.content_type.data = (u_char *) "application/vnd.apple.mpegurl";
} else if (len_) {
r->headers_out.content_type.len = sizeof ("text/html") - 1;
r->headers_out.content_type.data = (u_char *) "text/html";
} else {
r->headers_out.content_type.len = sizeof ("video/MP2T") - 1;
r->headers_out.content_type.data = (u_char *) "video/MP2T";
}
rc = ngx_http_send_header(r);
if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
ngx_log_error(NGX_LOG_ALERT, nlog, ngx_errno, ngx_close_file_n "ngx_http_send_header failed");
return rc;
}
return ngx_http_output_filter(r, bucket->first);
} else return NGX_HTTP_UNSUPPORTED_MEDIA_TYPE;
}
/*
* Bucket-sort algorithm.
*/
extern void bucketsort(int *array, int n)
{
int max; /* Maximum number. */
int i, j; /* Loop indexes. */
int range; /* Bucket range. */
struct minibucket *minib; /* Working mini-bucket. */
struct message *msg; /* Working message. */
struct bucket **todo; /* Todo buckets. */
struct bucket **done; /* Done buckets. */
uint64_t start, end; /* Timers. */
/* Setup slaves. */
open_noc_connectors();
spawn_slaves();
sync_slaves();
todo = smalloc(NUM_BUCKETS*sizeof(struct bucket *));
done = smalloc(NUM_BUCKETS*sizeof(struct bucket *));
for (i = 0; i < NUM_BUCKETS; i++)
{
done[i] = bucket_create();
todo[i] = bucket_create();
}
/* Find max number in the array. */
start = timer_get();
max = INT_MIN;
for (i = 0; i < n; i++)
{
/* Found. */
if (array[i] > max)
max = array[i];
}
/* Distribute numbers. */
range = max/NUM_BUCKETS;
for (i = 0; i < n; i++)
{
j = array[i]/range;
if (j >= NUM_BUCKETS)
j = NUM_BUCKETS - 1;
bucket_insert(&todo[j], array[i]);
}
end = timer_get();
master += timer_diff(start, end);
/* Sort buckets. */
j = 0;
for (i = 0; i < NUM_BUCKETS; i++)
{
while (bucket_size(todo[i]) > 0)
{
minib = bucket_pop(todo[i]);
/* Send message. */
msg = message_create(SORTWORK, i, minib->size);
message_send(outfd[j], msg);
message_destroy(msg);
/* Send data. */
communication +=
data_send(outfd[j], minib->elements, minib->size*sizeof(int));
minibucket_destroy(minib);
j++;
/*
* Slave processes are busy.
* So let's wait for results.
*/
if (j == nclusters)
{
/* Receive results. */
for (/* NOOP */ ; j > 0; j--)
{
/* Receive message. */
msg = message_receive(infd[nclusters - j]);
/* Receive mini-bucket. */
minib = minibucket_create();
minib->size = msg->u.sortresult.size;
communication += data_receive(infd[nclusters -j], minib->elements,
minib->size*sizeof(int));
bucket_push(done[msg->u.sortresult.id], minib);
message_destroy(msg);
}
}
}
}
/* Receive results. */
for (/* NOOP */ ; j > 0; j--)
{
/* Receive message. */
msg = message_receive(infd[j - 1]);
/* Receive bucket. */
minib = minibucket_create();
minib->size = msg->u.sortresult.size;
communication +=
data_receive(infd[j - 1], minib->elements, minib->size*sizeof(int));
bucket_push(done[msg->u.sortresult.id], minib);
message_destroy(msg);
}
start = timer_get();
rebuild_array(done, array);
end = timer_get();
master += timer_diff(start, end);
/* House keeping. */
for (i = 0; i < NUM_BUCKETS; i++)
{
bucket_destroy(todo[i]);
bucket_destroy(done[i]);
}
free(done);
free(todo);
join_slaves();
close_noc_connectors();
}
