Esempio n. 1
0
    void jacobi(
        std::size_t n
      , std::size_t iterations, std::size_t block_size
      , std::string output_filename)
    {
        typedef std::vector<double> vector;

        std::shared_ptr<vector> grid_new(new vector(n * n, 1));
        std::shared_ptr<vector> grid_old(new vector(n * n, 1));

        hpx::util::high_resolution_timer t;
        for(std::size_t i = 0; i < iterations; ++i)
        {
            // MSVC is unhappy if the OMP loop variable is unsigned
#pragma omp parallel for schedule(JACOBI_SMP_OMP_SCHEDULE)
            for(boost::int64_t y = 1; y < boost::int64_t(n-1); ++y)
            {
                      double * dst = &(*grid_new)[y * n];
                const double * src = &(*grid_new)[y * n];
                jacobi_kernel(
                    dst
                  , src
                  , n
                );
            }
            std::swap(grid_new, grid_old);
        }

        report_timing(n, iterations, t.elapsed());
        output_grid(output_filename, *grid_old, n);
   }
Esempio n. 2
0
 void jacobi_serial(size_t n, size_t iterations, std::string output_filename) {
     hpx::util::high_resolution_timer t;
     matrix1.resize(n*n);
     matrix2.resize(n*n);
     for(size_t i = 1; i < iterations; ++i) {
         matrix2[0] = ( matrix1[0] + matrix1[1] + matrix1[n] ) / 3;//LB
         for(size_t i = 1; i < n-1; i++) {//bot
             matrix2[i] = ( matrix1[i  ] + matrix1[i-1] +
                            matrix1[i+1] + matrix1[i+n] ) * .25;
         }
         matrix2[n-1] = ( matrix1[ n-1 ] + matrix1[n-2] + matrix1[n+n-1] ) / 3;//RB
 
         for(size_t i = 1; i < n-1; i++) {
             matrix2[i*n] = ( matrix1[ i   *n] + matrix1[ i   *n+1] + 
                              matrix1[(i-1)*n] + matrix1[(i+1)*n  ] 
                              ) * .25;//left
             for(size_t j = 1; j < n-1; j++) {
                 matrix2[i*n+j] = ( matrix1[ i   *n+j  ] +  
                                    matrix1[ i   *n+j-1] + matrix1[ i   *n+j+1] +
                                    matrix1[(i-1)*n+j  ] + matrix1[(i+1)*n+j  ]
                                    ) * .2; //mid
             }
             matrix2[i*n+n-1] = ( matrix1[i*n+n-1] + matrix1[i*n+n-2] + 
                                  matrix1[i*n-1] + matrix1[(i+2)*n-1] 
                                  ) * .25;//right
         }
         matrix2[(n-1)*n] = ( matrix1[(n-1)*n] + matrix1[(n-1)*n+1] +
                                                 matrix1[(n-2)*n] ) / 3;//TL
         for(size_t i = 1; i < n-1; i++) {//top
             matrix2[(n-1)*n + i] = ( matrix1[(n-1)*n+i] + matrix1[(n-1)*n+i-1] +
                                                           matrix1[(n-1)*n+i+1] +
                                                           matrix1[(n-2)*n+i  ] ) * .25;
         }
         matrix2[n*n-1] = ( matrix1[n*n-1] + matrix1[n*n-2] + matrix1[n*n-1-n] ) / 3;//TR
     }
     report_timing(n, iterations, t.elapsed());
     //output_grid(output_filename, *grid_old, n);
 }
Esempio n. 3
0
    void jacobi( size_t n , size_t iterations, size_t block_size, std::string output_filename) {
        hpx::util::high_resolution_timer t;

        vector< vector< vector< shared_future<block> > > > blockList(2);
        jacobi_init(blockList, n, block_size);

        size_t numBlocks = blockList[0].size();

        for(size_t i = 1; i < iterations; ++i) {
            const size_t prev = i%2;
            const size_t curr = (i+1)%2;
            blockList[curr][0][0] = dataflow(
                    jacobi_BL, blockList[prev][0][0],
                               blockList[prev][0][1],
                               blockList[prev][1][0] );
            for(size_t j = 1; j < numBlocks - 1; j++) {
                blockList[curr][j][0] = dataflow(
                        jacobi_left, blockList[prev][j  ][0],
                                     blockList[prev][j  ][1],
                                     blockList[prev][j-1][0],
                                     blockList[prev][j+1][0] );
            }
            blockList[curr][numBlocks-1][0] = dataflow(
                    jacobi_TL, blockList[prev][numBlocks-1][0],
                               blockList[prev][numBlocks-1][1],
                               blockList[prev][numBlocks-2][0] );

            for(size_t j = 1; j < numBlocks - 1; j++) {
                blockList[curr][0][j] = dataflow(
                        jacobi_bot, blockList[prev][0][j  ], 
                                    blockList[prev][0][j-1],
                                    blockList[prev][0][j+1],
                                    blockList[prev][1][j  ] );
                for(size_t k = 1; k < numBlocks - 1; k++) {
                    blockList[curr][j][k] = dataflow( 
                            jacobi_op, blockList[prev][k  ][j  ],
                                       blockList[prev][k  ][j-1],
                                       blockList[prev][k  ][j+1],
                                       blockList[prev][k-1][j  ],
                                       blockList[prev][k+1][j  ]);
                }
                blockList[curr][numBlocks-1][j] = dataflow(
                        jacobi_top, blockList[prev][numBlocks-1][j  ], 
                                    blockList[prev][numBlocks-1][j-1],
                                    blockList[prev][numBlocks-1][j+1],
                                    blockList[prev][numBlocks-2][j  ] );
            }
            blockList[curr][0][numBlocks-1] = dataflow(
                    jacobi_BR, blockList[prev][0][numBlocks-1],
                               blockList[prev][0][numBlocks-2],
                               blockList[prev][1][numBlocks-1]);
            for(size_t j = 1; j < numBlocks - 1; j++) {
            blockList[curr][j][numBlocks-1] = dataflow(
                    jacobi_left, blockList[prev][j ][numBlocks-1],
                                 blockList[prev][j ][numBlocks-2],
                                 blockList[prev][j-1][numBlocks-1],
                                 blockList[prev][j+1][numBlocks-1]);
            }
            blockList[curr][numBlocks-1][numBlocks-1] = dataflow(
                    jacobi_TR, blockList[prev][numBlocks-1][numBlocks-1],
                               blockList[prev][numBlocks-1][numBlocks-2],
                               blockList[prev][numBlocks-2][numBlocks-1]);
        }
        for(int i = 0; i < blockList[(n-1)%2].size(); i++) {
            hpx::wait_all(blockList[(n-1)%2][i]);
        }

        report_timing(n, iterations, t.elapsed());
        //output_grid(output_filename, *grid_old, n);
   }
Esempio n. 4
0
static int aio_event_thread(void *data)
{
	struct aio_threadinfo *tinfo = data;
	struct aio_output *output = tinfo->output;
	struct aio_threadinfo *other = &output->tinfo[2];
	int err = -ENOMEM;
	
	MARS_DBG("event thread has started.\n");
	//set_user_nice(current, -20);

	use_fake_mm();
	if (!current->mm)
		goto err;

	err = aio_start_thread(output, &output->tinfo[2], aio_sync_thread, 'y');
	if (unlikely(err < 0))
		goto err;

	while (!brick_thread_should_stop() || atomic_read(&tinfo->queued_sum) > 0) {
		mm_segment_t oldfs;
		int count;
		int i;
		struct timespec timeout = {
			.tv_sec = 1,
		};
		struct io_event events[MARS_MAX_AIO_READ];

		oldfs = get_fs();
		set_fs(get_ds());
		/* TODO: don't timeout upon termination.
		 * Probably we should submit a dummy request.
		 */
		count = sys_io_getevents(output->ctxp, 1, MARS_MAX_AIO_READ, events, &timeout);
		set_fs(oldfs);

		if (likely(count > 0)) {
			atomic_sub(count, &output->submit_count);
		}

		for (i = 0; i < count; i++) {
			struct aio_mref_aspect *mref_a = (void*)events[i].data;
			struct mref_object *mref;
			int err = events[i].res;

			if (!mref_a) {
				continue; // this was a dummy request
			}
			mref = mref_a->object;

			MARS_IO("AIO done %p pos = %lld len = %d rw = %d\n", mref, mref->ref_pos, mref->ref_len, mref->ref_rw);

			mapfree_set(output->mf, mref->ref_pos, mref->ref_pos + mref->ref_len);

			if (output->brick->o_fdsync
			   && err >= 0 
			   && mref->ref_rw != READ
			   && !mref->ref_skip_sync
			   && !mref_a->resubmit++) {
				// workaround for non-implemented AIO FSYNC operation
				if (output->mf &&
				    output->mf->mf_filp &&
				    output->mf->mf_filp->f_op &&
				    !output->mf->mf_filp->f_op->aio_fsync) {
					mars_trace(mref, "aio_fsync");
					_enqueue(other, mref_a, mref->ref_prio, true);
					continue;
				}
				err = aio_submit(output, mref_a, true);
				if (likely(err >= 0))
					continue;
			}

			_complete(output, mref_a, err);

		}
	}
	err = 0;

 err:
	MARS_DBG("event thread has stopped, err = %d\n", err);

	aio_stop_thread(output, 2, false);

	unuse_fake_mm();

	tinfo->terminated = true;
	wake_up_interruptible_all(&tinfo->terminate_event);
	return err;
}

#if 1
/* This should go to fs/open.c (as long as vfs_submit() is not implemented)
 */
#include <linux/fdtable.h>
void fd_uninstall(unsigned int fd)
{
	struct files_struct *files = current->files;
	struct fdtable *fdt;
	MARS_DBG("fd = %d\n", fd);
	if (unlikely(fd < 0)) {
		MARS_ERR("bad fd = %d\n", fd);
		return;
	}
	spin_lock(&files->file_lock);
	fdt = files_fdtable(files);
	rcu_assign_pointer(fdt->fd[fd], NULL);
	spin_unlock(&files->file_lock);
}
EXPORT_SYMBOL(fd_uninstall);
#endif

static
atomic_t ioctx_count = ATOMIC_INIT(0);

static
void _destroy_ioctx(struct aio_output *output)
{
	if (unlikely(!output))
		goto done;

	aio_stop_thread(output, 1, true);

	use_fake_mm();

	if (likely(output->ctxp)) {
		mm_segment_t oldfs;
		int err;

		MARS_DBG("ioctx count = %d destroying %p\n", atomic_read(&ioctx_count), (void*)output->ctxp);
		oldfs = get_fs();
		set_fs(get_ds());
		err = sys_io_destroy(output->ctxp);
		set_fs(oldfs);
		atomic_dec(&ioctx_count);
		MARS_DBG("ioctx count = %d status = %d\n", atomic_read(&ioctx_count), err);
		output->ctxp = 0;
	}

	if (likely(output->fd >= 0)) {
		MARS_DBG("destroying fd %d\n", output->fd);
		fd_uninstall(output->fd);
		put_unused_fd(output->fd);
		output->fd = -1;
	}

 done:
	if (likely(current->mm)) {
		unuse_fake_mm();
	}
}

static
int _create_ioctx(struct aio_output *output)
{
	struct file *file;
	mm_segment_t oldfs;
	int err = -EINVAL;

	CHECK_PTR_NULL(output, done);
	CHECK_PTR_NULL(output->mf, done);
	file = output->mf->mf_filp;
	CHECK_PTR_NULL(file, done);

	/* TODO: this is provisionary. We only need it for sys_io_submit()
	 * which uses userspace concepts like file handles.
	 * This should be accompanied by a future kernelsapce vfs_submit() or
	 * do_submit() which currently does not exist :(
	 */
	err = get_unused_fd();
	MARS_DBG("file %p '%s' new fd = %d\n", file, output->mf->mf_name, err);
	if (unlikely(err < 0)) {
		MARS_ERR("cannot get fd, err=%d\n", err);
		goto done;
	}
	output->fd = err;
	fd_install(err, file);

	use_fake_mm();

	err = -ENOMEM;
	if (unlikely(!current->mm)) {
		MARS_ERR("cannot fake mm\n");
		goto done;
	}

	MARS_DBG("ioctx count = %d old = %p\n", atomic_read(&ioctx_count), (void*)output->ctxp);
	output->ctxp = 0;

	oldfs = get_fs();
	set_fs(get_ds());
	err = sys_io_setup(MARS_MAX_AIO, &output->ctxp);
	set_fs(oldfs);
	if (likely(output->ctxp))
		atomic_inc(&ioctx_count);
	MARS_DBG("ioctx count = %d new = %p status = %d\n", atomic_read(&ioctx_count), (void*)output->ctxp, err);
	if (unlikely(err < 0)) {
		MARS_ERR("io_setup failed, err=%d\n", err);
		goto done;
	}
	
	err = aio_start_thread(output, &output->tinfo[1], aio_event_thread, 'e');
	if (unlikely(err < 0)) {
		MARS_ERR("could not start event thread\n");
		goto done;
	}

 done:
	if (likely(current->mm)) {
		unuse_fake_mm();
	}
	return err;
}

static int aio_submit_thread(void *data)
{
	struct aio_threadinfo *tinfo = data;
	struct aio_output *output = tinfo->output;
	struct file *file;
	int err = -EINVAL;

	MARS_DBG("submit thread has started.\n");

	file = output->mf->mf_filp;

	use_fake_mm();

	while (!brick_thread_should_stop() || atomic_read(&output->read_count) + atomic_read(&output->write_count) + atomic_read(&tinfo->queued_sum) > 0) {
		struct aio_mref_aspect *mref_a;
		struct mref_object *mref;
		int sleeptime;
		int status;

		wait_event_interruptible_timeout(
			tinfo->event,
			atomic_read(&tinfo->queued_sum) > 0,
			HZ / 4);

		mref_a = _dequeue(tinfo);
		if (!mref_a) {
			continue;
		}

		mref = mref_a->object;
		status = -EINVAL;
		CHECK_PTR(mref, error);

		mapfree_set(output->mf, mref->ref_pos, -1);

		if (mref->ref_rw) {
			insert_dirty(output, mref_a);
		}

		// check for reads exactly at EOF (special case)
		if (mref->ref_pos == mref->ref_total_size &&
		   !mref->ref_rw &&
		   mref->ref_timeout > 0) {
			loff_t total_size = i_size_read(file->f_mapping->host);
			loff_t len = total_size - mref->ref_pos;
			if (len > 0) {
				mref->ref_total_size = total_size;
				mref->ref_len = len;
			} else {
				if (!mref_a->start_jiffies) {
					mref_a->start_jiffies = jiffies;
				}
				if ((long long)jiffies - mref_a->start_jiffies <= mref->ref_timeout) {
					if (atomic_read(&tinfo->queued_sum) <= 0) {
						atomic_inc(&output->total_msleep_count);
						brick_msleep(1000 * 4 / HZ);
					}
					_enqueue(tinfo, mref_a, MARS_PRIO_LOW, true);
					continue;
				}
				MARS_DBG("ENODATA %lld\n", len);
				_complete(output, mref_a, -ENODATA);
				continue;
			}
		}

		sleeptime = 1;
		for (;;) {
			status = aio_submit(output, mref_a, false);

			if (likely(status != -EAGAIN)) {
				break;
			}
			atomic_inc(&output->total_delay_count);
			brick_msleep(sleeptime);
			if (sleeptime < 100) {
				sleeptime++;
			}
		}
	error:
		if (unlikely(status < 0)) {
			MARS_IO("submit_count = %d status = %d\n", atomic_read(&output->submit_count), status);
			_complete_mref(output, mref, status);
		}
	}

	MARS_DBG("submit thread has stopped, status = %d.\n", err);

	if (likely(current->mm)) {
		unuse_fake_mm();
	}

	tinfo->terminated = true;
	wake_up_interruptible_all(&tinfo->terminate_event);
	return err;
}

static int aio_get_info(struct aio_output *output, struct mars_info *info)
{
	struct file *file;
	loff_t min;
	loff_t max;

	if (unlikely(!output ||
		     !output->mf ||
		     !(file = output->mf->mf_filp) ||
		     !file->f_mapping ||
		     !file->f_mapping->host))
		return -EINVAL;

	info->tf_align = 1;
	info->tf_min_size = 1;

	/* Workaround for races in the page cache.
	 *
	 * It appears that concurrent reads and writes seem to
	 * result in inconsistent reads in some very rare cases, due to
	 * races. Sometimes, the inode claims that the file has been already
	 * appended by a write operation, but the data has not actually hit
	 * the page cache, such that a concurrent read gets NULL blocks.
	 */
	min = i_size_read(file->f_mapping->host);
	max = 0;

	if (!output->brick->is_static_device) {
		get_dirty(output, &min, &max);
	}

	info->current_size = min;
	MARS_DBG("determined file size = %lld\n", info->current_size);

	return 0;
}

//////////////// informational / statistics ///////////////

static noinline
char *aio_statistics(struct aio_brick *brick, int verbose)
{
	struct aio_output *output = brick->outputs[0];
	char *res = brick_string_alloc(4096);
	char *sync = NULL;
	int pos = 0;
	if (!res)
		return NULL;

	pos += report_timing(&timings[0], res + pos, 4096 - pos);
	pos += report_timing(&timings[1], res + pos, 4096 - pos);
	pos += report_timing(&timings[2], res + pos, 4096 - pos);

	snprintf(res + pos, 4096 - pos,
		 "total "
		 "reads = %d "
		 "writes = %d "
		 "allocs = %d "
		 "submits = %d "
		 "again = %d "
		 "delays = %d "
		 "msleeps = %d "
		 "fdsyncs = %d "
		 "fdsync_waits = %d "
		 "map_free = %d | "
		 "flying reads = %d "
		 "writes = %d "
		 "allocs = %d "
		 "submits = %d "
		 "q0 = %d "
		 "q1 = %d "
		 "q2 = %d "
		 "| total "
		 "q0 = %d "
		 "q1 = %d "
		 "q2 = %d "
		 "%s\n",
		 atomic_read(&output->total_read_count),
		 atomic_read(&output->total_write_count),
		 atomic_read(&output->total_alloc_count),
		 atomic_read(&output->total_submit_count),
		 atomic_read(&output->total_again_count),
		 atomic_read(&output->total_delay_count),
		 atomic_read(&output->total_msleep_count),
		 atomic_read(&output->total_fdsync_count),
		 atomic_read(&output->total_fdsync_wait_count),
		 atomic_read(&output->total_mapfree_count),
		 atomic_read(&output->read_count),
		 atomic_read(&output->write_count),
		 atomic_read(&output->alloc_count),
		 atomic_read(&output->submit_count),
		 atomic_read(&output->tinfo[0].queued_sum),
		 atomic_read(&output->tinfo[1].queued_sum),
		 atomic_read(&output->tinfo[2].queued_sum),
		 atomic_read(&output->tinfo[0].total_enqueue_count),
		 atomic_read(&output->tinfo[1].total_enqueue_count),
		 atomic_read(&output->tinfo[2].total_enqueue_count),
		 sync ? sync : "");
	
	if (sync)
		brick_string_free(sync);

	return res;
}

static noinline
void aio_reset_statistics(struct aio_brick *brick)
{
	struct aio_output *output = brick->outputs[0];
	int i;
	atomic_set(&output->total_read_count, 0);
	atomic_set(&output->total_write_count, 0);
	atomic_set(&output->total_alloc_count, 0);
	atomic_set(&output->total_submit_count, 0);
	atomic_set(&output->total_again_count, 0);
	atomic_set(&output->total_delay_count, 0);
	atomic_set(&output->total_msleep_count, 0);
	atomic_set(&output->total_fdsync_count, 0);
	atomic_set(&output->total_fdsync_wait_count, 0);
	atomic_set(&output->total_mapfree_count, 0);
	for (i = 0; i < 3; i++) {
		struct aio_threadinfo *tinfo = &output->tinfo[i];
		atomic_set(&tinfo->total_enqueue_count, 0);
	}
}


//////////////// object / aspect constructors / destructors ///////////////

static int aio_mref_aspect_init_fn(struct generic_aspect *_ini)
{
	struct aio_mref_aspect *ini = (void*)_ini;
	INIT_LIST_HEAD(&ini->io_head);
	INIT_LIST_HEAD(&ini->dirty_head);
	return 0;
}

static void aio_mref_aspect_exit_fn(struct generic_aspect *_ini)
{
	struct aio_mref_aspect *ini = (void*)_ini;
	CHECK_HEAD_EMPTY(&ini->dirty_head);
	CHECK_HEAD_EMPTY(&ini->io_head);
}

MARS_MAKE_STATICS(aio);

////////////////////// brick constructors / destructors ////////////////////

static int aio_brick_construct(struct aio_brick *brick)
{
	return 0;
}

static int aio_switch(struct aio_brick *brick)
{
	static int index;
	struct aio_output *output = brick->outputs[0];
	const char *path = output->brick->brick_path;
	int flags = O_RDWR | O_LARGEFILE;
	int status = 0;

	MARS_DBG("power.button = %d\n", brick->power.button);
	if (!brick->power.button)
		goto cleanup;

	if (brick->power.led_on || output->mf)
		goto done;

	mars_power_led_off((void*)brick, false);

	if (brick->o_creat) {
		flags |= O_CREAT;
		MARS_DBG("using O_CREAT on %s\n", path);
	}
	if (brick->o_direct) {
		flags |= O_DIRECT;
		MARS_DBG("using O_DIRECT on %s\n", path);
	}

	output->mf = mapfree_get(path, flags);
	if (unlikely(!output->mf)) {
		MARS_ERR("could not open file = '%s' flags = %d\n", path, flags);
		status = -ENOENT;
		goto err;
	} 

	output->index = ++index;

	status = _create_ioctx(output);
	if (unlikely(status < 0)) {
		MARS_ERR("could not create ioctx, status = %d\n", status);
		goto err;
	}

	status = aio_start_thread(output, &output->tinfo[0], aio_submit_thread, 's');
	if (unlikely(status < 0)) {
		MARS_ERR("could not start theads, status = %d\n", status);
		goto err;
	}

	MARS_DBG("opened file '%s'\n", path);
	mars_power_led_on((void*)brick, true);

done:
	return 0;

err:
	MARS_ERR("status = %d\n", status);
cleanup:
	if (brick->power.led_off) {
		goto done;
	}

	mars_power_led_on((void*)brick, false);

	aio_stop_thread(output, 0, false);

	_destroy_ioctx(output);

	mars_power_led_off((void*)brick,
			  (output->tinfo[0].thread == NULL &&
			   output->tinfo[1].thread == NULL &&
			   output->tinfo[2].thread == NULL));

	MARS_DBG("switch off led_off = %d status = %d\n", brick->power.led_off, status);
	if (brick->power.led_off) {
		if (output->mf) {
			MARS_DBG("closing file = '%s'\n", output->mf->mf_name);
			mapfree_put(output->mf);
			output->mf = NULL;
		}
	}
	return status;
}

static int aio_output_construct(struct aio_output *output)
{
	INIT_LIST_HEAD(&output->dirty_anchor);
	spin_lock_init(&output->dirty_lock);
	init_waitqueue_head(&output->fdsync_event);
	output->fd = -1;
	return 0;
}
Esempio n. 5
0
static void end_write_timing(unsigned length)
{
	timing_stats.write_time += (jiffies - start_time);
	timing_stats.write_bytes += length + 2;
	report_timing();
}
Esempio n. 6
0
    void jacobi(
        std::size_t n
      , std::size_t iterations, std::size_t block_size
      , std::string output_filename)
    {
        typedef std::vector<double> vector;

        boost::shared_ptr<vector> grid_new(new vector(n * n, 1));
        boost::shared_ptr<vector> grid_old(new vector(n * n, 1));

        typedef std::vector<hpx::shared_future<void> > deps_vector;

        std::size_t n_block = static_cast<std::size_t>(std::ceil(double(n)/block_size));


        boost::shared_ptr<deps_vector> deps_new(
            new deps_vector(n_block, hpx::make_ready_future()));
        boost::shared_ptr<deps_vector> deps_old(
            new deps_vector(n_block, hpx::make_ready_future()));

        hpx::util::high_resolution_timer t;
        for(std::size_t i = 0; i < iterations; ++i)
        {
            for(std::size_t y = 1, j = 0; y < n -1; y += block_size, ++j)
            {
                std::size_t y_end = (std::min)(y + block_size, n - 1);
                std::vector<hpx::shared_future<void> > trigger;
                trigger.reserve(3);
                trigger.push_back((*deps_old)[j]);
                if(j > 0) trigger.push_back((*deps_old)[j-1]);
                if(j + 1 < n_block) trigger.push_back((*deps_old)[j+1]);

                /*
                 * FIXME: dataflow seems to have some raceconditions
                 * left
                (*deps_new)[j]
                    = hpx::dataflow(
                        hpx::util::bind(
                            jacobi_kernel_wrap
                          , range(y, y_end)
                          , n
                          , boost::ref(*grid_new)
                          , boost::cref(*grid_old)
                        )
                      , trigger
                    );
                */

                (*deps_new)[j] = hpx::when_all(std::move(trigger)).then(
                    hpx::launch::async,
                    hpx::util::bind(
                        jacobi_kernel_wrap
                      , range(y, y_end)
                      , n
                      , boost::ref(*grid_new)
                      , boost::cref(*grid_old)
                    )
                );
            }

            std::swap(grid_new, grid_old);
            std::swap(deps_new, deps_old);
        }
        hpx::wait_all(*deps_new);
        hpx::wait_all(*deps_old);

        report_timing(n, iterations, t.elapsed());
        output_grid(output_filename, *grid_old, n);
   }