Exemplo n.º 1
0
// unbounded_channels <= 0 means no, > 0 means yes
// -1 means advance.
void check_channel(char threadsafe, qio_chtype_t type, int64_t start, int64_t len, int64_t chunksz, qio_hint_t file_hints, qio_hint_t ch_hints, char unbounded_channels, char reopen)
{
  qio_file_t* f;
  qio_channel_t* writing;
  qio_channel_t* reading;
  int64_t offset;
  int64_t usesz;
  int64_t end = start + len;
  int err;
  unsigned char* chunk;
  unsigned char* got_chunk;
  int64_t k;
  ssize_t amt_written;
  ssize_t amt_read;
  char* chhints;
  char* fhints;
  FILE* writefp = NULL;
  FILE* readfp = NULL;
  int memory;
  int64_t ch_end = end;
  char filename[128];
  int fd = -1;

  strcpy(filename,"/tmp/qio_testXXXXXX");

  if( unbounded_channels > 0 ) ch_end = INT64_MAX;

  ch_hints = (ch_hints & ~ QIO_CHTYPEMASK) | type;

  memory = 0;

  if( (file_hints & QIO_METHODMASK) == QIO_METHOD_MEMORY ||
      (ch_hints & QIO_METHODMASK) == QIO_METHOD_MEMORY ) {
    memory = 1;
  }
  if( memory ) {
    file_hints = (file_hints & ~ QIO_METHODMASK ) | QIO_METHOD_MEMORY;
    ch_hints = (ch_hints & ~ QIO_METHODMASK ) | QIO_METHOD_MEMORY;
  }
  if( memory && type == QIO_CH_ALWAYS_UNBUFFERED ) return;
  if( memory && reopen ) return;
  if( (ch_hints & QIO_METHODMASK) == QIO_METHOD_FREADFWRITE ) {
    if( (file_hints & QIO_METHODMASK) != QIO_METHOD_FREADFWRITE ) return;
  }
  if( (ch_hints & QIO_METHODMASK) == QIO_METHOD_MMAP ) {
    if( (file_hints & QIO_METHODMASK) != QIO_METHOD_MMAP ) return;
  }

  fhints = qio_hints_to_string(file_hints);
  chhints = qio_hints_to_string(ch_hints);
  printf("check_channel(threadsafe=%i, type=%i, start=%lli, len=%lli, chunksz=%lli, file_hints=%s, ch_hints=%s, unbounded=%i, reopen=%i)\n",
         (int) threadsafe,
         (int) type,
         (long long int) start,
         (long long int) len,
         (long long int) chunksz,
         fhints,
         chhints,
         (int) unbounded_channels,
         (int) reopen );
  free(fhints);
  free(chhints);

  chunk = malloc(chunksz);
  got_chunk = malloc(chunksz);

  assert(chunk);
  assert(got_chunk);

  if( memory ) {
    err = qio_file_open_mem_ext(&f, NULL, QIO_FDFLAG_READABLE|QIO_FDFLAG_WRITEABLE|QIO_FDFLAG_SEEKABLE, file_hints, NULL);
    assert(!err);
  } else {
    // Open a temporary file.
    if( reopen ) {
      fd = mkstemp(filename);
      close(fd);
      err = qio_file_open_access(&f, filename, "w", file_hints, NULL);
      assert(!err);
    } else {
      err = qio_file_open_tmp(&f, file_hints, NULL);
      assert(!err);
    }

    // Rewind the file
    if (f->fp ) {
      int got;
      got = fseek(f->fp, start, SEEK_SET);
      assert( got == 0 );
    } else {
      off_t off;

      err = sys_lseek(f->fd, start, SEEK_SET, &off);
      assert(!err);
    }
  }

  // Create a "write to file" channel.
  err = qio_channel_create(&writing, f, ch_hints, 0, 1, start, ch_end, NULL);
  assert(!err);

  // Write stuff to the file.
  for( offset = start; offset < end; offset += usesz ) {
    usesz = chunksz;
    if( offset + usesz > end ) usesz = end - offset;
    // Fill chunk.
    fill_testdata(offset, usesz, chunk);
    // Write chunk.
    if( writefp ) {
      amt_written = fwrite(chunk, 1, usesz, writefp);
    } else {
      err = qio_channel_write(threadsafe, writing, chunk, usesz, &amt_written);
      assert(!err);
    }
    assert(amt_written == usesz);

  }

  // Attempt to write 1 more byte; we should get EEOF
  // if we've restricted the range of the channel.
  // Write chunk.
  if( unbounded_channels > 0) {
    // do nothing
  } else {
    if( writefp ) {
      int got;
      got = fflush(writefp);
      assert(got == 0);
      amt_written = fwrite(chunk, 1, 1, writefp);
      // fwrite might buffer on its own.
      if( amt_written != 0 ) {
        got = fflush(writefp);
        assert(got == EOF);
      }
      assert(errno == EEOF);
    } else {
      if( unbounded_channels < 0 ) {
        int times = -unbounded_channels;
        for( int z = 0; z < times; z++ ) {
          err = qio_channel_advance(threadsafe, writing, 1);
          assert( !err );
        }
      }

      err = qio_channel_write(threadsafe, writing, chunk, 1, &amt_written);
      assert(amt_written == 0);
      assert( err == EEOF );
    }
  }

  qio_channel_release(writing);

  // Reopen the file if we're doing reopen
  if( reopen ) {
    // Close the file.
    qio_file_release(f);
    err = qio_file_open_access(&f, filename, "r", file_hints, NULL);
    assert(!err);
  }

  // Check that the file is the right length.
  if( !memory ) {
    struct stat stats;
    err = sys_fstat(f->fd, &stats);
    assert(!err);
    assert(stats.st_size == end);
  }

  // That was fun. Now start at the beginning of the file
  // and read the data.
  
  // Rewind the file 
  if( !memory ) {
    off_t off;

    sys_lseek(f->fd, start, SEEK_SET, &off);
    assert(!err);
  }

  // Read the data.
  //err = qio_channel_init_file(&reading, type, f, ch_hints, 1, 0, start, end);
  err = qio_channel_create(&reading, f, ch_hints, 1, 0, start, ch_end, NULL);
  assert(!err);

  // Read stuff from the file.
  for( offset = start; offset < end; offset += usesz ) {
    usesz = chunksz;
    if( offset + usesz > end ) usesz = end - offset;
    // Fill chunk.
    fill_testdata(offset, usesz, chunk);
    memset(got_chunk, 0xff, usesz);
    // Read chunk.
    if( readfp ) {
      amt_read = fread(got_chunk, 1, usesz, readfp);
    } else {
      err = qio_channel_read(threadsafe, reading, got_chunk, usesz, &amt_read);
      assert( err == EEOF || err == 0);
    }
    assert(amt_read == usesz);

    // Compare chunk.
    for( k = 0; k < usesz; k++ ) {
      assert(got_chunk[k] == chunk[k]);
    }
  }

  if( readfp ) {
    amt_read = fread(got_chunk, 1, 1, readfp);
    assert( amt_read == 0 );
    assert( feof(readfp) );
  } else {
    if( unbounded_channels < 0 ) {
      int times = -unbounded_channels;
      for( int z = 0; z < times; z++ ) {
        err = qio_channel_advance(threadsafe, reading, 1);
        assert( !err );
      }

      err = qio_channel_advance(threadsafe, reading, -unbounded_channels);
      assert( !err );
    }

    err = qio_channel_read(threadsafe, reading, got_chunk, 1, &amt_read);
    assert( err == EEOF );
  }

  qio_channel_release(reading);
  //err = qio_channel_destroy(&reading);

  // Close the file.
  qio_file_release(f);

  if( reopen ) {
    unlink(filename);
  }

  free(chunk);
  free(got_chunk);
}
void check_mark(char* writepattern, char* readpattern)
{
  qio_file_t* f;
  qio_channel_t* writing;
  qio_channel_t* reading;
  int64_t offset;
  int32_t got;

  err_t err;
  int i;

  err = qio_file_open_tmp(&f, 0, NULL);
  assert(!err);

  err = qio_channel_create(&writing, f, 0, 0, 1, 0, INT64_MAX, NULL);
  assert(!err);

  err = qio_channel_offset(true, writing, &offset);
  assert(!err);
  assert(offset == 0);

  for( i = 0; writepattern[i]; i++ ) {
    switch (writepattern[i]) {
      case 'M':
        err = qio_channel_mark(true, writing);
        break;
      case 'R':
        err = qio_channel_revert(true, writing);
        break;
      case 'C':
        err = qio_channel_commit(true, writing);
        break;
      case 'A':
        err = qio_channel_advance(true, writing, 1);
        break;
      default:
        err = qio_channel_write_byte(true, writing, writepattern[i]);
    }
    assert(!err);
  }

  qio_channel_release(writing);

  // Read the data a byte at a time.
  err = qio_channel_create(&reading, f, 0, 1, 0, 0, INT64_MAX, NULL);
  assert(!err);

  for( i = 0; readpattern[i]; i++ ) {
    switch (readpattern[i]) {
      case 'M':
        err = qio_channel_mark(true, reading);
        break;
      case 'R':
        err = qio_channel_revert(true, reading);
        break;
      case 'C':
        err = qio_channel_commit(true, reading);
        break;
      case 'A':
        err = qio_channel_advance(true, reading, 1);
        break;
      default:
        got = qio_channel_read_byte(true, reading);
        if( got >= 0 ) assert(got == readpattern[i]);
        else err = -got;
    }
    assert(!err);
  }

  got = qio_channel_read_byte(true, reading);
  assert( got == - EEOF );

  qio_channel_release(reading);

  // Close the file.
  qio_file_release(f);
}
Exemplo n.º 3
0
// commit input, sending any data to the subprocess.
// once input is sent, close input channel and file.
// While sending that data, read output and error channels,
// buffering up the read data.
// once output/error reads EOF, close output/error file (not channel
// since output channel has the buffered data).
qioerr qio_proc_communicate(
    const int threadsafe,
    qio_channel_t* input,
    qio_channel_t* output,
    qio_channel_t* error) {

  qioerr err = 0;
  int rc = 0;
  bool do_input;
  bool do_output;
  bool do_error;
  bool input_ready;
  bool output_ready;
  bool error_ready;
  fd_set rfds, wfds, efds;
  int nfds = 1;

  int input_fd = -1;
  int output_fd = -1;
  int error_fd = -1;

  if( threadsafe ) {
    // lock all three channels.
    // but unlock them immediately and set them to NULL
    // if they are already closed.
    if( input ) {
      err = qio_lock(&input->lock);
      if( err ) return err;
      if( qio_channel_isclosed(false, input) ) {
        qio_unlock(&input->lock);
        input = NULL;
      }
    }
    if( output ) {
      err = qio_lock(&output->lock);
      if( err ) {
        if( input ) qio_unlock(&input->lock);
        return err;
      }
      if( qio_channel_isclosed(false, output) ) {
        qio_unlock(&output->lock);
        output = NULL;
      }
    }
    if( error ) {
      err = qio_lock(&error->lock);
      if( err ) {
        if( input ) qio_unlock(&input->lock);
        if( output ) qio_unlock(&output->lock);
        return err;
      }
      if( qio_channel_isclosed(false, error) ) {
        qio_unlock(&error->lock);
        error = NULL;
      }
    }
  }

  if( input ) {
    input_fd = input->file->fd;
    if( nfds <= input_fd ) nfds = input_fd + 1;
  }
  if( output ) {
    output_fd = output->file->fd;
    if( nfds <= output_fd ) nfds = output_fd + 1;
  }
  if( error ) {
    error_fd = error->file->fd;
    if( nfds <= error_fd ) nfds = error_fd + 1;
  }

  // Adjust all three pipes to be non-blocking.
  if( input_fd != -1 ) {
    rc = fcntl(input_fd, F_SETFL, O_NONBLOCK);
    if( rc == -1 ) {
      err = qio_int_to_err(errno);
    }
  }
  if( output_fd != -1 ) {
    rc = fcntl(output_fd, F_SETFL, O_NONBLOCK);
    if( rc == -1 ) {
      err = qio_int_to_err(errno);
    }
  }
  if( error_fd != -1 ) {
    rc = fcntl(error_fd, F_SETFL, O_NONBLOCK);
    if( rc == -1 ) {
      err = qio_int_to_err(errno);
    }
  }

  // mark the output and error channels so that we
  // can just keep advancing to read while buffering
  // up all data. Before returning, we'll revert them
  // so that this buffered data can be read again and
  // update end_pos so that the channel knows not to
  // call read again. Then, we can close the file
  // descriptor.
  if( output ) {
    qio_channel_mark(false, output);
  }
  if( error ) {
    qio_channel_mark(false, error);
  }

  do_input = (input != NULL);
  do_output = (output != NULL);
  do_error = (error != NULL);

  while( do_input || do_output || do_error ) {

    // Now call select to wait for one of the descriptors to
    // become ready.

    FD_ZERO(&rfds);
    FD_ZERO(&wfds);
    FD_ZERO(&efds);

    if( do_input && input_fd != -1 ) {
      FD_SET(input_fd, &wfds);
      FD_SET(input_fd, &efds);
    }
    if( do_output && output_fd != -1 ) {
      FD_SET(output_fd, &rfds);
      FD_SET(output_fd, &efds);
    }
    if( do_error && error_fd != -1 ) {
      FD_SET(error_fd, &rfds);
      FD_SET(error_fd, &efds);
    }

    input_ready = false;
    output_ready = false;
    error_ready = false;

    // Run select to wait for something
    if( do_input || do_output || do_error ) {
      // TODO -- use sys_select so threading can interact
      struct timeval t;
      t.tv_sec = 0;
      t.tv_usec = 10;
      rc = select(nfds, &rfds, &wfds, &efds, &t);
      if (rc > 0) {
        // read ready file descriptors
        input_ready = input_fd != -1 && FD_ISSET(input_fd, &wfds);
        output_ready = output_fd != -1 && FD_ISSET(output_fd, &rfds);
        error_ready = error_fd != -1 && FD_ISSET(error_fd, &rfds);
      }
      // Ignore EAGAIN and EINTR
      if (rc == EAGAIN || rc == EINTR) rc = 0;
    }

    if( rc == -1 ) {
      err = qio_int_to_err(errno);
      break;
    }

    if( do_input && input_ready ) {
      err = _qio_channel_flush_qio_unlocked(input);
      if( !err ) {
        do_input = false;
        // Close input channel.
        err = qio_channel_close(false, input);
      }
      if( qio_err_to_int(err) == EAGAIN ) err = 0;
      if( err ) break;
    }

    if( do_output && output_ready ) {
      // read some into our buffer.
      err = qio_channel_advance(false, output, qbytes_iobuf_size);
      if( qio_err_to_int(err) == EEOF ) {
        qio_file_t* output_file = qio_channel_get_file(output);

        do_output = false;
        // close the output file (not channel), in case closing output
        // causes the program to output on stderr, e.g.
        if( output_file )
          err = qio_file_close(output_file);
        // Set the output channel maximum position
        // This prevents a read on output from trying to get
        // more data from the (now closed) file.
        output->end_pos = qio_channel_offset_unlocked(output);
      }
      if( qio_err_to_int(err) == EAGAIN ) err = 0;
      if( err ) break;
    }

    if( do_error && error_ready ) {
      // read some into our buffer.
      err = qio_channel_advance(false, error, qbytes_iobuf_size);
      if( qio_err_to_int(err) == EEOF ) {
        qio_file_t* error_file = qio_channel_get_file(error);

        do_error = false;
        // close the error file (not channel)
        if( error_file )
          err = qio_file_close(error_file);
        // Set the error channel maximum position
        error->end_pos = qio_channel_offset_unlocked(error);
      }
      if( qio_err_to_int(err) == EAGAIN ) err = 0;
      if( err ) break;
    }

    chpl_task_yield();
  }

  // we could close the file descriptors at this point,
  // but we don't because we don't want to modify
  // the file descriptor of the file since it's
  // constant (and not protected by file's lock).
  // The pipes will be closed when the channels are destroyed.

  // We marked the output and error channels so that we
  // can just keep advancing to read while buffering
  // up all data. Before returning, we'll revert them
  // so that this buffered data can be read again and
  // update end_pos so that the channel knows not to
  // call read again. Then, we can close the file
  // descriptor.

  // revert the output and error channels
  if( output) qio_channel_revert_unlocked(output);
  if( error ) qio_channel_revert_unlocked(error);


  if( threadsafe ) {
    // unlock all three channels.
    if( error ) qio_unlock(&error->lock);
    if( output ) qio_unlock(&output->lock);
    if( input ) qio_unlock(&input->lock);
  }

  return err;
}