DSO_PUBLIC int64_t
lfp_parse_memsize(const char *s, enum lfp_memsize_measure_unit default_unit)
{
int64_t default_multiplier, multiplier, amount;
SYSCHECK(EINVAL, s == NULL || *s == '\0');
SYSGUARD(default_multiplier = compute_multiplier(default_unit));
char *endptr = NULL;
lfp_set_errno(0);
SYSGUARD(amount = strtoll(s, &endptr, 10));
SYSCHECK(ERANGE, amount < 0);
if (amount == 0) {
return 0;
} else if (*endptr) {
if (strcasecmp(endptr, "KB") == 0) {
multiplier = 1000LL;
} else if (strcasecmp(endptr, "K") == 0 ||
strcasecmp(endptr, "KiB") == 0) {
multiplier = 1024LL;
} else if (strcasecmp(endptr, "MB") == 0) {
multiplier = 1000LL*1000LL;
} else if (strcasecmp(endptr, "M") == 0 ||
strcasecmp(endptr, "MiB") == 0) {
multiplier = 1024LL*1024LL;
} else if (strcasecmp(endptr, "GB") == 0) {
multiplier = 1000LL*1000LL*1000LL;
} else if (strcasecmp(endptr, "G") == 0 ||
strcasecmp(endptr, "GiB") == 0) {
multiplier = 1024LL*1024LL*1024LL;
} else if (strcasecmp(endptr, "TB") == 0) {
multiplier = 1000LL*1000LL*1000LL*1000LL;
} else if (strcasecmp(endptr, "T") == 0 ||
strcasecmp(endptr, "TiB") == 0) {
multiplier = 1024LL*1024LL*1024LL*1024LL;
} else if (strcasecmp(endptr, "PB") == 0) {
multiplier = 1000LL*1000LL*1000LL*1000LL*1000LL;
} else if (strcasecmp(endptr, "P") == 0 ||
strcasecmp(endptr, "PiB") == 0) {
multiplier = 1024LL*1024LL*1024LL*1024LL*1024LL;
} else if (strcasecmp(endptr, "EB") == 0) {
multiplier = 1000LL*1000LL*1000LL*1000LL*1000LL*1000LL;
} else if (strcasecmp(endptr, "E") == 0 ||
strcasecmp(endptr, "EiB") == 0) {
multiplier = 1024LL*1024LL*1024LL*1024LL*1024LL*1024LL;
} else {
SYSERR(EINVAL);
}
} else {
multiplier = default_multiplier;
}
// Check for overflow
if (amount > (INT64_MAX / multiplier)) {
SYSERR(ERANGE);
} else {
return amount * multiplier;
}
}
void THPStorage_(writeFileRaw)(THStorage *self, io fd)
{
real *data;
int64_t size = self->size;
#ifndef THC_GENERIC_FILE
data = self->data;
#else
std::unique_ptr<char[]> cpu_data(new char[size * sizeof(real)]);
data = (real*)cpu_data.get();
THCudaCheck(cudaMemcpy(data, self->data, size * sizeof(real), cudaMemcpyDeviceToHost));
#endif
ssize_t result = doWrite(fd, &size, sizeof(int64_t));
if (result != sizeof(int64_t))
throw std::system_error(result, std::system_category());
// fast track for bytes and little endian
if (sizeof(real) == 1 || THP_nativeByteOrder() == THPByteOrder::THP_LITTLE_ENDIAN) {
char *bytes = (char *) data;
int64_t remaining = sizeof(real) * size;
while (remaining > 0) {
// we write and read in 1GB blocks to avoid bugs on some OSes
ssize_t result = doWrite(fd, bytes, THMin(remaining, 1073741824));
if (result < 0)
throw std::system_error(result, std::system_category());
bytes += result;
remaining -= result;
}
if (remaining != 0)
throw std::system_error(result, std::system_category());
} else {
int64_t buffer_size = std::min(size, (int64_t)5000);
std::unique_ptr<uint8_t[]> le_buffer(new uint8_t[buffer_size * sizeof(real)]);
for (int64_t i = 0; i < size; i += buffer_size) {
size_t to_convert = std::min(size - i, buffer_size);
if (sizeof(real) == 2) {
THP_encodeInt16Buffer((uint8_t*)le_buffer.get(),
(const int16_t*)data + i,
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
} else if (sizeof(real) == 4) {
THP_encodeInt32Buffer((uint8_t*)le_buffer.get(),
(const int32_t*)data + i,
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
} else if (sizeof(real) == 8) {
THP_encodeInt64Buffer((uint8_t*)le_buffer.get(),
(const int64_t*)data + i,
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
}
SYSCHECK(doWrite(fd, le_buffer.get(), to_convert * sizeof(real)));
}
}
}
THStorage * THPStorage_(readFileRaw)(io file, THStorage *_storage)
{
real *data;
int64_t size;
ssize_t result = doRead(file, &size, sizeof(int64_t));
if (result == 0)
throw std::runtime_error("unexpected EOF. The file might be corrupted.");
if (result != sizeof(int64_t))
throw std::system_error(result, std::system_category());
THStoragePtr storage;
if (_storage == nullptr) {
storage = THStorage_(newWithSize)(LIBRARY_STATE size);
} else {
THPUtils_assert(_storage->size == size,
"storage has wrong size: expected %ld got %ld",
size, _storage->size);
storage = _storage;
}
#ifndef THC_GENERIC_FILE
data = storage->data;
#else
std::unique_ptr<char[]> cpu_data(new char[size * sizeof(real)]);
data = (real*)cpu_data.get();
#endif
// fast track for bytes and little endian
if (sizeof(real) == 1 || THP_nativeByteOrder() == THPByteOrder::THP_LITTLE_ENDIAN) {
char *bytes = (char *) data;
int64_t remaining = sizeof(real) * storage->size;
while (remaining > 0) {
// we write and read in 1GB blocks to avoid bugs on some OSes
ssize_t result = doRead(file, bytes, THMin(remaining, 1073741824));
if (result == 0) // 0 means EOF, which is also an error
throw std::runtime_error("unexpected EOF. The file might be corrupted.");
if (result < 0)
throw std::system_error(result, std::system_category());
bytes += result;
remaining -= result;
}
if (remaining != 0)
throw std::system_error(result, std::system_category());
} else {
int64_t buffer_size = std::min(size, (int64_t)5000);
std::unique_ptr<uint8_t[]> le_buffer(new uint8_t[buffer_size * sizeof(real)]);
for (int64_t i = 0; i < size; i += buffer_size) {
size_t to_convert = std::min(size - i, buffer_size);
SYSCHECK(doRead(file, le_buffer.get(), sizeof(real) * to_convert));
if (sizeof(real) == 2) {
THP_decodeInt16Buffer((int16_t*)data + i,
le_buffer.get(),
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
} else if (sizeof(real) == 4) {
THP_decodeInt32Buffer((int32_t*)data + i,
le_buffer.get(),
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
} else if (sizeof(real) == 8) {
THP_decodeInt64Buffer((int64_t*)data + i,
le_buffer.get(),
THPByteOrder::THP_LITTLE_ENDIAN,
to_convert);
}
}
}
#ifdef THC_GENERIC_FILE
THCudaCheck(cudaMemcpy(storage->data, data, size * sizeof(real), cudaMemcpyHostToDevice));
#endif
return storage.release();
}
