std::vector<ImportDescriptor*> FitsImporter::getImportDescriptors(const std::string& fname)
{
std::string filename = fname;
std::vector<std::vector<std::string> >& errors = mErrors[fname];
std::vector<std::vector<std::string> >& warnings = mWarnings[fname];
errors.clear();
warnings.clear();
int status=0;
std::vector<ImportDescriptor*> descriptors;
FitsFileResource pFile(filename);
if (!pFile.isValid())
{
errors.resize(1);
errors[0].push_back(pFile.getStatus());
RETURN_DESCRIPTORS;
}
int hduCnt = 0;
int specificHdu = 0;
int hdu = 1;
if (!splitFilename(filename, hduCnt, specificHdu, hdu, pFile, errors, warnings))
{
RETURN_DESCRIPTORS;
}
errors.resize(hduCnt+1);
warnings.resize(hduCnt+1);
for(; hdu <= hduCnt; ++hdu)
{
std::string datasetName = filename + "[" + StringUtilities::toDisplayString(hdu) + "]";
int hduType;
CHECK_FITS(fits_movabs_hdu(pFile, hdu, &hduType, &status), hdu, false, continue);
ImportDescriptorResource pImportDescriptor(static_cast<ImportDescriptor*>(NULL));
FactoryResource<DynamicObject> pMetadata;
VERIFYRV(pMetadata.get() != NULL, descriptors);
{ // scope
std::vector<std::string> comments;
char pCard[81];
char pValue[81];
char pComment[81];
int nkeys = 0;
CHECK_FITS(fits_get_hdrspace(pFile, &nkeys, NULL, &status), hdu, true, ;);
for(int keyidx = 1; keyidx <= nkeys; ++keyidx)
{
CHECK_FITS(fits_read_keyn(pFile, keyidx, pCard, pValue, pComment, &status), hdu, true, continue);
std::string name = StringUtilities::stripWhitespace(std::string(pCard));
std::string val = StringUtilities::stripWhitespace(std::string(pValue));
std::string comment = StringUtilities::stripWhitespace(std::string(pComment));
if (!val.empty())
{
pMetadata->setAttributeByPath("FITS/" + name, val);
}
else if (!comment.empty())
{
comments.push_back(comment);
}
}
if (!comments.empty())
{
// ideally, this would add a multi-line string but Opticks doesn't display this properly
// pMetadata->setAttributeByPath("FITS/COMMENT", StringUtilities::join(comments, "\n"));
for(unsigned int idx = 0; idx < comments.size(); ++idx)
{
pMetadata->setAttributeByPath("FITS/COMMENT/" + StringUtilities::toDisplayString(idx), comments[idx]);
}
}
}
switch(hduType)
{
case IMAGE_HDU:
{
pImportDescriptor = ImportDescriptorResource(datasetName, TypeConverter::toString<RasterElement>());
VERIFYRV(pImportDescriptor.get() != NULL, descriptors);
EncodingType fileEncoding;
InterleaveFormatType interleave(BSQ);
unsigned int rows=0;
unsigned int cols=0;
unsigned int bands=1;
int bitpix;
int naxis;
long axes[3];
CHECK_FITS(fits_get_img_param(pFile, 3, &bitpix, &naxis, axes, &status), hdu, false, continue);
switch(bitpix)
{
case BYTE_IMG:
fileEncoding = INT1UBYTE;
break;
case SHORT_IMG:
fileEncoding = INT2SBYTES;
break;
case LONG_IMG:
fileEncoding = INT4SBYTES;
break;
case FLOAT_IMG:
fileEncoding = FLT4BYTES;
break;
case DOUBLE_IMG:
fileEncoding = FLT8BYTES;
break;
default:
warnings[hdu].push_back("Unsupported BITPIX value " + StringUtilities::toDisplayString(bitpix) + ".");
continue;
}
EncodingType dataEncoding = checkForOverflow(fileEncoding, pMetadata.get(), hdu, errors, warnings);
if (naxis == 1)
{
// 1-D data is a signature
pImportDescriptor = ImportDescriptorResource(datasetName, TypeConverter::toString<Signature>());
pMetadata->setAttributeByPath(METADATA_SIG_ENCODING, dataEncoding);
pMetadata->setAttributeByPath(METADATA_SIG_LENGTH, axes[0]);
RasterUtilities::generateAndSetFileDescriptor(pImportDescriptor->getDataDescriptor(), filename,
StringUtilities::toDisplayString(hdu), BIG_ENDIAN_ORDER);
// add units
SignatureDataDescriptor* pSigDd =
dynamic_cast<SignatureDataDescriptor*>(pImportDescriptor->getDataDescriptor());
if (pSigDd != NULL)
{
FactoryResource<Units> pUnits;
pUnits->setUnitName("Custom");
pUnits->setUnitType(CUSTOM_UNIT);
pSigDd->setUnits("Reflectance", pUnits.get());
}
break; // leave switch()
}
else if (naxis == 2)
{
cols = axes[0];
rows = axes[1];
}
else if (naxis == 3)
{
cols = axes[0];
rows = axes[1];
bands = axes[2];
}
else
{
errors[hdu].push_back(StringUtilities::toDisplayString(naxis) + " axis data not supported.");
}
RasterDataDescriptor* pDataDesc = RasterUtilities::generateRasterDataDescriptor(
datasetName, NULL, rows, cols, bands, interleave, dataEncoding, IN_MEMORY);
pImportDescriptor->setDataDescriptor(pDataDesc);
if (specificHdu == 0 && hdu == 1 && naxis == 2 && (axes[0] <= 5 || axes[1] <= 5))
{
// use 5 as this is a good top end for the number of astronomical band pass filters
// in general usage. this is not in a spec anywhere and is derived from various sample
// FITS files for different astronomical instruments.
//
// There's a good chance this is really a spectrum. (0th HDU)
// We'll create an import descriptor for the spectrum version of this
// And disable the raster descriptor by default
pImportDescriptor->setImported(false);
ImportDescriptorResource pSigDesc(datasetName, TypeConverter::toString<SignatureLibrary>());
DynamicObject* pSigMetadata = pSigDesc->getDataDescriptor()->getMetadata();
pSigMetadata->merge(pMetadata.get());
std::vector<double> centerWavelengths;
unsigned int cnt = (axes[0] <= 5) ? axes[1] : axes[0];
double startVal = StringUtilities::fromDisplayString<double>(
dv_cast<std::string>(pMetadata->getAttributeByPath("FITS/MINWAVE"), "0.0"));
double endVal = StringUtilities::fromDisplayString<double>(
dv_cast<std::string>(pMetadata->getAttributeByPath("FITS/MAXWAVE"), "0.0"));
double incr = (endVal == 0.0) ? 1.0 : ((endVal - startVal) / static_cast<double>(cnt));
centerWavelengths.reserve(cnt);
for (unsigned int idx = 0; idx < cnt; idx++)
{
centerWavelengths.push_back(startVal + (idx * incr));
}
pSigMetadata->setAttributeByPath(CENTER_WAVELENGTHS_METADATA_PATH, centerWavelengths);
// Units
std::string unitsName = dv_cast<std::string>(pMetadata->getAttributeByPath("FITS/BUNIT"), std::string());
if (!unitsName.empty())
{
FactoryResource<Units> units;
units->setUnitName(unitsName);
units->setUnitType(RADIANCE);
SignatureDataDescriptor* pSigDd =
dynamic_cast<SignatureDataDescriptor*>(pSigDesc->getDataDescriptor());
if (pSigDd != NULL)
{
pSigDd->setUnits("Reflectance", units.get());
}
}
RasterUtilities::generateAndSetFileDescriptor(pSigDesc->getDataDescriptor(),
filename, StringUtilities::toDisplayString(hdu), BIG_ENDIAN_ORDER);
// If units are not available, set custom units into the data descriptor so that the user can
// modify them - this must occur after the call to RasterUtilities::generateAndSetFileDescriptor()
// so that the file descriptor will still display no defined units
if (unitsName.empty())
{
FactoryResource<Units> units;
units->setUnitName("Custom");
units->setUnitType(CUSTOM_UNIT);
SignatureDataDescriptor* pSigDd = dynamic_cast<SignatureDataDescriptor*>(pSigDesc->getDataDescriptor());
if (pSigDd != NULL)
{
pSigDd->setUnits("Reflectance", units.get());
}
}
descriptors.push_back(pSigDesc.release());
}
RasterFileDescriptor* pFileDescriptor = dynamic_cast<RasterFileDescriptor*>(
RasterUtilities::generateAndSetFileDescriptor(pDataDesc, filename,
StringUtilities::toDisplayString(hdu), BIG_ENDIAN_ORDER));
if (pFileDescriptor != NULL)
{
unsigned int bitsPerElement = RasterUtilities::bytesInEncoding(fileEncoding) * 8;
pFileDescriptor->setBitsPerElement(bitsPerElement);
}
break; // leave switch()
}
case ASCII_TBL:
case BINARY_TBL:
warnings[hdu].push_back("Tables not supported. [HDU " + StringUtilities::toDisplayString(hdu) + "]");
continue;
default:
warnings[hdu].push_back("HDU " + StringUtilities::toDisplayString(hdu) + " is an unknown type.");
continue;
}
pImportDescriptor->getDataDescriptor()->setMetadata(pMetadata.release());
pImportDescriptor->setImported(errors[hdu].empty());
descriptors.push_back(pImportDescriptor.release());
}
/**
* Recursively normalizes the given input object.
*
* Input: A subject with predicates and possibly embedded other subjects.
* Output: Either a map of normalized subjects OR a tree of normalized subjects.
*
* Normalization Algorithm:
*
* Replace the existing context if the input has '#'.
*
* For each key-value:
* 1. Split the key on a colon and look for prefix in the context. If found,
* expand the key to an IRI, else it is already an IRI, add <>, save the
* new predicate to add to the output.
* 2. If value is a Map, then it is a subject, set the predicate to the
* subject's '@' IRI value and recurse into it. Else goto #3.
* 3. Look up the key in the context to find type coercion info. If not found,
* goto #4, else goto #5.
* 4. Check the value for an integer, double, or boolean. If matched, set
* type according to xsd types. If not matched, look for <>, if found,
* do CURIE vs. IRI check like #1 and create appropriate value.
* 5. If type coercion entry is a string, encode the value using the specific
* type. If it is an array, check the type in order of preference. If an
* unrecognized type (non-xsd, non-IRI) is provided, throw an exception.
*
* @param ctx the context to use (changes during recursion as necessary).
* @param in the input object.
* @param subjects a map of normalized subjects.
* @param out a tree normalized objects.
* @param bnodeId the last blank node ID used.
*
* @return true on success, false on failure with exception set.
*/
static bool _normalize(
DynamicObject ctx, DynamicObject& in,
DynamicObject* subjects, DynamicObject* out, int& bnodeId)
{
bool rval = true;
// FIXME: validate context (check for non-xsd types in type coercion arrays)
if(!in.isNull())
{
// update context
if(in->hasMember("#"))
{
ctx = in["#"];
}
// vars for normalization state
DynamicObject subject(NULL);
if(subjects != NULL)
{
subject = DynamicObject();
subject->setType(Map);
// assign blank node ID as needed
if(!in->hasMember("@"))
{
string bnodeKey = _createBlankNodeId(++bnodeId);
subject["@"] = bnodeKey.c_str();
}
}
string nKey;
// iterate over key-values
DynamicObjectIterator i = in.getIterator();
while(rval && i->hasNext())
{
DynamicObject& value = i->next();
const char* key = i->getName();
// skip context keys
if(key[0] == '#')
{
continue;
}
// get normalized key
nKey = _normalizeValue(ctx, key, RDF_TYPE_IRI, NULL, NULL);
// put values in an array for single code path
DynamicObject values;
values->setType(Array);
if(value->getType() == Array)
{
values.merge(value, true);
// preserve array structure when not using subjects map
if(out != NULL)
{
(*out)[nKey.c_str()]->setType(Array);
}
}
else
{
values->append(value);
}
// normalize all values
DynamicObjectIterator vi = values.getIterator();
while(rval && vi->hasNext())
{
DynamicObject v = vi->next();
if(v->getType() == Map)
{
if(subjects != NULL)
{
// get a normalized subject for the value
string vSubject;
if(v->hasMember("@"))
{
// normalize existing subject
vSubject = _normalizeValue(
ctx, v["@"], RDF_TYPE_IRI, NULL, NULL);
}
else
{
// generate the next blank node ID in order to preserve
// the blank node embed in the code below
vSubject = _createBlankNodeId(bnodeId + 1);
}
// determine if value is a blank node
bool isBNode = _isBlankNode(v);
// update non-blank node subject (use value's subject IRI)
if(!isBNode)
{
_setPredicate(subject, nKey.c_str(), vSubject.c_str());
}
// recurse
rval = _normalize(ctx, v, subjects, out, bnodeId);
// preserve embedded blank node
if(rval && isBNode)
{
// remove embed from top-level subjects
DynamicObject embed = (*subjects)[vSubject.c_str()];
(*subjects)->removeMember(vSubject.c_str());
embed->removeMember("@");
_setEmbed(subject, nKey.c_str(), embed);
}
}
else
{
// update out and recurse
DynamicObject next = (*out)[nKey.c_str()];
if(value->getType() == Array)
{
next = next->append();
}
else
{
next->setType(Map);
}
rval = _normalize(ctx, v, subjects, &next, bnodeId);
}
}
else
{
_setPredicate(
(subjects != NULL) ? subject : *out, nKey.c_str(),
_normalizeValue(ctx, v, RDF_TYPE_UNKNOWN, key, NULL).c_str());
}
}
}
// add subject to map
if(subjects != NULL)
{
(*subjects)[subject["@"]->getString()] = subject;
}
}
return rval;
}
bool JsonLd::normalize(DynamicObject& in, DynamicObject& out)
{
bool rval = true;
// initialize output
out->setType(Map);
out->clear();
// create map to store subjects
DynamicObject subjects;
subjects->setType(Map);
// initialize context
DynamicObject ctx(NULL);
if(in->hasMember("#"))
{
ctx = in["#"];
}
// put all subjects in an array for single code path
DynamicObject input;
input->setType(Array);
if(in->hasMember("@") && in["@"]->getType() == Array)
{
input.merge(in["@"], true);
}
else
{
input->append(in);
}
// do normalization
int bnodeId = 0;
DynamicObjectIterator i = input.getIterator();
while(rval && i->hasNext())
{
rval = _normalize(ctx, i->next(), &subjects, NULL, bnodeId);
}
// build output
if(rval)
{
// single subject
if(subjects->length() == 1)
{
DynamicObject subject = subjects.first();
out.merge(subject, false);
// FIXME: will need to check predicates for blank nodes as well...
// and fail if they aren't embeds?
// strip blank node '@'
if(_isBlankNode(out))
{
out->removeMember("@");
}
}
// multiple subjects
else
{
DynamicObject& array = out["@"];
array->setType(Array);
i = subjects.getIterator();
while(i->hasNext())
{
DynamicObject& next = i->next();
// FIXME: will need to check predicates for blank nodes as well...
// and fail if they aren't embeds?
// strip blank node '@'
if(_isBlankNode(next))
{
next->removeMember("@");
}
array->append(next);
}
}
}
return rval;
}
std::vector<ImportDescriptor*> LandsatGeotiffImporter::createImportDescriptors(const std::string& filename,
const DynamicObject* pImageMetadata,
Landsat::LandsatImageType type)
{
std::string suffix;
if (type == Landsat::LANDSAT_VNIR)
{
suffix = "vnir";
}
else if (type == Landsat::LANDSAT_PAN)
{
suffix = "pan";
}
else if (type == Landsat::LANDSAT_TIR)
{
suffix = "tir";
}
std::vector<ImportDescriptor*> descriptors;
std::string spacecraft = dv_cast<std::string>(
pImageMetadata->getAttributeByPath("LANDSAT_MTL/L1_METADATA_FILE/PRODUCT_METADATA/SPACECRAFT_ID"), "");
std::vector<std::string> bandNames = Landsat::getSensorBandNames(spacecraft, type);
if (bandNames.empty())
{
//this spacecraft and iamge type
//isn't meant to have any bands, so terminate early
//e.g. spacecraft == "Landsat5" && type == Landsat::LANDSAT_PAN
return descriptors;
}
std::vector<unsigned int> validBands;
std::vector<std::string> bandFiles = Landsat::getGeotiffBandFilenames(
pImageMetadata, filename, type, validBands);
if (bandFiles.empty())
{
mWarnings.push_back("Unable to locate band files for " + suffix + " product.");
return descriptors;
}
ImportDescriptorResource pImportDescriptor(filename + "-" + suffix,
TypeConverter::toString<RasterElement>(), NULL, false);
if (pImportDescriptor.get() == NULL)
{
return descriptors;
}
RasterDataDescriptor* pDescriptor = dynamic_cast<RasterDataDescriptor*>(pImportDescriptor->getDataDescriptor());
if (pDescriptor == NULL)
{
return descriptors;
}
pDescriptor->setProcessingLocation(ON_DISK);
DynamicObject* pMetadata = pDescriptor->getMetadata();
pMetadata->merge(pImageMetadata);
FactoryResource<RasterFileDescriptor> pFileDescriptorRes;
pDescriptor->setFileDescriptor(pFileDescriptorRes.get());
RasterFileDescriptor* pFileDescriptor = dynamic_cast<RasterFileDescriptor*>(pDescriptor->getFileDescriptor());
pFileDescriptor->setFilename(filename);
std::string tiffFile = bandFiles[0];
if (!Landsat::parseBasicsFromTiff(tiffFile, pDescriptor))
{
mWarnings.push_back("Unable to parse basic information about image from tiff file for " + suffix + " product.");
return descriptors;
}
if (pDescriptor->getBandCount() != 1 || pDescriptor->getDataType() != INT1UBYTE)
{
mWarnings.push_back("Improperly formatted tiff file for " + suffix + " product.");
return descriptors;
}
pDescriptor->setInterleaveFormat(BSQ); //one tiff file per band.
pFileDescriptor->setInterleaveFormat(BSQ);
std::vector<DimensionDescriptor> bands = RasterUtilities::generateDimensionVector(
bandFiles.size(), true, false, true);
pDescriptor->setBands(bands);
pFileDescriptor->setBands(bands);
pDescriptor->setBadValues(std::vector<int>(1, 0));
pFileDescriptor->setDatasetLocation(suffix);
//special metadata here
Landsat::fixMtlMetadata(pMetadata, type, validBands);
std::vector<std::string> defaultImport = OptionsLandsatImport::getSettingDefaultImport();
bool fallbackToDn = false;
descriptors.push_back(pImportDescriptor.release());
if (type == Landsat::LANDSAT_VNIR)
{
//attempt to display true-color
DimensionDescriptor redBand = RasterUtilities::findBandWavelengthMatch(0.630, 0.690, pDescriptor);
DimensionDescriptor greenBand = RasterUtilities::findBandWavelengthMatch(0.510, 0.590, pDescriptor);
DimensionDescriptor blueBand = RasterUtilities::findBandWavelengthMatch(0.410, 0.490, pDescriptor);
if (redBand.isValid() && greenBand.isValid() && blueBand.isValid())
{
pDescriptor->setDisplayMode(RGB_MODE);
pDescriptor->setDisplayBand(RED, redBand);
pDescriptor->setDisplayBand(GREEN, greenBand);
pDescriptor->setDisplayBand(BLUE, blueBand);
}
}
std::vector<std::pair<double, double> > radianceFactors = Landsat::determineRadianceConversionFactors(
pMetadata, type, validBands);
bool shouldDefaultImportRadiance =
std::find(defaultImport.begin(), defaultImport.end(), suffix + "-Radiance") != defaultImport.end();
if (radianceFactors.size() == bandFiles.size())
{
//we have enough to create radiance import descriptor
RasterDataDescriptor* pRadianceDescriptor = dynamic_cast<RasterDataDescriptor*>(
pDescriptor->copy(filename + "-" + suffix + "-radiance", NULL));
if (pRadianceDescriptor != NULL)
{
pRadianceDescriptor->setDataType(FLT4BYTES);
pRadianceDescriptor->setValidDataTypes(std::vector<EncodingType>(1, pRadianceDescriptor->getDataType()));
pRadianceDescriptor->setBadValues(std::vector<int>(1, -100));
FactoryResource<Units> pUnits;
pUnits->setUnitType(RADIANCE);
pUnits->setUnitName("w/(m^2*sr*um)");
pUnits->setScaleFromStandard(1.0);
pRadianceDescriptor->setUnits(pUnits.get());
FileDescriptor* pRadianceFileDescriptor = pRadianceDescriptor->getFileDescriptor();
if (pRadianceFileDescriptor != NULL)
{
pRadianceFileDescriptor->setDatasetLocation(suffix + "-radiance");
ImportDescriptorResource pRadianceImportDescriptor(pRadianceDescriptor,
shouldDefaultImportRadiance);
descriptors.push_back(pRadianceImportDescriptor.release());
}
}
}
else if (shouldDefaultImportRadiance)
{
fallbackToDn = true;
}
std::vector<double> reflectanceFactors = Landsat::determineReflectanceConversionFactors(
pMetadata, type, validBands);
bool shouldDefaultImportReflectance =
std::find(defaultImport.begin(), defaultImport.end(), suffix + "-Reflectance") != defaultImport.end();
if (radianceFactors.size() == bandFiles.size() && reflectanceFactors.size() == bandFiles.size())
{
//we have enough to create reflectance import descriptor
RasterDataDescriptor* pReflectanceDescriptor = dynamic_cast<RasterDataDescriptor*>(
pDescriptor->copy(filename + "-" + suffix + "-reflectance", NULL));
if (pReflectanceDescriptor != NULL)
{
pReflectanceDescriptor->setDataType(INT2SBYTES);
pReflectanceDescriptor->setValidDataTypes(
std::vector<EncodingType>(1, pReflectanceDescriptor->getDataType()));
pReflectanceDescriptor->setBadValues(std::vector<int>(1, std::numeric_limits<short>::max()));
FactoryResource<Units> pUnits;
pUnits->setUnitType(REFLECTANCE);
pUnits->setUnitName("Reflectance");
pUnits->setScaleFromStandard(1/10000.0);
pReflectanceDescriptor->setUnits(pUnits.get());
FileDescriptor* pReflectanceFileDescriptor = pReflectanceDescriptor->getFileDescriptor();
if (pReflectanceFileDescriptor != NULL)
{
pReflectanceFileDescriptor->setDatasetLocation(suffix + "-reflectance");
ImportDescriptorResource pReflectanceImportDescriptor(pReflectanceDescriptor,
shouldDefaultImportReflectance);
descriptors.push_back(pReflectanceImportDescriptor.release());
}
}
}
else if (shouldDefaultImportReflectance)
{
fallbackToDn = true;
}
double K1 = 0.0;
double K2 = 0.0;
bool haveTemperatureFactors = Landsat::getTemperatureConstants(pMetadata, type,
K1, K2);
bool shouldDefaultImportTemperature =
std::find(defaultImport.begin(), defaultImport.end(), suffix + "-Temperature") != defaultImport.end();
if (radianceFactors.size() == bandFiles.size() && haveTemperatureFactors)
{
//we have enough to create temperature import descriptor
RasterDataDescriptor* pTemperatureDescriptor = dynamic_cast<RasterDataDescriptor*>(
pDescriptor->copy(filename + "-" + suffix + "-temperature", NULL));
if (pTemperatureDescriptor != NULL)
{
pTemperatureDescriptor->setDataType(FLT4BYTES);
pTemperatureDescriptor->setValidDataTypes(
std::vector<EncodingType>(1, pTemperatureDescriptor->getDataType()));
pTemperatureDescriptor->setBadValues(std::vector<int>(1, -1));
FactoryResource<Units> pUnits;
pUnits->setUnitType(EMISSIVITY);
pUnits->setUnitName("K");
pUnits->setScaleFromStandard(1.0);
pTemperatureDescriptor->setUnits(pUnits.get());
FileDescriptor* pTemperatureFileDescriptor = pTemperatureDescriptor->getFileDescriptor();
if (pTemperatureFileDescriptor != NULL)
{
pTemperatureFileDescriptor->setDatasetLocation(suffix + "-temperature");
ImportDescriptorResource pTemperatureImportDescriptor(pTemperatureDescriptor,
shouldDefaultImportTemperature);
descriptors.push_back(pTemperatureImportDescriptor.release());
}
}
}
else if (shouldDefaultImportTemperature)
{
fallbackToDn = true;
}
if (fallbackToDn ||
std::find(defaultImport.begin(), defaultImport.end(), suffix + "-DN") != defaultImport.end())
{
pImportDescriptor->setImported(true);
}
return descriptors;
}
