void
showImage(typename TImage::Pointer image)
{
using std::cout;
using std::endl;
using std::setw;
image->Update();
itk::ImageRegionIterator<TImage> it(image,
image->GetLargestPossibleRegion());
int i = 1;
for (it = it.Begin(); !it.IsAtEnd(); ++it, i++)
{
std::cout << std::setw(4) << it.Get() << " ";
if (i % (image->GetLargestPossibleRegion().GetSize()[0]) == 0)
{
cout << endl;
}
if (i
% (image->GetLargestPossibleRegion().GetSize()[0]
* image->GetLargestPossibleRegion().GetSize()[1]) == 0)
{
cout << endl;
}
}
}
void
CreateImage(typename TImage::Pointer image)
{
typename TImage::IndexType start;
start.Fill(0);
typename TImage::SizeType size;
size.Fill(s);
typename TImage::RegionType region(start, size);
image->SetRegions(region);
image->Allocate();
itk::ImageRegionIterator<TImage> it(image,
image->GetLargestPossibleRegion());
it.GoToBegin();
int i = 0;
typename TImage::PixelType vv = 1;
for (it = it.Begin(); !it.IsAtEnd(); ++it)
{
vv++;
it.Set(i++ % 3 == 0 ? 0 : vv);
}
}
void CreateImage(const typename TImage::Pointer & image)
{
// This function creates a 2D image consisting of a black background,
// a large square of a non-zero pixel value, and a single "erroneous" pixel
// near the square.
typename TImage::IndexType corner = {{0,0,0}};
typename TImage::SizeType size = {{20,20,20}};
typename TImage::RegionType region(corner, size);
image->SetRegions(region);
image->Allocate();
image->FillBuffer(0);
// Make a square
for(int r = 4; r < 10; r++){
for(int c = 4; c < 10; c++){
for(int h=4;h < 10;h++){
typename TImage::IndexType pixelIndex = {{r,c,h}};
image->SetPixel(pixelIndex, 1);
}
}
}
// typename TImage::IndexType pixelIndex = {{102, 102}};
// image->SetPixel(pixelIndex, 50);
}
void BDSInpainting<TImage>::Inpaint(TPatchMatchFunctor* const patchMatchFunctor,
TCompositor* const compositor)
{
assert(this->Image);
assert(this->InpaintingMask);
ConstructValidPatchCentersImage();
// Initialize the output with the input
typename TImage::Pointer currentImage = TImage::New();
ITKHelpers::DeepCopy(this->Image.GetPointer(), currentImage.GetPointer());
// Allocate the initial NNField
NNFieldType::Pointer nnField =
NNFieldType::New();
nnField->SetRegions(currentImage->GetLargestPossibleRegion());
nnField->Allocate();
// Initialize the NNField in the target region
typedef SSD<TImage> PatchDistanceFunctorType;
PatchDistanceFunctorType patchDistanceFunctor;
patchDistanceFunctor.SetImage(currentImage);
patchMatchFunctor->SetValidPatchCentersImage(this->ValidPatchCentersImage);
std::vector<itk::Index<2> > pixelsToProcess = this->InpaintingMask->GetHolePixels();
patchMatchFunctor->SetTargetPixels(pixelsToProcess);
patchMatchFunctor->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetPropagationFunctor()->SetPatchDistanceFunctor(&patchDistanceFunctor);
patchMatchFunctor->GetPropagationFunctor()->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetRandomSearchFunctor()->SetImage(this->Image);
patchMatchFunctor->GetRandomSearchFunctor()->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetRandomSearchFunctor()->SetPatchDistanceFunctor(&patchDistanceFunctor);
compositor->SetPatchRadius(this->PatchRadius);
compositor->SetTargetMask(this->InpaintingMask);
compositor->SetImage(currentImage);
compositor->SetNearestNeighborField(patchMatchFunctor->GetNNField());
for(unsigned int iteration = 0; iteration < this->Iterations; ++iteration)
{
// Run PatchMatch to compute the NNField
patchMatchFunctor->Compute();
std::stringstream ssNNFieldFileName;
ssNNFieldFileName << "BDS_" << iteration << "_NNField.mha";
PatchMatchHelpers::WriteNNField(patchMatchFunctor->GetNNField(), ssNNFieldFileName.str());
// Update the target pixels
compositor->Composite();
ITKHelpers::DeepCopy(compositor->GetOutput(), currentImage.GetPointer());
}
ITKHelpers::DeepCopy(currentImage.GetPointer(), this->Output.GetPointer());
}
void ManualPatchSelectionDialog<TImage>::slot_UpdateSource(const itk::ImageRegion<2>& sourceRegion,
const itk::ImageRegion<2>& targetRegion)
{
// This function needs the targetRegion because this is the region of the Mask that is used to mask the source patch.
// std::cout << "Update source." << std::endl;
if(!this->Image->GetLargestPossibleRegion().IsInside(sourceRegion))
{
std::cerr << "Source region is outside the image!" << std::endl;
return;
}
if(MaskImage->CountHolePixels(sourceRegion) > 0)
{
std::cerr << "The source patch must not have any hole pixels!" << std::endl;
btnAccept->setVisible(false);
}
else
{
btnAccept->setVisible(true);
}
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedSourceImage = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), sourceRegion);
QGraphicsPixmapItem* item = this->SourcePatchScene->addPixmap(QPixmap::fromImage(maskedSourceImage));
gfxSource->fitInView(item);
// Refresh the image
//ITKVTKHelpers::ITKImageToVTKRGBImage(this->Image, this->ImageLayer.ImageData);
unsigned char green[3] = {0, 255, 0};
MaskOperations::ITKImageToVTKImageMasked(tempImage, this->MaskImage,
this->ImageLayer.ImageData, green);
// Outline the source patch
unsigned char blue[3] = {0, 0, 255};
ITKVTKHelpers::OutlineRegion(this->ImageLayer.ImageData, sourceRegion, blue);
this->qvtkWidget->GetRenderWindow()->Render();
}
void SmoothedClassProbabilites< TImage>
::GenerateData()
{
typename TImage::Pointer out = this->GetOutput(0);
out->SetRegions(this->GetInput(0)->GetLargestPossibleRegion());
out->Allocate();
for(unsigned int i = 0 ; i < this->GetNumberOfInputs(); i++)
{
auto gf = itk::DiscreteGaussianImageFilter<TImage,TImage>::New();
gf->SetInput(this->GetInput(i));
gf->SetVariance(this->m_Sigma);
gf->Update();
ImageRegionConstIterator<TImage> git(gf->GetOutput(),gf->GetOutput()->GetLargestPossibleRegion());
ImageRegionIterator<TImage> maskiter(m_MaskImage, m_MaskImage->GetLargestPossibleRegion());
ImageRegionIterator<TImage> outputIter(out, out->GetLargestPossibleRegion());
while (!outputIter.IsAtEnd())
{
if(maskiter.Value() > 0 ){
if(git.Value() > outputIter.Value())
outputIter.Set(i);
}else
{
outputIter.Set(0);
}
++git;
++outputIter;
++maskiter;
}
}
}
ManualPatchSelectionDialog<TImage>::ManualPatchSelectionDialog(TImage* const image, Mask* const mask,
const itk::ImageRegion<2>& targetRegion)
: Image(image), MaskImage(mask), TargetRegion(targetRegion)
{
// Allow the type itkImageRegion to be used in signals/slots.
qRegisterMetaType<itk::ImageRegion<2> >("itkImageRegion");
this->setupUi(this);
this->ImageLayer.ImageSlice->SetDragable(false);
this->ImageLayer.ImageSlice->SetPickable(false);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::DeepCopy(image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(tempImage->GetNumberOfComponentsPerPixel());
zeroPixel.Fill(0);
mask->ApplyToImage(tempImage.GetPointer(), zeroPixel);
ITKVTKHelpers::ITKVectorImageToVTKImageFromDimension(tempImage.GetPointer(), this->ImageLayer.ImageData);
SetupScenes();
this->Renderer = vtkSmartPointer<vtkRenderer>::New();
this->InteractorStyle = vtkSmartPointer<InteractorStyleImageWithDrag>::New();
this->qvtkWidget->GetRenderWindow()->AddRenderer(this->Renderer);
this->Renderer->AddViewProp(ImageLayer.ImageSlice);
// Per the comment in InteractorStyleImageWithDrag, the next 3 lines must be in this order
this->InteractorStyle->SetCurrentRenderer(this->Renderer);
this->qvtkWidget->GetRenderWindow()->GetInteractor()->SetInteractorStyle(this->InteractorStyle);
this->InteractorStyle->Init();
this->PatchSelector = new MovablePatch(this->TargetRegion.GetSize()[0]/2, this->InteractorStyle, this->gfxSource);
// slot_UpdateImage();
this->Renderer->ResetCamera();
this->qvtkWidget->GetRenderWindow()->Render();
// this->Camera = new ImageCamera(this->Renderer);
connect(this->PatchSelector, SIGNAL(signal_PatchMoved()), this, SLOT(slot_PatchMoved()));
}
void ManualPatchSelectionDialog<TImage>::slot_UpdateTarget(const itk::ImageRegion<2>& region)
{
// std::cout << "Update target." << std::endl;
unsigned char red[3] = {255, 0, 0};
ITKVTKHelpers::OutlineRegion(this->ImageLayer.ImageData, region, red);
this->qvtkWidget->GetRenderWindow()->Render();
// Masked target patch
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedTargetImage = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), region);
QGraphicsPixmapItem* maskedItem = this->TargetPatchScene->addPixmap(QPixmap::fromImage(maskedTargetImage));
gfxTarget->fitInView(maskedItem);
}
void TopPatchesDialog<TImage>::SetQueryNode(const Node& queryNode)
{
this->QueryNode = queryNode;
itk::Index<2> queryIndex = ITKHelpers::CreateIndex(queryNode);
itk::ImageRegion<2> queryRegion = ITKHelpers::GetRegionInRadiusAroundPixel(queryIndex, PatchHalfWidth);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedQueryPatch = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), queryRegion);
MaskedQueryPatchItem = this->QueryPatchScene->addPixmap(QPixmap::fromImage(maskedQueryPatch));
// We do this here because we could potentially call SetQueryNode after the widget is constructed as well.
gfxQueryPatch->fitInView(MaskedQueryPatchItem);
}
//BasicViewerWidget<TImage>::BasicViewerWidget(TImage* const image, Mask* const mask) :
BasicViewerWidget<TImage>::BasicViewerWidget(typename TImage::Pointer image, Mask::Pointer mask) :
Image(image), MaskImage(mask)
{
qRegisterMetaType<itk::ImageRegion<2> >("itkImageRegion");
this->setupUi(this);
int dims[3];
this->ImageLayer.ImageData->GetDimensions(dims);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::DeepCopy(image.GetPointer(), tempImage.GetPointer());
typename TImage::PixelType zeroPixel(tempImage->GetNumberOfComponentsPerPixel());
zeroPixel = itk::NumericTraits<typename TImage::PixelType>::ZeroValue(zeroPixel);
mask->ApplyToImage(tempImage.GetPointer(), zeroPixel);
ITKVTKHelpers::ITKVectorImageToVTKImageFromDimension(tempImage.GetPointer(), this->ImageLayer.ImageData);
// if(chkScaleImage->isChecked())
// {
// VTKHelpers::ScaleImage(this->ImageLayer.ImageData);
// }
this->ImageDimension[0] = dims[0];
this->ImageDimension[1] = dims[1];
this->ImageDimension[2] = dims[2];
SetupScenes();
// this->InteractorStyle = vtkSmartPointer<InteractorStyleImageWithDrag>::New();
this->InteractorStyle = vtkSmartPointer<vtkInteractorStyleImage>::New();
this->Renderer = vtkSmartPointer<vtkRenderer>::New();
this->qvtkWidget->GetRenderWindow()->AddRenderer(this->Renderer);
this->Renderer->AddViewProp(ImageLayer.ImageSlice);
this->InteractorStyle->SetCurrentRenderer(this->Renderer);
this->qvtkWidget->GetRenderWindow()->GetInteractor()->SetInteractorStyle(this->InteractorStyle);
// this->InteractorStyle->Init();
// this->Camera = new ImageCamera(this->Renderer);
this->ItkVtkCamera = new ITKVTKCamera(this->InteractorStyle, this->Renderer,
this->qvtkWidget->GetRenderWindow());
this->ItkVtkCamera->SetCameraPositionPNG();
}
/** The mask is inpainted with ValidValue in this function. */
void FinishVertex(VertexDescriptorType targetNode, VertexDescriptorType sourceNode)// __attribute__((optimize(0))) // This supposedly makes this function build in debug mode (-g -O0) when the rest of the program is built in -O3 or similar)
{
// Mark this pixel and the area around it as filled, and mark the mask in this region as filled.
// Determine the new boundary, and setup the nodes in the boundary queue.
// std::cout << "InpaintingVisitor::FinishVertex()" << std::endl;
// Construct the region around the vertex
itk::Index<2> indexToFinish = ITKHelpers::CreateIndex(targetNode);
itk::ImageRegion<2> regionToFinishFull = ITKHelpers::GetRegionInRadiusAroundPixel(indexToFinish, this->PatchHalfWidth);
// Copy this region so that we can change (crop) the regionToFinish and still have a copy of the original region
itk::ImageRegion<2> regionToFinish = regionToFinishFull;
// Make sure the region is entirely inside the image
// (because we allow target patches to not be entirely inside the image to handle the case where
// the hole boundary is near the image boundary)
regionToFinish.Crop(this->Image->GetLargestPossibleRegion());
if(this->DebugImages)
{
ITKHelpers::WriteImage(this->MaskImage, Helpers::GetSequentialFileName("Mask_Before", this->NumberOfFinishedPatches, "png", 3));
}
// Mark all the pixels in this region as filled in the mask.
ITKHelpers::SetRegionToConstant(this->MaskImage, regionToFinish, this->MaskImage->GetValidValue());
if(this->DebugImages)
{
ITKHelpers::WriteImage(this->MaskImage, Helpers::GetSequentialFileName("Mask_After", this->NumberOfFinishedPatches, "png", 3));
typename TImage::PixelType red;
red.Fill(0);
red[0] = 255;
typename TImage::PixelType green;
green.Fill(0);
green[1] = 255;
typename TImage::Pointer patchesCopiedImage = TImage::New();
ITKHelpers::DeepCopy(this->Image, patchesCopiedImage.GetPointer());
ITKHelpers::OutlineRegion(patchesCopiedImage.GetPointer(), regionToFinish, red);
itk::Index<2> sourceRegionCenter = ITKHelpers::CreateIndex(sourceNode);
itk::ImageRegion<2> sourceRegion = ITKHelpers::GetRegionInRadiusAroundPixel(sourceRegionCenter, this->PatchHalfWidth);
sourceRegion = ITKHelpers::CropRegionAtPosition(sourceRegion, this->Image->GetLargestPossibleRegion(), regionToFinishFull);
ITKHelpers::OutlineRegion(patchesCopiedImage.GetPointer(), sourceRegion, green);
ITKHelpers::WriteRGBImage(patchesCopiedImage.GetPointer(), Helpers::GetSequentialFileName("PatchesCopied", this->NumberOfFinishedPatches, "png", 3));
}
// Update the priority function. This must be done AFTER the mask is filled,
// as some of the Priority functors only compute things on the hole boundary, or only
// use data from the valid region of the image (indicated by valid pixels in the mask).
this->PriorityFunction->Update(sourceNode, targetNode, this->NumberOfFinishedPatches);
// Initialize (if requested) all vertices in the newly filled region because they may now be valid source nodes.
// (You may not want to do this in some cases (i.e. if the descriptors needed cannot be
// computed on newly filled regions))
if(this->AllowNewPatches)
{
itk::ImageRegionConstIteratorWithIndex<TImage> gridIterator(Image, regionToFinish);
while(!gridIterator.IsAtEnd())
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(gridIterator.GetIndex());
InitializeVertex(v);
++gridIterator;
}
}
// Add pixels that are on the new boundary to the queue, and mark other pixels as not in the queue.
itk::ImageRegionConstIteratorWithIndex<Mask> imageIterator(this->MaskImage, regionToFinish);
typedef typename TBoundaryNodeQueue::HandleType HandleType;
while(!imageIterator.IsAtEnd())
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(imageIterator.GetIndex());
if(this->MaskImage->HasHoleNeighbor(imageIterator.GetIndex()))
{
float priority = this->PriorityFunction->ComputePriority(imageIterator.GetIndex());
this->BoundaryNodeQueue.push_or_update(v, priority);
}
else
{
this->BoundaryNodeQueue.mark_as_invalid(v);
}
++imageIterator;
}
// std::cout << "FinishVertex after traversing finishing region there are "
// << BoostHelpers::CountValidQueueNodes(BoundaryNodeQueue, BoundaryStatusMap)
// << " valid nodes in the queue." << std::endl;
// Sometimes pixels that are not in the finishing region that were boundary pixels are no longer
// boundary pixels after the filling. Check for these.
// E.g. (H=hole, B=boundary, V=valid, Q=query, F=filled, N=new boundary,
// R=old boundary pixel that needs to be removed because it is no longer on the boundary)
// Before filling
/* V V V B H H H H
* V V V B H H H H
* V V V B H H H H
* V V V B B Q B B
* V V V V V V V V
*/
// After filling
/* V V V B H H H H
* V V V B H H H H
* V V V B F F F H
* V V V B F F F B
* V V V V F F F V
*/
// New boundary
/* V V V B H H H H
* V V V B H H H H
* V V V B N N N H
* V V V R F F N B
* V V V V F F F V
*/
// Expand the region
itk::ImageRegion<2> expandedRegion = ITKHelpers::GetRegionInRadiusAroundPixel(indexToFinish, PatchHalfWidth + 1);
// Make sure the region is entirely inside the image (to allow for target regions near the image boundary)
expandedRegion.Crop(this->Image->GetLargestPossibleRegion());
std::vector<itk::Index<2> > boundaryPixels = ITKHelpers::GetBoundaryPixels(expandedRegion);
for(unsigned int i = 0; i < boundaryPixels.size(); ++i)
{
// the region (the entire image) can be omitted, as this function automatically checks if the pixels are inside the image
if(!this->MaskImage->HasHoleNeighbor(boundaryPixels[i]))
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(boundaryPixels[i]);
put(this->BoundaryNodeQueue.BoundaryStatusMap, v, false);
}
}
// std::cout << "FinishVertex after removing stale nodes outside finishing region there are "
// << BoostHelpers::CountValidQueueNodes(BoundaryNodeQueue, BoundaryStatusMap)
// << " valid nodes in the queue." << std::endl;
this->NumberOfFinishedPatches++;
// std::cout << "Leave InpaintingVisitor::FinishVertex()" << std::endl;
} // end FinishVertex
typename TImage::Pointer modelBasedImageToImageRegistration(std::string referenceFilename,
std::string targetFilename,
typename TStatisticalModelType::Pointer model,
std::string outputDfFilename,
unsigned numberOfIterations){
typedef itk::ImageFileReader<TImage> ImageReaderType;
typedef itk::InterpolatingStatisticalDeformationModelTransform<TRepresenter, double, VImageDimension> TransformType;
typedef itk::LBFGSOptimizer OptimizerType;
typedef itk::ImageRegistrationMethod<TImage, TImage> RegistrationFilterType;
typedef itk::WarpImageFilter< TImage, TImage, TVectorImage > WarperType;
typedef itk::LinearInterpolateImageFunction< TImage, double > InterpolatorType;
typename ImageReaderType::Pointer referenceReader = ImageReaderType::New();
referenceReader->SetFileName(referenceFilename.c_str());
referenceReader->Update();
typename TImage::Pointer referenceImage = referenceReader->GetOutput();
referenceImage->Update();
typename ImageReaderType::Pointer targetReader = ImageReaderType::New();
targetReader->SetFileName(targetFilename.c_str());
targetReader->Update();
typename TImage::Pointer targetImage = targetReader->GetOutput();
targetImage->Update();
// do the fitting
typename TransformType::Pointer transform = TransformType::New();
transform->SetStatisticalModel(model);
transform->SetIdentity();
// Setting up the fitting
OptimizerType::Pointer optimizer = OptimizerType::New();
optimizer->MinimizeOn();
optimizer->SetMaximumNumberOfFunctionEvaluations(numberOfIterations);
typedef IterationStatusObserver ObserverType;
ObserverType::Pointer observer = ObserverType::New();
optimizer->AddObserver( itk::IterationEvent(), observer );
typename TMetricType::Pointer metric = TMetricType::New();
typename InterpolatorType::Pointer interpolator = InterpolatorType::New();
typename RegistrationFilterType::Pointer registration = RegistrationFilterType::New();
registration->SetInitialTransformParameters(transform->GetParameters());
registration->SetMetric(metric);
registration->SetOptimizer( optimizer );
registration->SetTransform( transform );
registration->SetInterpolator( interpolator );
registration->SetFixedImage( targetImage );
registration->SetFixedImageRegion(targetImage->GetBufferedRegion() );
registration->SetMovingImage( referenceImage );
try {
std::cout << "Performing registration... " << std::flush;
registration->Update();
std::cout << "[done]" << std::endl;
} catch ( itk::ExceptionObject& o ) {
std::cout << "caught exception " << o << std::endl;
}
typename TVectorImage::Pointer df = model->DrawSample(transform->GetCoefficients());
// write deformation field
if(outputDfFilename.size()>0){
typename itk::ImageFileWriter<TVectorImage>::Pointer df_writer = itk::ImageFileWriter<TVectorImage>::New();
df_writer->SetFileName(outputDfFilename);
df_writer->SetInput(df);
df_writer->Update();
}
// warp reference
std::cout << "Warping reference... " << std::flush;
typename WarperType::Pointer warper = WarperType::New();
warper->SetInput(referenceImage );
warper->SetInterpolator( interpolator );
warper->SetOutputSpacing( targetImage->GetSpacing() );
warper->SetOutputOrigin( targetImage->GetOrigin() );
warper->SetOutputDirection( targetImage->GetDirection() );
warper->SetDisplacementField( df );
warper->Update();
std::cout << "[done]" << std::endl;
return warper->GetOutput();
}
void LidarInpaintingRGBTextureVerification(TImage* const originalImage, Mask* const mask,
const unsigned int patchHalfWidth, const unsigned int numberOfKNN,
float slightBlurVariance = 1.0f, unsigned int searchRadius = 1000)
{
typedef TImage RGBDxDyImageType;
itk::ImageRegion<2> fullRegion = originalImage->GetLargestPossibleRegion();
// Extract the RGB image
typedef itk::Image<itk::CovariantVector<float, 3>, 2> RGBImageType;
std::vector<unsigned int> firstThreeChannels = {0,1,2};
RGBImageType::Pointer rgbImage = RGBImageType::New();
ITKHelpers::ExtractChannels(originalImage, firstThreeChannels, rgbImage.GetPointer());
// Blur the image for gradient computation stability (Criminisi's data term)
RGBImageType::Pointer blurredRGBImage = RGBImageType::New();
float blurVariance = 2.0f;
MaskOperations::MaskedBlur(rgbImage.GetPointer(), mask, blurVariance, blurredRGBImage.GetPointer());
ITKHelpers::WriteRGBImage(blurredRGBImage.GetPointer(), "BlurredRGBImage.png");
// Blur the image slightly so that the SSD comparisons are not so noisy
typename TImage::Pointer slightlyBlurredRGBDxDyImage = TImage::New();
MaskOperations::MaskedBlur(originalImage, mask, slightBlurVariance, slightlyBlurredRGBDxDyImage.GetPointer());
ITKHelpers::WriteImage(slightlyBlurredRGBDxDyImage.GetPointer(), "SlightlyBlurredRGBDxDyImage.mha");
// Create the graph
typedef ImagePatchPixelDescriptor<TImage> ImagePatchPixelDescriptorType;
typedef boost::grid_graph<2> VertexListGraphType;
// We can't make this a signed type (size_t versus int) because we allow negative
boost::array<std::size_t, 2> graphSideLengths = { { fullRegion.GetSize()[0],
fullRegion.GetSize()[1] } };
VertexListGraphType graph(graphSideLengths);
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
typedef boost::graph_traits<VertexListGraphType>::vertex_iterator VertexIteratorType;
// Get the index map
typedef boost::property_map<VertexListGraphType, boost::vertex_index_t>::const_type IndexMapType;
IndexMapType indexMap(get(boost::vertex_index, graph));
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType, IndexMapType> ImagePatchDescriptorMapType;
ImagePatchDescriptorMapType imagePatchDescriptorMap(num_vertices(graph), indexMap);
// Create the patch inpainter.
typedef PatchInpainter<TImage> OriginalImageInpainterType;
OriginalImageInpainterType originalImagePatchInpainter(patchHalfWidth, originalImage, mask);
originalImagePatchInpainter.SetDebugImages(true);
originalImagePatchInpainter.SetImageName("RGB");
// Create an inpainter for the blurred image.
typedef PatchInpainter<RGBImageType> BlurredImageInpainterType;
BlurredImageInpainterType blurredRGBImagePatchInpainter(patchHalfWidth, blurredRGBImage, mask);
// Create an inpainter for the slightly blurred image.
typedef PatchInpainter<TImage> SlightlyBlurredRGBDxDyImageImageInpainterType;
SlightlyBlurredRGBDxDyImageImageInpainterType slightlyBlurredRGBDxDyImageImagePatchInpainter(patchHalfWidth, slightlyBlurredRGBDxDyImage, mask);
// Create a composite inpainter. (Note: the mask is inpainted in InpaintingVisitor::FinishVertex)
CompositePatchInpainter inpainter;
inpainter.AddInpainter(&originalImagePatchInpainter);
inpainter.AddInpainter(&blurredRGBImagePatchInpainter);
inpainter.AddInpainter(&slightlyBlurredRGBDxDyImageImagePatchInpainter);
// Create the priority function
typedef PriorityCriminisi<RGBImageType> PriorityType;
PriorityType priorityFunction(blurredRGBImage, mask, patchHalfWidth);
// priorityFunction.SetDebugLevel(1);
// Queue
typedef IndirectPriorityQueue<VertexListGraphType> BoundaryNodeQueueType;
BoundaryNodeQueueType boundaryNodeQueue(graph);
// Create the descriptor visitor (used for SSD comparisons).
typedef ImagePatchDescriptorVisitor<VertexListGraphType, TImage, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
// ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(originalImage, mask,
// imagePatchDescriptorMap, patchHalfWidth); // Use the non-blurred image for the SSD comparisons
ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(slightlyBlurredRGBDxDyImage, mask,
imagePatchDescriptorMap, patchHalfWidth); // Use the slightly blurred image for the SSD comparisons.
typedef DefaultAcceptanceVisitor<VertexListGraphType> AcceptanceVisitorType;
AcceptanceVisitorType acceptanceVisitor;
// Create the inpainting visitor. (The mask is inpainted in FinishVertex)
typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType, TImage>
InpaintingVisitorType;
InpaintingVisitorType inpaintingVisitor(mask, boundaryNodeQueue,
imagePatchDescriptorVisitor, acceptanceVisitor,
&priorityFunction, patchHalfWidth, "InpaintingVisitor", originalImage);
inpaintingVisitor.SetDebugImages(true); // This produces PatchesCopied* images showing where patches were copied from/to at each iteration
// inpaintingVisitor.SetAllowNewPatches(false);
inpaintingVisitor.SetAllowNewPatches(true); // we can do this as long as we use one of the LinearSearchKNNProperty*Reuse (like LinearSearchKNNPropertyLimitLocalReuse) in the steps below
InitializePriority(mask, boundaryNodeQueue, &priorityFunction);
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<InpaintingVisitorType, VertexDescriptorType>(mask, &inpaintingVisitor);
std::cout << "PatchBasedInpaintingNonInteractive: There are " << boundaryNodeQueue.CountValidNodes()
<< " nodes in the boundaryNodeQueue" << std::endl;
#define DUseWeightedDifference
#ifdef DUseWeightedDifference
// The absolute value of the depth derivative range is usually about [0,12], so to make
// it comparable to to the color image channel range of [0,255], we multiply by 255/12 ~= 20.
// float depthDerivativeWeight = 20.0f;
// This should not be computed "by eye" by looking at the Dx and Dy channels of the PTX scan, because there are typically
// huge depth discontinuties around the objects that are going to be inpainted. We'd have to look at the masked version of this
// image to determine the min/max values of the unmasked pixels. They will be much smaller than the min/max values of the original
// image, which will make the depth derivative weights much higher (~100 or so)
// std::vector<typename TImage::PixelType> channelMins = ITKHelpers::ComputeMinOfAllChannels(originalImage);
// std::vector<typename TImage::PixelType> channelMaxs = ITKHelpers::ComputeMaxOfAllChannels(originalImage);
typename TImage::PixelType channelMins;
ITKHelpers::ComputeMinOfAllChannels(originalImage, channelMins);
typename TImage::PixelType channelMaxs;
ITKHelpers::ComputeMaxOfAllChannels(originalImage, channelMaxs);
float minX = fabs(channelMins[3]);
float maxX = fabs(channelMaxs[3]);
float maxValueX = std::max(minX, maxX);
std::cout << "maxValueX = " << maxValueX << std::endl;
float depthDerivativeWeightX = 255.0f / maxValueX;
std::cout << "Computed depthDerivativeWeightX = " << depthDerivativeWeightX << std::endl;
float minY = fabs(channelMins[4]);
float maxY = fabs(channelMaxs[4]);
float maxValueY = std::max(minY, maxY);
std::cout << "maxValueY = " << maxValueY << std::endl;
float depthDerivativeWeightY = 255.0f / maxValueY;
std::cout << "Computed depthDerivativeWeightY = " << depthDerivativeWeightY << std::endl;
// Full pixels
std::vector<float> weights = {1.0f, 1.0f, 1.0f, depthDerivativeWeightX, depthDerivativeWeightY};
typedef WeightedSumSquaredPixelDifference<typename TImage::PixelType> FullPixelDifferenceType;
FullPixelDifferenceType fullPixelDifferenceFunctor(weights);
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
FullPixelDifferenceType > FullPatchDifferenceType;
FullPatchDifferenceType fullPatchDifferenceFunctor(fullPixelDifferenceFunctor);
// First 3 channels only pixels
typedef RGBSSD<typename TImage::PixelType> RGBPixelDifferenceType;
RGBPixelDifferenceType rgbPixelDifferenceFunctor;
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
RGBPixelDifferenceType > RGBPatchDifferenceType;
RGBPatchDifferenceType rgbPatchDifferenceFunctor(rgbPixelDifferenceFunctor);
#else
// Use an unweighted pixel difference
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
SumSquaredPixelDifference<typename TImage::PixelType> > PatchDifferenceType;
PatchDifferenceType patchDifferenceFunctor;
#endif
//#define DAllowReuse // comment/uncomment this line to toggle allowing patches to be used as the source patch more than once
#ifdef DAllowReuse
// Create the first (KNN) neighbor finder
typedef LinearSearchKNNProperty<ImagePatchDescriptorMapType, PatchDifferenceType> KNNSearchType;
KNNSearchType linearSearchKNN(imagePatchDescriptorMap, numberOfKNN, patchDifferenceFunctor);
#else
// Full pixel search
typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, FullPatchDifferenceType, RGBImageType> FullKNNSearchType;
FullKNNSearchType fullSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN,
fullPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
rgbImage.GetPointer());
fullSearchKNN.SetDebugImages(true);
// RGB-only search
typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, RGBPatchDifferenceType, RGBImageType> RGBKNNSearchType;
RGBKNNSearchType rgbSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN,
rgbPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
rgbImage.GetPointer());
rgbSearchKNN.SetDebugImages(true);
// Combine
typedef LinearSearchKNNPropertyCombine<FullKNNSearchType, RGBKNNSearchType> KNNSearchType;
KNNSearchType linearSearchKNN(fullSearchKNN, rgbSearchKNN);
#endif
// Setup the second (1-NN) neighbor finder
typedef std::vector<VertexDescriptorType>::iterator VertexDescriptorVectorIteratorType;
// This is templated on TImage because we need it to write out debug patches from this searcher (since we are not using an RGB image to compute the histograms)
// typedef LinearSearchBestTexture<ImagePatchDescriptorMapType, RGBImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the histogram of the gradient magnitudes of a scalar represetnation of the image (e.g. magnitude image)
// typedef LinearSearchBestLidarTextureDerivatives<ImagePatchDescriptorMapType, RGBImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the concatenated histograms of the absolute value of the derivatives of each channel
typedef LinearSearchBestLidarRGBTextureGradient<ImagePatchDescriptorMapType, RGBDxDyImageType,
VertexDescriptorVectorIteratorType, RGBImageType> BestSearchType; // Use the concatenated histograms of the gradient magnitudes of each channel. This RGBDxDyImageType must match the rgbDxDyImage provided below
// BestSearchType linearSearchBest(imagePatchDescriptorMap, rgbDxDyImage.GetPointer(), mask); // use non-blurred for texture sorting
Debug debug;
// debug.SetDebugOutputs(true);
// debug.SetDebugImages(true);
// linearSearchBest.SetDebugOutputs(true);
// linearSearchBest.SetDebugImages(true);
// use slightly blurred for texture sorting
BestSearchType linearSearchBest(imagePatchDescriptorMap, slightlyBlurredRGBDxDyImage.GetPointer(),
mask, rgbImage.GetPointer(), debug);
linearSearchBest.WritePatches = true;
// Setup the two step neighbor finder
TwoStepNearestNeighbor<KNNSearchType, BestSearchType>
twoStepNearestNeighbor(linearSearchKNN, linearSearchBest);
// #define DFullSearch // comment/uncomment this line to set the search region
#ifdef DFullSearch
// Perform the inpainting (full search)
InpaintingAlgorithm(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter);
#else
NeighborhoodSearch<VertexDescriptorType, ImagePatchDescriptorMapType> neighborhoodSearch(originalImage->GetLargestPossibleRegion(),
searchRadius, imagePatchDescriptorMap);
// Perform the inpainting (local search)
InpaintingAlgorithmWithLocalSearch(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter, neighborhoodSearch);
#endif
}
void itkSegmentLV< TImage>
::ThreadedGenerateData(const OutputImageRegionType& outputRegionForThread,
ThreadIdType threadId)
{
typename TImage::ConstPointer input = this->GetInput();
typename TImage::Pointer output = this->GetOutput();
//caster2->SetInput(input);
ConnectedRegionGrowing->SetInput( input );
ConnectedRegionGrowing->SetLower( lowerThreshold );
ConnectedRegionGrowing->SetUpper( upperThreshold );
ConnectedRegionGrowing->SetRadius( radius );
ConnectedRegionGrowing->SetSeed( index );
ConnectedRegionGrowing->SetReplaceValue( 255 );
binaryErode->SetKernel( structuringElement );
binaryDilate->SetKernel( structuringElement );
binaryDilate->SetInput( ConnectedRegionGrowing->GetOutput());
binaryErode->SetInput( binaryDilate->GetOutput() );
sbs->SetInput(binaryErode->GetOutput() );
// sbs->SetInput(ConnectedRegionGrowing->GetOutput() );
sbs->SetInputFilter(connected);
sbs->SetOutputFilter(rescaleFilter);
labelShapeKeepNObjectsImageFilter->SetInput( connected->GetOutput() );
labelShapeKeepNObjectsImageFilter->SetBackgroundValue( 0 );
labelShapeKeepNObjectsImageFilter->SetNumberOfObjects( 1 );
labelShapeKeepNObjectsImageFilter->SetAttribute( LabelShapeKeepNObjectsImageFilterType::
LabelObjectType::NUMBER_OF_PIXELS);
rescaleFilter->SetOutputMinimum(0);
rescaleFilter->SetOutputMaximum(itk::NumericTraits<signed short>::max());
rescaleFilter->SetInput(labelShapeKeepNObjectsImageFilter->GetOutput());
sbs->Update();
this->AllocateOutputs();
ImageAlgorithm::Copy(sbs->GetOutput(), output.GetPointer(), output->GetRequestedRegion(),
output->GetRequestedRegion() );
}
void SampleGradHistogram(double *pt, double radius, typename TImage::Pointer image, arma::ivec& samples)
{
typedef TImage ImageType;
int radius_pix[3]; //radius in voxels, to be calculated
radius_pix[0] = std::ceil(radius / image->GetSpacing()[0]);
radius_pix[1] = std::ceil(radius / image->GetSpacing()[1]);
radius_pix[2] = std::ceil(radius / image->GetSpacing()[2]);
#ifdef DEBUG_MESSAGES_HOG
std::cout << "Requested point: " << pt[0] << ", " << pt[1] << ", " << pt[2] << std::endl;
std::cout << "Neighborhood size in mm: " << radius << std::endl;
std::cout << "Neighborhood size in pix: " << radius_pix[0] << ", " << radius_pix[1] << ", " << radius_pix[2] << std::endl;
#endif
if (radius_pix[0] == 0 || radius_pix[1] == 0 || radius_pix[2] == 0) {
std::cout << "One of the neighborhood dimensions is zero. Please correct the radius. Aborting." << std::endl;
return;
}
//transform the point from physical s1pace
typename ImageType::PointType point;
point[0] = pt[0];
point[1] = pt[1];
point[2] = pt[2];
typename ImageType::IndexType pt_pix;
image->TransformPhysicalPointToIndex(point, pt_pix);
//define the region around the point of interest
typename ImageType::IndexType rgn_idx = {
{pt_pix[0] - radius_pix[0], pt_pix[1] - radius_pix[1], pt_pix[2] - radius_pix[2]}};
typename ImageType::SizeType rgn_size = {
{2 * radius_pix[0] + 1, 2 * radius_pix[1] + 1, 2 * radius_pix[2] + 1}};
//crop the region so that it is inside
rgn_idx[0] = std::max(rgn_idx[0], image->GetLargestPossibleRegion().GetIndex(0));
rgn_idx[1] = std::max(rgn_idx[1], image->GetLargestPossibleRegion().GetIndex(1));
rgn_idx[2] = std::max(rgn_idx[2], image->GetLargestPossibleRegion().GetIndex(2));
//set it first as a corner for comparison and then undo that operation
rgn_size[0] = std::min(rgn_size[0]+rgn_idx[0], image->GetLargestPossibleRegion().GetSize(0))-rgn_idx[0];
rgn_size[1] = std::min(rgn_size[1]+rgn_idx[1], image->GetLargestPossibleRegion().GetSize(1))-rgn_idx[1];
rgn_size[2] = std::min(rgn_size[2]+rgn_idx[2], image->GetLargestPossibleRegion().GetSize(2))-rgn_idx[2];
typename ImageType::RegionType window(rgn_idx, rgn_size);
#ifdef DEBUG_MESSAGES_HOG
std::cout << "Region: " << window << std::endl;
#endif
samples.set_size(rgn_size[0]*rgn_size[1]*rgn_size[2]);
samples.zeros();
// itk::ImageRegionConstIterator<typename GradientFilterType::OutputImageType> imageIterator(gradient_filter->GetOutput(), window);
itk::ImageRegionConstIterator<ImageType> imageIterator(image, window);
imageIterator.GoToBegin();
int i=0;
while (!imageIterator.IsAtEnd()) {
// Get the value of the current pixel
typename ImageType::PixelType val = imageIterator.Get();
//std::cout << val << std::endl;
samples[i++] = val;
++imageIterator;
}
}
void SampleStatistics(double *pt, double radius, typename TImage::Pointer image, arma::vec& features)
{
features.set_size(4); //just four features for now
features.zeros();
typedef TImage ImageType;
int radius_pix[3]; //radius in voxels, to be calculated
radius_pix[0] = std::ceil(radius / image->GetSpacing()[0]);
radius_pix[1] = std::ceil(radius / image->GetSpacing()[1]);
radius_pix[2] = std::ceil(radius / image->GetSpacing()[2]);
#ifdef DEBUG_MESSAGES_HOG
std::cout << "Requested point: " << pt[0] << ", " << pt[1] << ", " << pt[2] << std::endl;
std::cout << "Neighborhood size in mm: " << radius << std::endl;
std::cout << "Neighborhood size in pix: " << radius_pix[0] << ", " << radius_pix[1] << ", " << radius_pix[2] << std::endl;
#endif
if (radius_pix[0] == 0 || radius_pix[1] == 0 || radius_pix[2] == 0) {
std::cout << "One of the neighborhood dimensions is zero. Please correct the radius. Aborting." << std::endl;
return;
}
//transform the point from physical s1pace
typename ImageType::PointType point;
point[0] = pt[0];
point[1] = pt[1];
point[2] = pt[2];
typename ImageType::IndexType pt_pix;
image->TransformPhysicalPointToIndex(point, pt_pix);
//define the region around the point of interest
typename ImageType::IndexType rgn_idx = {
{pt_pix[0] - radius_pix[0], pt_pix[1] - radius_pix[1], pt_pix[2] - radius_pix[2]}};
typename ImageType::SizeType rgn_size = {
{2 * radius_pix[0] + 1, 2 * radius_pix[1] + 1, 2 * radius_pix[2] + 1}};
//crop the region so that it is inside
rgn_idx[0] = std::max(rgn_idx[0], image->GetLargestPossibleRegion().GetIndex(0));
rgn_idx[1] = std::max(rgn_idx[1], image->GetLargestPossibleRegion().GetIndex(1));
rgn_idx[2] = std::max(rgn_idx[2], image->GetLargestPossibleRegion().GetIndex(2));
//set it first as a corner for comparison and then undo that operation
rgn_size[0] = std::min(rgn_size[0]+rgn_idx[0], image->GetLargestPossibleRegion().GetSize(0))-rgn_idx[0];
rgn_size[1] = std::min(rgn_size[1]+rgn_idx[1], image->GetLargestPossibleRegion().GetSize(1))-rgn_idx[1];
rgn_size[2] = std::min(rgn_size[2]+rgn_idx[2], image->GetLargestPossibleRegion().GetSize(2))-rgn_idx[2];
typename ImageType::RegionType window(rgn_idx, rgn_size);
#ifdef DEBUG_MESSAGES_HOG
std::cout << "Region: " << window << std::endl;
#endif
itk::ImageRegionConstIterator<ImageType> imageIterator(image, window);
imageIterator.GoToBegin();
const int nvoxels = rgn_size[0]*rgn_size[1]*rgn_size[2];
arma::vec voxels;
voxels.set_size(nvoxels);
voxels.zeros();
long int i=0;
while (!imageIterator.IsAtEnd()) {
// Get the value of the current pixel
typename ImageType::PixelType val = imageIterator.Get();
//std::cout << val << std::endl;
voxels[i++] = val;
++imageIterator;
}
//do the statistics
const double mean = arma::mean(voxels);
const double std = arma::stddev(voxels);
double E3 = 0; //Accumulate expectation of (X-u)^3
double E4 = 0; //Accumulate expectation of (X-u)^4
for(int i=0; i<voxels.size(); i++)
{
const double diff = voxels[i] - mean;
const double k = diff*diff*diff/nvoxels;
E3 += k; //cubed
E4 += diff*k; //4th power
}
features[0] = mean;
features[1] = std;
features[2] = E3/(std*std*std); //skewness
features[3] = E3/(std*std*std*std); //kurtosis
}
void ManualPatchSelectionDialog<TImage>::slot_UpdateResult(const itk::ImageRegion<2>& sourceRegion,
const itk::ImageRegion<2>& targetRegion)
{
assert(sourceRegion.GetSize() == targetRegion.GetSize());
if(!this->Image->GetLargestPossibleRegion().IsInside(sourceRegion))
{
std::cerr << "Source region is outside the image!" << std::endl;
return;
}
QImage qimage(sourceRegion.GetSize()[0], sourceRegion.GetSize()[1], QImage::Format_RGB888);
if(MaskImage->CountHolePixels(sourceRegion) > 0)
{
//std::cerr << "The source patch must not have any hole pixels!" << std::endl;
//btnAccept->setVisible(false);
qimage.fill(Qt::green);
}
else
{
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
if(this->Image->GetNumberOfComponentsPerPixel() != 3)
{
ITKHelpers::ScaleAllChannelsTo255(tempImage.GetPointer());
}
itk::ImageRegionIterator<TImage> sourceIterator(tempImage, sourceRegion);
itk::ImageRegionIterator<TImage> targetIterator(tempImage, targetRegion);
itk::ImageRegionIterator<Mask> maskIterator(MaskImage, targetRegion);
typename TImage::Pointer resultPatch = TImage::New();
resultPatch->SetNumberOfComponentsPerPixel(Image->GetNumberOfComponentsPerPixel());
itk::ImageRegion<2> resultPatchRegion;
resultPatchRegion.SetSize(sourceRegion.GetSize());
resultPatch->SetRegions(resultPatchRegion);
resultPatch->Allocate();
while(!maskIterator.IsAtEnd())
{
ITKHelpers::FloatVectorImageType::PixelType pixel;
if(MaskImage->IsHole(maskIterator.GetIndex()))
{
pixel = sourceIterator.Get();
}
else
{
pixel = targetIterator.Get();
}
itk::Offset<2> offset = sourceIterator.GetIndex() - sourceRegion.GetIndex();
itk::Index<2> offsetIndex;
offsetIndex[0] = offset[0];
offsetIndex[1] = offset[1];
resultPatch->SetPixel(offsetIndex, pixel);
++sourceIterator;
++targetIterator;
++maskIterator;
} // end iterator loop
qimage = ITKQtHelpers::GetQImageColor(resultPatch.GetPointer(), resultPatch->GetLargestPossibleRegion());
} // end else
this->ResultPatchScene->clear();
QGraphicsPixmapItem* item = this->ResultPatchScene->addPixmap(QPixmap::fromImage(qimage));
gfxResult->fitInView(item);
}
