void BDSInpainting<TImage>::Inpaint(TPatchMatchFunctor* const patchMatchFunctor,
TCompositor* const compositor)
{
assert(this->Image);
assert(this->InpaintingMask);
ConstructValidPatchCentersImage();
// Initialize the output with the input
typename TImage::Pointer currentImage = TImage::New();
ITKHelpers::DeepCopy(this->Image.GetPointer(), currentImage.GetPointer());
// Allocate the initial NNField
NNFieldType::Pointer nnField =
NNFieldType::New();
nnField->SetRegions(currentImage->GetLargestPossibleRegion());
nnField->Allocate();
// Initialize the NNField in the target region
typedef SSD<TImage> PatchDistanceFunctorType;
PatchDistanceFunctorType patchDistanceFunctor;
patchDistanceFunctor.SetImage(currentImage);
patchMatchFunctor->SetValidPatchCentersImage(this->ValidPatchCentersImage);
std::vector<itk::Index<2> > pixelsToProcess = this->InpaintingMask->GetHolePixels();
patchMatchFunctor->SetTargetPixels(pixelsToProcess);
patchMatchFunctor->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetPropagationFunctor()->SetPatchDistanceFunctor(&patchDistanceFunctor);
patchMatchFunctor->GetPropagationFunctor()->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetRandomSearchFunctor()->SetImage(this->Image);
patchMatchFunctor->GetRandomSearchFunctor()->SetPatchRadius(this->PatchRadius);
patchMatchFunctor->GetRandomSearchFunctor()->SetPatchDistanceFunctor(&patchDistanceFunctor);
compositor->SetPatchRadius(this->PatchRadius);
compositor->SetTargetMask(this->InpaintingMask);
compositor->SetImage(currentImage);
compositor->SetNearestNeighborField(patchMatchFunctor->GetNNField());
for(unsigned int iteration = 0; iteration < this->Iterations; ++iteration)
{
// Run PatchMatch to compute the NNField
patchMatchFunctor->Compute();
std::stringstream ssNNFieldFileName;
ssNNFieldFileName << "BDS_" << iteration << "_NNField.mha";
PatchMatchHelpers::WriteNNField(patchMatchFunctor->GetNNField(), ssNNFieldFileName.str());
// Update the target pixels
compositor->Composite();
ITKHelpers::DeepCopy(compositor->GetOutput(), currentImage.GetPointer());
}
ITKHelpers::DeepCopy(currentImage.GetPointer(), this->Output.GetPointer());
}
//BasicViewerWidget<TImage>::BasicViewerWidget(TImage* const image, Mask* const mask) :
BasicViewerWidget<TImage>::BasicViewerWidget(typename TImage::Pointer image, Mask::Pointer mask) :
Image(image), MaskImage(mask)
{
qRegisterMetaType<itk::ImageRegion<2> >("itkImageRegion");
this->setupUi(this);
int dims[3];
this->ImageLayer.ImageData->GetDimensions(dims);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::DeepCopy(image.GetPointer(), tempImage.GetPointer());
typename TImage::PixelType zeroPixel(tempImage->GetNumberOfComponentsPerPixel());
zeroPixel = itk::NumericTraits<typename TImage::PixelType>::ZeroValue(zeroPixel);
mask->ApplyToImage(tempImage.GetPointer(), zeroPixel);
ITKVTKHelpers::ITKVectorImageToVTKImageFromDimension(tempImage.GetPointer(), this->ImageLayer.ImageData);
// if(chkScaleImage->isChecked())
// {
// VTKHelpers::ScaleImage(this->ImageLayer.ImageData);
// }
this->ImageDimension[0] = dims[0];
this->ImageDimension[1] = dims[1];
this->ImageDimension[2] = dims[2];
SetupScenes();
// this->InteractorStyle = vtkSmartPointer<InteractorStyleImageWithDrag>::New();
this->InteractorStyle = vtkSmartPointer<vtkInteractorStyleImage>::New();
this->Renderer = vtkSmartPointer<vtkRenderer>::New();
this->qvtkWidget->GetRenderWindow()->AddRenderer(this->Renderer);
this->Renderer->AddViewProp(ImageLayer.ImageSlice);
this->InteractorStyle->SetCurrentRenderer(this->Renderer);
this->qvtkWidget->GetRenderWindow()->GetInteractor()->SetInteractorStyle(this->InteractorStyle);
// this->InteractorStyle->Init();
// this->Camera = new ImageCamera(this->Renderer);
this->ItkVtkCamera = new ITKVTKCamera(this->InteractorStyle, this->Renderer,
this->qvtkWidget->GetRenderWindow());
this->ItkVtkCamera->SetCameraPositionPNG();
}
void ManualPatchSelectionDialog<TImage>::slot_UpdateSource(const itk::ImageRegion<2>& sourceRegion,
const itk::ImageRegion<2>& targetRegion)
{
// This function needs the targetRegion because this is the region of the Mask that is used to mask the source patch.
// std::cout << "Update source." << std::endl;
if(!this->Image->GetLargestPossibleRegion().IsInside(sourceRegion))
{
std::cerr << "Source region is outside the image!" << std::endl;
return;
}
if(MaskImage->CountHolePixels(sourceRegion) > 0)
{
std::cerr << "The source patch must not have any hole pixels!" << std::endl;
btnAccept->setVisible(false);
}
else
{
btnAccept->setVisible(true);
}
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedSourceImage = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), sourceRegion);
QGraphicsPixmapItem* item = this->SourcePatchScene->addPixmap(QPixmap::fromImage(maskedSourceImage));
gfxSource->fitInView(item);
// Refresh the image
//ITKVTKHelpers::ITKImageToVTKRGBImage(this->Image, this->ImageLayer.ImageData);
unsigned char green[3] = {0, 255, 0};
MaskOperations::ITKImageToVTKImageMasked(tempImage, this->MaskImage,
this->ImageLayer.ImageData, green);
// Outline the source patch
unsigned char blue[3] = {0, 0, 255};
ITKVTKHelpers::OutlineRegion(this->ImageLayer.ImageData, sourceRegion, blue);
this->qvtkWidget->GetRenderWindow()->Render();
}
ManualPatchSelectionDialog<TImage>::ManualPatchSelectionDialog(TImage* const image, Mask* const mask,
const itk::ImageRegion<2>& targetRegion)
: Image(image), MaskImage(mask), TargetRegion(targetRegion)
{
// Allow the type itkImageRegion to be used in signals/slots.
qRegisterMetaType<itk::ImageRegion<2> >("itkImageRegion");
this->setupUi(this);
this->ImageLayer.ImageSlice->SetDragable(false);
this->ImageLayer.ImageSlice->SetPickable(false);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::DeepCopy(image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(tempImage->GetNumberOfComponentsPerPixel());
zeroPixel.Fill(0);
mask->ApplyToImage(tempImage.GetPointer(), zeroPixel);
ITKVTKHelpers::ITKVectorImageToVTKImageFromDimension(tempImage.GetPointer(), this->ImageLayer.ImageData);
SetupScenes();
this->Renderer = vtkSmartPointer<vtkRenderer>::New();
this->InteractorStyle = vtkSmartPointer<InteractorStyleImageWithDrag>::New();
this->qvtkWidget->GetRenderWindow()->AddRenderer(this->Renderer);
this->Renderer->AddViewProp(ImageLayer.ImageSlice);
// Per the comment in InteractorStyleImageWithDrag, the next 3 lines must be in this order
this->InteractorStyle->SetCurrentRenderer(this->Renderer);
this->qvtkWidget->GetRenderWindow()->GetInteractor()->SetInteractorStyle(this->InteractorStyle);
this->InteractorStyle->Init();
this->PatchSelector = new MovablePatch(this->TargetRegion.GetSize()[0]/2, this->InteractorStyle, this->gfxSource);
// slot_UpdateImage();
this->Renderer->ResetCamera();
this->qvtkWidget->GetRenderWindow()->Render();
// this->Camera = new ImageCamera(this->Renderer);
connect(this->PatchSelector, SIGNAL(signal_PatchMoved()), this, SLOT(slot_PatchMoved()));
}
void ManualPatchSelectionDialog<TImage>::slot_UpdateTarget(const itk::ImageRegion<2>& region)
{
// std::cout << "Update target." << std::endl;
unsigned char red[3] = {255, 0, 0};
ITKVTKHelpers::OutlineRegion(this->ImageLayer.ImageData, region, red);
this->qvtkWidget->GetRenderWindow()->Render();
// Masked target patch
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedTargetImage = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), region);
QGraphicsPixmapItem* maskedItem = this->TargetPatchScene->addPixmap(QPixmap::fromImage(maskedTargetImage));
gfxTarget->fitInView(maskedItem);
}
void TopPatchesDialog<TImage>::SetQueryNode(const Node& queryNode)
{
this->QueryNode = queryNode;
itk::Index<2> queryIndex = ITKHelpers::CreateIndex(queryNode);
itk::ImageRegion<2> queryRegion = ITKHelpers::GetRegionInRadiusAroundPixel(queryIndex, PatchHalfWidth);
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
typename TImage::PixelType zeroPixel(3);
zeroPixel.Fill(0);
this->MaskImage->ApplyToImage(tempImage.GetPointer(), zeroPixel);
QImage maskedQueryPatch = ITKQtHelpers::GetQImageColor(tempImage.GetPointer(), queryRegion);
MaskedQueryPatchItem = this->QueryPatchScene->addPixmap(QPixmap::fromImage(maskedQueryPatch));
// We do this here because we could potentially call SetQueryNode after the widget is constructed as well.
gfxQueryPatch->fitInView(MaskedQueryPatchItem);
}
/** The mask is inpainted with ValidValue in this function. */
void FinishVertex(VertexDescriptorType targetNode, VertexDescriptorType sourceNode)// __attribute__((optimize(0))) // This supposedly makes this function build in debug mode (-g -O0) when the rest of the program is built in -O3 or similar)
{
// Mark this pixel and the area around it as filled, and mark the mask in this region as filled.
// Determine the new boundary, and setup the nodes in the boundary queue.
// std::cout << "InpaintingVisitor::FinishVertex()" << std::endl;
// Construct the region around the vertex
itk::Index<2> indexToFinish = ITKHelpers::CreateIndex(targetNode);
itk::ImageRegion<2> regionToFinishFull = ITKHelpers::GetRegionInRadiusAroundPixel(indexToFinish, this->PatchHalfWidth);
// Copy this region so that we can change (crop) the regionToFinish and still have a copy of the original region
itk::ImageRegion<2> regionToFinish = regionToFinishFull;
// Make sure the region is entirely inside the image
// (because we allow target patches to not be entirely inside the image to handle the case where
// the hole boundary is near the image boundary)
regionToFinish.Crop(this->Image->GetLargestPossibleRegion());
if(this->DebugImages)
{
ITKHelpers::WriteImage(this->MaskImage, Helpers::GetSequentialFileName("Mask_Before", this->NumberOfFinishedPatches, "png", 3));
}
// Mark all the pixels in this region as filled in the mask.
ITKHelpers::SetRegionToConstant(this->MaskImage, regionToFinish, this->MaskImage->GetValidValue());
if(this->DebugImages)
{
ITKHelpers::WriteImage(this->MaskImage, Helpers::GetSequentialFileName("Mask_After", this->NumberOfFinishedPatches, "png", 3));
typename TImage::PixelType red;
red.Fill(0);
red[0] = 255;
typename TImage::PixelType green;
green.Fill(0);
green[1] = 255;
typename TImage::Pointer patchesCopiedImage = TImage::New();
ITKHelpers::DeepCopy(this->Image, patchesCopiedImage.GetPointer());
ITKHelpers::OutlineRegion(patchesCopiedImage.GetPointer(), regionToFinish, red);
itk::Index<2> sourceRegionCenter = ITKHelpers::CreateIndex(sourceNode);
itk::ImageRegion<2> sourceRegion = ITKHelpers::GetRegionInRadiusAroundPixel(sourceRegionCenter, this->PatchHalfWidth);
sourceRegion = ITKHelpers::CropRegionAtPosition(sourceRegion, this->Image->GetLargestPossibleRegion(), regionToFinishFull);
ITKHelpers::OutlineRegion(patchesCopiedImage.GetPointer(), sourceRegion, green);
ITKHelpers::WriteRGBImage(patchesCopiedImage.GetPointer(), Helpers::GetSequentialFileName("PatchesCopied", this->NumberOfFinishedPatches, "png", 3));
}
// Update the priority function. This must be done AFTER the mask is filled,
// as some of the Priority functors only compute things on the hole boundary, or only
// use data from the valid region of the image (indicated by valid pixels in the mask).
this->PriorityFunction->Update(sourceNode, targetNode, this->NumberOfFinishedPatches);
// Initialize (if requested) all vertices in the newly filled region because they may now be valid source nodes.
// (You may not want to do this in some cases (i.e. if the descriptors needed cannot be
// computed on newly filled regions))
if(this->AllowNewPatches)
{
itk::ImageRegionConstIteratorWithIndex<TImage> gridIterator(Image, regionToFinish);
while(!gridIterator.IsAtEnd())
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(gridIterator.GetIndex());
InitializeVertex(v);
++gridIterator;
}
}
// Add pixels that are on the new boundary to the queue, and mark other pixels as not in the queue.
itk::ImageRegionConstIteratorWithIndex<Mask> imageIterator(this->MaskImage, regionToFinish);
typedef typename TBoundaryNodeQueue::HandleType HandleType;
while(!imageIterator.IsAtEnd())
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(imageIterator.GetIndex());
if(this->MaskImage->HasHoleNeighbor(imageIterator.GetIndex()))
{
float priority = this->PriorityFunction->ComputePriority(imageIterator.GetIndex());
this->BoundaryNodeQueue.push_or_update(v, priority);
}
else
{
this->BoundaryNodeQueue.mark_as_invalid(v);
}
++imageIterator;
}
// std::cout << "FinishVertex after traversing finishing region there are "
// << BoostHelpers::CountValidQueueNodes(BoundaryNodeQueue, BoundaryStatusMap)
// << " valid nodes in the queue." << std::endl;
// Sometimes pixels that are not in the finishing region that were boundary pixels are no longer
// boundary pixels after the filling. Check for these.
// E.g. (H=hole, B=boundary, V=valid, Q=query, F=filled, N=new boundary,
// R=old boundary pixel that needs to be removed because it is no longer on the boundary)
// Before filling
/* V V V B H H H H
* V V V B H H H H
* V V V B H H H H
* V V V B B Q B B
* V V V V V V V V
*/
// After filling
/* V V V B H H H H
* V V V B H H H H
* V V V B F F F H
* V V V B F F F B
* V V V V F F F V
*/
// New boundary
/* V V V B H H H H
* V V V B H H H H
* V V V B N N N H
* V V V R F F N B
* V V V V F F F V
*/
// Expand the region
itk::ImageRegion<2> expandedRegion = ITKHelpers::GetRegionInRadiusAroundPixel(indexToFinish, PatchHalfWidth + 1);
// Make sure the region is entirely inside the image (to allow for target regions near the image boundary)
expandedRegion.Crop(this->Image->GetLargestPossibleRegion());
std::vector<itk::Index<2> > boundaryPixels = ITKHelpers::GetBoundaryPixels(expandedRegion);
for(unsigned int i = 0; i < boundaryPixels.size(); ++i)
{
// the region (the entire image) can be omitted, as this function automatically checks if the pixels are inside the image
if(!this->MaskImage->HasHoleNeighbor(boundaryPixels[i]))
{
VertexDescriptorType v = Helpers::ConvertFrom<VertexDescriptorType, itk::Index<2> >(boundaryPixels[i]);
put(this->BoundaryNodeQueue.BoundaryStatusMap, v, false);
}
}
// std::cout << "FinishVertex after removing stale nodes outside finishing region there are "
// << BoostHelpers::CountValidQueueNodes(BoundaryNodeQueue, BoundaryStatusMap)
// << " valid nodes in the queue." << std::endl;
this->NumberOfFinishedPatches++;
// std::cout << "Leave InpaintingVisitor::FinishVertex()" << std::endl;
} // end FinishVertex
void ManualPatchSelectionDialog<TImage>::slot_UpdateResult(const itk::ImageRegion<2>& sourceRegion,
const itk::ImageRegion<2>& targetRegion)
{
assert(sourceRegion.GetSize() == targetRegion.GetSize());
if(!this->Image->GetLargestPossibleRegion().IsInside(sourceRegion))
{
std::cerr << "Source region is outside the image!" << std::endl;
return;
}
QImage qimage(sourceRegion.GetSize()[0], sourceRegion.GetSize()[1], QImage::Format_RGB888);
if(MaskImage->CountHolePixels(sourceRegion) > 0)
{
//std::cerr << "The source patch must not have any hole pixels!" << std::endl;
//btnAccept->setVisible(false);
qimage.fill(Qt::green);
}
else
{
typename TImage::Pointer tempImage = TImage::New();
ITKHelpers::ConvertTo3Channel(this->Image, tempImage.GetPointer());
if(this->Image->GetNumberOfComponentsPerPixel() != 3)
{
ITKHelpers::ScaleAllChannelsTo255(tempImage.GetPointer());
}
itk::ImageRegionIterator<TImage> sourceIterator(tempImage, sourceRegion);
itk::ImageRegionIterator<TImage> targetIterator(tempImage, targetRegion);
itk::ImageRegionIterator<Mask> maskIterator(MaskImage, targetRegion);
typename TImage::Pointer resultPatch = TImage::New();
resultPatch->SetNumberOfComponentsPerPixel(Image->GetNumberOfComponentsPerPixel());
itk::ImageRegion<2> resultPatchRegion;
resultPatchRegion.SetSize(sourceRegion.GetSize());
resultPatch->SetRegions(resultPatchRegion);
resultPatch->Allocate();
while(!maskIterator.IsAtEnd())
{
ITKHelpers::FloatVectorImageType::PixelType pixel;
if(MaskImage->IsHole(maskIterator.GetIndex()))
{
pixel = sourceIterator.Get();
}
else
{
pixel = targetIterator.Get();
}
itk::Offset<2> offset = sourceIterator.GetIndex() - sourceRegion.GetIndex();
itk::Index<2> offsetIndex;
offsetIndex[0] = offset[0];
offsetIndex[1] = offset[1];
resultPatch->SetPixel(offsetIndex, pixel);
++sourceIterator;
++targetIterator;
++maskIterator;
} // end iterator loop
qimage = ITKQtHelpers::GetQImageColor(resultPatch.GetPointer(), resultPatch->GetLargestPossibleRegion());
} // end else
this->ResultPatchScene->clear();
QGraphicsPixmapItem* item = this->ResultPatchScene->addPixmap(QPixmap::fromImage(qimage));
gfxResult->fitInView(item);
}
void itkSegmentLV< TImage>
::ThreadedGenerateData(const OutputImageRegionType& outputRegionForThread,
ThreadIdType threadId)
{
typename TImage::ConstPointer input = this->GetInput();
typename TImage::Pointer output = this->GetOutput();
//caster2->SetInput(input);
ConnectedRegionGrowing->SetInput( input );
ConnectedRegionGrowing->SetLower( lowerThreshold );
ConnectedRegionGrowing->SetUpper( upperThreshold );
ConnectedRegionGrowing->SetRadius( radius );
ConnectedRegionGrowing->SetSeed( index );
ConnectedRegionGrowing->SetReplaceValue( 255 );
binaryErode->SetKernel( structuringElement );
binaryDilate->SetKernel( structuringElement );
binaryDilate->SetInput( ConnectedRegionGrowing->GetOutput());
binaryErode->SetInput( binaryDilate->GetOutput() );
sbs->SetInput(binaryErode->GetOutput() );
// sbs->SetInput(ConnectedRegionGrowing->GetOutput() );
sbs->SetInputFilter(connected);
sbs->SetOutputFilter(rescaleFilter);
labelShapeKeepNObjectsImageFilter->SetInput( connected->GetOutput() );
labelShapeKeepNObjectsImageFilter->SetBackgroundValue( 0 );
labelShapeKeepNObjectsImageFilter->SetNumberOfObjects( 1 );
labelShapeKeepNObjectsImageFilter->SetAttribute( LabelShapeKeepNObjectsImageFilterType::
LabelObjectType::NUMBER_OF_PIXELS);
rescaleFilter->SetOutputMinimum(0);
rescaleFilter->SetOutputMaximum(itk::NumericTraits<signed short>::max());
rescaleFilter->SetInput(labelShapeKeepNObjectsImageFilter->GetOutput());
sbs->Update();
this->AllocateOutputs();
ImageAlgorithm::Copy(sbs->GetOutput(), output.GetPointer(), output->GetRequestedRegion(),
output->GetRequestedRegion() );
}
void LidarInpaintingRGBTextureVerification(TImage* const originalImage, Mask* const mask,
const unsigned int patchHalfWidth, const unsigned int numberOfKNN,
float slightBlurVariance = 1.0f, unsigned int searchRadius = 1000)
{
typedef TImage RGBDxDyImageType;
itk::ImageRegion<2> fullRegion = originalImage->GetLargestPossibleRegion();
// Extract the RGB image
typedef itk::Image<itk::CovariantVector<float, 3>, 2> RGBImageType;
std::vector<unsigned int> firstThreeChannels = {0,1,2};
RGBImageType::Pointer rgbImage = RGBImageType::New();
ITKHelpers::ExtractChannels(originalImage, firstThreeChannels, rgbImage.GetPointer());
// Blur the image for gradient computation stability (Criminisi's data term)
RGBImageType::Pointer blurredRGBImage = RGBImageType::New();
float blurVariance = 2.0f;
MaskOperations::MaskedBlur(rgbImage.GetPointer(), mask, blurVariance, blurredRGBImage.GetPointer());
ITKHelpers::WriteRGBImage(blurredRGBImage.GetPointer(), "BlurredRGBImage.png");
// Blur the image slightly so that the SSD comparisons are not so noisy
typename TImage::Pointer slightlyBlurredRGBDxDyImage = TImage::New();
MaskOperations::MaskedBlur(originalImage, mask, slightBlurVariance, slightlyBlurredRGBDxDyImage.GetPointer());
ITKHelpers::WriteImage(slightlyBlurredRGBDxDyImage.GetPointer(), "SlightlyBlurredRGBDxDyImage.mha");
// Create the graph
typedef ImagePatchPixelDescriptor<TImage> ImagePatchPixelDescriptorType;
typedef boost::grid_graph<2> VertexListGraphType;
// We can't make this a signed type (size_t versus int) because we allow negative
boost::array<std::size_t, 2> graphSideLengths = { { fullRegion.GetSize()[0],
fullRegion.GetSize()[1] } };
VertexListGraphType graph(graphSideLengths);
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
typedef boost::graph_traits<VertexListGraphType>::vertex_iterator VertexIteratorType;
// Get the index map
typedef boost::property_map<VertexListGraphType, boost::vertex_index_t>::const_type IndexMapType;
IndexMapType indexMap(get(boost::vertex_index, graph));
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType, IndexMapType> ImagePatchDescriptorMapType;
ImagePatchDescriptorMapType imagePatchDescriptorMap(num_vertices(graph), indexMap);
// Create the patch inpainter.
typedef PatchInpainter<TImage> OriginalImageInpainterType;
OriginalImageInpainterType originalImagePatchInpainter(patchHalfWidth, originalImage, mask);
originalImagePatchInpainter.SetDebugImages(true);
originalImagePatchInpainter.SetImageName("RGB");
// Create an inpainter for the blurred image.
typedef PatchInpainter<RGBImageType> BlurredImageInpainterType;
BlurredImageInpainterType blurredRGBImagePatchInpainter(patchHalfWidth, blurredRGBImage, mask);
// Create an inpainter for the slightly blurred image.
typedef PatchInpainter<TImage> SlightlyBlurredRGBDxDyImageImageInpainterType;
SlightlyBlurredRGBDxDyImageImageInpainterType slightlyBlurredRGBDxDyImageImagePatchInpainter(patchHalfWidth, slightlyBlurredRGBDxDyImage, mask);
// Create a composite inpainter. (Note: the mask is inpainted in InpaintingVisitor::FinishVertex)
CompositePatchInpainter inpainter;
inpainter.AddInpainter(&originalImagePatchInpainter);
inpainter.AddInpainter(&blurredRGBImagePatchInpainter);
inpainter.AddInpainter(&slightlyBlurredRGBDxDyImageImagePatchInpainter);
// Create the priority function
typedef PriorityCriminisi<RGBImageType> PriorityType;
PriorityType priorityFunction(blurredRGBImage, mask, patchHalfWidth);
// priorityFunction.SetDebugLevel(1);
// Queue
typedef IndirectPriorityQueue<VertexListGraphType> BoundaryNodeQueueType;
BoundaryNodeQueueType boundaryNodeQueue(graph);
// Create the descriptor visitor (used for SSD comparisons).
typedef ImagePatchDescriptorVisitor<VertexListGraphType, TImage, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
// ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(originalImage, mask,
// imagePatchDescriptorMap, patchHalfWidth); // Use the non-blurred image for the SSD comparisons
ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(slightlyBlurredRGBDxDyImage, mask,
imagePatchDescriptorMap, patchHalfWidth); // Use the slightly blurred image for the SSD comparisons.
typedef DefaultAcceptanceVisitor<VertexListGraphType> AcceptanceVisitorType;
AcceptanceVisitorType acceptanceVisitor;
// Create the inpainting visitor. (The mask is inpainted in FinishVertex)
typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType, TImage>
InpaintingVisitorType;
InpaintingVisitorType inpaintingVisitor(mask, boundaryNodeQueue,
imagePatchDescriptorVisitor, acceptanceVisitor,
&priorityFunction, patchHalfWidth, "InpaintingVisitor", originalImage);
inpaintingVisitor.SetDebugImages(true); // This produces PatchesCopied* images showing where patches were copied from/to at each iteration
// inpaintingVisitor.SetAllowNewPatches(false);
inpaintingVisitor.SetAllowNewPatches(true); // we can do this as long as we use one of the LinearSearchKNNProperty*Reuse (like LinearSearchKNNPropertyLimitLocalReuse) in the steps below
InitializePriority(mask, boundaryNodeQueue, &priorityFunction);
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<InpaintingVisitorType, VertexDescriptorType>(mask, &inpaintingVisitor);
std::cout << "PatchBasedInpaintingNonInteractive: There are " << boundaryNodeQueue.CountValidNodes()
<< " nodes in the boundaryNodeQueue" << std::endl;
#define DUseWeightedDifference
#ifdef DUseWeightedDifference
// The absolute value of the depth derivative range is usually about [0,12], so to make
// it comparable to to the color image channel range of [0,255], we multiply by 255/12 ~= 20.
// float depthDerivativeWeight = 20.0f;
// This should not be computed "by eye" by looking at the Dx and Dy channels of the PTX scan, because there are typically
// huge depth discontinuties around the objects that are going to be inpainted. We'd have to look at the masked version of this
// image to determine the min/max values of the unmasked pixels. They will be much smaller than the min/max values of the original
// image, which will make the depth derivative weights much higher (~100 or so)
// std::vector<typename TImage::PixelType> channelMins = ITKHelpers::ComputeMinOfAllChannels(originalImage);
// std::vector<typename TImage::PixelType> channelMaxs = ITKHelpers::ComputeMaxOfAllChannels(originalImage);
typename TImage::PixelType channelMins;
ITKHelpers::ComputeMinOfAllChannels(originalImage, channelMins);
typename TImage::PixelType channelMaxs;
ITKHelpers::ComputeMaxOfAllChannels(originalImage, channelMaxs);
float minX = fabs(channelMins[3]);
float maxX = fabs(channelMaxs[3]);
float maxValueX = std::max(minX, maxX);
std::cout << "maxValueX = " << maxValueX << std::endl;
float depthDerivativeWeightX = 255.0f / maxValueX;
std::cout << "Computed depthDerivativeWeightX = " << depthDerivativeWeightX << std::endl;
float minY = fabs(channelMins[4]);
float maxY = fabs(channelMaxs[4]);
float maxValueY = std::max(minY, maxY);
std::cout << "maxValueY = " << maxValueY << std::endl;
float depthDerivativeWeightY = 255.0f / maxValueY;
std::cout << "Computed depthDerivativeWeightY = " << depthDerivativeWeightY << std::endl;
// Full pixels
std::vector<float> weights = {1.0f, 1.0f, 1.0f, depthDerivativeWeightX, depthDerivativeWeightY};
typedef WeightedSumSquaredPixelDifference<typename TImage::PixelType> FullPixelDifferenceType;
FullPixelDifferenceType fullPixelDifferenceFunctor(weights);
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
FullPixelDifferenceType > FullPatchDifferenceType;
FullPatchDifferenceType fullPatchDifferenceFunctor(fullPixelDifferenceFunctor);
// First 3 channels only pixels
typedef RGBSSD<typename TImage::PixelType> RGBPixelDifferenceType;
RGBPixelDifferenceType rgbPixelDifferenceFunctor;
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
RGBPixelDifferenceType > RGBPatchDifferenceType;
RGBPatchDifferenceType rgbPatchDifferenceFunctor(rgbPixelDifferenceFunctor);
#else
// Use an unweighted pixel difference
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
SumSquaredPixelDifference<typename TImage::PixelType> > PatchDifferenceType;
PatchDifferenceType patchDifferenceFunctor;
#endif
//#define DAllowReuse // comment/uncomment this line to toggle allowing patches to be used as the source patch more than once
#ifdef DAllowReuse
// Create the first (KNN) neighbor finder
typedef LinearSearchKNNProperty<ImagePatchDescriptorMapType, PatchDifferenceType> KNNSearchType;
KNNSearchType linearSearchKNN(imagePatchDescriptorMap, numberOfKNN, patchDifferenceFunctor);
#else
// Full pixel search
typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, FullPatchDifferenceType, RGBImageType> FullKNNSearchType;
FullKNNSearchType fullSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN,
fullPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
rgbImage.GetPointer());
fullSearchKNN.SetDebugImages(true);
// RGB-only search
typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, RGBPatchDifferenceType, RGBImageType> RGBKNNSearchType;
RGBKNNSearchType rgbSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN,
rgbPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
rgbImage.GetPointer());
rgbSearchKNN.SetDebugImages(true);
// Combine
typedef LinearSearchKNNPropertyCombine<FullKNNSearchType, RGBKNNSearchType> KNNSearchType;
KNNSearchType linearSearchKNN(fullSearchKNN, rgbSearchKNN);
#endif
// Setup the second (1-NN) neighbor finder
typedef std::vector<VertexDescriptorType>::iterator VertexDescriptorVectorIteratorType;
// This is templated on TImage because we need it to write out debug patches from this searcher (since we are not using an RGB image to compute the histograms)
// typedef LinearSearchBestTexture<ImagePatchDescriptorMapType, RGBImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the histogram of the gradient magnitudes of a scalar represetnation of the image (e.g. magnitude image)
// typedef LinearSearchBestLidarTextureDerivatives<ImagePatchDescriptorMapType, RGBImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the concatenated histograms of the absolute value of the derivatives of each channel
typedef LinearSearchBestLidarRGBTextureGradient<ImagePatchDescriptorMapType, RGBDxDyImageType,
VertexDescriptorVectorIteratorType, RGBImageType> BestSearchType; // Use the concatenated histograms of the gradient magnitudes of each channel. This RGBDxDyImageType must match the rgbDxDyImage provided below
// BestSearchType linearSearchBest(imagePatchDescriptorMap, rgbDxDyImage.GetPointer(), mask); // use non-blurred for texture sorting
Debug debug;
// debug.SetDebugOutputs(true);
// debug.SetDebugImages(true);
// linearSearchBest.SetDebugOutputs(true);
// linearSearchBest.SetDebugImages(true);
// use slightly blurred for texture sorting
BestSearchType linearSearchBest(imagePatchDescriptorMap, slightlyBlurredRGBDxDyImage.GetPointer(),
mask, rgbImage.GetPointer(), debug);
linearSearchBest.WritePatches = true;
// Setup the two step neighbor finder
TwoStepNearestNeighbor<KNNSearchType, BestSearchType>
twoStepNearestNeighbor(linearSearchKNN, linearSearchBest);
// #define DFullSearch // comment/uncomment this line to set the search region
#ifdef DFullSearch
// Perform the inpainting (full search)
InpaintingAlgorithm(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter);
#else
NeighborhoodSearch<VertexDescriptorType, ImagePatchDescriptorMapType> neighborhoodSearch(originalImage->GetLargestPossibleRegion(),
searchRadius, imagePatchDescriptorMap);
// Perform the inpainting (local search)
InpaintingAlgorithmWithLocalSearch(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter, neighborhoodSearch);
#endif
}
