*fXAux = fX;
imprL = true;
if (debug == DEBUG_LEVEL4_){
printf("\tLocalMelhorou "); Util::printX(xBestAux,n);
printf(" = %lf \n\n", *fXAux);
}
}
h /= 2.0;
}
Util::copy(x, xBestAux, n);
return imprL;
}*/
// Etapa de exploração do DTS
bool Dts::explorationSearch(double *x, double *fXAux, double h){
bool imprL = false;
double *dir = new double[n];;
double fAnt, *xAnt = new double[n];;
rSTR = h/2.0;
rTR = h/4.0;
*fXAux = func->calc(x);
//fAnt = *fXAux;
//Util::copy(xAnt, x, n);
if (debug == DEBUG_LEVEL4_){
printf("X ");
Util::printX(x, n);
printf(" = %lf \n", *fXAux);
}
void LidarInpaintingHSVTextureVerification(TImage* const originalImage, Mask* const mask,
const unsigned int patchHalfWidth, const unsigned int numberOfKNN,
float slightBlurVariance = 1.0f, unsigned int searchRadius = 1000,
float localRegionSizeMultiplier = 3.0f, float maxAllowedUsedPixelsRatio = 0.5f)
{
itk::ImageRegion<2> fullRegion = originalImage->GetLargestPossibleRegion();
// Extract the RGB image
typedef itk::Image<itk::CovariantVector<float, 3>, 2> RGBImageType;
std::vector<unsigned int> firstThreeChannels = {0,1,2};
RGBImageType::Pointer rgbImage = RGBImageType::New();
ITKHelpers::ExtractChannels(originalImage, firstThreeChannels, rgbImage.GetPointer());
// Create the HSV image
typedef itk::Image<itk::CovariantVector<float, 3>, 2> HSVImageType;
HSVImageType::Pointer hsvImage = HSVImageType::New();
ITKVTKHelpers::ConvertRGBtoHSV(rgbImage.GetPointer(), hsvImage.GetPointer());
ITKHelpers::WriteImage(hsvImage.GetPointer(), "HSVImage.mha");
// Stack the HSV image with the original rest of the channels
typedef itk::Image<itk::CovariantVector<float, 5>, 2> HSVDxDyImageType;
HSVDxDyImageType::Pointer hsvDxDyImage = HSVDxDyImageType::New();
ITKHelpers::DeepCopy(originalImage, hsvDxDyImage.GetPointer());
ITKHelpers::ReplaceChannels(hsvDxDyImage.GetPointer(), firstThreeChannels, hsvImage.GetPointer());
// Blur the image for gradient computation stability (Criminisi's data term)
RGBImageType::Pointer blurredRGBImage = RGBImageType::New();
float blurVariance = 2.0f;
MaskOperations::MaskedBlur(rgbImage.GetPointer(), mask, blurVariance, blurredRGBImage.GetPointer());
ITKHelpers::WriteRGBImage(blurredRGBImage.GetPointer(), "BlurredRGBImage.png");
// Blur the image slightly so that the SSD comparisons are not so noisy
typename HSVDxDyImageType::Pointer slightlyBlurredHSVDxDyImage = TImage::New();
MaskOperations::MaskedBlur(hsvDxDyImage.GetPointer(), mask, slightBlurVariance, slightlyBlurredHSVDxDyImage.GetPointer());
ITKHelpers::WriteImage(slightlyBlurredHSVDxDyImage.GetPointer(), "SlightlyBlurredHSVDxDyImage.mha");
// Create the graph
typedef ImagePatchPixelDescriptor<TImage> ImagePatchPixelDescriptorType;
typedef boost::grid_graph<2> VertexListGraphType;
// We can't make this a signed type (size_t versus int) because we allow negative
boost::array<std::size_t, 2> graphSideLengths = { { fullRegion.GetSize()[0],
fullRegion.GetSize()[1] } };
VertexListGraphType graph(graphSideLengths);
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
typedef boost::graph_traits<VertexListGraphType>::vertex_iterator VertexIteratorType;
// Get the index map
typedef boost::property_map<VertexListGraphType, boost::vertex_index_t>::const_type IndexMapType;
IndexMapType indexMap(get(boost::vertex_index, graph));
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType, IndexMapType> ImagePatchDescriptorMapType;
ImagePatchDescriptorMapType imagePatchDescriptorMap(num_vertices(graph), indexMap);
// Create the patch inpainter.
typedef PatchInpainter<TImage> OriginalImageInpainterType;
OriginalImageInpainterType originalImagePatchInpainter(patchHalfWidth, originalImage, mask);
originalImagePatchInpainter.SetDebugImages(true);
originalImagePatchInpainter.SetImageName("RGB");
// Create an inpainter for the HSV image.
typedef PatchInpainter<HSVImageType> HSVImageInpainterType;
HSVImageInpainterType hsvImagePatchInpainter(patchHalfWidth, hsvImage, mask);
// Create an inpainter for the RGB image.
typedef PatchInpainter<RGBImageType> RGBImageInpainterType;
RGBImageInpainterType rgbImagePatchInpainter(patchHalfWidth, rgbImage, mask);
// Create an inpainter for the blurred image.
typedef PatchInpainter<RGBImageType> BlurredImageInpainterType;
BlurredImageInpainterType blurredRGBImagePatchInpainter(patchHalfWidth, blurredRGBImage, mask);
// Create an inpainter for the slightly blurred image.
typedef PatchInpainter<TImage> SlightlyBlurredHSVDxDyImageImageInpainterType;
SlightlyBlurredHSVDxDyImageImageInpainterType slightlyBlurredHSVDxDyImageImagePatchInpainter(patchHalfWidth, slightlyBlurredHSVDxDyImage, mask);
// Create a composite inpainter. (Note: the mask is inpainted in InpaintingVisitor::FinishVertex)
CompositePatchInpainter inpainter;
inpainter.AddInpainter(&originalImagePatchInpainter);
inpainter.AddInpainter(&hsvImagePatchInpainter);
inpainter.AddInpainter(&blurredRGBImagePatchInpainter);
inpainter.AddInpainter(&slightlyBlurredHSVDxDyImageImagePatchInpainter);
inpainter.AddInpainter(&rgbImagePatchInpainter);
// Create the priority function
typedef PriorityCriminisi<RGBImageType> PriorityType;
PriorityType priorityFunction(blurredRGBImage, mask, patchHalfWidth);
// priorityFunction.SetDebugLevel(1);
// Queue
typedef IndirectPriorityQueue<VertexListGraphType> BoundaryNodeQueueType;
BoundaryNodeQueueType boundaryNodeQueue(graph);
// Create the descriptor visitor (used for SSD comparisons).
typedef ImagePatchDescriptorVisitor<VertexListGraphType, TImage, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
// ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(originalImage, mask,
// imagePatchDescriptorMap, patchHalfWidth); // Use the non-blurred image for the SSD comparisons
ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(slightlyBlurredHSVDxDyImage, mask,
imagePatchDescriptorMap, patchHalfWidth); // Use the slightly blurred HSV image for the SSD comparisons. Make sure to use a *HSVSSD difference functor so the H differences are treated appropriately!
typedef DefaultAcceptanceVisitor<VertexListGraphType> AcceptanceVisitorType;
AcceptanceVisitorType acceptanceVisitor;
// Create the inpainting visitor. (The mask is inpainted in FinishVertex)
typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType, TImage>
InpaintingVisitorType;
InpaintingVisitorType inpaintingVisitor(mask, boundaryNodeQueue,
imagePatchDescriptorVisitor, acceptanceVisitor,
&priorityFunction, patchHalfWidth, "InpaintingVisitor", originalImage);
inpaintingVisitor.SetDebugImages(true); // This produces PatchesCopied* images showing where patches were copied from/to at each iteration
// inpaintingVisitor.SetAllowNewPatches(false);
inpaintingVisitor.SetAllowNewPatches(true); // we can do this as long as we use one of the LinearSearchKNNProperty*Reuse (like LinearSearchKNNPropertyLimitLocalReuse) in the steps below
InitializePriority(mask, boundaryNodeQueue, &priorityFunction);
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<InpaintingVisitorType, VertexDescriptorType>(mask, &inpaintingVisitor);
std::cout << "PatchBasedInpaintingNonInteractive: There are " << boundaryNodeQueue.CountValidNodes()
<< " nodes in the boundaryNodeQueue" << std::endl;
#define DUseWeightedDifference
#ifdef DUseWeightedDifference
// The absolute value of the depth derivative range is usually about [0,12], so to make
// it comparable to to the color image channel range of [0,255], we multiply by 255/12 ~= 20.
// float depthDerivativeWeight = 20.0f;
// This should not be computed "by eye" by looking at the Dx and Dy channels of the PTX scan, because there are typically
// huge depth discontinuties around the objects that are going to be inpainted. We'd have to look at the masked version of this
// image to determine the min/max values of the unmasked pixels. They will be much smaller than the min/max values of the original
// image, which will make the depth derivative weights much higher (~100 or so)
// std::vector<typename TImage::PixelType> channelMins = ITKHelpers::ComputeMinOfAllChannels(originalImage);
// std::vector<typename TImage::PixelType> channelMaxs = ITKHelpers::ComputeMaxOfAllChannels(originalImage);
typename TImage::PixelType channelMins;
ITKHelpers::ComputeMinOfAllChannels(originalImage, channelMins);
typename TImage::PixelType channelMaxs;
ITKHelpers::ComputeMaxOfAllChannels(originalImage, channelMaxs);
float minX = fabs(channelMins[3]);
float maxX = fabs(channelMaxs[3]);
float maxValueX = std::max(minX, maxX);
std::cout << "maxValueX = " << maxValueX << std::endl;
float depthDerivativeWeightX = 255.0f / maxValueX;
std::cout << "Computed depthDerivativeWeightX = " << depthDerivativeWeightX << std::endl;
float minY = fabs(channelMins[4]);
float maxY = fabs(channelMaxs[4]);
float maxValueY = std::max(minY, maxY);
std::cout << "maxValueY = " << maxValueY << std::endl;
float depthDerivativeWeightY = 255.0f / maxValueY;
std::cout << "Computed depthDerivativeWeightY = " << depthDerivativeWeightY << std::endl;
// Use all channels
std::vector<float> weights = {1.0f, 1.0f, 1.0f, depthDerivativeWeightX, depthDerivativeWeightY};
// typedef WeightedSumSquaredPixelDifference<typename TImage::PixelType> PixelDifferenceType;
typedef WeightedHSVSSDFull<typename TImage::PixelType> FullPixelDifferenceType;
FullPixelDifferenceType fullPixelDifferenceFunctor(weights);
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
FullPixelDifferenceType > FullPatchDifferenceType;
FullPatchDifferenceType fullPatchDifferenceFunctor(fullPixelDifferenceFunctor);
// Use only the first 3 channels
typedef HSVSSD<typename TImage::PixelType> First3PixelDifferenceType;
First3PixelDifferenceType first3PixelDifferenceFunctor;
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
First3PixelDifferenceType > First3PatchDifferenceType;
First3PatchDifferenceType first3PatchDifferenceFunctor(first3PixelDifferenceFunctor);
#else
// Use an unweighted pixel difference
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
SumSquaredPixelDifference<typename TImage::PixelType> > PatchDifferenceType;
PatchDifferenceType patchDifferenceFunctor;
#endif
//#define DAllowReuse // comment/uncomment this line to toggle allowing patches to be used as the source patch more than once
#ifdef DAllowReuse
// Create the first (KNN) neighbor finder
typedef LinearSearchKNNProperty<ImagePatchDescriptorMapType, PatchDifferenceType> KNNSearchType;
KNNSearchType linearSearchKNN(imagePatchDescriptorMap, numberOfKNN, patchDifferenceFunctor);
#else
typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, FullPatchDifferenceType, RGBImageType> KNNSearchType;
KNNSearchType linearSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN, localRegionSizeMultiplier, maxAllowedUsedPixelsRatio,
fullPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
rgbImage.GetPointer());
linearSearchKNN.SetDebugImages(true);
linearSearchKNN.SetDebugScreenOutputs(true);
#endif
//#else // This works the best, but is less useful for demonstrations
// typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, FullPatchDifferenceType, RGBImageType> FullPixelKNNSearchType;
// FullPixelKNNSearchType fullPixelSearchKNN(imagePatchDescriptorMap, mask, numberOfKNN, localRegionSizeMultiplier, maxAllowedUsedPixelsRatio,
// fullPatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
// rgbImage.GetPointer());
// fullPixelSearchKNN.SetDebugImages(true);
// fullPixelSearchKNN.SetDebugScreenOutputs(true);
// typedef LinearSearchKNNPropertyLimitLocalReuse<ImagePatchDescriptorMapType, First3PatchDifferenceType, RGBImageType> First3PixelKNNSearchType;
// First3PixelKNNSearchType first3SearchKNN(imagePatchDescriptorMap, mask, numberOfKNN, localRegionSizeMultiplier, maxAllowedUsedPixelsRatio,
// first3PatchDifferenceFunctor, inpaintingVisitor.GetSourcePixelMapImage(),
// rgbImage.GetPointer());
// first3SearchKNN.SetDebugScreenOutputs(true);
//// typedef LinearSearchKNNProperty<ImagePatchDescriptorMapType, First3PatchDifferenceType> First3PixelKNNSearchType;
//// First3PixelKNNSearchType first3SearchKNN(imagePatchDescriptorMap, numberOfKNN,
//// first3PatchDifferenceFunctor);
//// first3SearchKNN.SetDebugImages(true);
// typedef LinearSearchKNNPropertyCombine<FullPixelKNNSearchType, First3PixelKNNSearchType> KNNSearchType;
// KNNSearchType linearSearchKNN(fullPixelSearchKNN, first3SearchKNN);
//#endif
// Setup the second (1-NN) neighbor finder
typedef std::vector<VertexDescriptorType>::iterator VertexDescriptorVectorIteratorType;
// This is templated on TImage because we need it to write out debug patches from this searcher (since we are not using an RGB image to compute the histograms)
// typedef LinearSearchBestTexture<ImagePatchDescriptorMapType, HSVImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the histogram of the gradient magnitudes of a scalar represetnation of the image (e.g. magnitude image)
// typedef LinearSearchBestLidarTextureDerivatives<ImagePatchDescriptorMapType, HSVImageType,
// VertexDescriptorVectorIteratorType, TImage> BestSearchType; // Use the concatenated histograms of the absolute value of the derivatives of each channel
// typedef LinearSearchBestLidarHSVTextureGradient<ImagePatchDescriptorMapType, HSVDxDyImageType,
// VertexDescriptorVectorIteratorType, RGBImageType> BestSearchType; // Use the concatenated histograms of the gradient magnitudes of each channel. This HSVDxDyImageType must match the hsvDxDyImage provided below
typedef LinearSearchBestLidarHSVTextureGradientWithSort<ImagePatchDescriptorMapType, HSVDxDyImageType,
VertexDescriptorVectorIteratorType, RGBImageType> BestSearchType; // Use the concatenated histograms of the gradient magnitudes of each channel. This HSVDxDyImageType must match the hsvDxDyImage provided below. Also sort the patches for demonstrative output purposes.
// BestSearchType linearSearchBest(imagePatchDescriptorMap, hsvDxDyImage.GetPointer(), mask); // use non-blurred for texture sorting
Debug bestSearchTypeDebug;
bestSearchTypeDebug.SetDebugScreenOutputs(true);
// bestSearchTypeDebug.SetDebugImages(true);
// linearSearchBest.SetDebugOutputs(true);
// linearSearchBest.SetDebugImages(true);
// use slightly blurred for texture sorting
BestSearchType linearSearchBest(imagePatchDescriptorMap, slightlyBlurredHSVDxDyImage.GetPointer(),
mask, rgbImage.GetPointer(), bestSearchTypeDebug);
// linearSearchBest.SetDebugImages(false);
linearSearchBest.SetDebugImages(true); // This produces BestPatch* images showing the list of the top K patches that were passed to the BestSearch functor
// Setup the two step neighbor finder
TwoStepNearestNeighbor<KNNSearchType, BestSearchType>
twoStepNearestNeighbor(linearSearchKNN, linearSearchBest);
// #define DFullSearch // comment/uncomment this line to set the search region
#ifdef DFullSearch
// Perform the inpainting (full search)
InpaintingAlgorithm(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter);
#else
NeighborhoodSearch<VertexDescriptorType, ImagePatchDescriptorMapType> neighborhoodSearch(originalImage->GetLargestPossibleRegion(),
searchRadius, imagePatchDescriptorMap);
// Perform the inpainting (local search)
bool algorithmDebug = true;
InpaintingAlgorithmWithLocalSearch(graph, inpaintingVisitor, &boundaryNodeQueue,
twoStepNearestNeighbor, &inpainter, neighborhoodSearch, algorithmDebug);
#endif
}
// Run with: Data/trashcan.mha Data/trashcan_mask.mha 15 filled.mha
int main(int argc, char *argv[])
{
// Verify arguments
if(argc != 6)
{
std::cerr << "Required arguments: image.mha imageMask.mha patchHalfWidth neighborhoodRadius output.mha" << std::endl;
std::cerr << "Input arguments: ";
for(int i = 1; i < argc; ++i)
{
std::cerr << argv[i] << " ";
}
return EXIT_FAILURE;
}
// Parse arguments
std::string imageFileName = argv[1];
std::string maskFileName = argv[2];
std::stringstream ssPatchRadius;
ssPatchRadius << argv[3];
unsigned int patchHalfWidth = 0;
ssPatchRadius >> patchHalfWidth;
// The percent of the image size to use as the neighborhood (0 - 1)
std::stringstream ssNeighborhoodPercent;
ssNeighborhoodPercent << argv[4];
float neighborhoodPercent = 0;
ssNeighborhoodPercent >> neighborhoodPercent;
std::string outputFileName = argv[5];
// Output arguments
std::cout << "Reading image: " << imageFileName << std::endl;
std::cout << "Reading mask: " << maskFileName << std::endl;
std::cout << "Patch half width: " << patchHalfWidth << std::endl;
std::cout << "Neighborhood percent: " << neighborhoodPercent << std::endl;
std::cout << "Output: " << outputFileName << std::endl;
typedef itk::Image<itk::CovariantVector<int, 3>, 2> ImageType;
typedef itk::ImageFileReader<ImageType> ImageReaderType;
ImageReaderType::Pointer imageReader = ImageReaderType::New();
imageReader->SetFileName(imageFileName);
imageReader->Update();
ImageType::Pointer image = ImageType::New();
ITKHelpers::DeepCopy(imageReader->GetOutput(), image.GetPointer());
Mask::Pointer mask = Mask::New();
mask->Read(maskFileName);
std::cout << "hole pixels: " << mask->CountHolePixels() << std::endl;
std::cout << "valid pixels: " << mask->CountValidPixels() << std::endl;
typedef ImagePatchPixelDescriptor<ImageType> ImagePatchPixelDescriptorType;
// Create the graph
typedef boost::grid_graph<2> VertexListGraphType;
boost::array<std::size_t, 2> graphSideLengths = { { imageReader->GetOutput()->GetLargestPossibleRegion().GetSize()[0],
imageReader->GetOutput()->GetLargestPossibleRegion().GetSize()[1] } };
// VertexListGraphType graph(graphSideLengths);
std::shared_ptr<VertexListGraphType> graph(new VertexListGraphType(graphSideLengths));
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
//ImagePatchDescriptorMapType smallImagePatchDescriptorMap(num_vertices(graph), indexMap);
// Create the patch inpainter. The inpainter needs to know the status of each pixel to determine if they should be inpainted.
typedef PatchInpainter<ImageType> ImageInpainterType;
std::shared_ptr<ImageInpainterType> imagePatchInpainter(new
ImageInpainterType(patchHalfWidth, image, mask));
// Create the priority function
typedef PriorityRandom PriorityType;
std::shared_ptr<PriorityType> priorityFunction(new PriorityType);
// typedef PriorityCriminisi<ImageType> PriorityType;
// std::shared_ptr<PriorityType> priorityFunction(new PriorityType(image, mask, patchHalfWidth));
typedef IndirectPriorityQueue<VertexListGraphType> BoundaryNodeQueueType;
std::shared_ptr<BoundaryNodeQueueType> boundaryNodeQueue(new BoundaryNodeQueueType(*graph));
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType, BoundaryNodeQueueType::IndexMapType> ImagePatchDescriptorMapType;
// ImagePatchDescriptorMapType imagePatchDescriptorMap(num_vertices(graph), indexMap);
std::shared_ptr<ImagePatchDescriptorMapType> imagePatchDescriptorMap(new
ImagePatchDescriptorMapType(num_vertices(*graph), *(boundaryNodeQueue->GetIndexMap())));
// Create the descriptor visitor
typedef ImagePatchDescriptorVisitor<VertexListGraphType, ImageType, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
// ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(image, mask, imagePatchDescriptorMap, patchHalfWidth);
std::shared_ptr<ImagePatchDescriptorVisitorType> imagePatchDescriptorVisitor(new
ImagePatchDescriptorVisitorType(image.GetPointer(), mask,
imagePatchDescriptorMap, patchHalfWidth));
/* ImagePatchDescriptorVisitor(TImage* const in_image, Mask* const in_mask,
std::shared_ptr<TDescriptorMap> in_descriptorMap,
const unsigned int in_half_width) : */
typedef ImagePatchDifference<ImagePatchPixelDescriptorType, SumAbsolutePixelDifference<ImageType::PixelType> >
ImagePatchDifferenceType;
// typedef CompositeDescriptorVisitor<VertexListGraphType> CompositeDescriptorVisitorType;
// CompositeDescriptorVisitorType compositeDescriptorVisitor;
// compositeDescriptorVisitor.AddVisitor(imagePatchDescriptorVisitor);
// Create the descriptor visitor
// typedef CompositeAcceptanceVisitor<VertexListGraphType> CompositeAcceptanceVisitorType;
// CompositeAcceptanceVisitorType compositeAcceptanceVisitor;
typedef DefaultAcceptanceVisitor<VertexListGraphType> AcceptanceVisitorType;
std::shared_ptr<AcceptanceVisitorType> acceptanceVisitor(new AcceptanceVisitorType);
// typedef AlwaysAccept<VertexListGraphType> AcceptanceVisitorType;
// AcceptanceVisitorType acceptanceVisitor;
// If the hole is less than 15% of the patch, always accept the initial best match
// HoleSizeAcceptanceVisitor<VertexListGraphType> holeSizeAcceptanceVisitor(mask, patchHalfWidth, .15);
// compositeAcceptanceVisitor.AddOverrideVisitor(&holeSizeAcceptanceVisitor);
// AllQuadrantHistogramCompareAcceptanceVisitor<VertexListGraphType, ImageType>
// allQuadrantHistogramCompareAcceptanceVisitor(image, mask, patchHalfWidth, 8.0f); // Crazy low
// compositeAcceptanceVisitor.AddRequiredPassVisitor(&allQuadrantHistogramCompareAcceptanceVisitor);
// template <typename TGraph, typename TBoundaryNodeQueue,
// typename TDescriptorVisitor, typename TAcceptanceVisitor, typename TPriority>
typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType>
InpaintingVisitorType;
std::shared_ptr<InpaintingVisitorType> inpaintingVisitor(new InpaintingVisitorType(mask, boundaryNodeQueue,
imagePatchDescriptorVisitor, acceptanceVisitor,
priorityFunction, patchHalfWidth, "InpaintingVisitor"));
// typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
// ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType>
// InpaintingVisitorType;
// std::shared_ptr<InpaintingVisitorType> inpaintingVisitor(new InpaintingVisitorType(mask, boundaryNodeQueue,
// imagePatchDescriptorVisitor, acceptanceVisitor,
// priorityFunction, patchHalfWidth, "InpaintingVisitor"));
// typedef DebugVisitor<VertexListGraphType, ImageType, BoundaryStatusMapType, BoundaryNodeQueueType> DebugVisitorType;
// DebugVisitorType debugVisitor(image, mask, patchHalfWidth, boundaryStatusMap, boundaryNodeQueue);
LoggerVisitor<VertexListGraphType> loggerVisitor("log.txt");
InitializePriority(mask, boundaryNodeQueue.get(), priorityFunction.get());
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<InpaintingVisitorType, VertexDescriptorType>(mask, inpaintingVisitor.get());
// For debugging we use LinearSearchBestProperty instead of DefaultSearchBest because it can output the difference value.
typedef LinearSearchBestProperty<ImagePatchDescriptorMapType,
ImagePatchDifferenceType > BestSearchType;
std::shared_ptr<BestSearchType> linearSearchBest(new BestSearchType(*imagePatchDescriptorMap));
typedef NeighborhoodSearch<VertexDescriptorType, ImagePatchDescriptorMapType> NeighborhoodSearchType;
NeighborhoodSearchType neighborhoodSearch(image->GetLargestPossibleRegion(), image->GetLargestPossibleRegion().GetSize()[0] * neighborhoodPercent, *imagePatchDescriptorMap);
InpaintingAlgorithmWithLocalSearch<VertexListGraphType, InpaintingVisitorType,
BoundaryNodeQueueType, NeighborhoodSearchType,
ImageInpainterType, BestSearchType>(graph, inpaintingVisitor, boundaryNodeQueue,
linearSearchBest, imagePatchInpainter, neighborhoodSearch);
// If the output filename is a png file, then use the RGBImage writer so that it is first
// casted to unsigned char. Otherwise, write the file directly.
if(Helpers::GetFileExtension(outputFileName) == "png")
{
ITKHelpers::WriteRGBImage(image.GetPointer(), outputFileName);
}
else
{
ITKHelpers::WriteImage(image.GetPointer(), outputFileName);
}
return EXIT_SUCCESS;
}
// Run with: Data/trashcan.mha Data/trashcan_mask.mha 15 filled.mha
int main(int argc, char *argv[])
{
// Verify arguments
if(argc != 5)
{
std::cerr << "Required arguments: image.mha imageMask.mha patchHalfWidth output.mha" << std::endl;
std::cerr << "Input arguments: ";
for(int i = 1; i < argc; ++i)
{
std::cerr << argv[i] << " ";
}
return EXIT_FAILURE;
}
// Setup the GUI system
QApplication app( argc, argv );
// Parse arguments
std::string imageFilename = argv[1];
std::string maskFilename = argv[2];
std::stringstream ssPatchRadius;
ssPatchRadius << argv[3];
unsigned int patchHalfWidth = 0;
ssPatchRadius >> patchHalfWidth;
std::string outputFilename = argv[4];
// Output arguments
std::cout << "Reading image: " << imageFilename << std::endl;
std::cout << "Reading mask: " << maskFilename << std::endl;
std::cout << "Patch half width: " << patchHalfWidth << std::endl;
std::cout << "Output: " << outputFilename << std::endl;
typedef FloatVectorImageType ImageType;
typedef itk::ImageFileReader<ImageType> ImageReaderType;
ImageReaderType::Pointer imageReader = ImageReaderType::New();
imageReader->SetFileName(imageFilename);
imageReader->Update();
ImageType* image = imageReader->GetOutput();
itk::ImageRegion<2> fullRegion = imageReader->GetOutput()->GetLargestPossibleRegion();
Mask::Pointer mask = Mask::New();
mask->Read(maskFilename);
std::cout << "hole pixels: " << mask->CountHolePixels() << std::endl;
std::cout << "valid pixels: " << mask->CountValidPixels() << std::endl;
std::cout << "image has " << image->GetNumberOfComponentsPerPixel() << " components." << std::endl;
typedef ImagePatchPixelDescriptor<ImageType> ImagePatchPixelDescriptorType;
// Create the graph
typedef boost::grid_graph<2> VertexListGraphType;
boost::array<std::size_t, 2> graphSideLengths = { { fullRegion.GetSize()[0],
fullRegion.GetSize()[1] } };
VertexListGraphType graph(graphSideLengths);
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
// Get the index map
typedef boost::property_map<VertexListGraphType, boost::vertex_index_t>::const_type IndexMapType;
IndexMapType indexMap(get(boost::vertex_index, graph));
// Create the priority map
typedef boost::vector_property_map<float, IndexMapType> PriorityMapType;
PriorityMapType priorityMap(num_vertices(graph), indexMap);
// Create the boundary status map. A node is on the current boundary if this property is true.
// This property helps the boundaryNodeQueue because we can mark here if a node has become no longer
// part of the boundary, so when the queue is popped we can check this property to see if it should
// actually be processed.
typedef boost::vector_property_map<bool, IndexMapType> BoundaryStatusMapType;
BoundaryStatusMapType boundaryStatusMap(num_vertices(graph), indexMap);
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType, IndexMapType> ImagePatchDescriptorMapType;
ImagePatchDescriptorMapType imagePatchDescriptorMap(num_vertices(graph), indexMap);
//ImagePatchDescriptorMapType smallImagePatchDescriptorMap(num_vertices(graph), indexMap);
// Create the patch inpainter. The inpainter needs to know the status of each
// pixel to determine if they should be inpainted.
typedef MaskImagePatchInpainter InpainterType;
MaskImagePatchInpainter patchInpainter(patchHalfWidth, mask);
// Create the priority function
// typedef PriorityRandom PriorityType;
// PriorityType priorityFunction;
typedef PriorityOnionPeel PriorityType;
PriorityType priorityFunction(mask, patchHalfWidth);
// Create the boundary node queue. The priority of each node is used to order the queue.
typedef boost::vector_property_map<std::size_t, IndexMapType> IndexInHeapMap;
IndexInHeapMap index_in_heap(indexMap);
// Create the priority compare functor (we want the highest priority nodes to be first in the queue)
typedef std::greater<float> PriorityCompareType;
PriorityCompareType lessThanFunctor;
typedef boost::d_ary_heap_indirect<VertexDescriptorType, 4, IndexInHeapMap, PriorityMapType, PriorityCompareType>
BoundaryNodeQueueType;
BoundaryNodeQueueType boundaryNodeQueue(priorityMap, index_in_heap, lessThanFunctor);
// Create the descriptor visitor
typedef ImagePatchDescriptorVisitor<VertexListGraphType, ImageType, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(image, mask, imagePatchDescriptorMap, patchHalfWidth);
//ImagePatchDescriptorVisitorType imagePatchDescriptorVisitor(cielabImage,
// mask, imagePatchDescriptorMap, patchHalfWidth);
typedef ImagePatchDifference<ImagePatchPixelDescriptorType, SumAbsolutePixelDifference<ImageType::PixelType> >
ImagePatchDifferenceType;
ImagePatchDifferenceType imagePatchDifferenceFunction;
// typedef WeightedSumAbsolutePixelDifference<ImageType::PixelType> PixelDifferenceFunctorType;
// PixelDifferenceFunctorType pixelDifferenceFunctor;
// std::vector<float> weights;
// weights.push_back(1.0f);
// weights.push_back(1.0f);
// weights.push_back(1.0f);
// float gradientWeight = 500.0f;
// weights.push_back(gradientWeight);
// weights.push_back(gradientWeight);
// pixelDifferenceFunctor.Weights = weights;
// std::cout << "Weights: ";
// OutputHelpers::OutputVector(pixelDifferenceFunctor.Weights);
// typedef ImagePatchDifference<ImagePatchPixelDescriptorType, PixelDifferenceFunctorType >
// ImagePatchDifferenceType;
//
// ImagePatchDifferenceType imagePatchDifferenceFunction(pixelDifferenceFunctor);
typedef CompositeDescriptorVisitor<VertexListGraphType> CompositeDescriptorVisitorType;
CompositeDescriptorVisitorType compositeDescriptorVisitor;
compositeDescriptorVisitor.AddVisitor(&imagePatchDescriptorVisitor);
typedef CompositeAcceptanceVisitor<VertexListGraphType> CompositeAcceptanceVisitorType;
CompositeAcceptanceVisitorType compositeAcceptanceVisitor;
typedef InpaintingVisitor<VertexListGraphType, ImageType, BoundaryNodeQueueType,
CompositeDescriptorVisitorType, CompositeAcceptanceVisitorType, PriorityType,
PriorityMapType, BoundaryStatusMapType>
InpaintingVisitorType;
InpaintingVisitorType inpaintingVisitor(image, mask, boundaryNodeQueue,
compositeDescriptorVisitor, compositeAcceptanceVisitor, priorityMap,
&priorityFunction, patchHalfWidth,
boundaryStatusMap, outputFilename);
typedef DisplayVisitor<VertexListGraphType, ImageType> DisplayVisitorType;
DisplayVisitorType displayVisitor(image, mask, patchHalfWidth);
typedef DebugVisitor<VertexListGraphType, ImageType, BoundaryStatusMapType, BoundaryNodeQueueType>
DebugVisitorType;
DebugVisitorType debugVisitor(image, mask, patchHalfWidth, boundaryStatusMap, boundaryNodeQueue);
LoggerVisitor<VertexListGraphType> loggerVisitor("log.txt");
PaintPatchVisitor<VertexListGraphType, ImageType> inpaintRGBVisitor(image,
mask.GetPointer(), patchHalfWidth);
typedef CompositeInpaintingVisitor<VertexListGraphType> CompositeInpaintingVisitorType;
CompositeInpaintingVisitorType compositeInpaintingVisitor;
compositeInpaintingVisitor.AddVisitor(&inpaintingVisitor);
//compositeInpaintingVisitor.AddVisitor(&inpaintRGBVisitor);
compositeInpaintingVisitor.AddVisitor(&displayVisitor);
compositeInpaintingVisitor.AddVisitor(&debugVisitor);
//compositeInpaintingVisitor.AddVisitor(&loggerVisitor);
InitializePriority(mask, boundaryNodeQueue, priorityMap, &priorityFunction, boundaryStatusMap);
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<CompositeInpaintingVisitorType, VertexDescriptorType>(mask, &compositeInpaintingVisitor);
// Create the nearest neighbor finders
typedef LinearSearchKNNProperty<ImagePatchDescriptorMapType,
ImagePatchDifferenceType > KNNSearchType;
KNNSearchType knnSearch(imagePatchDescriptorMap, 50000, 1, imagePatchDifferenceFunction);
// For debugging we use LinearSearchBestProperty instead of DefaultSearchBest
// because it can output the difference value.
typedef LinearSearchBestProperty<ImagePatchDescriptorMapType,
ImagePatchDifferenceType > BestSearchType;
BestSearchType bestSearch(imagePatchDescriptorMap, imagePatchDifferenceFunction);
BasicViewerWidget<ImageType> basicViewerWidget(image, mask);
basicViewerWidget.show();
// These connections are Qt::BlockingQueuedConnection because the algorithm quickly
// goes on to fill the hole, and since we are sharing the image memory, we want to make sure these things are
// refreshed at the right time, not after the hole has already been filled
// (this actually happens, it is not just a theoretical thing).
QObject::connect(&displayVisitor, SIGNAL(signal_RefreshImage()), &basicViewerWidget, SLOT(slot_UpdateImage()),
Qt::BlockingQueuedConnection);
QObject::connect(&displayVisitor, SIGNAL(signal_RefreshSource(const itk::ImageRegion<2>&,
const itk::ImageRegion<2>&)),
&basicViewerWidget, SLOT(slot_UpdateSource(const itk::ImageRegion<2>&,
const itk::ImageRegion<2>&)),
Qt::BlockingQueuedConnection);
QObject::connect(&displayVisitor, SIGNAL(signal_RefreshTarget(const itk::ImageRegion<2>&)),
&basicViewerWidget, SLOT(slot_UpdateTarget(const itk::ImageRegion<2>&)),
Qt::BlockingQueuedConnection);
QObject::connect(&displayVisitor, SIGNAL(signal_RefreshResult(const itk::ImageRegion<2>&,
const itk::ImageRegion<2>&)),
&basicViewerWidget, SLOT(slot_UpdateResult(const itk::ImageRegion<2>&,
const itk::ImageRegion<2>&)),
Qt::BlockingQueuedConnection);
// TopPatchesDialog<ImageType> topPatchesDialog(image, mask, patchHalfWidth, &basicViewerWidget);
// typedef VisualSelectionBest<ImageType> ManualSearchType;
// ManualSearchType manualSearchBest(image, mask, patchHalfWidth, &topPatchesDialog);
typedef DefaultSearchBest ManualSearchType;
DefaultSearchBest manualSearchBest;
// By specifying the radius as the image size/8, we are searching up to 1/4 of the image each time
typedef NeighborhoodSearch<VertexDescriptorType> NeighborhoodSearchType;
NeighborhoodSearchType neighborhoodSearch(fullRegion, fullRegion.GetSize()[0]/8);
// Run the remaining inpainting
QtConcurrent::run(boost::bind(InpaintingAlgorithmWithLocalSearch<
VertexListGraphType, CompositeInpaintingVisitorType, BoundaryStatusMapType,
BoundaryNodeQueueType, NeighborhoodSearchType, KNNSearchType, BestSearchType,
ManualSearchType, InpainterType>,
graph, compositeInpaintingVisitor, &boundaryStatusMap, &boundaryNodeQueue,
neighborhoodSearch, knnSearch, bestSearch, boost::ref(manualSearchBest),
patchInpainter));
return app.exec();
}
void ClassicalImageInpainting(typename itk::SmartPointer<TImage> originalImage, Mask* const mask,
const unsigned int patchHalfWidth)
{
itk::ImageRegion<2> fullRegion = originalImage->GetLargestPossibleRegion();
// Blur the image
typedef TImage BlurredImageType; // Usually the blurred image is the same type as the original image.
typename BlurredImageType::Pointer blurredImage = BlurredImageType::New();
float blurVariance = 2.0f;
MaskOperations::MaskedBlur(originalImage.GetPointer(), mask, blurVariance, blurredImage.GetPointer());
// ITKHelpers::WriteRGBImage(blurredImage.GetPointer(), "BlurredImage.png");
typedef ImagePatchPixelDescriptor<TImage> ImagePatchPixelDescriptorType;
// Create the graph
typedef boost::grid_graph<2> VertexListGraphType;
boost::array<std::size_t, 2> graphSideLengths = { { fullRegion.GetSize()[0],
fullRegion.GetSize()[1] } };
std::shared_ptr<VertexListGraphType> graph(new VertexListGraphType(graphSideLengths));
typedef boost::graph_traits<VertexListGraphType>::vertex_descriptor VertexDescriptorType;
typedef boost::graph_traits<VertexListGraphType>::vertex_iterator VertexIteratorType;
// Queue
typedef IndirectPriorityQueue<VertexListGraphType> BoundaryNodeQueueType;
std::shared_ptr<BoundaryNodeQueueType> boundaryNodeQueue(new BoundaryNodeQueueType(*graph));
// Create the descriptor map. This is where the data for each pixel is stored.
typedef boost::vector_property_map<ImagePatchPixelDescriptorType,
BoundaryNodeQueueType::IndexMapType> ImagePatchDescriptorMapType;
std::shared_ptr<ImagePatchDescriptorMapType> imagePatchDescriptorMap(new
ImagePatchDescriptorMapType(num_vertices(*graph), *(boundaryNodeQueue->GetIndexMap())));
// Create the patch inpainter.
typedef PatchInpainter<TImage> OriginalImageInpainterType;
std::shared_ptr<OriginalImageInpainterType> originalImagePatchInpainter(new
OriginalImageInpainterType(patchHalfWidth, originalImage, mask));
// Show the inpainted image at each iteration
// originalImagePatchInpainter.SetDebugImages(true);
// originalImagePatchInpainter.SetImageName("RGB");
// Create an inpainter for the blurred image.
typedef PatchInpainter<BlurredImageType> BlurredImageInpainterType;
std::shared_ptr<BlurredImageInpainterType> blurredImagePatchInpainter(new
BlurredImageInpainterType(patchHalfWidth, blurredImage, mask));
// Create a composite inpainter.
std::shared_ptr<CompositePatchInpainter> inpainter(new CompositePatchInpainter);
inpainter->AddInpainter(originalImagePatchInpainter);
inpainter->AddInpainter(blurredImagePatchInpainter);
// Create the priority function
typedef PriorityCriminisi<BlurredImageType> PriorityType;
std::shared_ptr<PriorityType> priorityFunction(new PriorityType(blurredImage, mask, patchHalfWidth));
// Create the descriptor visitor
typedef ImagePatchDescriptorVisitor<VertexListGraphType, TImage, ImagePatchDescriptorMapType>
ImagePatchDescriptorVisitorType;
std::shared_ptr<ImagePatchDescriptorVisitorType> imagePatchDescriptorVisitor(new
ImagePatchDescriptorVisitorType(originalImage.GetPointer(), mask,
imagePatchDescriptorMap, patchHalfWidth));
typedef DefaultAcceptanceVisitor<VertexListGraphType> AcceptanceVisitorType;
std::shared_ptr<AcceptanceVisitorType> acceptanceVisitor(new AcceptanceVisitorType);
// Create the inpainting visitor
typedef InpaintingVisitor<VertexListGraphType, BoundaryNodeQueueType,
ImagePatchDescriptorVisitorType, AcceptanceVisitorType, PriorityType>
InpaintingVisitorType;
std::shared_ptr<InpaintingVisitorType> inpaintingVisitor(new InpaintingVisitorType(mask, boundaryNodeQueue,
imagePatchDescriptorVisitor, acceptanceVisitor,
priorityFunction, patchHalfWidth, "InpaintingVisitor"));
inpaintingVisitor->SetAllowNewPatches(false);
// inpaintingVisitor.SetDebugImages(true); // Write PatchesCopied images that show the source and target patch at each iteration
InitializePriority(mask, boundaryNodeQueue.get(), priorityFunction.get());
// Initialize the boundary node queue from the user provided mask image.
InitializeFromMaskImage<InpaintingVisitorType, VertexDescriptorType>(mask, inpaintingVisitor.get());
// Create the nearest neighbor finder
typedef ImagePatchDifference<ImagePatchPixelDescriptorType,
SumSquaredPixelDifference<typename TImage::PixelType> > PatchDifferenceType;
// Create the best patch searcher
typedef LinearSearchBestProperty<ImagePatchDescriptorMapType,
PatchDifferenceType> BestSearchType;
std::shared_ptr<BestSearchType> linearSearchBest(new BestSearchType(*imagePatchDescriptorMap));
// By specifying the radius as the image size/8, we are searching up to 1/4 of the image each time
typedef NeighborhoodSearch<VertexDescriptorType, ImagePatchDescriptorMapType> NeighborhoodSearchType;
NeighborhoodSearchType neighborhoodSearch(fullRegion, fullRegion.GetSize()[0]/8, *imagePatchDescriptorMap);
// Perform the inpainting
InpaintingAlgorithmWithLocalSearch<VertexListGraphType, InpaintingVisitorType,
BoundaryNodeQueueType, NeighborhoodSearchType,
CompositePatchInpainter, BestSearchType>(graph, inpaintingVisitor, boundaryNodeQueue,
linearSearchBest, inpainter, neighborhoodSearch);
/*template <typename TVertexListGraph, typename TInpaintingVisitor,
typename TPriorityQueue, typename TSearchRegion,
typename TPatchInpainter, typename TBestPatchFinder>
InpaintingAlgorithmWithLocalSearch(TVertexListGraph& graph,
TInpaintingVisitor visitor,
TPriorityQueue* boundaryNodeQueue,
TBestPatchFinder bestPatchFinder,
TPatchInpainter* patchInpainter,
TSearchRegion& searchRegion) */
}
