www.pudn.com > bsiftC_.rar > ScaleSpace.cs
/* ScaleSpace.cs
*
* Key feature identification functionality, octave and scale-space handling.
*
* (C) Copyright 2004 -- Sebastian Nowozin (nowozin@cs.tu-berlin.de)
*
* "This software is provided for non-commercial use only. The University of
* British Columbia has applied for a patent on the SIFT algorithm in the
* United States. Commercial applications of this software may require a
* license from the University of British Columbia."
* For more information, see the LICENSE file supplied with the distribution.
*/
using System;
using System.Collections;
public class OctavePyramid
{
public OctavePyramid ()
{
}
// Holds DScaleSpace objects, ordered by descending image size.
ArrayList octaves;
public int Count {
get {
return (octaves.Count);
}
}
public DScaleSpace this[int idx] {
get {
return ((DScaleSpace) octaves[idx]);
}
}
// Build the largest possible number of octaves, each holding
// 'levelsPerOctave' scales in scale-space. Each octave is downscaled by
// 0.5 and the scales in each octave represent a sigma change of
// 'octaveSigma' to 2.0 * 'octaveSigma'. If minSize is greater zero, every
// scale-space with less than minSize effective pixels in x-dimension will
// be discarded.
//
// Return the number of octaves build
public int BuildOctaves (ImageMap source, double scale,
int levelsPerOctave, double octaveSigma, int minSize)
{
octaves = new ArrayList ();
DScaleSpace downSpace = null;
ImageMap prev = source;
while (prev != null && prev.XDim >= minSize && prev.YDim >= minSize) {
DScaleSpace dsp = new DScaleSpace ();
Console.WriteLine ("Building octave, ({0}, {1})", prev.XDim, prev.YDim);
// Create both the gaussian filtered images and the DoG maps
dsp.BuildGaussianMaps (prev, scale, levelsPerOctave, octaveSigma);
dsp.BuildDiffMaps ();
octaves.Add (dsp);
prev = dsp.LastGaussianMap.ScaleHalf ();
if (downSpace != null)
downSpace.Up = dsp;
dsp.Down = downSpace;
downSpace = dsp;
scale *= 2.0;
}
return (octaves.Count);
}
}
public class DScaleSpace
{
//int s;
DScaleSpace down = null;
DScaleSpace up = null;
internal DScaleSpace Down {
get {
return (down);
}
set {
down = value;
}
}
internal DScaleSpace Up {
get {
return (up);
}
set {
up = value;
}
}
// The original gaussian blurred source image this level was started with.
// Needed for keypoint generation.
ImageMap baseImg;
double basePixScale;
// The smoothed gaussian images, all the base (lower scale) relative to
// the DoG spaces below.
ImageMap[] imgScaled;
public ImageMap GetGaussianMap (int idx)
{
return (imgScaled[idx]);
}
private ImageMap[] magnitudes;
private ImageMap[] directions;
// The last created gaussian map, with 2 \sigma blur. (For use with next
// octave).
public ImageMap LastGaussianMap {
get {
// position s = Length - 2 has: D(k^s * sigma) = D(2 * sigma)
if (imgScaled.Length < 2)
throw (new Exception ("bu keneng: too few gaussian maps"));
return (imgScaled[imgScaled.Length - 2]);
}
}
// The DoG spaces.
ImageMap[] spaces;
public DScaleSpace ()
{
}
public int Count {
get {
return (spaces.Length);
}
}
// Return a single DoG map
public ImageMap this[int idx] {
get {
return (spaces[idx]);
}
}
// Generate keypoints from localized peak list.
// TODO: description
// scaleCount: number of scales (3)
public ArrayList GenerateKeypoints (ArrayList localizedPeaks,
int scaleCount, double octaveSigma)
{
ArrayList keypoints = new ArrayList ();
foreach (ScalePoint sp in localizedPeaks) {
// Generate zero or more keypoints from the scale point locations.
// TODO: make the values configurable
ArrayList thisPointKeys = GenerateKeypointSingle (basePixScale,
sp, 36, 0.8, scaleCount, octaveSigma);
// Generate the feature descriptor.
thisPointKeys = CreateDescriptors (thisPointKeys,
magnitudes[sp.Level], directions[sp.Level], 2.0, 4, 8, 0.2);
// Only copy over those keypoints that have been successfully
// assigned a descriptor (feature vector).
foreach (Keypoint kp in thisPointKeys) {
if (kp.HasFV == false)
throw (new Exception ("should not happen"));
// Transform the this level image relative coordinate system
// to the original image coordinates by multiplying with the
// current img scale (which starts with either 0.5 or 1.0 and
// then always doubles: 2.0, 4.0, ..)
// Note that the kp coordinates are not used for processing by
// the detection methods and this has to be the last step.
// Also transform the relative-to-image scale to an
// absolute-to-original-image scale.
kp.X *= kp.ImgScale;
kp.Y *= kp.ImgScale;
kp.Scale *= kp.ImgScale;
keypoints.Add (kp);
}
}
return (keypoints);
}
// Assign each feature point one or more standardized orientations.
// (section 5 in Lowe's paper)
//
// We use an orientation histogram with 36 bins, 10 degrees each. For
// this, every pixel (x,y) lieing in a circle of 'squareDim' diameter
// within in a 'squareDim' sized field within the image L ('gaussImg') is
// examined and two measures calculated:
//
// m = \sqrt{ (L_{x+1,y} - L_{x-1,y})^2 + (L_{x,y+1} - L_{x,y-1})^2 }
// theta = tan^{-1} ( \frac{ L_{x,y+1} - L_{x,y-1} }
// { L_{x+1,y} - L_{x-1,y} } )
//
// Where m is the gradient magnitude around the pixel and theta is the
// gradient orientation. The 'imgScale' value is the octave scale,
// starting with 1.0 at the finest-detail octave, and doubling every
// octave. The gradient orientations are discreetized to 'binCount'
// directions (should be: 36). For every peak orientation that lies within
// 'peakRelThresh' of the maximum peak value, a keypoint location is
// added (should be: 0.8).
//
// Note that 'space' is the gaussian smoothed original image, not the
// difference-of-gaussian one used for peak-search.
private ArrayList GenerateKeypointSingle (double imgScale, ScalePoint point,
int binCount, double peakRelThresh, int scaleCount,
double octaveSigma)
{
// The relative estimated keypoint scale. The actual absolute keypoint
// scale to the original image is yielded by the product of imgScale.
// But as we operate in the current octave, the size relative to the
// anchoring images is missing the imgScale factor.
double kpScale = octaveSigma *
Math.Pow (2.0, (point.Level + point.Local.ScaleAdjust) / scaleCount);
// Lowe03, "A gaussian-weighted circular window with a \sigma three
// times that of the scale of the keypoint".
//
// With \sigma = 3.0 * kpScale, the square dimension we have to
// consider is (3 * \sigma) (until the weight becomes very small).
double sigma = 3.0 * kpScale;
int radius = (int) (3.0 * sigma / 2.0 + 0.5);
int radiusSq = radius * radius;
ImageMap magnitude = magnitudes[point.Level];
ImageMap direction = directions[point.Level];
// As the point may lie near the border, build the rectangle
// coordinates we can still reach, minus the border pixels, for which
// we do not have gradient information available.
int xMin = Math.Max (point.X - radius, 1);
int xMax = Math.Min (point.X + radius, magnitude.XDim - 1);
int yMin = Math.Max (point.Y - radius, 1);
int yMax = Math.Min (point.Y + radius, magnitude.YDim - 1);
// Precompute 1D gaussian divisor (2 \sigma^2) in:
// G(r) = e^{-\frac{r^2}{2 \sigma^2}}
double gaussianSigmaFactor = 2.0 * sigma * sigma;
double[] bins = new double[binCount];
// Build the direction histogram
for (int y = yMin ; y < yMax ; ++y) {
for (int x = xMin ; x < xMax ; ++x) {
// Only consider pixels in the circle, else we might skew the
// orientation histogram by considering more pixels into the
// corner directions
int relX = x - point.X;
int relY = y - point.Y;
if (IsInCircle (relX, relY, radiusSq) == false)
continue;
// The gaussian weight factor.
double gaussianWeight = Math.Exp
(- ((relX * relX + relY * relY) / gaussianSigmaFactor));
// find the closest bin and add the direction
int binIdx = FindClosestRotationBin (binCount, direction[x, y]);
bins[binIdx] += magnitude[x, y] * gaussianWeight;
}
}
// As there may be succeeding histogram entries like this:
// ( ..., 0.4, 0.3, 0.4, ... ) where the real peak is located at the
// middle of this three entries, we can improve the distinctiveness of
// the bins by applying an averaging pass.
//
// TODO: is this really the best method? (we also loose a bit of
// information. Maybe there is a one-step method that conserves more)
AverageWeakBins (bins, binCount);
// find the maximum peak in gradient orientation
double maxGrad = 0.0;
int maxBin = 0;
for (int b = 0 ; b < binCount ; ++b) {
if (bins[b] > maxGrad) {
maxGrad = bins[b];
maxBin = b;
}
}
// First determine the real interpolated peak high at the maximum bin
// position, which is guaranteed to be an absolute peak.
//
// XXX: should we use the estimated peak value as reference for the
// 0.8 check or the original bin-value?
double maxPeakValue, maxDegreeCorrection;
InterpolateOrientation (bins[maxBin == 0 ? (binCount - 1) : (maxBin - 1)],
bins[maxBin], bins[(maxBin + 1) % binCount],
out maxDegreeCorrection, out maxPeakValue);
// Now that we know the maximum peak value, we can find other keypoint
// orientations, which have to fulfill two criterias:
//
// 1. They must be a local peak themselves. Else we might add a very
// similar keypoint orientation twice (imagine for example the
// values: 0.4 1.0 0.8, if 1.0 is maximum peak, 0.8 is still added
// with the default threshhold, but the maximum peak orientation
// was already added).
// 2. They must have at least peakRelThresh times the maximum peak
// value.
bool[] binIsKeypoint = new bool[binCount];
for (int b = 0 ; b < binCount ; ++b) {
binIsKeypoint[b] = false;
// The maximum peak of course is
if (b == maxBin) {
binIsKeypoint[b] = true;
continue;
}
// Local peaks are, too, in case they fulfill the threshhold
if (bins[b] < (peakRelThresh * maxPeakValue))
continue;
int leftI = (b == 0) ? (binCount - 1) : (b - 1);
int rightI = (b + 1) % binCount;
if (bins[b] <= bins[leftI] || bins[b] <= bins[rightI])
continue; // no local peak
binIsKeypoint[b] = true;
}
// All the valid keypoint bins are now marked in binIsKeypoint, now
// build them.
ArrayList keypoints = new ArrayList ();
// find other possible locations
double oneBinRad = (2.0 * Math.PI) / binCount;
for (int b = 0 ; b < binCount ; ++b) {
if (binIsKeypoint[b] == false)
continue;
int bLeft = (b == 0) ? (binCount - 1) : (b - 1);
int bRight = (b + 1) % binCount;
// Get an interpolated peak direction and value guess.
double peakValue;
double degreeCorrection;
if (InterpolateOrientation (bins[bLeft], bins[b], bins[bRight],
out degreeCorrection, out peakValue) == false)
{
throw (new InvalidOperationException ("BUG: Parabola fitting broken"));
}
// [-1.0 ; 1.0] -> [0 ; binrange], and add the fixed absolute bin
// position.
// We subtract PI because bin 0 refers to 0, binCount-1 bin refers
// to a bin just below 2PI, so -> [-PI ; PI]. Note that at this
// point we determine the canonical descriptor anchor angle. It
// does not matter where we set it relative to the peak degree,
// but it has to be constant. Also, if the output of this
// implementation is to be matched with other implementations it
// must be the same constant angle (here: -PI).
double degree = (b + degreeCorrection) * oneBinRad - Math.PI;
if (degree < -Math.PI)
degree += 2.0 * Math.PI;
else if (degree > Math.PI)
degree -= 2.0 * Math.PI;
Keypoint kp = new Keypoint (imgScaled[point.Level],
point.X + point.Local.FineX,
point.Y + point.Local.FineY,
imgScale, kpScale, degree);
keypoints.Add (kp);
}
return (keypoints);
}
// Fit a parabol to the three points (-1.0 ; left), (0.0 ; middle) and
// (1.0 ; right).
//
// Formulas:
// f(x) = a (x - c)^2 + b
//
// c is the peak offset (where f'(x) is zero), b is the peak value.
//
// In case there is an error false is returned, otherwise a correction
// value between [-1 ; 1] is returned in 'degreeCorrection', where -1
// means the peak is located completely at the left vector, and -0.5 just
// in the middle between left and middle and > 0 to the right side. In
// 'peakValue' the maximum estimated peak value is stored.
private bool InterpolateOrientation (double left, double middle,
double right, out double degreeCorrection, out double peakValue)
{
double a = ((left + right) - 2.0 * middle) / 2.0;
degreeCorrection = peakValue = Double.NaN;
// Not a parabol
if (a == 0.0)
return (false);
double c = (((left - middle) / a) - 1.0) / 2.0;
double b = middle - c * c * a;
if (c < -0.5 || c > 0.5)
throw (new InvalidOperationException
("InterpolateOrientation: off peak ]-0.5 ; 0.5["));
degreeCorrection = c;
peakValue = b;
return (true);
}
// Find the bin out of 'binCount' bins that matches the 'angle' closest.
// 'angle' fulfills -PI <= angle <= PI. Bin 0 is assigned to -PI, the
// binCount-1 bin refers to just below PI.
//
// Return the index of the closest bin.
private int FindClosestRotationBin (int binCount, double angle)
{
angle += Math.PI;
angle /= 2.0 * Math.PI;
// calculate the aligned bin
angle *= binCount;
int idx = (int) angle;
if (idx == binCount)
idx = 0;
return (idx);
}
// Average the content of the direction bins.
private void AverageWeakBins (double[] bins, int binCount)
{
// TODO: make some tests what number of passes is the best. (its clear
// one is not enough, as we may have something like
// ( 0.4, 0.4, 0.3, 0.4, 0.4 ))
for (int sn = 0 ; sn < 4 ; ++sn) {
double firstE = bins[0];
double last = bins[binCount - 1];
for (int sw = 0 ; sw < binCount ; ++sw) {
double cur = bins[sw];
double next = (sw == (binCount - 1)) ?
firstE : bins[(sw + 1) % binCount];
bins[sw] = (last + cur + next) / 3.0;
last = cur;
}
}
}
// Create the descriptor vector for a list of keypoints.
//
// keypoints: The list of keypoints to be processed. Everything but the
// descriptor must be filled in already.
// magnitude/direction: The precomputed gradient magnitude and direction
// maps.
// considerScaleFactor: The downscale factor, which describes the amount
// of pixels in the circular region relative to the keypoint scale.
// Low values means few pixels considered, large values extend the
// range. (Use values between 1.0 and 6.0)
// descDim: The dimension size of the output descriptor. There will be
// descDim * descDim * directionCount elements in the feature vector.
// directionCount: The dimensionality of the low level gradient vectors.
// fvGradHicap: The feature vector gradient length hi-cap threshhold.
// (Should be: 0.2)
//
// Some parts modelled after Alexandre Jenny's Matlab implementation.
//
// Return a list of survivors, which a descriptor was created for
// successfully.
private ArrayList CreateDescriptors (ArrayList keypoints,
ImageMap magnitude, ImageMap direction,
double considerScaleFactor, int descDim, int directionCount,
double fvGradHicap)
{
if (keypoints.Count <= 0)
return (keypoints);
considerScaleFactor *= ((Keypoint) keypoints[0]).Scale;
double dDim05 = ((double) descDim) / 2.0;
// Now calculate the radius: We consider pixels in a square with
// dimension 'descDim' plus 0.5 in each direction. As the feature
// vector elements at the diagonal borders are most distant from the
// center pixel we have scale up with sqrt(2).
int radius = (int) (((descDim + 1.0) / 2) *
Math.Sqrt (2.0) * considerScaleFactor + 0.5);
// Instead of modifying the original list, we just copy the keypoints
// that received a descriptor.
ArrayList survivors = new ArrayList ();
// Precompute the sigma for the "center-most, border-less" gaussian
// weighting.
// (We are operating to dDim05, CV book tells us G(x), x > 3 \sigma
// negligible, but this range seems much shorter!?)
//
// In Lowe03, page 15 it says "A Gaussian weighting function with
// \sigma equal to one half the width of the descriptor window is
// used", so we just use his advice.
double sigma2Sq = 2.0 * dDim05 * dDim05;
foreach (Keypoint kp in keypoints)
{
// The angle to rotate with: negate the orientation.
double angle = -kp.Orientation;
kp.CreateVector (descDim, descDim, directionCount);
//Console.WriteLine (" FV allocated");
for (int y = -radius ; y < radius ; ++y) {
for (int x = -radius ; x < radius ; ++x) {
// Rotate and scale
double yR = Math.Sin (angle) * x +
Math.Cos (angle) * y;
double xR = Math.Cos (angle) * x -
Math.Sin (angle) * y;
yR /= considerScaleFactor;
xR /= considerScaleFactor;
// Now consider all (xR, yR) that are anchored within
// (- descDim/2 - 0.5 ; -descDim/2 - 0.5) to
// (descDim/2 + 0.5 ; descDim/2 + 0.5),
// as only those can influence the FV.
if (yR >= (dDim05 + 0.5) || xR >= (dDim05 + 0.5) ||
xR <= -(dDim05 + 0.5) || yR <= -(dDim05 + 0.5))
continue;
int currentX = (int) (x + kp.X + 0.5);
int currentY = (int) (y + kp.Y + 0.5);
if (currentX < 1 || currentX >= (magnitude.XDim - 1) ||
currentY < 1 || currentY >= (magnitude.YDim - 1))
continue;
/*
Console.WriteLine (" ({0},{1}) by angle {2} -> ({3},{4})",
x, y, angle, xR, yR);
*/
// Weight the magnitude relative to the center of the
// whole FV. We do not need a normalizing factor now, as
// we normalize the whole FV later anyway (see below).
// xR, yR are each in -(dDim05 + 0.5) to (dDim05 + 0.5)
// range
double magW = Math.Exp (-(xR * xR + yR * yR) / sigma2Sq) *
magnitude[currentX, currentY];
// Anchor to (-1.0, -1.0)-(dDim + 1.0, dDim + 1.0), where
// the FV points are located at (x, y)
yR += dDim05 - 0.5;
xR += dDim05 - 0.5;
// Build linear interpolation weights:
// A B
// C D
//
// The keypoint is located between A, B, C and D.
int[] xIdx = new int[2];
int[] yIdx = new int[2];
int[] dirIdx = new int[2];
double[] xWeight = new double[2];
double[] yWeight = new double[2];
double[] dirWeight = new double[2];
if (xR >= 0) {
xIdx[0] = (int) xR;
xWeight[0] = (1.0 - (xR - xIdx[0]));
}
if (yR >= 0) {
yIdx[0] = (int) yR;
yWeight[0] = (1.0 - (yR - yIdx[0]));
}
if (xR < (descDim - 1)) {
xIdx[1] = (int) (xR + 1.0);
xWeight[1] = xR - xIdx[1] + 1.0;
}
if (yR < (descDim - 1)) {
yIdx[1] = (int) (yR + 1.0);
yWeight[1] = yR - yIdx[1] + 1.0;
}
// Rotate the gradient direction by the keypoint
// orientation, then normalize to [-pi ; pi] range.
double dir = direction[currentX, currentY] - kp.Orientation;
if (dir <= Math.PI)
dir += Math.PI;
if (dir > Math.PI)
dir -= Math.PI;
double idxDir = (dir * directionCount) /
(2.0 * Math.PI);
if (idxDir < 0.0)
idxDir += 8.0;
dirIdx[0] = (int) idxDir;
dirIdx[1] = (dirIdx[0] + 1) % directionCount;
dirWeight[0] = 1.0 - (idxDir - dirIdx[0]);
dirWeight[1] = idxDir - dirIdx[0];
/*
Console.WriteLine (" ({0},{1}) yields:", xR, yR);
Console.WriteLine (" x<{0},{1}>*({2},{3})",
xIdx[0], xIdx[1], xWeight[0], xWeight[1]);
Console.WriteLine (" y<{0},{1}>*({2},{3})",
yIdx[0], yIdx[1], yWeight[0], yWeight[1]);
Console.WriteLine (" dir<{0},{1}>*({2},{3})",
dirIdx[0], dirIdx[1], dirWeight[0], dirWeight[1]);
Console.WriteLine (" weighting m * w: {0} * {1}",
magnitude[currentX, currentY], Math.Exp (-(xR * xR +
yR * yR) / sigma2Sq));
*/
for (int iy = 0 ; iy < 2 ; ++iy) {
for (int ix = 0 ; ix < 2 ; ++ix) {
for (int id = 0 ; id < 2 ; ++id) {
kp.FVSet (xIdx[ix], yIdx[iy], dirIdx[id],
kp.FVGet (xIdx[ix], yIdx[iy], dirIdx[id]) +
xWeight[ix] * yWeight[iy] * dirWeight[id] * magW);
}
}
}
}
}
// Normalize and hicap the feature vector, as recommended on page
// 16 in Lowe03.
CapAndNormalizeFV (kp, fvGradHicap);
survivors.Add (kp);
}
return (survivors);
}
// Threshhold and normalize feature vector.
// Note that the feature vector as a whole is normalized (Lowe's paper is
// a bit unclear at that point).
private void CapAndNormalizeFV (Keypoint kp, double fvGradHicap)
{
// Straight normalization
double norm = 0.0;
for (int n = 0 ; n < kp.FVLinearDim ; ++n)
norm += Math.Pow (kp.FVLinearGet (n), 2.0);
norm = Math.Sqrt (norm);
if (norm == 0.0)
throw (new InvalidOperationException
("CapAndNormalizeFV cannot normalize with norm = 0.0"));
for (int n = 0 ; n < kp.FVLinearDim ; ++n)
kp.FVLinearSet (n, kp.FVLinearGet (n) / norm);
// Hicap after normalization
for (int n = 0 ; n < kp.FVLinearDim ; ++n) {
if (kp.FVLinearGet (n) > fvGradHicap) {
kp.FVLinearSet (n, fvGradHicap);
}
}
// Renormalize again
norm = 0.0;
for (int n = 0 ; n < kp.FVLinearDim ; ++n)
norm += Math.Pow (kp.FVLinearGet (n), 2.0);
norm = Math.Sqrt (norm);
for (int n = 0 ; n < kp.FVLinearDim ; ++n)
kp.FVLinearSet (n, kp.FVLinearGet (n) / norm);
}
// Simple helper predicate to tell if (rX, rY) is within a circle of
// \sqrt{radiusSq} radius, assuming the circle center is (0, 0).
private bool IsInCircle (int rX, int rY, int radiusSq)
{
rX *= rX;
rY *= rY;
if ((rX + rY) <= radiusSq)
return (true);
return (false);
}
// Remove peaks by peak magnitude and peak edge response. Find the
// sub-pixel local offset by interpolation.
//
// Sub-pixel localization and peak magnitude:
// After this method returns, every peak has a relative localization
// offset and its peak magnitude available in 'peak.Local'. The peak
// magnitude value must be above 'dValueLoThresh' for the point to
// survive. Usual values might lie in the range 0.0 (no filtering) to
// 0.03 (Lowe/Brown's recommendation). We normally use a value around
// 0.0001 to 0.00025 (and Brown's values seem quite large to me). The
// scaleAdjustThresh value is explained in LoweDetector.cs.
//
// Edge filtering:
// 'edgeRatio' denotes the required hi-threshhold for the ratio between
// the principle curvatures. Small values (1.5 to 3.0) will filter most
// points, leaving only the most corner-like points. Larger values (3.0 to
// 10.0) will remove the points which lie on a straight edge, whose
// position might be more vulnerable to noise.
//
// Return a filtered list of ScalePoint elements, with only the remaining
// survivors.
public ArrayList FilterAndLocalizePeaks (ArrayList peaks, double edgeRatio,
double dValueLoThresh, double scaleAdjustThresh, int relocationMaximum)
{
ArrayList filtered = new ArrayList ();
int[,] processedMap = new int[spaces[0].XDim, spaces[0].YDim];
foreach (ScalePoint peak in peaks) {
if (IsTooEdgelike (spaces[peak.Level], peak.X, peak.Y, edgeRatio))
continue;
// When the localization hits some problem, i.e. while moving the
// point a border is reached, then skip this point.
if (LocalizeIsWeak (peak, relocationMaximum, processedMap))
continue;
// Now we approximated the exact sub-pixel peak position.
// Comment the following line out to get a number of image files
// which show the located peak in the closest DoG scale.
/*DEBUGSaveRectangle (spaces[peak.Level], peak.X, peak.Y,
String.Format ("rect_{0}.png", peak.Local.DValue);
*/
/*Console.WriteLine ("peak.Local.ScaleAdjust = {0}",
peak.Local.ScaleAdjust);*/
if (Math.Abs (peak.Local.ScaleAdjust) > scaleAdjustThresh)
continue;
// Additional local pixel information is now available, threshhold
// the D(^x)
// Console.WriteLine ("{0} {1} {2} # == DVALUE", peak.Y, peak.X, peak.Local.DValue);
if (Math.Abs (peak.Local.DValue) <= dValueLoThresh)
continue;
/*Console.WriteLine ("{0} {1} {2} {3} # FILTERLOCALIZE",
peak.Y, peak.X, peak.Local.ScaleAdjust, peak.Local.DValue);*/
// its edgy enough, add it
filtered.Add (peak);
}
return (filtered);
}
// Return true if the point is not suitable, either because it lies on a
// border pixel or the Hessian matrix cannot be inverted.
// If false is returned, the pixel is suitable for localization and
// additional localization information has been put into 'point.Local'.
// No more than 'steps' corrections are made.
private bool LocalizeIsWeak (ScalePoint point, int steps, int[,] processed)
{
bool needToAdjust = true;
int adjusted = steps;
while (needToAdjust) {
int x = point.X;
int y = point.Y;
// Points we cannot say anything about, as they lie on the border
// of the scale space
if (point.Level <= 0 || point.Level >= (spaces.Length - 1))
return (true);
ImageMap space = spaces[point.Level];
if (x <= 0 || x >= (space.XDim - 1))
return (true);
if (y <= 0 || y >= (space.YDim - 1))
return (true);
double dp;
SimpleMatrix adj = GetAdjustment (point, point.Level, x, y, out dp);
// Get adjustments and check if we require further adjustments due
// to pixel level moves. If so, turn the adjustments into real
// changes and continue the loop. Do not adjust the plane, as we
// are usually quite low on planes in thie space and could not do
// further adjustments from the top/bottom planes.
double adjS = adj[0, 0];
double adjY = adj[1, 0];
double adjX = adj[2, 0];
if (Math.Abs (adjX) > 0.5 || Math.Abs (adjY) > 0.5) {
// Already adjusted the last time, give up
if (adjusted == 0) {
//Console.WriteLine ("too many adjustments, returning");
return (true);
}
adjusted -= 1;
// Check that just one pixel step is needed, otherwise discard
// the point
double distSq = adjX * adjX + adjY * adjY;
if (distSq > 2.0)
return (true);
point.X = (int) (point.X + adjX + 0.5);
point.Y = (int) (point.Y + adjY + 0.5);
//point.Level = (int) (point.Level + adjS + 0.5);
/*Console.WriteLine ("moved point by ({0},{1}: {2}) to ({3},{4}: {5})",
adjX, adjY, adjS, point.X, point.Y, point.Level);*/
continue;
}
/* for processing with gnuplot
*
Console.WriteLine ("{0} {1} # POINT LEVEL {2}", point.X,
point.Y, basePixScale);
Console.WriteLine ("{0} {1} {2} # ADJ POINT LEVEL {3}",
adjS, adjX, adjY, basePixScale);
*/
// Check if we already have a keypoint within this octave for this
// pixel position in order to avoid dupes. (Maybe we can move this
// check earlier after any adjustment, so we catch dupes earlier).
// If its not in there, mark it for later searches.
//
// FIXME: check why there does not seem to be a dupe at all
if (processed[point.X, point.Y] != 0)
return (true);
processed[point.X, point.Y] = 1;
// Save final sub-pixel adjustments.
PointLocalInformation local = new PointLocalInformation (adjS, adjX, adjY);
//local.DValue = dp;
local.DValue = space[point.X, point.Y] + 0.5 * dp;
point.Local = local;
needToAdjust = false;
}
return (false);
}
private bool IsTooEdgelike (ImageMap space, int x, int y, double r)
{
double D_xx, D_yy, D_xy;
// Calculate the Hessian H elements [ D_xx, D_xy ; D_xy , D_yy ]
D_xx = space[x + 1, y] + space[x - 1, y] - 2.0 * space[x, y];
D_yy = space[x, y + 1] + space[x, y - 1] - 2.0 * space[x, y];
D_xy = 0.25 * ((space[x + 1, y + 1] - space[x + 1, y - 1]) -
(space[x - 1, y + 1] - space[x - 1, y - 1]));
// page 13 in Lowe's paper
double TrHsq = D_xx + D_yy;
TrHsq *= TrHsq;
double DetH = D_xx * D_yy - (D_xy * D_xy);
double r1sq = (r + 1.0);
r1sq *= r1sq;
// BUG: this can invert < to >, uhh: if ((TrHsq * r) < (DetH * r1sq))
if ((TrHsq / DetH) < (r1sq / r)) {
/*Console.WriteLine ("{0} {1} {2} {3} {4} # EDGETEST",
y, x, (TrHsq * r), (DetH * r1sq),
(TrHsq / DetH) / (r1sq / r));*/
return (false);
}
return (true);
}
// Return adjustment (scale, y, x) on success,
// return null on failure
// TODO: integrate this
private SimpleMatrix GetAdjustment (ScalePoint point,
int level, int x, int y, out double dp)
{
/*Console.WriteLine ("GetAdjustment (point, {0}, {1}, {2}, out double dp)",
level, x, y);*/
dp = 0.0;
if (point.Level <= 0 || point.Level >= (spaces.Length - 1))
throw (new ArgumentException ("point.Level is not within [bottom-1;top-1] range"));
ImageMap below = spaces[level - 1];
ImageMap current = spaces[level];
ImageMap above = spaces[level + 1];
SimpleMatrix H = new SimpleMatrix (3, 3);
H[0, 0] = below[x, y] - 2 * current[x, y] + above[x, y];
H[0, 1] = H[1, 0] = 0.25 * (above[x, y + 1] - above[x, y - 1] -
(below[x, y + 1] - below[x, y - 1]));
H[0, 2] = H[2, 0] = 0.25 * (above[x + 1, y] - above[x - 1, y] -
(below[x + 1, y] - below[x - 1, y]));
H[1, 1] = current[x, y - 1] - 2 * current[x, y] + current[x, y + 1];
H[1, 2] = H[2, 1] = 0.25 * (current[x + 1, y + 1] - current[x - 1, y + 1] -
(current[x + 1, y - 1] - current[x - 1, y - 1]));
H[2, 2] = current[x - 1, y] - 2 * current[x, y] + current[x + 1, y];
SimpleMatrix d = new SimpleMatrix (3, 1);
d[0, 0] = 0.5 * (above[x, y] - below[x, y]);
d[1, 0] = 0.5 * (current[x, y + 1] - current[x, y - 1]);
d[2, 0] = 0.5 * (current[x + 1, y] - current[x - 1, y]);
SimpleMatrix b = (SimpleMatrix) d.Clone ();
b.Negate ();
// Solve: A x = b
H.SolveLinear (b);
dp = b.Dot (d);
return (b);
}
// Peak localization in scale-space.
//
// From Lowe's paper its not really clear whether we always need three
// neighbourhood spaces or should also search only one or two spaces. As
// figure 3 might suggest the later, we do it like this.
//
// Return an arraylist holding ScalePoint elements.
public ArrayList FindPeaks (double dogThresh)
{
Console.WriteLine (" FindPeaks: scale {0:N2}, testing {1} levels",
basePixScale, Count - 2);
ArrayList peaks = new ArrayList ();
ImageMap current, above, below;
// Search the D(k * sigma) to D(2 * sigma) spaces
for (int level = 1 ; level < (Count - 1) ; ++level)
{
current = this[level];
below = this[level - 1];
above = this[level + 1];
/*
Console.WriteLine ("below/current/above: {0} {1} {2}",
below == null ? "-" : "X",
current == null ? "-" : "X",
above == null ? "-" : "X");
Console.WriteLine ("peak-search at level {0}", level);
*/
peaks.AddRange (FindPeaksThreeLevel (below, current, above,
level, dogThresh));
below = current;
}
return (peaks);
}
private ArrayList FindPeaksThreeLevel (ImageMap below, ImageMap current,
ImageMap above, int curLev, double dogThresh)
{
ArrayList peaks = new ArrayList ();
for (int y = 1 ; y < (current.YDim - 1) ; ++y) {
for (int x = 1 ; x < (current.XDim - 1) ; ++x) {
bool cIsMax = true;
bool cIsMin = true;
double c = current[x, y]; // Center value
if (Math.Abs (c) <= dogThresh)
continue;
CheckMinMax (current, c, x, y, ref cIsMin, ref cIsMax, true);
CheckMinMax (below, c, x, y, ref cIsMin, ref cIsMax, false);
CheckMinMax (above, c, x, y, ref cIsMin, ref cIsMax, false);
if (cIsMin == false && cIsMax == false)
continue;
//Console.WriteLine ("{0} {1} {2} # DOG", y, x, c);
peaks.Add (new ScalePoint (x, y, curLev));
}
}
return (peaks);
}
// Check if a pixel ('x', 'y') with value 'c' is minimum or maximum in the
// 'layer' image map. Except for the center, and its above/below planes
// corresponding pixels, use a strong > and < check (because the if is
// inverted it looks like >= and <=).
private void CheckMinMax (ImageMap layer, double c, int x, int y,
ref bool IsMin, ref bool IsMax, bool cLayer)
{
if (layer == null)
return;
if (IsMin == true) {
if (layer[x - 1, y - 1] <= c ||
layer[x, y - 1] <= c ||
layer[x + 1, y - 1] <= c ||
layer[x - 1, y] <= c ||
// note here its just < instead of <=
(cLayer ? false : (layer[x, y] < c)) ||
layer[x + 1, y] <= c ||
layer[x - 1, y + 1] <= c ||
layer[x, y + 1] <= c ||
layer[x + 1, y + 1] <= c)
IsMin = false;
}
if (IsMax == true) {
if (layer[x - 1, y - 1] >= c ||
layer[x, y - 1] >= c ||
layer[x + 1, y - 1] >= c ||
layer[x - 1, y] >= c ||
// note here its just > instead of >=
(cLayer ? false : (layer[x, y] > c)) ||
layer[x + 1, y] >= c ||
layer[x - 1, y + 1] >= c ||
layer[x, y + 1] >= c ||
layer[x + 1, y + 1] >= c)
IsMax = false;
}
}
static public double SToK (int s)
{
return (Math.Pow (2.0, 1.0 / s));
}
// Precompute all gradient magnitude and direction planes for one octave.
public void GenerateMagnitudeAndDirectionMaps ()
{
// We leave the first entry to null, and ommit the last. This way, the
// magnitudes and directions maps have the same index as the
// imgScaled maps they below to.
magnitudes = new ImageMap[Count - 1];
directions = new ImageMap[Count - 1];
// Build the maps, omitting the border pixels, as we cannot build
// gradient information there.
for (int s = 1 ; s < (Count - 1) ; ++s) {
magnitudes[s] = new ImageMap (imgScaled[s].XDim, imgScaled[s].YDim);
directions[s] = new ImageMap (imgScaled[s].XDim, imgScaled[s].YDim);
for (int y = 1 ; y < (imgScaled[s].YDim - 1) ; ++y) {
for (int x = 1 ; x < (imgScaled[s].XDim - 1) ; ++x) {
// gradient magnitude m
magnitudes[s][x, y] = Math.Sqrt (
Math.Pow (imgScaled[s][x + 1, y] -
imgScaled[s][x - 1, y], 2.0) +
Math.Pow (imgScaled[s][x, y + 1] -
imgScaled[s][x, y - 1], 2.0));
// gradient direction theta
directions[s][x, y] = Math.Atan2
(imgScaled[s][x, y + 1] - imgScaled[s][x, y - 1],
imgScaled[s][x + 1, y] - imgScaled[s][x - 1, y]);
}
}
}
}
public void ClearMagnitudeAndDirectionMaps ()
{
magnitudes = directions = null;
}
// Build a set of Difference-of-Gaussian (DoG) maps from the gaussian
// blurred images.
// This method has to be called after BuildGaussianMaps has completed.
public void BuildDiffMaps ()
{
// Generate DoG maps. The maps are organized like this:
// 0: D(sigma)
// 1: D(k * sigma)
// 2: D(k^2 * sigma)
// ...
// s: D(k^s * sigma) = D(2 * sigma)
// s+1: D(k * 2 * sigma)
//
// So, we can start peak searching at 1 to s, and have a DoG map into
// each direction.
spaces = new ImageMap[imgScaled.Length - 1];
// After the loop completes, we have used (s + 1) maps, yielding s
// D-gaussian maps in the range of sigma to 2*sigma, as k^s = 2, which
// is defined as one octave.
for (int sn = 0 ; sn < spaces.Length ; ++sn) {
// XXX: order correct? It should not really matter as we search
// for both minimums and maximums, but still, what is recommended?
// (otherwise maybe the gradient directions are inverted?)
spaces[sn] = imgScaled[sn + 1] - imgScaled[sn];
}
}
// Incrementally blur the input image first so it reaches the next octave.
public void BuildGaussianMaps (ImageMap first, double firstScale,
int scales, double sigma)
{
// We need one more gaussian blurred image than the number of DoG
// maps. But for the minima/maxima pixel search, we need two more. See
// BuildDiffMaps.
imgScaled = new ImageMap[scales + 1 + 1 + 1];
this.basePixScale = firstScale;
// Convolve first image with the octaveSigma. Previously we got this
// wrong, but thanks to Alexandre Jenny, this got fixed. Thanks!
GaussianConvolution gauss = new GaussianConvolution (sigma);
ImageMap prev = imgScaled[0] = gauss.Convolve (first);
// Ln1(x, y, k^{p+1}) = G(x, y, k) * Ln0(x, y, k^p).
for (int scI = 1 ; scI < imgScaled.Length ; ++scI) {
gauss = new GaussianConvolution (sigma);
prev = imgScaled[scI] = gauss.Convolve (prev);
sigma *= SToK (scales);
}
}
/*
private void DEBUGSaveRectangle (ImageMap map, int x, int y,
string filename)
{
int x1, x2, y1, y2;
int xo = x, yo = y;
x1 = x - 10;
x2 = x + 10;
y1 = y - 10;
y2 = y + 10;
if (x1 < 0)
x1 = 0;
if (y1 < 0)
y1 = 0;
if (x2 >= map.XDim)
x2 = map.XDim - 1;
if (y2 >= map.YDim)
y2 = map.YDim - 1;
Gdk.Pixbuf pbuf = new Gdk.Pixbuf (Gdk.Colorspace.Rgb, false, 8,
x2 - x1, y2 - y1);
pbuf.Fill (0x000000);
// FIXME: temporary workaround GTK# brokenness
unsafe {
byte *pixels = (byte *) pbuf.Pixels;
for (y = y1 ; y < y2 ; ++y) {
for (x = x1 ; x < x2 ; ++x) {
byte grayVal = (byte) (map[x, y] * 255.0);
for (int n = 0 ; n < 3 ; ++n) {
pixels[(y-y1) * pbuf.Rowstride +
(x-x1) * pbuf.NChannels + n] = grayVal;
}
}
}
pixels[(yo-y1) * pbuf.Rowstride + (xo-x1) * pbuf.NChannels + 0] = 255;
pixels[(yo-y1) * pbuf.Rowstride + (xo-x1) * pbuf.NChannels + 1] = 0;
pixels[(yo-y1) * pbuf.Rowstride + (xo-x1) * pbuf.NChannels + 2] = 0;
}
pbuf = pbuf.ScaleSimple ((x2-x1) * 16, (y2-y1) * 16, Gdk.InterpType.Nearest);
Console.WriteLine ("saving under: {0}", filename);
pbuf.Savev (filename, "png", null, null);
}
*/
}
// A single point in scale space, used in keypoint creation to describe an
// exact position in scalespace and additional information about that point.
// Should not be used outside.
internal class ScalePoint
{
int x, y;
int level;
public int X {
get {
return (x);
}
set {
x = value;
}
}
public int Y {
get {
return (y);
}
set {
y = value;
}
}
public int Level {
get {
return (level);
}
set {
level = value;
}
}
// Sub-pixel level information from the Localization step are put here
PointLocalInformation local;
public PointLocalInformation Local {
get {
return (local);
}
set {
local = value;
}
}
private ScalePoint ()
{
}
public ScalePoint (int x, int y, int level)
{
this.x = x;
this.y = y;
this.level = level;
}
}
internal class PointLocalInformation
{
// Sub-pixel offset relative from this point. In the range of [-0.5 ; 0.5]
double fineX, fineY;
public double FineX {
get {
return (fineX);
}
}
public double FineY {
get {
return (fineY);
}
}
// Relative scale adjustment to the base image scale
double scaleAdjust;
public double ScaleAdjust {
get {
return (scaleAdjust);
}
set {
scaleAdjust = value;
}
}
double dValue;
public double DValue {
get {
return (dValue);
}
set {
dValue = value;
}
}
private PointLocalInformation ()
{
}
public PointLocalInformation (double fineS, double fineX, double fineY)
{
this.fineX = fineX;
this.fineY = fineY;
this.scaleAdjust = fineS;
}
}
// A single keypoint, the final result of keypoint creation. Contains the
// keypoint descriptor and position.
public class Keypoint
{
public Keypoint ()
{
}
ImageMap image;
public ImageMap Image {
get {
return (image);
}
}
double x, y;
double imgScale; // The scale of the image the keypoint was found in
// The absolute keypoint scale, where 1.0 is the original input image
double scale;
double orientation;
public double X {
get {
return (x);
}
set {
x = value;
}
}
public double Y {
get {
return (y);
}
set {
y = value;
}
}
public double ImgScale {
get {
return (imgScale);
}
set {
imgScale = value;
}
}
public double Scale {
get {
return (scale);
}
set {
scale = value;
}
}
public double Orientation {
get {
return (orientation);
}
set {
orientation = value;
}
}
// The actual keypoint descriptor.
bool hasFV = false;
public bool HasFV {
get {
return (hasFV);
}
set {
hasFV = value;
}
}
double[] featureVector;
public double[] FV {
get {
return (featureVector);
}
set {
featureVector = value;
}
}
public double FVGet (int xI, int yI, int oI)
{
return (featureVector[(xI * yDim * oDim) + (yI * oDim) + oI]);
}
public void FVSet (int xI, int yI, int oI, double value)
{
featureVector[(xI * yDim * oDim) + (yI * oDim) + oI] = value;
}
public int FVLinearDim
{
get {
return (featureVector.Length);
}
}
public double FVLinearGet (int idx)
{
return (featureVector[idx]);
}
public void FVLinearSet (int idx, double value)
{
featureVector[idx] = value;
}
public void CreateLinearVector (int dim)
{
featureVector = new double[dim];
}
private int xDim, yDim, oDim;
public void CreateVector (int xDim, int yDim, int oDim)
{
hasFV = true;
this.xDim = xDim;
this.yDim = yDim;
this.oDim = oDim;
featureVector = new double[yDim * xDim * oDim];
}
// Keypoint constructor.
//
// image: The smoothed gaussian image the keypoint was located in.
// x, y: The subpixel level coordinates of the keypoint.
// imgScale: The scale of the gaussian image, with 1.0 being the original
// detail scale (source image), and doubling at each octave.
// kpScale: The scale of the keypoint.
// orientation: Orientation degree in the range of [-PI ; PI] of the
// keypoint.
//
// First add a keypoint, then use 'MakeDescriptor' to generate the local
// image descriptor for this keypoint.
public Keypoint (ImageMap image, double x, double y, double imgScale,
double kpScale, double orientation)
{
this.image = image;
this.x = x;
this.y = y;
this.imgScale = imgScale;
this.scale = kpScale;
this.orientation = orientation;
}
public int DimensionCount {
get {
return (FVLinearDim);
}
}
public double GetDimensionElement (int dim)
{
return (FVLinearGet (dim));
}
}