001package org.openimaj.demos.sandbox.tldcpp.detector; 002 003import java.util.Random; 004 005import org.openimaj.image.FImage; 006import org.openimaj.math.geometry.shape.Rectangle; 007 008/** 009 * The ensemble classifier implements a forest of random binary pixel 010 * comparisons. Each tree in the forest is made up of several features. 011 * Each feature is a randomly selected pair of pixels (at the same location 012 * across scales) within a window. The boolean comparison of the pixels 013 * sets the value of a bit of the feature int. This means technically each 014 * feature can only be 32 decisions wide, 13 seems to work well though 015 * @author Sina Samangooei (ss@ecs.soton.ac.uk) 016 * 017 */ 018public class EnsembleClassifier { 019 private float[][] img; 020 021 /** 022 * Whether this classifier is enabled 023 */ 024 public boolean enabled; 025 026 /** 027 * The number of trees, each with {@link #numFeatures} features (default = 10) 028 */ 029 public int numTrees; 030 /** 031 * The number of bits per feature (each feature is in fact held as an int of bits, default = 13) 032 */ 033 public int numFeatures; 034 035 /** 036 * The number of scales, not configurable, dependant on the scales searched in {@link DetectorCascade} 037 */ 038 private int numScales; 039 private Rectangle[] scales; 040 041 private int[][] windowOffsets; 042 private int[][] featureOffsets; 043 private float[] features; 044 045 int numIndices; 046 047 private float [] posteriors; 048 private int [] positives; 049 private int [] negatives; 050 051 DetectionResult detectionResult; 052 053 private Random rand; 054 055 EnsembleClassifier() 056 { 057 numTrees=10; 058 numFeatures = 13; 059 enabled = true; 060 rand = new Random(); 061 } 062 063 064 void init() { 065 numIndices = (int) Math.pow(2.0f, numFeatures); 066 067 initFeatureLocations(); 068 initFeatureOffsets(); 069 initPosteriors(); 070 } 071 072 void release() { 073 features = null; 074 featureOffsets = null; 075 posteriors = null; 076 positives = null; 077 negatives = null; 078 } 079 080 /* 081 * Generates random measurements in the format <x1,y1,x2,y2> 082 */ 083 void initFeatureLocations() { 084 int size = 2 * 2 * numFeatures * numTrees; 085 086 features = new float[size]; 087 088 for(int i=0; i < size; i++) { 089 features[i] = rand.nextFloat(); 090 } 091 092 } 093 094 /** 095 * Creates the offsets which represent the pixels which are compared within a bounding box 096 * for each feature within each tree at each scale. 097 * 098 * This process just picks two random pixels within the bounding box based on the known size 099 * of the bounding box at a particular scale. 100 * 101 * How the two random pixels are compared to form the feature itself is described in 102 * {@link #calcFernFeature(int, int)} 103 */ 104 void initFeatureOffsets() { 105 106 featureOffsets= new int[numScales*numTrees*numFeatures*2][2]; 107// int *off = featureOffsets; 108 int offIndex = 0; 109 110 for (int k = 0; k < numScales; k++){ 111 Rectangle scale = scales[k]; 112 for (int i = 0; i < numTrees; i++) { 113 for (int j = 0; j < numFeatures; j++) { 114// float *currentFeature = features + (4*numFeatures)*i +4*j; 115// *off++ = sub2idx((scale.width-1)*currentFeature[0]+1,(scale.height-1)*currentFeature[1]+1,imgWidthStep); //We add +1 because the index of the bounding box points to x-1, y-1 116// *off++ = sub2idx((scale.width-1)*currentFeature[2]+1,(scale.height-1)*currentFeature[3]+1,imgWidthStep); 117 // float *currentFeature = features + (4*numFeatures)*i +4*j; 118 int currentFeatureIndex = (4*numFeatures)*i +4*j; 119 featureOffsets[offIndex++] = new int[]{ 120 (int) ((scale.width-1)*features[currentFeatureIndex + 0]+1), 121 (int) ((scale.height-1)*features[currentFeatureIndex + 1]+1) 122 }; //We add +1 because the index of the bounding box points to x-1, y-1 123 featureOffsets[offIndex++] = new int[]{ 124 (int) ((scale.width-1)*features[currentFeatureIndex + 2]+1), 125 (int) ((scale.height-1)*features[currentFeatureIndex + 3]+1) 126 }; 127 } 128 } 129 } 130 } 131 132 void initPosteriors() { 133 posteriors = new float[numTrees * numIndices]; 134 positives = new int[numTrees * numIndices]; 135 negatives = new int[numTrees * numIndices]; 136 137 for (int i = 0; i<numTrees; i++) { 138 for(int j = 0; j < numIndices; j++) { 139 posteriors[i*numIndices + j] = 0; 140 positives[i*numIndices + j] = 0; 141 negatives[i*numIndices + j] = 0; 142 } 143 } 144 } 145 146 /** 147 * @param img just sets the image internally read to be used to calculate the feature 148 */ 149 public void nextIteration(FImage img) { 150 if(!enabled) return; 151 152 this.img = img.pixels; 153 } 154 155 /** 156 * calculate the value for each feature of a tree at a scale for a given window. 157 * 158 * The value of each feature is an int whose bits look like this: 159 * 01110010101. 160 * 161 * 1 if the first random pixel for that feature is bigger than the second random pixel 162 * 0 otherwise! 163 * WHOA 164 * @param windowIdx 165 * @param treeIdx 166 * @return the int feature of the tree for the window 167 */ 168 int calcFernFeature(int windowIdx, int treeIdx) { 169 170 int index = 0; 171// int *bbox = windowOffsets+ windowIdx* TLD_WINDOW_OFFSET_SIZE; 172 int bboxIndex = windowIdx* DetectorCascade.TLD_WINDOW_OFFSET_SIZE; 173// int *off = featureOffsets + bbox[4] + treeIdx*2*numFeatures; //bbox[4] is pointer to features for the current scale 174 int offIndex = windowOffsets[bboxIndex + 4][0] + treeIdx*2*numFeatures; 175 for (int i=0; i<numFeatures; i++) { 176// int fp0 = img[bbox[0] + off[0]]; 177// int fp1 = img[bbox[0] + off[1]]; 178// if (fp0>fp1) { index |= 1;} 179// off += 2; 180 index<<=1; 181 int[] bbox0 = windowOffsets[bboxIndex]; 182 int[] off0 = featureOffsets[offIndex]; 183 int[] off1 = featureOffsets[offIndex+1]; 184 float fp0 = img[bbox0[1] + off0[1]][bbox0[0] + off0[0]]; 185 float fp1 = img[bbox0[1] + off1[1]][bbox0[0] + off1[0]]; 186 if (fp0>fp1) { index |= 1;} 187 offIndex += 2; 188 } 189 return index; 190 } 191 192 /** 193 * For a window index, calculate the feature values for each tree. Store them in 194 * the featureVector array from the featureVectorIndex for a length of numTrees. 195 * 196 * @param windowIdx 197 * @param featureVector 198 * @param featureVectorIndex 199 */ 200 public void calcFeatureVector(int windowIdx, int[] featureVector, int featureVectorIndex) { 201 for(int i = 0; i < numTrees; i++) { 202 featureVector[featureVectorIndex + i] = calcFernFeature(windowIdx, i); 203 } 204 } 205 206 /** 207 * sum the probability calculated for each tree with the calculated feature value (which 208 * is in fact the index into the posteriors array) 209 * @param featureVector 210 * @param featureVectorIndex 211 * @return the sum confidence from each tree of this feature assignment 212 */ 213 public float calcConfidence(int [] featureVector, int featureVectorIndex) { 214 float conf = 0.0f; 215 216 for(int i = 0; i < numTrees; i++) { 217 conf += posteriors[i * numIndices + featureVector[featureVectorIndex + i]]; 218 } 219 220 return conf; 221 } 222 223 /** 224 * Fill the feature vector values for each tree for a given window. Use the features 225 * for each tree for this window is to calculate the confidence of this window. 226 * @param windowIdx 227 */ 228 void classifyWindow(int windowIdx) { 229// int* featureVector = detectionResult->featureVectors + numTrees * windowIdx; 230 int featureVectorIndex = numTrees * windowIdx; 231 calcFeatureVector(windowIdx, detectionResult.featureVectors, featureVectorIndex); 232 233 detectionResult.posteriors[windowIdx] = calcConfidence(detectionResult.featureVectors, featureVectorIndex); 234 } 235 236 /** 237 * find the proability of this window and return true if this window is more 238 * than 50% likely. 239 * @param i 240 * @return whether this window is likely to match given the previously seen positive examples 241 */ 242 public boolean filter(int i) { 243 if(!enabled) return true; 244 245 classifyWindow(i); 246 if(detectionResult.posteriors[i] < 0.5) return false; 247 return true; 248 } 249 250 /** 251 * An array index is calculated from treeIdx (the y) and the calculated 252 * feature index (the x) so: (y * width) + x where width is the maximum feature 253 * index (2 ^ numverOfFeatures). 254 * 255 * The posterior for each arrayIndex is calculated by doing: 256 * nPositive / (nPositive + nNegative). So it will be higher the more positive 257 * examples are seen for this particular feature in this particular tree. 258 * 259 * The logic is that things that are similar will be positive in the same tree/feature 260 * so they will have higher counts. 261 * 262 * 263 * @param treeIdx 264 * @param idx 265 * @param positive 266 * @param amount 267 */ 268 public void updatePosterior(int treeIdx, int idx, boolean positive, int amount) { 269 int arrayIndex = treeIdx * numIndices + idx; 270 if(positive) 271 { 272 positives[arrayIndex] += amount; 273 } 274 else 275 { 276 negatives[arrayIndex] += amount; 277 } 278 posteriors[arrayIndex] = ((float) positives[arrayIndex]) / (positives[arrayIndex] + negatives[arrayIndex]) / (float)numTrees; 279 } 280 281 /** 282 * Update the calculated feature value in every tree. Remembering that the feature value is 283 * calculated based on underlying values of the what the image is doing in the window 284 * at a few random pixels. So if two images match a featureIndex in a given tree 285 * they match a numIndices pixels 286 * @param featureVector 287 * @param featureIndex 288 * @param positive 289 * @param amount 290 */ 291 public void updatePosteriors(int []featureVector,int featureIndex, boolean positive, int amount) { 292 293 for (int i = 0; i < numTrees; i++) { 294 295 int idx = featureVector[featureIndex+i]; 296 updatePosterior(i, idx, positive, amount); 297 298 } 299 } 300 301 /** 302 * Calculate the confidence of the given feature vector at a given index (which is 303 * the feature vector of a particular window). 304 * 305 * If the window is particularly low confidence but is meant to be positive, then update 306 * the posteriors with its values. 307 * 308 * If the window is particularly high confidence but is meant to be negative, then update 309 * the posteriors also. 310 * 311 * @param img 312 * @param positive 313 * @param featureVector 314 * @param featureIndex 315 */ 316 public void learn(FImage img, boolean positive, int [] featureVector, int featureIndex) { 317 if(!enabled) return; 318 319 float conf = calcConfidence(featureVector,featureIndex); 320 321 //Update if positive patch and confidence < 0.5 or negative and conf > 0.5 322 if((positive && conf < 0.5) || (!positive && conf > 0.5)) { 323 updatePosteriors(featureVector, featureIndex, positive,1); 324 } 325 326 } 327 328 /** 329 * @param numScales the number of scales must be set dynamically 330 */ 331 void setNumScales(int numScales) { 332 this.numScales = numScales; 333 } 334 335 336 /** 337 * @param windowOffsets the location of each window 338 */ 339 public void setWindowOffsets(int[][] windowOffsets) { 340 this.windowOffsets = windowOffsets; 341 } 342 343 344 /** 345 * @param scales the scale rectangles (i.e. the window sizes at each scale) 346 */ 347 public void setScales(Rectangle[] scales) { 348 this.scales = scales; 349 } 350 351}