Commit b4583f2d authored by lafabregue's avatar lafabregue

Add masks in data sampling

parent aa6c4d76
......@@ -47,7 +47,12 @@ public class IntArrayMask implements Mask {
public IntArrayMask(final int[] mask, final int id, final boolean ignore) {
super();
this.mask = mask;
this.card = IntArrayMask.objectInMask(mask, id);
int countInMask = IntArrayMask.objectInMask(mask, id);
if (ignore) {
this.card = mask.length - countInMask;
} else {
this.card = countInMask;
}
this.id = id;
this.ignore = ignore;
}
......@@ -73,7 +78,7 @@ public class IntArrayMask implements Mask {
}
@Override
public boolean isMaksed(int index) {
public boolean isMasked(int index) {
boolean isId = false;
if (mask[index] == id) {
isId = true;
......@@ -85,6 +90,30 @@ public class IntArrayMask implements Mask {
return !isId;
}
@Override
public int getCardinalityInInterval(int start, int end) {
int nb = 0;
int cursor = start;
if (ignore) {
while (cursor < end) {
if (mask[cursor] == id) {
nb++;
}
cursor++;
}
} else {
while (cursor < end) {
if (mask[cursor] != id) {
nb++;
}
cursor++;
}
}
return nb;
}
@Override
public IntMaskIterator getIncludedIndexes() {
......@@ -112,12 +141,12 @@ public class IntArrayMask implements Mask {
int last = end;
if (ignore) {
while(mask[last] == id && last > 0) {
while(mask[last] == id && last > start) {
last--;
}
} else {
while(mask[last] != id && last > 0) {
while(mask[last] != id && last > start) {
last--;
}
}
......
......@@ -12,7 +12,7 @@ public interface Mask{
*
* @return true if it is masked, false otherwise
*/
public boolean isMaksed(int index);
public boolean isMasked(int index);
/**
* Return an Iterator that retrieves all indexes not masked
......@@ -23,7 +23,12 @@ public interface Mask{
/**
* Return an Iterator that retrieves all indexes not masked
* between two index
* between two indexes
*
* @param start
* the index of the start boundary
* @param end
* the index of the end boundary
*
* @return the Iterator
*/
......@@ -45,14 +50,27 @@ public interface Mask{
public int getOverallCarinality();
/**
* Get the index of the last index to including an unmasked pixel
* Get the index of the last index to including an unmasked element
* within a given boundary
*
* @param start
* the index of the start boundary
* @param end
* the index of the end boundary
* @return the pixel index
*
* @return the element index
*/
public int getLastIndex(int start, int end);
/**
* Return the number of unmasked element between two indexes
*
* @param start
* the index of the start boundary
* @param end
* the index of the end boundary
*
* @return the count of unmasked elements
*/
public int getCardinalityInInterval(int start, int end);
}
package jcl.data.mask;
import java.util.Iterator;
public class MaskIterator implements Iterator<Integer> {
private int cursor;
private final int end;
private final Mask mask;
public MaskIterator(final Mask mask) {
this.mask = mask;
this.cursor = 0;
this.end = mask.getLastIndex(0, mask.getOverallCarinality()-1);
}
public MaskIterator(Mask mask, final int start, final int end) {
this.mask = mask;
this.cursor = start;
this.end = mask.getLastIndex(start, end);
}
public boolean hasNext() {
return this.cursor < end;
}
public Integer next() {
while(cursor < end) {
cursor++;
if (!mask.isMasked(cursor)) {
return new Integer(cursor);
}
}
return -1;
}
public Integer previous() {
while(cursor > 0) {
cursor--;
if(!mask.isMasked(cursor)) {
return new Integer(cursor);
}
}
return -1;
}
public Integer current() {
return new Integer(cursor);
}
public void remove() {
throw new UnsupportedOperationException();
}
}
package jcl.data.mask;
import java.util.Iterator;
public class MultiIDIntArrayMask implements Mask {
private final int[] mask;
......@@ -47,16 +45,21 @@ public class MultiIDIntArrayMask implements Mask {
public MultiIDIntArrayMask(final int[] mask, final int[] id, final boolean ignore) {
super();
this.mask = mask;
this.card = MultiIDIntArrayMask.objectInMask(mask, id);
int countInMask = MultiIDIntArrayMask.objectInMask(mask, id);
if (ignore) {
this.card = mask.length - countInMask;
} else {
this.card = countInMask;
}
this.id = id;
this.ignore = ignore;
}
/**
* Return the number of times a value appears in an array
* Return the number of times a set of values appears in an array
*
* @param mask
* the array to analyse
* the array to analyze
* @param id
* the values searched
*
......@@ -75,7 +78,7 @@ public class MultiIDIntArrayMask implements Mask {
}
@Override
public boolean isMaksed(int index) {
public boolean isMasked(int index) {
boolean isId = false;
for(int e : id) {
if (mask[index] == e) {
......@@ -92,13 +95,27 @@ public class MultiIDIntArrayMask implements Mask {
}
@Override
public MultiIDIntMaskIterator getIncludedIndexes() {
return new MultiIDIntMaskIterator(mask, id, ignore);
public MaskIterator getIncludedIndexes() {
return new MaskIterator(this);
}
@Override
public MultiIDIntMaskIterator getIncludedIndexes(int start, int end) {
return new MultiIDIntMaskIterator(mask, id, ignore, start, end);
public MaskIterator getIncludedIndexes(int start, int end) {
return new MaskIterator(this, start, end);
}
@Override
public int getCardinalityInInterval(int start, int end) {
int nb = 0;
int cursor = start;
while (cursor < end) {
if (isMasked(cursor)) {
nb++;
}
cursor++;
}
return nb;
}
@Override
......@@ -116,173 +133,17 @@ public class MultiIDIntArrayMask implements Mask {
public int getLastIndex(int start, int end) {
int last = end;
if (ignore) {
for(int e : id) {
if (mask[last] != e) {
return last;
}
}
while (last > 0) {
last--;
for(int e : id) {
if (mask[last] != e) {
return last;
}
}
}
} else {
for(int e : id) {
if (mask[last] == e) {
return last;
}
}
if (!isMasked(last))
return last;
while(last > 0) {
last--;
for(int e : id) {
if (mask[last] == e) {
return last;
}
}
while(last > 0) {
last--;
if (!isMasked(last)) {
return last;
}
}
return last;
}
public static final class MultiIDIntMaskIterator implements Iterator<Integer> {
private int cursor;
private final int end;
private final int[] mask;
private final int[] id;
private final boolean ignore;
public MultiIDIntMaskIterator(final int[] mask, final int[] id, final boolean ignore) {
this.id = id;
this.mask = mask;
this.ignore = ignore;
this.cursor = 0;
this.end = getLastIndex(0, mask.length-1);
}
public MultiIDIntMaskIterator(final int[] mask, final int[] id, final boolean ignore,
final int start, final int end) {
this.id = id;
this.mask = mask;
this.ignore = ignore;
this.cursor = start;
this.end = getLastIndex(start, end);
}
/**
* Get the index of the last index to including an unmasked pixel
* within a given boundary
*
* @param start
* the index of the start boundary
* @param end
* the index of the end boundary
* @return the pixel index
*/
private int getLastIndex(int start, int end) {
int last = end;
if (ignore) {
for(int e : id) {
if (mask[last] != e) {
return last;
}
}
while (last > 0) {
last--;
for(int e : id) {
if (mask[last] != e) {
return last;
}
}
}
} else {
for(int e : id) {
if (mask[last] == e) {
return last;
}
}
while(last > 0) {
last--;
for(int e : id) {
if (mask[last] == e) {
return last;
}
}
}
}
return last;
}
public boolean hasNext() {
return this.cursor < end;
}
public Integer next() {
if (ignore) {
while(cursor < end) {
cursor++;
for(int e : id) {
if(mask[cursor] != e) {
return new Integer(cursor);
}
}
}
} else {
while(cursor < end) {
cursor++;
for(int e : id) {
if(mask[cursor] == e) {
return new Integer(cursor);
}
}
}
}
return -1;
}
public Integer previous() {
if (ignore) {
while(cursor > 0) {
cursor--;
for(int e : id) {
if(mask[cursor] != e) {
return new Integer(cursor);
}
}
}
} else {
while(cursor > 0) {
cursor--;
for(int e : id) {
if(mask[cursor] == e) {
return new Integer(cursor);
}
}
}
}
return -1;
}
public Integer current() {
return new Integer(cursor);
}
public void remove() {
throw new UnsupportedOperationException();
}
}
}
......@@ -3,6 +3,7 @@ package jcl.data.sampling;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
......@@ -52,8 +53,12 @@ public class ImageSampler extends Sampler {
private int startX = 0;
private int startY = 0;
// TODO consider the mandatory pixels in the data construction, they should be added in the sample
// and the function should then only generated sizeByCount - mandatoriesIndexs.length objects
private int[] mandatoriesIndexes = null;
private List<Mask> maskSequence = null;
/**
* Constructor from a sequence of images
*
......@@ -184,6 +189,11 @@ public class ImageSampler extends Sampler {
@Override
public List<DataObject> getDataObjects() {
// if a mask is applied we use it
if (mask != null) {
return getDataObjects(mask);
}
if (mandatoriesIndexes != null && mandatoriesIndexes.length > sizeByCount) {
throw new IndexOutOfBoundsException("The number of mandatory pixels"
+ " is bigger than the expeted sample size");
......@@ -198,6 +208,7 @@ public class ImageSampler extends Sampler {
@Override
public List<DataObject> getDataObjects(final Mask mask) {
int mandatoryPixelCount = unmaskedPixelsCount(mandatoriesIndexes, mask);
if (mandatoryPixelCount > sizeByCount) {
throw new IndexOutOfBoundsException("The number of mandatory pixels"
......@@ -215,9 +226,13 @@ public class ImageSampler extends Sampler {
}
private int unmaskedPixelsCount(int[] mandatoriesIndexes, Mask mask) {
if (mandatoriesIndexes == null) {
return 0;
}
int count = 0;
for(int i : mandatoriesIndexes) {
if (!mask.isMaksed(i)) {
if (!mask.isMasked(i)) {
count++;
}
}
......@@ -237,6 +252,13 @@ public class ImageSampler extends Sampler {
switch (getSamplingMethod()) {
case Sampler.RANDOM_METHOD_TYPE :
selectedIndexes = RandomizeTools.getReservoirSamplingIndex(getDataSize(), sizeByCount);
// we sort the array to get an optimal access to the buffered image reader
selectedIndexes.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1.compareTo(o2);
}
});
break;
case Sampler.INTERVAL_METHOD_TYPE :
selectedIndexes = new ArrayList<Integer>();
......@@ -287,6 +309,13 @@ public class ImageSampler extends Sampler {
switch (getSamplingMethod()) {
case Sampler.RANDOM_METHOD_TYPE :
selectedIndexes = RandomizeTools.getReservoirSamplingIndex(getDataSize(), sizeByCount, mask);
// we sort the array to get an optimal access to the buffered image reader
selectedIndexes.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1.compareTo(o2);
}
});
break;
case Sampler.INTERVAL_METHOD_TYPE :
selectedIndexes = new ArrayList<Integer>();
......@@ -341,6 +370,13 @@ public class ImageSampler extends Sampler {
switch (getSamplingMethod()) {
case Sampler.RANDOM_METHOD_TYPE :
selectedIndexes = RandomizeTools.getReservoirSamplingIndex(getDataSize(), sizeByCount);
// we sort the array to get an optimal access to the buffered image reader
selectedIndexes.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1.compareTo(o2);
}
});
break;
case Sampler.INTERVAL_METHOD_TYPE :
selectedIndexes = new ArrayList<Integer>();
......@@ -355,29 +391,39 @@ public class ImageSampler extends Sampler {
x = getStartX() + index % imageWidth;
y = getStartY() + index / imageWidth;
result.add(new DataObject(1));
List<double[]> values = new ArrayList<double[]>();
try {
if (geoTranslateVectors != null) {
// compute geo coordinate from first image
double geoX = geoTranslateVectors.get(0)[0] + ((double)x)*geoTranslateVectors.get(0)[2];
double geoY = geoTranslateVectors.get(0)[1] + ((double)y)*geoTranslateVectors.get(0)[3];
double[][] buffer = new double[readers.size()][channelCount];
for(int j= 0 ; j < readers.size() ; j++) {
// reverse compute the image coordinate
int imgX = (int) ((geoX - geoTranslateVectors.get(j)[0]) /geoTranslateVectors.get(j)[2]);
int imgY = (int) ((geoY - geoTranslateVectors.get(j)[1]) /geoTranslateVectors.get(j)[3]);;
buffer[j] = readers.get(j).getPixel(imgX, imgY);
if (maskSequence == null || maskSequence.get(j) == null ||
!maskSequence.get(j).isMasked(imgX+imgY*readers.get(j).getImageWidth())) {
values.add(readers.get(j).getPixel(imgX, imgY));
}
}
double[][] buffer = new double[values.size()][channelCount];
buffer = values.toArray(buffer);
result.get(i).setAttribute(
0,
new AttributeMultiDimSequence(Arrays.copyOf(
buffer, buffer.length)));
} else {
double[][] buffer = new double[readers.size()][channelCount];
for(int j= 0 ; j < readers.size() ; j++) {
buffer[j] = readers.get(j).getPixel(x, y);
if (maskSequence == null || maskSequence.get(j) == null ||
!maskSequence.get(j).isMasked(x+y*readers.get(j).getImageWidth())) {
values.add(readers.get(j).getPixel(x, y));
}
}
double[][] buffer = new double[values.size()][channelCount];
buffer = values.toArray(buffer);
result.get(i).setAttribute(
0,
new AttributeMultiDimSequence(Arrays.copyOf(
......@@ -414,6 +460,13 @@ public class ImageSampler extends Sampler {
switch (getSamplingMethod()) {
case Sampler.RANDOM_METHOD_TYPE :
selectedIndexes = RandomizeTools.getReservoirSamplingIndex(getDataSize(), sizeByCount, mask);
// we sort the array to get an optimal access to the buffered image reader
selectedIndexes.sort(new Comparator<Integer>() {
@Override
public int compare(Integer o1, Integer o2) {
return o1.compareTo(o2);
}
});
break;
case Sampler.INTERVAL_METHOD_TYPE :
selectedIndexes = new ArrayList<Integer>();
......@@ -440,29 +493,40 @@ public class ImageSampler extends Sampler {
x = getStartX() + ((int) index) % imageWidth;
y = getStartY() + ((int) index) / imageWidth;
result.add(new DataObject(1));
List<double[]> values = new ArrayList<double[]>();
try {
if (geoTranslateVectors != null) {
// compute geo coordinate from first image
double geoX = geoTranslateVectors.get(0)[0] + ((double)x)*geoTranslateVectors.get(0)[2];
double geoY = geoTranslateVectors.get(0)[1] + ((double)y)*geoTranslateVectors.get(0)[3];
double[][] buffer = new double[readers.size()][channelCount];
for(int j= 0 ; j < readers.size() ; j++) {
// reverse compute the image coordinate
int imgX = (int) ((geoX - geoTranslateVectors.get(j)[0]) /geoTranslateVectors.get(j)[2]);
int imgY = (int) ((geoY - geoTranslateVectors.get(j)[1]) /geoTranslateVectors.get(j)[3]);;
buffer[j] = readers.get(j).getPixel(imgX, imgY);
if (maskSequence == null || maskSequence.get(j) == null ||
!maskSequence.get(j).isMasked(imgX+imgY*readers.get(j).getImageWidth())) {
values.add(readers.get(j).getPixel(imgX, imgY));
}
}
double[][] buffer = new double[values.size()][channelCount];
buffer = values.toArray(buffer);
result.get(i).setAttribute(
0,
new AttributeMultiDimSequence(Arrays.copyOf(
buffer, buffer.length)));
} else {
double[][] buffer = new double[readers.size()][channelCount];
for(int j= 0 ; j < readers.size() ; j++) {
buffer[j] = readers.get(j).getPixel(x, y);
if (maskSequence == null || maskSequence.get(j) == null ||
!maskSequence.get(j).isMasked(x+y*readers.get(j).getImageWidth())) {
values.add(readers.get(j).getPixel(x, y));
}
}
double[][] buffer = new double[values.size()][channelCount];
buffer = values.toArray(buffer);
result.get(i).setAttribute(
0,
new AttributeMultiDimSequence(Arrays.copyOf(
......@@ -631,21 +695,21 @@ public class ImageSampler extends Sampler {
@Override
protected void updateCount() {
sizeByCount = (int) Math.floor(dataSize * sizeByPercent);
sizeByCount = (int) Math.floor(dataUnmaskedSize * sizeByPercent);
if (sizeByCount > 0) {
offset = dataSize / sizeByCount;
offset = dataUnmaskedSize / sizeByCount;
} else {
offset = dataSize;
offset = dataUnmaskedSize;
}
}
@Override
protected void updatePercentage() {
sizeByPercent = (double) sizeByCount / (double) dataSize;
sizeByPercent = (double) sizeByCount / (double) dataUnmaskedSize;
if (sizeByCount > 0) {
offset = dataSize / sizeByCount;
offset = dataUnmaskedSize / sizeByCount;
} else {
offset = dataSize;
offset = dataUnmaskedSize;
}
}
......@@ -819,6 +883,32 @@ public class ImageSampler extends Sampler {
return getDataObject(index);
}
/**
* Return the mask list used to generate the data
* It does not affect the whole data functions.
*
* This is only use for sequence of images (size bigger then one).
*
* @return the list of Mask
*/
public List<Mask> getMaskSequence() {
return maskSequence;
}
/**
* Set a list of mask to use to generate the data.
* The list must have the same size that the set of images in the sample.
* It does not affect the whole data functions.
*
* This is only use for sequence of images (size bigger then one).
*
* @param maskList
* the list of Mask
*/
public void setMaskSequence(List<Mask> maskList) {
this.maskSequence = maskList;
}
/**
* <p>
* The ImageSampler Iterator for the whole data
......
......@@ -23,12 +23,18 @@ public abstract class Sampler implements MemoryFlush, Cloneable, Serializable {
/** Number of element in the external data */
protected int dataSize;
/** Number of element in the external data that are unmasked */
protected int dataUnmaskedSize;
/** Fix the percent of data to use */
protected double sizeByPercent = -1;
/** Fix a defined number of element to use */
protected int sizeByCount = -1;
/** A mask use when the sample is generated */
protected Mask mask = null;
/** Indicates which methode to use to genreate the sample */
private int samplingMethod = INTERVAL_METHOD_TYPE;
......@@ -80,7 +86,7 @@ public abstract class Sampler implements MemoryFlush, Cloneable, Serializable {
this.sizeByPercent = sizeByPercent;
updateCount();
}
/**
* Return the count of element used for the extraction
*
......@@ -112,7 +118,17 @@ public abstract class Sampler implements MemoryFlush, Cloneable, Serializable {
public int getDataSize() {
return dataSize;
}
/**
* Return the number of elements in source data used for the sampling
* without the masked elements if any
*
* @return the size of the unmasked data
*/