Skip to content

GitLab

M
MultiCube
Commit 81816bb8 authored by lafabregue's avatar lafabregue
Browse files
Options

Modification of test files

parent bae79103
Hide whitespace changes
Inline Side-by-side
Showing with 443 additions and 11 deletions
src/test/TestA2CNESIterative.java
View file @ 81816bb8
......@@ -24,14 +24,11 @@ import jcl.clustering.constraints.Constraint;
import jcl.clustering.constraints.MustLinkConstraint;
import jcl.data.Data;
import jcl.data.Model;
import jcl.data.SimpleData;
import jcl.data.attribute.AttributeMultiDimSequence;
import jcl.data.distance.Distance;
import jcl.data.distance.DistanceParameter;
import jcl.data.distance.EmptyDistanceParameter;
import jcl.data.distance.MetaDistance;
import jcl.data.distance.MetaDistanceEuclidean;
import jcl.data.distance.NumericalEuclideanDistance;
import jcl.data.distance.sequential.ParameterDTW;
import jcl.data.mask.IntArrayMask;
import jcl.data.mask.Mask;
......@@ -52,8 +49,8 @@ public class TestA2CNESIterative {
final String datasetName = "FacesUCR";
final String datasetPath = "FacesUCR";
final String dataPath = "~/A2CNES/";
final String resultPath = "~/A2CNES/results_iter/";
final String dataPath = System.getProperty("user.home")+"/A2CNES/";
final String resultPath = System.getProperty("user.home")+"/A2CNES/results_iter/";
final int nInf = 12;
final int nSup = 17;
final int ag1_seeds = 16;
......@@ -61,7 +58,7 @@ public class TestA2CNESIterative {
final int ag3_seeds = 24;
final int nb_iter = 15;
// String testResultPath = "~/A2CNES/Train_results/";
// String testResultPath = System.getProperty("user.home")+"/A2CNES/Train_results/";
// Data dataTrain = TestA2CNES.getDataFromFile(dataPath+datasetPath+"/train/"+datasetName+".data", '\t', "train", null);
final Data dataTest = TestA2CNES.getDataFromFile(dataPath+datasetPath+"/test/"+datasetName+".data", '\t', "test", null);
......@@ -164,6 +161,7 @@ public class TestA2CNESIterative {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(
Paths.get(dataPath+datasetName+"/train/"), "*0.1_*")) {
Iterator<Path> iter = dirStream.iterator();
final String rand = Integer.toString((int) (Math.random()*1000));
while(iter.hasNext()) {
final Vector<Constraint> constraints = new Vector<Constraint>();
String filename = iter.next().getFileName().toString();
......@@ -220,7 +218,6 @@ public class TestA2CNESIterative {
Thread t = null;
t = new Thread() {
@Override
public void run() {
......@@ -234,7 +231,7 @@ public class TestA2CNESIterative {
FileWriter fw = null;
try {
fw = new FileWriter("log_cst"+classif.getName()+".log", true);
fw = new FileWriter("log/"+rand+"log_cst"+classif.getName()+".log", true);
} catch (IOException e) {
e.printStackTrace();
}
......@@ -242,7 +239,7 @@ public class TestA2CNESIterative {
PrintWriter out = new PrintWriter(bw);
Distance[] distances = new Distance[1]; // a distance is set for every attribute
distances[0] = jcl.data.distance.sequential.DistanceDTW.getInstance(); // second attribute (sequential) compared with the DTW distance
distances[0] = jcl.data.distance.sequential.DistanceDTWMD.getInstance(); // second attribute (sequential) compared with the DTW distance
MetaDistance metaDistance = MetaDistanceEuclidean.getInstance(); // defines the way the two scores are combined (possibility to weight)
Model model = new Model(distances, metaDistance);
......@@ -268,9 +265,9 @@ public class TestA2CNESIterative {
}
Mask satisfactionMask = new IntArrayMask(satisifiedMap, 1, true);
extractAndAddConstraints(subset, constraints, subsetSize, satisfactionMask);
Vector<Constraint> subset = extractConstraints(constraints, subsetSize, satisfactionMask);
out.println("------- new iter : "+ (i+1) + " --------");
for (Constraint c : constraints) {
for (Constraint c : subset) {
if (c instanceof MustLinkConstraint) {
MustLinkConstraint ml = (MustLinkConstraint) c;
out.println(c.toString()+";"+
......
src/test/TestA2CNESIterativeSelectedCst.java 0 → 100644
View file @ 81816bb8
package test;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import jcl.Classification;
import jcl.clustering.constraints.CannotLinkConstraint;
import jcl.clustering.constraints.Constraint;
import jcl.clustering.constraints.MustLinkConstraint;
import jcl.data.Data;
import jcl.data.Model;
import jcl.data.attribute.AttributeMultiDimSequence;
import jcl.data.distance.Distance;
import jcl.data.distance.DistanceParameter;
import jcl.data.distance.MetaDistance;
import jcl.data.distance.MetaDistanceEuclidean;
import jcl.data.distance.sequential.ParameterDTW;
import jcl.data.mask.IntArrayMask;
import jcl.data.mask.Mask;
import jcl.io.results.CSVResultWriter;
import jcl.learning.methods.monostrategy.kmeans.ParametersKmeans;
import jcl.learning.methods.multistrategy.samarah.HybridClassification;
import jcl.learning.methods.multistrategy.samarah.SamarahConfig;
import jcl.utils.RandomizeTools;
import jcl.weights.ClassificationWeights;
import jcl.weights.GlobalWeights;
import multiCube.tools.image.ImageHelper;
import mustic.gui.ClassificationFrame;
import mustic.utils.io.CSVUtils;
public class TestA2CNESIterativeSelectedCst {
public static void main(String[] args) {
HybridClassification classification = new HybridClassification();
final String datasetName = "FacesUCR";
final String datasetPath = "FacesUCR";
final String dataPath = System.getProperty("user.home")+"/A2CNES/";
final String resultPath = System.getProperty("user.home")+"/A2CNES/results_iter/";
final int nInf = 12;
final int nSup = 17;
final int ag1_seeds = 16;
final int ag2_seeds = 20;
final int ag3_seeds = 24;
final int nb_iter = 15;
// String testResultPath = System.getProperty("user.home")+"/A2CNES/Train_results/";
// Data dataTrain = TestA2CNES.getDataFromFile(dataPath+datasetPath+"/train/"+datasetName+".data", '\t', "train", null);
final Data dataTest = TestA2CNES.getDataFromFile(dataPath+datasetPath+"/test/"+datasetName+".data", '\t', "test", null);
// AttributeMultiDimSequence.setMode(AttributeMultiDimSequence.EUCLIDIEN);
AttributeMultiDimSequence.setMode(AttributeMultiDimSequence.DTW_BARYCENTRE);
final DateTime startTime = DateTime.now();
// parametre pour la precision des conflits
final double minC = 0.9;
// parametre pour les classes representatives
final double pcr = 0.2;
// parametre qualite/similitude
final double ps = 0.2;
final double pq = 1.0 - ps;
final double constraintsWgt = 0;
final double valueKExtern = 60;
final double kExtern = valueKExtern * ((100-constraintsWgt)/100);
final double kIntern = (100 - valueKExtern) * ((100-constraintsWgt)/100);
final int solutionType = 3;
final double classRatio = 0.01;
final int criterion = 0;
final double degradation = 0.9;
final int unificationType = SamarahConfig.WITHOUT_UNCLASSIFIED_UNIFICATION;
classification.setParameters(nInf, nSup, minC, ps, pq, pcr);
classification.setAdvancedParameters(degradation, classRatio, solutionType, kIntern,
kExtern, unificationType, criterion, constraintsWgt);
ClassificationWeights weights = new GlobalWeights(dataTest);
final Vector<Thread> threadList = new Vector<Thread>();
final Vector<Classification> classifList = new Vector<Classification>();
// we search for all constraints files
// <<<< START FOR PARAM config
// for (int i = 0 ; i < 10 ; i++) {
// HybridClassification classif = (HybridClassification) classification.clone();
// if (i % 2 == 1 ) {
// classification.setParameters(nInf, nSup, minC, 0.4, 0.6, pcr);
// }
// if (i <= 5)
// classif.addAgent(new ParametersKmeans(8, 25, weights), dataTrain);
// if (i <= 7)
// classif.addAgent(new ParametersKmeans(10, 25, weights), dataTrain);
// classif.addAgent(new ParametersKmeans(13, 25, weights), dataTrain);
// classif.addAgent(new ParametersKmeans(16, 25, weights), dataTrain);
// if (i >= 2)
// classif.addAgent(new ParametersKmeans(19, 25, weights), dataTrain);
// if (i >= 4)
// classif.addAgent(new ParametersKmeans(22, 25, weights), dataTrain);
//// classif.addAgent(new ParametersKmeans(3, 10, weights), dataTest);
//// classif.addAgent(new ParametersKmeans(4, 10, weights), dataTest);
//// classif.addAgent(new ParametersKmeans(6, 10, weights), dataTest);
//
// classif.setName(testResultPath+datasetName+"/clusteing"+i);
// classif.setData(dataTrain);
// >>>> END FOR PARAM config1
// for (int i = 0 ; i < 20 ; i++) {
// HybridClassification classif = (HybridClassification) classification.clone();
// classif.addAgent(new ParametersKmeans(13, 20, weights), dataTest);
// classif.addAgent(new ParametersKmeans(16, 20, weights), dataTest);
// classif.addAgent(new ParametersKmeans(19, 20, weights), dataTest);
//
// classif.setName(resultPath+datasetPath+"/clustering"+i);
// classif.setData(dataTest);
//// >>>> REPLACE PARAM
// Thread t = null;
//
// ClassificationImage classificationImage = new ClassificationImage(testSession,
// Messages.getString("ClassifierPanel.73") + DataSession.nbClustering, //$NON-NLS-1$
// true);
//
//
// t = new ClassificationThread(classif, classificationImage.getProgressBar(),
// classificationImage, null);
// t.start();
// try {
// testSession.associatedFrame.setMaximum(true);
// } catch (PropertyVetoException e1) {}
// testSession.addClassifier(classificationImage);
// try {
// testSession.associatedFrame.setSelected(true);
// testSession.associatedFrame.toFront();
// } catch (PropertyVetoException e) {
// e.printStackTrace();
// }
// classificationImage.setVisible(true);
//
// threadList.add(t);
// classifList.add(classif);
// }
// <<<< CONSTRAINTS
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(
Paths.get(dataPath+datasetName+"/train/"), "*0.1_*")) {
Iterator<Path> iter = dirStream.iterator();
final String rand = Integer.toString((int) (Math.random()*1000));
while(iter.hasNext()) {
final Vector<Constraint> constraintsCL = new Vector<Constraint>();
final Vector<Constraint> constraintsML = new Vector<Constraint>();
final Vector<Double> distancesCL = new Vector<Double>();
final Vector<Double> distancesML = new Vector<Double>();
String filename = iter.next().getFileName().toString();
System.out.println(filename);
CSVUtils reader = null;
try {
reader = new CSVUtils(new File(dataPath+datasetName+"/test/"+filename), '\t');
List<String> line;
while((line = reader.readNext()) != null) {
switch (Integer.parseInt(line.get(line.size()-1))) {
case Constraint.MUST_LINK_TYPE :
try {
int index1 = Integer.parseInt(line.get(0))-1;
int index2 = Integer.parseInt(line.get(1))-1;
MustLinkConstraint ml = new MustLinkConstraint(index1, index2);
double mldist = dataTest.getDataObject(ml.getFirstIndex())
.distance(dataTest.getDataObject(ml.getSecondIndex()));
boolean insert = false;
for(int i = 0 ; i < constraintsML.size() ; i++) {
if (mldist < distancesML.get(i)) {
constraintsML.add(i, ml);
distancesML.add(i, mldist);
insert = true;
break;
}
}
if (!insert) {
constraintsML.add(ml);
distancesML.add(mldist);
}
} catch(Exception e) {
e.printStackTrace();
}
break;
case Constraint.CANNOT_LINK_TYPE :
try {
int index1 = Integer.parseInt(line.get(0))-1;
int index2 = Integer.parseInt(line.get(1))-1;
CannotLinkConstraint cl = new CannotLinkConstraint(index1, index2);
double cldist = dataTest.getDataObject(cl.getFirstIndex())
.distance(dataTest.getDataObject(cl.getSecondIndex()));
boolean insert = false;
for(int i = 0 ; i < constraintsCL.size() ; i++) {
if (cldist > distancesCL.get(i)) {
constraintsCL.add(i, cl);
distancesCL.add(i, cldist);
insert = true;
break;
}
}
if (!insert) {
constraintsCL.add(cl);
distancesCL.add(cldist);
}
} catch(Exception e) {
e.printStackTrace();
}
break;
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
}
final int subsetSize = (int) Math.ceil(constraintsCL.size() * 0.04);
for(int i = 0 ; i < 1; i++) {
final HybridClassification classif = (HybridClassification) classification.clone();
Data currentData = (Data) dataTest.clone();
final Vector<Constraint> subset = new Vector<Constraint>();
// extractAndAddConstraints(subset, constraints, subsetSize, null);
// currentData.updateAndSetConstraintsToSample(subset);
classif.addAgent(new ParametersKmeans(ag1_seeds, nb_iter, weights), currentData);
classif.addAgent(new ParametersKmeans(ag2_seeds, nb_iter, weights), currentData);
classif.addAgent(new ParametersKmeans(ag3_seeds, nb_iter, weights), currentData);
final String path_to_add = resultPath + datasetName + "/";
classif.setName(ImageHelper.stripExtension(filename)+"-"+i+
".clustering");
classif.setData(currentData);
Thread t = null;
t = new Thread() {
@Override
public void run() {
classif.classify();
try {
new CSVResultWriter(classif, path_to_add + classif.getName()).write();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("First classification completed");
FileWriter fw = null;
try {
fw = new FileWriter("log/"+rand+"log_cst"+classif.getName()+".log", true);
} catch (IOException e) {
e.printStackTrace();
}
BufferedWriter bw = new BufferedWriter(fw);
PrintWriter out = new PrintWriter(bw);
Distance[] distances = new Distance[1]; // a distance is set for every attribute
distances[0] = jcl.data.distance.sequential.DistanceDTWMD.getInstance(); // second attribute (sequential) compared with the DTW distance
MetaDistance metaDistance = MetaDistanceEuclidean.getInstance(); // defines the way the two scores are combined (possibility to weight)
Model model = new Model(distances, metaDistance);
int seqLength = ((AttributeMultiDimSequence) dataTest.getOneDataObject().getAttribute(0)).sequence.length;
DistanceParameter[] distanceParameters = new DistanceParameter[1];
distanceParameters[0] = new ParameterDTW(new double[seqLength][seqLength]); //but yes for DTW (requires a matrix to work in)
for (int i = 0 ; i < 5 ; i++) {
int[] clustMap = classif.getClusteringResult().getClusterMap();
int[] satisifiedMapCL = new int[constraintsCL.size()];
int[] satisifiedMapML = new int[constraintsML.size()];
for (int j = 0 ; j < constraintsML.size() ; j++) {
satisifiedMapML[j] = 0;
MustLinkConstraint ml = (MustLinkConstraint) constraintsML.get(j);
if (clustMap[ml.getFirstIndex()] != clustMap[ml.getSecondIndex()])
satisifiedMapML[j] = 1;
}
for (int j = 0 ; j < constraintsCL.size() ; j++) {
satisifiedMapCL[j] = 0;
CannotLinkConstraint cl = (CannotLinkConstraint) constraintsCL.get(j);
if (clustMap[cl.getFirstIndex()] == clustMap[cl.getSecondIndex()])
satisifiedMapCL[j] = 1;
}
Mask satisfactionMaskCL = new IntArrayMask(satisifiedMapCL, 1, true);
Vector<Constraint> subsetCL = extractConstraintsRandom(constraintsCL, subsetSize, satisfactionMaskCL);
Mask satisfactionMaskML = new IntArrayMask(satisifiedMapML, 1, true);
Vector<Constraint> subset = extractConstraintsRandom(constraintsML, subsetSize, satisfactionMaskML);
subset.addAll(constraintsCL);
out.println("------- new iter : "+ (i+1) + " --------");
for (Constraint c : subset) {
if (c instanceof MustLinkConstraint) {
MustLinkConstraint ml = (MustLinkConstraint) c;
out.println(c.toString()+";"+
Constraint.marginalSilhouetteScore(
ml.getFirstIndex(), classif.getClusteringResult(),
model, distanceParameters)+";"+
Constraint.marginalSilhouetteScore(
ml.getSecondIndex(), classif.getClusteringResult(),
model, distanceParameters)
);
} else {
CannotLinkConstraint cl = (CannotLinkConstraint) c;
out.println(c.toString()+";"+
Constraint.marginalSilhouetteScore(
cl.getFirstIndex(), classif.getClusteringResult(),
model, distanceParameters)+";"+
Constraint.marginalSilhouetteScore(
cl.getSecondIndex(), classif.getClusteringResult(),
model, distanceParameters)
);
}
}
classif.setAdvancedParameters(degradation, classRatio, solutionType, kIntern,
kExtern, unificationType, criterion, 95);
classif.newIteration(subset);
try {
new CSVResultWriter(classif, path_to_add + classif.getName()+"_"+(i+1)).write();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Iteration "+(i+1)+" completed");
}
try {
out.close();
bw.close();
fw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
};
t.start();
threadList.add(t);
classifList.add(classif);
}
}
} catch (IOException e2) {
e2.printStackTrace();
}
// >>>> CONSTRAINTS
new Thread() {
@Override
public void run() {
try {
for (Thread thread : threadList) {
thread.join();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
DateTimeFormatter formatter = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss");
System.out.println("Start at "+ formatter.print(startTime));
System.out.println("wrote final results at "+ formatter.print(DateTime.now()));
}
}.start();
ClassificationFrame.getInstance().updateClassifierPanel();
ClassificationFrame.getInstance().dispose();
}
private static void extractAndAddConstraintsRandom(Vector<Constraint> subset, Vector<Constraint> constraints,
int subsetSize, Mask satisfactionMask) {
subset.addAll(extractConstraintsRandom(constraints, subsetSize, satisfactionMask));
}
private static Vector<Constraint> extractConstraintsRandom(Vector<Constraint> constraints, int subsetSize, Mask satisfactionMask) {
Vector<Constraint> result = new Vector<Constraint>();
if (subsetSize > constraints.size()) {
subsetSize = constraints.size();
}
List<Integer> indexes = null;
if (satisfactionMask == null) {
indexes = RandomizeTools.getReservoirSamplingIndex(constraints.size(), subsetSize);
} else {
if (satisfactionMask.getCarinality() < subsetSize) {
subsetSize = satisfactionMask.getCarinality();
}
indexes = RandomizeTools.getReservoirSamplingIndex(constraints.size(), subsetSize, satisfactionMask);
}
for (int i = 0 ; i < indexes.size() ; i++) {
result.add(constraints.get(indexes.get(i)));
}
constraints.removeAll(result);
return result;
}
private static Vector<Constraint> extractConstraintsByOrder(Vector<Constraint> constraints, int subsetSize, Mask satisfactionMask) {
Vector<Constraint> result = new Vector<Constraint>();
if (subsetSize > constraints.size()) {
subsetSize = constraints.size();
}
if (satisfactionMask == null) {
for (int i = 0 ; i < subsetSize ; i++) {
result.add(constraints.get(i));
}
} else {
if (satisfactionMask.getCarinality() < subsetSize) {
subsetSize = satisfactionMask.getCarinality();
}
Iterator<Integer> iter = satisfactionMask.getIncludedIndexes();
for (int i = 0 ; i < subsetSize ; i++) {
result.add(constraints.get(iter.next()));
}
}
constraints.removeAll(result);
return result;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment