Uploading whole of Java Information Dynamics toolkit for the first time.

java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java

@@ -0,0 +1,90 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MeasurementDistribution;
+
+public interface ActiveInfoStorageCalculator {
+
+	public static final String K_PROP_NAME = "k_HISTORY";
+
+	/**
+	 * Initialise the calculator using the existing or default value of k
+	 * 
+	 */
+	public void initialise() throws Exception;
+
+	/**
+	 * Initialise the calculator
+	 * 
+	 * @param k Length of past history to consider
+	 */
+	public void initialise(int k) throws Exception;
+	
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+
+	public void setObservations(double observations[]) throws Exception;
+	
+	/**
+	 * Elect to add in the observations from several disjoint time series.
+	 *
+	 */
+	public void startAddObservations();
+	
+	/**
+	 * Add some more observations.
+	 * Note that the array must not be over-written by the user
+	 *  until after finaliseAddObservations() has been called.
+	 * 
+	 * @param observations
+	 */
+	public void addObservations(double[] observations) throws Exception;
+
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param observations
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[] observations,
+			int startTime, int numTimeSteps) throws Exception ;
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations();
+	
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * valid is a time series (with time indices the same as destination)
+	 *  indicating whether the observation at that point is valid.
+	 * 
+	 * @param source observations for the source variable
+	 * @param destValid
+	 */
+	public void setObservations(double[] observations,
+			boolean[] valid) throws Exception;
+
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalOfPreviousObservations() throws Exception;
+
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
+	
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception;
+
+	public void setDebug(boolean debug);
+	
+	public double getLastAverage();
+
+	public int getNumObservations();
+}

java/source/infodynamics/measures/continuous/ChannelCalculator.java

@@ -0,0 +1,53 @@
+package infodynamics.measures.continuous;
+
+/**
+ * An interface for calculators computing measures from a source to a destination
+ * for single variables in each.
+ * 
+ * 
+ * @author Joseph Lizier, jlizier at gmail.com
+ *
+ */
+public interface ChannelCalculator extends ChannelCalculatorCommon {
+
+	public void initialise() throws Exception;
+
+	public void setObservations(double source[], double destination[]) throws Exception;
+	
+	/**
+	 * Add some more observations.
+	 * Note that the arrays source and destination must not be over-written by the user
+	 *  until after finaliseAddObservations() has been called.
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void addObservations(double[] source, double[] destination) throws Exception;
+
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[] source, double[] destination,
+			int startTime, int numTimeSteps) throws Exception ;
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[] source, double[] destination,
+			boolean[] sourceValid, boolean[] destValid) throws Exception;
+
+}

java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java

@@ -0,0 +1,76 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * An abstract  interface for calculators computing measures from a source to a destination.
+ * 
+ * 
+ * @author Joseph Lizier, jlizier at gmail.com
+ *
+ */
+public abstract interface ChannelCalculatorCommon {
+
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+
+	/**
+	 * Elect to add in the observations from several disjoint time series.
+	 *
+	 */
+	public void startAddObservations();
+	
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations();
+	
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalOfPreviousObservations() throws Exception;
+
+	/**
+	 * <p>Compute the significance of obtaining the given average TE from the given observations.</p>
+	 * 
+	 * <p>This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 * </p>
+	 * 
+	 * <p>Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * </p>
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
+	
+	/**
+	 * <p>As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.</p>
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * 		If the length of each permutation in newOrderings
+	 * 		is not equal to numObservations, an Exception is thrown. 
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception;
+
+	public void setDebug(boolean debug);
+	
+	public double getLastAverage();
+
+	public int getNumObservations();
+
+}

java/source/infodynamics/measures/continuous/ChannelCalculatorMultiVariate.java

@@ -0,0 +1,57 @@
+package infodynamics.measures.continuous;
+
+/**
+ * An interface for calculators computing measures from a source to a destination
+ * for multiple joint variables in each.
+ * 
+ * 
+ * @author Joseph Lizier, jlizier at gmail.com
+ *
+ */
+public interface ChannelCalculatorMultiVariate extends ChannelCalculatorCommon {
+
+	/**
+	 * Initialise the calculator
+	 *
+	 */
+	public void initialise(int destDimensions, int sourceDimensions) throws Exception;
+
+	public void setObservations(double[][] source, double[][] destination) throws Exception;
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception;
+	
+	public void addObservations(double[][] source, double[][] destination,
+			int startTime, int numTimeSteps) throws Exception;
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[][] source, double[][] destination,
+			boolean[] sourceValid, boolean[] destValid) throws Exception;
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid, with one
+	 *  validity supplied for each sub-variable.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[][] source, double[][] destination,
+			boolean[][] sourceValid, boolean[][] destValid) throws Exception;
+
+}

java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariateWithDiscrete.java

@@ -0,0 +1,44 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * A conditional mutual information calculator between a joint set of continuous variables, 
+ *  and a discrete variable, conditioned on another discrete variable. 
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ *
+ */
+public interface ConditionalMutualInfoCalculatorMultiVariateWithDiscrete {
+
+	/**
+	 * 
+	 * 
+	 * @param dimensions the number of joint continuous variables
+	 * @param base the base of the discrete variable
+	 * @param condBase the base of the discrete variable to condition on
+	 * @throws Exception
+	 */
+	public void initialise(int dimensions, int base, int condBase) throws Exception;
+	
+	public void setProperty(String propertyName, String propertyValue);
+	
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations, int[] conditionedObservations) throws Exception;
+
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] contStates,
+			int[] discreteStates, int[] conditionedStates) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception;
+
+	public void setDebug(boolean debug);
+
+	public double getLastAverage();
+	
+	public int getNumObservations();
+
+}

java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSource.java

@@ -0,0 +1,45 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * A conditional mutual information calculator between a joint set of continuous variables, 
+ *  and a discrete variable, conditioned on another continuous variable vector. 
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ *
+ */
+public interface ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSource {
+
+	/**
+	 * 
+	 * 
+	 * @param dimensions the number of joint continuous variables
+	 * @param base the base of the discrete variable
+	 * @param dimensionsCond the number of joint continuous variables
+	 *     to condition on
+	 * @throws Exception
+	 */
+	public void initialise(int dimensions, int base, int dimensionsCond) throws Exception;
+	
+	public void setProperty(String propertyName, String propertyValue);
+	
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations, double[][] conditionedObservations) throws Exception;
+
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] contStates,
+			int[] discreteStates, double[][] conditionedStates) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception;
+
+	public void setDebug(boolean debug);
+
+	public double getLastAverage();
+	
+	public int getNumObservations();
+
+}

java/source/infodynamics/measures/continuous/EntropyCalculator.java

@@ -0,0 +1,25 @@
+package infodynamics.measures.continuous;
+
+public interface EntropyCalculator {
+
+	/**
+	 * Initialise the calculator using the existing or default value of calculator-specific parameters
+	 * 
+	 */
+	public void initialise() throws Exception;
+	
+	public void setObservations(double observations[]);
+	
+	public double computeAverageLocalOfObservations();
+
+	public void setDebug(boolean debug);
+
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+}

java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java

@@ -0,0 +1,27 @@
+package infodynamics.measures.continuous;
+
+/**
+ * Interface for computing entropy for
+ *  a multi-variate values (independent of underlying technique)
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public interface EntropyCalculatorMultiVariate {
+
+	public void initialise(int dimensions);
+	
+	public void setObservations(double observations[][]);
+	
+	public double computeAverageLocalOfObservations();
+	
+	public double[] computeLocalUsingPreviousObservations(double states[][]) throws Exception;
+
+	public double[] computeLocalOfPreviousObservations();
+	
+	public double getLastAverage();
+	
+	public void setDebug(boolean debug);
+	
+}

java/source/infodynamics/measures/continuous/MultiInfoCalculator.java

@@ -0,0 +1,35 @@
+package infodynamics.measures.continuous;
+
+public interface MultiInfoCalculator {
+
+	public void initialise(int dimensions);
+
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+	
+	public void setObservations(double observations[][]) throws Exception;
+	
+	public void startIndividualObservations();
+	
+	public void addObservation(double observation[]);
+	
+	public void endIndividualObservations() throws Exception;
+
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalOfPreviousObservations()
+		throws Exception;
+	
+	public double[] computeLocalUsingPreviousObservations(double states[][])
+		throws Exception;
+
+	public void setDebug(boolean debug);
+	
+	public double getLastAverage();
+}

java/source/infodynamics/measures/continuous/MultiInfoCalculatorWithMarginals.java

@@ -0,0 +1,49 @@
+package infodynamics.measures.continuous;
+
+/**
+ * Adds methods for computing average and local marginal entropies, joint entropy, 
+ *  and info distance.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public interface MultiInfoCalculatorWithMarginals {
+
+	public double computeAverageJointEntropy();
+
+	public double computeAverageMarginalEntropy(int variableIndex);
+
+	/**
+	 * I'm not sure whether info distance is defined properly for multi-info in addition
+	 *  to mutual info - I should check this.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageInfoDistanceOfObservations();
+
+	public double[] computeLocalJointEntropyOfPreviousObservations()
+		throws Exception;
+
+	public double[] computeLocalJointEntropyUsingPreviousObservations(double states[][])
+		throws Exception;
+
+	public double[] computeLocalMarginalEntropyOfPreviousObservations(int variableIndex);
+
+	public double[] computeLocalMarginalEntropyUsingPreviousObservations(double states[][], int variableIndex)
+		throws Exception;
+
+	/**
+	 * I'm not sure whether info distance is defined properly for multi-info in addition
+	 *  to mutual info - I should check this.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalInfoDistanceOfPreviousObservations()
+		throws Exception;
+
+	public double[] computeLocalInfoDistanceUsingPreviousObservations(double states[][])
+		throws Exception;
+
+}

java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariate.java

@@ -0,0 +1,22 @@
+package infodynamics.measures.continuous;
+
+public interface MutualInfoCalculatorMultiVariate extends ChannelCalculatorMultiVariate {
+
+	/**
+	 * Time difference between data1 and data2 - assumed to be >= 0
+	 */ 
+	public static final String PROP_TIME_DIFF = "TIME_DIFF";
+
+	/**
+	 * Compute the mutual information if the second variable were ordered as per the ordering
+	 *  specified in newOrdering
+	 * 
+	 * @param newOrdering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] newOrdering) throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double states1[][], double states2[][])
+		throws Exception;
+}

java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariateWithDiscrete.java

@@ -0,0 +1,28 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MeasurementDistribution;
+
+public interface MutualInfoCalculatorMultiVariateWithDiscrete {
+
+	public void initialise(int dimensions, int base) throws Exception;
+	
+	public void setProperty(String propertyName, String propertyValue);
+	
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations) throws Exception;
+
+	public double computeAverageLocalOfObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] contStates, int[] discreteStates) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
+
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception;
+
+	public void setDebug(boolean debug);
+
+	public double getLastAverage();
+	
+	public int getNumObservations();
+
+}

java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariateWithMarginals.java

@@ -0,0 +1,36 @@
+package infodynamics.measures.continuous;
+
+public interface MutualInfoCalculatorMultiVariateWithMarginals
+	extends MutualInfoCalculatorMultiVariate {
+
+	public double computeAverageJointEntropy();
+
+	public double computeAverageEntropyOfObservation1();
+
+	public double computeAverageEntropyOfObservation2();
+
+	public double computeAverageInfoDistanceOfObservations();
+
+	public double[] computeLocalJointEntropyOfPreviousObservations()
+		throws Exception;
+
+	public double[] computeLocalJointEntropyUsingPreviousObservations(double states1[][], double states2[][])
+		throws Exception;
+
+	public double[] computeLocalEntropy1OfPreviousObservations();
+
+	public double[] computeLocalEntropy1UsingPreviousObservations(double states[][])
+		throws Exception;
+
+	public double[] computeLocalEntropy2OfPreviousObservations();
+
+	public double[] computeLocalEntropy2UsingPreviousObservations(double states[][])
+		throws Exception;
+
+	public double[] computeLocalInfoDistanceOfPreviousObservations()
+		throws Exception;
+
+	public double[] computeLocalInfoDistanceUsingPreviousObservations(double states1[][], double states2[][])
+		throws Exception;
+
+}

java/source/infodynamics/measures/continuous/TransferEntropyCalculator.java

@@ -0,0 +1,9 @@
+package infodynamics.measures.continuous;
+
+public interface TransferEntropyCalculator extends ChannelCalculator {
+
+	public static final String K_PROP_NAME = "k_HISTORY";
+	
+	public void initialise(int k) throws Exception;
+	
+}

java/source/infodynamics/measures/continuous/TransferEntropyCalculatorMultiVariate.java

@@ -0,0 +1,16 @@
+package infodynamics.measures.continuous;
+
+/**
+ * <p>
+ * Interface to define computers for transfer entropy between a multi-variate source
+ * and destination.
+ * </p>
+ * 
+ * @author Joseph Lizier; joseph.lizier at gmail.com
+ *
+ */
+public interface TransferEntropyCalculatorMultiVariate extends ChannelCalculatorMultiVariate {
+
+	public void initialise(int k, int destDimensions, int sourceDimensions) throws Exception;
+	
+}

java/source/infodynamics/measures/continuous/TransferEntropyCommon.java

@@ -0,0 +1,289 @@
+package infodynamics.measures.continuous;
+
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+
+
+/**
+ * <p>Base class for implementations of the transfer entropy,
+ * e.g. kernel estimation, Kraskov style extensions.
+ * It implements some common code to be used across transfer entropy calculators
+ * </p>
+ * 
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ *
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ */
+public abstract class TransferEntropyCommon implements
+		TransferEntropyCalculator {
+
+	/**
+	 * Length of past history to consider
+	 */
+	protected int k;
+	protected int totalObservations = 0;
+	protected boolean debug = false;
+	protected double lastAverage;
+
+	/**
+	 * Storage for source observations for addObservsations
+	 */
+	protected Vector<double[]> vectorOfSourceObservations;
+	/**
+	 * Storage for destination observations for addObservsations
+	 */
+	protected Vector<double[]> vectorOfDestinationObservations;
+	
+	protected boolean addedMoreThanOneObservationSet;
+
+	/**
+	 * Initialise the calculator, and call initialise() to complete
+	 * 
+	 * @param k Length of past history to consider
+	 */
+	public void initialise(int k) throws Exception {
+		this.k = k;
+		addedMoreThanOneObservationSet = false;
+		initialise();
+	}
+
+	/**
+	 * Set the observations to compute the probabilities from 
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void setObservations(double[] source, double[] destination) throws Exception {
+		startAddObservations();
+		addObservations(source, destination);
+		finaliseAddObservations();
+	}
+
+	/**
+	 * Elect to add in the observations from several disjoint time series.
+	 *
+	 */
+	public void startAddObservations() {
+		vectorOfSourceObservations = new Vector<double[]>();
+		vectorOfDestinationObservations = new Vector<double[]>();
+	}
+	
+	/**
+	 * Add some more observations.
+	 * Note that the arrays source and destination must not be over-written by the user
+	 *  until after finaliseAddObservations() has been called.
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void addObservations(double[] source, double[] destination) throws Exception {
+		if (vectorOfSourceObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if (source.length != destination.length) {
+			throw new Exception(String.format("Source and destination lengths (%d and %d) must match!",
+					source.length, destination.length));
+		}
+		if (source.length <= k) {
+			// we won't be taking any observations here
+			return;
+		}
+		vectorOfSourceObservations.add(source);
+		vectorOfDestinationObservations.add(destination);
+	}
+
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[] source, double[] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		if (vectorOfSourceObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if (numTimeSteps <= k) {
+			// We won't be taking any observations here
+			return;
+		}
+		double[] sourceToAdd = new double[numTimeSteps];
+		System.arraycopy(source, startTime, sourceToAdd, 0, numTimeSteps);
+		vectorOfSourceObservations.add(sourceToAdd);
+		double[] destToAdd = new double[numTimeSteps];
+		System.arraycopy(destination, startTime, destToAdd, 0, numTimeSteps);
+		vectorOfDestinationObservations.add(destToAdd);
+	}
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[] source, double[] destination,
+			boolean[] sourceValid, boolean[] destValid) throws Exception {
+		
+		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+		
+		// We've found the set of start and end times for this pair
+		startAddObservations();
+		for (int[] timePair : startAndEndTimePairs) {
+			int startTime = timePair[0];
+			int endTime = timePair[1];
+			addObservations(source, destination, startTime, endTime - startTime + 1);
+		}
+		finaliseAddObservations();
+	}
+
+	/**
+	 * Compute a vector of start and end pairs of time points, between which we have
+	 *  valid series of both source and destinations.
+	 * 
+	 * Made public so it can be used if one wants to compute the number of
+	 *  observations prior to setting the observations.
+	 * 
+	 * @param sourceValid
+	 * @param destValid
+	 * @return
+	 */
+	public Vector<int[]> computeStartAndEndTimePairs(boolean[] sourceValid, boolean[] destValid) {
+		// Scan along the data avoiding invalid values
+		int startTime = 0;
+		int endTime = 0;
+		boolean lookingForStart = true;
+		Vector<int[]> startAndEndTimePairs = new Vector<int[]>();
+		for (int t = 0; t < destValid.length; t++) {
+			if (lookingForStart) {
+				// Precondition: startTime holds a candidate start time
+				if (destValid[t]) {
+					// This point is OK at the destination
+					if (t - startTime < k) {
+						// We're still checking the past history only, so
+						continue;
+					} else {
+						// We've got the full past history ok, so check the source also
+						if (sourceValid[t - 1]) {
+							// source is good to go also
+							// set a candidate endTime
+							endTime = t;
+							lookingForStart = false;
+							if (t == destValid.length - 1) {
+								// we need to terminate now
+								int[] timePair = new int[2];
+								timePair[0] = startTime;
+								timePair[1] = endTime;
+								startAndEndTimePairs.add(timePair);
+								// System.out.printf("t_s=%d, t_e=%d\n", startTime, endTime);
+							}
+						} else {
+							// source was not good to go, so try moving along one time point
+							startTime++;
+						}
+					}
+				} else {
+					// We need to keep looking.
+					// Move the potential start time to the next point
+					startTime = t + 1;
+				}
+			} else {
+				// Precondition: startTime holds the start time for this set, 
+				//  endTime holds a candidate end time
+				// Check if we can include the current time step
+				boolean terminateSequence = false;
+				if (destValid[t] && sourceValid[t - 1]) {
+					// We can extend
+					endTime = t;
+				} else {
+					terminateSequence = true;
+				}
+				if (t == destValid.length - 1) {
+					// we need to terminate the sequence anyway
+					terminateSequence = true;
+				}
+				if (terminateSequence) {
+					// This section is done
+					int[] timePair = new int[2];
+					timePair[0] = startTime;
+					timePair[1] = endTime;
+					startAndEndTimePairs.add(timePair);
+					// System.out.printf("t_s=%d, t_e=%d\n", startTime, endTime);
+					lookingForStart = true;
+					startTime = t + 1;
+				}
+			}
+		}
+		return startAndEndTimePairs;
+	}
+	
+	/**
+	 * Generate a vector for each time step, containing the past k states of the destination.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return array of vectors for each time step
+	 */
+	protected double[][] makeJointVectorForPast(double[] destination) {
+		try {
+			// We want one less delay vector here - we don't need the last k point,
+			//  because there is no next state for these.
+			return MatrixUtils.makeDelayEmbeddingVector(destination, k, k-1, destination.length - k);
+		} catch (Exception e) {
+			// The parameters for the above call should be fine, so we don't expect to
+			//  throw an Exception here - embed in a RuntimeException if it occurs 
+			throw new RuntimeException(e);
+		}
+	}
+	
+	/**
+	 * Generate a vector for each time step, containing the past k states of
+	 *  the destination, and the current state.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return
+	 */
+	protected double[][] makeJointVectorForNextPast(double[] destination) {
+		// We want all delay vectors here
+		return MatrixUtils.makeDelayEmbeddingVector(destination, k+1);
+	}
+	
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(K_PROP_NAME)) {
+			k = Integer.parseInt(propertyValue);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
+	public int getNumObservations() {
+		return totalObservations;
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/ActiveInfoStorageCalculatorCorrelationIntegrals.java

@@ -0,0 +1,278 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.ActiveInfoStorageCalculator;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+
+
+public abstract class ActiveInfoStorageCalculatorCorrelationIntegrals
+	implements ActiveInfoStorageCalculator {
+
+	/**
+	 * Length of past history to consider
+	 */
+	protected int k = 1;
+	protected int totalObservations = 0;
+	protected boolean debug = false;
+	protected double lastAverage;
+	
+	/**
+	 * Storage for source observations for addObservsations
+	 */
+	protected Vector<double[]> vectorOfObservations;
+	
+	protected boolean addedMoreThanOneObservationSet;
+
+	public ActiveInfoStorageCalculatorCorrelationIntegrals() {
+	}
+
+	/**
+	 * Initialise the calculator using the existing value of k
+	 * 
+	 */
+	public void initialise() throws Exception {
+		initialise(k);
+	}
+
+	/**
+	 * Initialise the calculator
+	 * 
+	 * @param k Length of past history to consider
+	 */
+	public void initialise(int k) throws Exception {
+		this.k = k;
+		addedMoreThanOneObservationSet = false;
+	}
+	
+	/**
+	 * Set the observations to compute the probabilities from 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double[] observations) throws Exception {
+		startAddObservations();
+		addObservations(observations);
+		finaliseAddObservations();
+	}
+
+	/**
+	 * Elect to add in the observations from several disjoint time series.
+	 *
+	 */
+	public void startAddObservations() {
+		vectorOfObservations = new Vector<double[]>();
+	}
+	
+	/**
+	 * Add some more observations.
+	 * Note that the arrays must not be over-written by the user
+	 *  until after finaliseAddObservations() has been called.
+	 * 
+	 * @param observations
+	 */
+	public void addObservations(double[] observations) throws Exception {
+		if (vectorOfObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if (observations.length <= k) {
+			// we won't be taking any observations here
+			return;
+		}
+		vectorOfObservations.add(observations);
+	}
+
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param observations
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[] observations,
+			int startTime, int numTimeSteps) throws Exception {
+		if (vectorOfObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if (numTimeSteps <= k) {
+			// We won't be taking any observations here
+			return;
+		}
+		double[] obsToAdd = new double[numTimeSteps];
+		System.arraycopy(observations, startTime, obsToAdd, 0, numTimeSteps);
+		vectorOfObservations.add(obsToAdd);
+	}
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * valid is a time series (with time indices the same as observations)
+	 *  indicating whether the observation at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param observation observations for the source variable
+	 * @param valid
+	 */
+	public void setObservations(double[] observations,
+			boolean[] valid) throws Exception {
+		
+		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(valid);
+		
+		// We've found the set of start and end times for this pair
+		startAddObservations();
+		for (int[] timePair : startAndEndTimePairs) {
+			int startTime = timePair[0];
+			int endTime = timePair[1];
+			addObservations(observations, startTime, endTime - startTime + 1);
+		}
+		finaliseAddObservations();
+	}
+
+	/**
+	 * Compute a vector of start and end pairs of time points, between which we have
+	 *  valid series of the observations.
+	 * 
+	 * Made public so it can be used if one wants to compute the number of
+	 *  observations prior to setting the observations.
+	 * 
+	 * @param valid
+	 * @return
+	 */
+	public Vector<int[]> computeStartAndEndTimePairs(boolean[] valid) {
+		// Scan along the data avoiding invalid values
+		int startTime = 0;
+		int endTime = 0;
+		boolean lookingForStart = true;
+		Vector<int[]> startAndEndTimePairs = new Vector<int[]>();
+		for (int t = 0; t < valid.length; t++) {
+			if (lookingForStart) {
+				// Precondition: startTime holds a candidate start time
+				if (valid[t]) {
+					// This point is OK at the destination
+					if (t - startTime < k) {
+						// We're still checking the past history only, so
+						continue;
+					} else {
+						// We've got the full past history ok
+						// set a candidate endTime
+						endTime = t;
+						lookingForStart = false;
+						if (t == valid.length - 1) {
+							// we need to terminate now
+							int[] timePair = new int[2];
+							timePair[0] = startTime;
+							timePair[1] = endTime;
+							startAndEndTimePairs.add(timePair);
+							// System.out.printf("t_s=%d, t_e=%d\n", startTime, endTime);
+						}
+					}
+				} else {
+					// We need to keep looking.
+					// Move the potential start time to the next point
+					startTime = t + 1;
+				}
+			} else {
+				// Precondition: startTime holds the start time for this set, 
+				//  endTime holds a candidate end time
+				// Check if we can include the current time step
+				boolean terminateSequence = false;
+				if (valid[t]) {
+					// We can extend
+					endTime = t;
+				} else {
+					terminateSequence = true;
+				}
+				if (t == valid.length - 1) {
+					// we need to terminate the sequence anyway
+					terminateSequence = true;
+				}
+				if (terminateSequence) {
+					// This section is done
+					int[] timePair = new int[2];
+					timePair[0] = startTime;
+					timePair[1] = endTime;
+					startAndEndTimePairs.add(timePair);
+					// System.out.printf("t_s=%d, t_e=%d\n", startTime, endTime);
+					lookingForStart = true;
+					startTime = t + 1;
+				}
+			}
+		}
+		return startAndEndTimePairs;
+	}
+	
+	/**
+	 * Generate a vector for each time step, containing the past k states of the destination.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return array of vectors for each time step
+	 */
+	protected double[][] makeJointVectorForPast(double[] destination) {
+		try {
+			// We want one less delay vector here - we don't need the last k point,
+			//  because there is no next state for these.
+			return MatrixUtils.makeDelayEmbeddingVector(destination, k, k-1, destination.length - k);
+		} catch (Exception e) {
+			// The parameters for the above call should be fine, so we don't expect to
+			//  throw an Exception here - embed in a RuntimeException if it occurs 
+			throw new RuntimeException(e);
+		}
+	}
+	
+	/**
+	 * Compute the next values into a vector
+	 * 
+	 * @param destination
+	 * @return
+	 */
+	protected double[][] makeJointVectorForNext(double[] destination) {
+		double[][] destNextVectors = new double[destination.length - k][1];
+		for (int t = k; t < destination.length; t++) {
+			destNextVectors[t - k][0] = destination[t];
+		}
+		return destNextVectors;
+	}
+	
+	/**
+	 * Generate a vector for each time step, containing the past k states of
+	 *  the destination, and the current state.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return
+	 */
+	protected double[][] makeJointVectorForNextPast(double[] destination) {
+		// We want all delay vectors here
+		return MatrixUtils.makeDelayEmbeddingVector(destination, k+1);
+	}
+	
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
+	public int getNumObservations() {
+		return totalObservations;
+	}
+	
+	public void setProperty(String propertyName, String propertyValue) {
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(K_PROP_NAME)) {
+			k = Integer.parseInt(propertyValue);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/ActiveInfoStorageCalculatorKernel.java

@@ -0,0 +1,235 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.ActiveInfoStorageCalculator;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * 
+ * <p>
+ * Implements an active information storage calculator using kernel estimation.
+ * </p> 
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>
+ * TODO Use only a single kernel estimator class for the joint space, and compute other
+ *  probabilities from this. This will save much time.
+ * </p>
+ * 
+ * <p>
+ * Matched against Oliver Obst's c++ implementation on 23/4/2010.
+ * </p>
+ * 
+ * @author Joseph Lizier
+ * @see ActiveInfoStorageCalculator
+ * @see ActiveInfoStorageCalculatorCorrelationIntegrals
+ *
+ */
+public class ActiveInfoStorageCalculatorKernel
+	extends ActiveInfoStorageCalculatorCorrelationIntegrals {
+
+	protected MutualInfoCalculatorMultiVariateKernel miKernel = null;
+
+	// Keep joint vectors so we don't need to regenerate them
+	protected double[][] destNextVectors;
+	protected double[][] destPastVectors;
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public ActiveInfoStorageCalculatorKernel() {
+		super();
+		miKernel = new MutualInfoCalculatorMultiVariateKernel();
+	}
+
+	/**
+	 * Initialises the calculator with the existing values for k and epsilon
+	 * 
+	 */
+	public void initialise() throws Exception {
+		initialise(k, epsilon);
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		this.k = k;
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		super.initialise(k);
+		this.epsilon = epsilon;
+		miKernel.initialise(k, 1, epsilon);
+		destPastVectors = null;
+		destNextVectors = null;
+	}
+
+	/**
+	 * Set properties for the calculator.
+	 * Can include any of the accepted values for
+	 * {@link MutualInfoCalculatorMultiVariateKernel#setProperty(String, String)}
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		super.setProperty(propertyName, propertyValue);
+		if (propertyName.equalsIgnoreCase(MutualInfoCalculatorMultiVariateKernel.EPSILON_PROP_NAME)) {
+			// Grab epsilon here - we need it for the initialisation:
+			epsilon = Double.parseDouble(propertyValue);
+			// It will get passed onto miKernel in our initialisation routine
+			if (debug) {
+				System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+			}
+		} else {
+			// Pass any other property through to the miKernel object
+			miKernel.setProperty(propertyName, propertyValue);
+		}
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] currentObservation : vectorOfObservations) {
+			totalObservations += currentObservation.length - k;
+		}
+		destPastVectors = new double[totalObservations][k];
+		destNextVectors = new double[totalObservations][1];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		for (double[] currentObservation : vectorOfObservations) {
+			double[][] currentDestPastVectors = makeJointVectorForPast(currentObservation);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					destPastVectors, startObservation, 0, currentDestPastVectors.length, k);
+			double[][] currentDestNextVectors = makeJointVectorForNext(currentObservation);
+			MatrixUtils.arrayCopy(currentDestNextVectors, 0, 0,
+					destNextVectors, startObservation, 0, currentDestNextVectors.length, 1);
+			startObservation += currentObservation.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimator
+		try {
+			miKernel.setObservations(destPastVectors, destNextVectors);
+		} catch (Exception e) {
+			// Should only throw where our vector sizes don't match
+			throw new RuntimeException(e);
+		}
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfObservations.size() > 1;
+		
+		// And clear the vector of observations
+		vectorOfObservations = null;
+	}
+	
+	/**
+	 * <p>Computes the average Active Info Storage for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		lastAverage = miKernel.computeAverageLocalOfObservations();
+		return lastAverage;
+	}
+
+	/**
+	 * Computes the local active information storage for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		double[] local = miKernel.computeLocalOfPreviousObservations();
+		lastAverage = miKernel.getLastAverage();
+		if (!addedMoreThanOneObservationSet) {
+			double[] localsToReturn = new double[local.length + k];
+			System.arraycopy(local, 0, localsToReturn, k, local.length);
+			return localsToReturn;
+		} else {
+			return local;
+		}
+	}
+
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *
+	 * Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		return miKernel.computeSignificance(numPermutationsToCheck);
+	}
+	
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 *  
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		
+		return miKernel.computeSignificance(newOrderings);
+	}
+
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		miKernel.setDebug(debug);
+	}
+}

java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java

@@ -0,0 +1,109 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.EntropyCalculator;
+
+public class EntropyCalculatorKernel implements EntropyCalculator {
+
+	protected KernelEstimatorSingleVariate svke = null;
+	protected int totalObservations = 0;
+	protected boolean debug = false;
+	protected double[] observations;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	public EntropyCalculatorKernel() {
+		svke = new KernelEstimatorSingleVariate();
+		svke.setDebug(debug);
+		svke.setNormalise(normalise);
+	}
+
+	public void initialise() {
+		initialise(epsilon);
+	}
+
+	public void initialise(double epsilon) {
+		this.epsilon = epsilon;
+		svke.initialise(epsilon);
+	}
+
+	/**
+	 * Set the observations for the PDFs.
+	 * Should only be called once, the last call contains the
+	 *  observations that are used (they are not accumulated). 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double observations[]) {
+		this.observations = observations;
+		svke.setObservations(observations);
+		totalObservations = observations.length;
+	}
+	
+	public double computeAverageLocalOfObservations() {
+		double entropy = 0.0;
+		for (int t = 0; t < observations.length; t++) {
+			double prob = svke.getProbability(observations[t]);
+			double cont = Math.log(prob);
+			entropy -= cont;
+			if (debug) {
+				System.out.println(t + ": p(" + observations[t] + ")= " +
+						prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " +
+						(entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+		if (svke != null) {
+			svke.setDebug(debug);
+		}
+	}
+	
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * These can include:
+	 * <ul>
+	 * 		<li>{@link #EPSILON_PROP_NAME}</li>
+	 * 		<li>{@link #NORMALISE_PROP_NAME}</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		boolean propertySet = true;
+
+		// TODO If we implement a dynamic correlation exclusion property,
+		//  then we will need to call getProbability(double, int) instead of
+		//  just getProbability(double) above.
+		
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			svke.setNormalise(normalise);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+}

java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java

@@ -0,0 +1,146 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
+
+/**
+ * Class to compute entropy for
+ *  a multi-variate values, using kernel estimates.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMultiVariate {
+
+	private KernelEstimatorMultiVariate mvke = null;
+	private int totalObservations = 0;
+	// private int dimensions = 0;
+	private boolean debug = false;
+	private double[][] observations = null;
+	private double lastEntropy;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	private static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+
+	public EntropyCalculatorMultiVariateKernel() {
+		mvke = new KernelEstimatorMultiVariate();
+		mvke.setDebug(debug);
+		mvke.setNormalise(normalise);
+		lastEntropy = 0.0;
+	}
+
+	/**
+	 * Initialises with the default value for epsilon
+	 */
+	public void initialise(int dimensions) {
+		initialise(dimensions, epsilon);
+	}
+
+	public void initialise(int dimensions, double epsilon) {
+		this.epsilon = epsilon;
+		mvke.initialise(dimensions, epsilon);
+		// this.dimensions = dimensions;
+		lastEntropy = 0.0;
+	}
+
+	/**
+	 * Set the observations for the PDFs.
+	 * Should only be called once, the last call contains the
+	 *  observations that are used (they are not accumulated). 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double observations[][]) {
+		mvke.setObservations(observations);
+		totalObservations = observations.length;
+		this.observations = observations;
+	}
+
+	public double computeAverageLocalOfObservations() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvke.getProbability(observations[b]);
+			double cont = Math.log(prob);
+			entropy -= cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + (-cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		lastEntropy = entropy / (double) totalObservations / Math.log(2.0);
+		return lastEntropy;
+	}
+
+	
+	public double[] computeLocalOfPreviousObservations() {
+		return computeLocalUsingPreviousObservations(observations);
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double states[][]) {
+		double entropy = 0.0;
+		double[] localEntropy = new double[states.length];
+		for (int b = 0; b < states.length; b++) {
+			double prob = mvke.getProbability(states[b]);
+			double cont = -Math.log(prob);
+			localEntropy[b] = cont;
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		entropy /= (double) totalObservations / Math.log(2.0);
+		return localEntropy;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+		mvke.setDebug(debug);
+	}
+
+	public double getLastAverage() {
+		return lastEntropy;
+	}
+	
+	/**
+	 * Allows the user to set properties for the underlying calculator implementation
+	 * These can include:
+	 * <ul>
+	 * 		<li>{@link #EPSILON_PROP_NAME}</li>
+	 * 		<li>{@link #NORMALISE_PROP_NAME}</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		boolean propertySet = true;
+
+		// TODO If we implement a dynamic correlation exclusion property,
+		//  then we will need to call getProbability(double, int) instead of
+		//  just getProbability(double) above.
+		
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			mvke.setNormalise(normalise);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+}

java/source/infodynamics/measures/continuous/kernel/KernelCount.java

@@ -0,0 +1,24 @@
+package infodynamics.measures.continuous.kernel;
+
+/**
+ * Structure to hold a kernel count, the total count it was taken from,
+ *  and an array of booleans indicating which time points got counted
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class KernelCount {
+	public int count;
+	public int totalObservationsForCount;
+	public boolean[] isCorrelatedWithArgument;
+	
+	public KernelCount(int count, int totalObservationsForCount, boolean[] isCorrelatedWithArgument) {
+		this.count = count;
+		this.totalObservationsForCount = totalObservationsForCount;
+		this.isCorrelatedWithArgument = isCorrelatedWithArgument;
+	}
+	
+	public KernelCount(int count, int totalObservationsForCount) {
+		this(count, totalObservationsForCount, null);
+	}
+}

java/source/infodynamics/measures/continuous/kernel/KernelEstimatorMultiVariate.java

@@ -0,0 +1,981 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Arrays;
+import java.util.Vector;
+import java.util.Hashtable;
+
+
+/**
+ * Class to maintain probability distribution function for
+ *  a single variable, using kernel estimates.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class KernelEstimatorMultiVariate {
+
+	protected double[] epsilon = null;
+	protected double[] epsilonInUse = null;
+	protected int dimensions = 1;
+	private int[] bins = null;
+	private boolean usingIntegerIndexBins = true;
+	private boolean useBins = true;
+	private double[] mins = null;
+	private int[] multipliers = null;
+	private int totalObservations = 0;
+	private Vector<TimeStampedObservation>[] observations = null;
+	private Hashtable<IntArray, Vector<TimeStampedObservation>> observationsByHash = null;
+	protected double[][] rawData = null;
+	private boolean debug;
+	
+	// Overriding classes should set this to true if they want a callback for
+	//  every point that is found to be correlated in time;
+	protected boolean makeCorrelatedPointAddedCallback = false;
+	
+	protected boolean normalise = true;
+	
+	private boolean excludeDynamicCorrelations = false;
+	private int timeProximityForDynamicCorrelationExclusion = 100;
+	
+	// Force the kernel estimator to compare each point to every other rather
+	//  than use fast bin techniques
+	private boolean forceCompareToAll = false;
+	
+	private final static int MAX_NUMBER_INTEGER_BINS = 1000000;
+	
+	/**
+	 * 
+	 * Private class to store a time-stamped data point.
+	 * This allows us to eliminate dynamic correlations later.
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	private class TimeStampedObservation {
+		public int timeStep;
+		public double[] observation;
+		
+		TimeStampedObservation(int time, double[] data) {
+			timeStep = time;
+			observation = data;
+		}
+	}
+	
+	/**
+	 * Wrapper class for an integer array so we can hash it properly
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	private class IntArray {
+		public int[] array;
+		
+		public IntArray(int[] array) {
+			this.array = array;
+		}
+
+		@Override
+		public int hashCode() {
+			return Arrays.hashCode(array);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (!IntArray.class.isInstance(obj)) {
+				return false;
+			}
+			IntArray other = (IntArray) obj;
+			return Arrays.equals(array, other.array);
+		}
+	}
+	
+	/**
+	 * Empty constructor
+	 *
+	 */
+	public KernelEstimatorMultiVariate() {
+	}
+
+	/**
+	 * Initialise the estimator before passing any observations in.
+	 *
+	 * @param epsilon
+	 * @param dimensions
+	 */
+	public void initialise(int dimensions, double epsilon) {
+		this.dimensions = dimensions;
+		this.epsilon = new double[dimensions];
+		for (int d = 0; d < dimensions; d++) {
+			this.epsilon[d] = epsilon;
+		}
+		finishInitialisation();
+	}
+
+	/**
+	 * Initialise the estimator before passing any observations in.
+	 *
+	 * @param epsilon
+	 */
+	public void initialise(double[] epsilon) {
+		dimensions = epsilon.length;
+		this.epsilon = new double[dimensions];
+		for (int d = 0; d < dimensions; d++) {
+			this.epsilon[d] = epsilon[d];
+		}
+		finishInitialisation();
+	}
+	
+	/**
+	 * Private method to provide common variable initialisation
+	 *
+	 */
+	private void finishInitialisation() {
+		observations = null;
+		observationsByHash = null;
+		bins = null;
+		useBins = true;
+		usingIntegerIndexBins = true;
+		mins = null;
+		multipliers = null;
+		rawData = null;
+		totalObservations = 0;
+		epsilonInUse = new double[dimensions];
+	}
+
+	/**
+	 * Each row of the data is an observation; each column of
+	 *  the row is a new variable in the multivariate observation.
+	 * 
+	 * @param data
+	 */
+	public void setObservations(double[][] data) {
+		totalObservations = data.length;
+		bins = new int[dimensions];
+		mins = new double[dimensions];
+		multipliers = new int[dimensions];
+		int multiDimBins = 1;
+		// If we are forcing n^2 comparisons, don't even consider using integer index bins.
+		// Otherwise, we will assume we're doing this unless we find there would be too 
+		//  many bins:
+		usingIntegerIndexBins = !forceCompareToAll;
+		useBins = true;
+		// Work out the max and min for each bin
+		for (int d = 0; d < dimensions; d++) {
+			mins[d] = MatrixUtils.min(data, d);
+			double max = MatrixUtils.max(data, d);
+			double std = 0.0;
+			if (normalise) {
+				// Compute what the epsilonInUse should be here:
+				//  it should expand with the standard deviation.
+				// This saves us from normalising all of the incoming data points!
+				std = MatrixUtils.stdDev(data, d);
+				epsilonInUse[d] = epsilon[d] * std;
+			} else {
+				epsilonInUse[d] = epsilon[d];
+			}
+			bins[d] = (int) Math.ceil((max - mins[d]) / epsilonInUse[d]);
+			if (bins[d] == 0) {
+				// This means the min and max are exactly the same:
+				//  for our purposes this is akin to requiring one bin here.
+				// If we leave it as zero bins, this will screw up
+				//  our calculation of the number of multidimensional bins
+				//  and cause errors later.
+				bins[d] = 1;
+			}
+			multipliers[d] = multiDimBins;
+			multiDimBins *= bins[d];
+			if (usingIntegerIndexBins && (multiDimBins > MAX_NUMBER_INTEGER_BINS)) {
+				// the number of bins has breached our maximum, so we'll use
+				//  a hash table instead of an integer array
+				usingIntegerIndexBins = false;
+			}
+			if (debug) {
+				System.out.println("Dim: " + d + " => Max: " + max + ", min: " + mins[d] +
+					", bins: " + bins[d] +
+					(normalise ? ", std: " + std : "") +
+					", eps: " + epsilonInUse[d]);
+			}
+		}
+		
+		observations = null;
+		observationsByHash = null;
+		rawData = data; // Store the raw data regardless of our
+						//  indexing approach - we may need to 
+						//  access it for dynamic correlation exclusion
+		if (usingIntegerIndexBins) {
+			// multiDimBins is the number of multi-dimensional bins we have
+
+			if (debug) {
+				System.out.println("Multidimensional bins: " + multiDimBins);
+			}
+			
+			// Create the bins
+			observations = (Vector<TimeStampedObservation>[]) new Vector[multiDimBins];
+			for (int v = 0; v < multiDimBins; v++) {
+				observations[v] = new Vector<TimeStampedObservation>();
+			}
+			
+			// Add each observation
+			for (int t = 0; t < data.length; t++) {
+				int bin = getMultiDimBinIndex(data[t]);
+				// Create the time-stamped observation to store:
+				TimeStampedObservation tso = new TimeStampedObservation(t, data[t]);
+				observations[bin].add(tso);
+			}
+
+		} else {
+			// Could be using either hash table style bins or no bins at all.
+			// Work out how many bins we might be trawling through if we used bins:
+			//  (need to take 3 ^ d, since we look at one bin either side for each dimension)
+			int possibleBinsToTrawl = 1;
+			for (int d = 0; d < dimensions; d++) {
+				if (possibleBinsToTrawl > (Integer.MAX_VALUE / 3)) {
+					// Next multiplication will overflow
+					possibleBinsToTrawl = Integer.MAX_VALUE;
+					break;
+				}
+				possibleBinsToTrawl *= 3;
+			}
+			
+			// We might be forcing n^2 comparisons between all points, or 
+			//  as a rule of thumb, check if there are less observations than possible bins to trawl
+			if (forceCompareToAll || (totalObservations < possibleBinsToTrawl)) {
+				useBins = false;
+				
+				// Simply store the raw data:
+				// rawData = data; - this is already done above
+				
+			} else {
+				// Using hash table
+				useBins = true;
+				
+				if (debug) {
+					System.out.println("Using array hash index bins");
+				}
+				
+				// Create the hashtable
+				observationsByHash = new Hashtable<IntArray, Vector<TimeStampedObservation>>();
+				
+				// Add each observation
+				for (int t = 0; t < data.length; t++) {
+					int[] multiDimBin = getMultiDimBinArray(data[t]);
+					IntArray intArrayMultiDimBin = new IntArray(multiDimBin);
+					Vector<TimeStampedObservation> hashedVector =
+						(Vector<TimeStampedObservation>) observationsByHash.get(intArrayMultiDimBin);
+					if (hashedVector == null) {
+						hashedVector = new Vector<TimeStampedObservation>();
+					}
+					// Create the time-stamped observation to store:
+					TimeStampedObservation tso = new TimeStampedObservation(t, data[t]);
+					hashedVector.add(tso);
+					// And put the observations for this multidimensional bin back in
+					observationsByHash.put(intArrayMultiDimBin, hashedVector);
+				}
+			}
+		}
+		
+		// Not sorting the observations, as this can only
+		//  be practically done for a single variable.
+	}
+	
+	/**
+	 * Each row of the data is an observation; each column of
+	 *  the row is a new variable in the multivariate observation.
+	 * This method signature allows the user to call setObservations for
+	 *  joint time series without combining them into a single joint time
+	 *  series (we do the combining for them).
+	 * 
+	 * @param data1
+	 * @param data2
+	 * @throws Exception When the length of the two arrays of observations do not match.
+	 */
+	public void setObservations(double[][] data1, double[][] data2) throws Exception {
+		int timeSteps = data1.length;
+		if ((data1 == null) || (data2 == null)) {
+			throw new Exception("Cannot have null data arguments");
+		}
+		if (data1.length != data2.length) {
+			throw new Exception("Length of data1 (" + data1.length + ") is not equal to the length of data2 (" +
+					data2.length + ")");
+		}
+		int data1Variables = data1[0].length;
+		int data2Variables = data2[0].length;
+		double[][] data = new double[timeSteps][data1Variables + data2Variables];
+		for (int t = 0; t < timeSteps; t++) {
+			System.arraycopy(data1[t], 0, data[t], 0, data1Variables);
+			System.arraycopy(data2[t], 0, data[t], data1Variables, data2Variables);
+		}
+		// Now defer to the normal setObservations method
+		setObservations(data);
+	}
+	
+	/**
+	 * This method signature allows the user to call setObservations for
+	 *  joint time series without combining them into a single joint time
+	 *  series (we do the combining for them).
+	 * 
+	 * @param data1
+	 * @param data2
+	 * @throws Exception When the length of the two arrays of observations do not match.
+	 */
+	public void setObservations(double[] data1, double[] data2) throws Exception {
+		int timeSteps = data1.length;
+		if ((data1 == null) || (data2 == null)) {
+			throw new Exception("Cannot have null data arguments");
+		}
+		if (data1.length != data2.length) {
+			throw new Exception("Length of data1 (" + data1.length + ") is not equal to the length of data2 (" +
+					data2.length + ")");
+		}
+		double[][] data = new double[timeSteps][2];
+		MatrixUtils.copyIntoColumn(data, 0, data1);
+		MatrixUtils.copyIntoColumn(data, 1, data2);
+		// Now defer to the normal setObservations method
+		setObservations(data);
+	}
+	
+	/**
+	 * Return the kernel estimate for the probability without any dynamic
+	 *  correlation exlcusion
+	 * 
+	 * @param observation
+	 * @return
+	 */
+	public double getProbability(double[] observation) {
+		if (useBins) {
+			return getProbabilityFromBins(observation, 0, false);
+		} else {
+			return getProbabilityComparingToAll(observation, 0, false);
+		}
+	}
+
+	/**
+	 * Return the kernel estimate for the probability with dynamic
+	 *  correlation exlcusion if it has been swithced on
+	 * 
+	 * @param observation
+	 * @param timeStep
+	 * @return
+	 */
+	public double getProbability(double[] observation, int timeStep) {
+		if (useBins) {
+			return getProbabilityFromBins(observation, timeStep, excludeDynamicCorrelations);
+		} else {
+			return getProbabilityComparingToAll(observation, timeStep, excludeDynamicCorrelations);
+		}
+	}
+
+	/**
+	 * Return the kernel estimate for the joint probability without any dynamic
+	 *  correlation exlcusion
+	 *  
+	 * @param observation1
+	 * @param observation2
+	 * @return
+	 */
+	public double getProbability(double[] observation1, double[] observation2) {
+		// Make a joint array
+		double[] jointObservation = new double[observation1.length + observation2.length];
+		System.arraycopy(observation1, 0, jointObservation, 0, observation1.length);
+		System.arraycopy(observation2, 0, jointObservation, observation1.length, observation2.length);
+		return getProbability(jointObservation);
+	}
+
+	/**
+	 * Return the kernel estimate for the joint probability with dynamic
+	 *  correlation exlcusion if it has been swithced on
+	 *  
+	 * @param observation1
+	 * @param observation2
+	 * @param timeStep
+	 * @return
+	 */
+	public double getProbability(double[] observation1, double[] observation2, int timeStep) {
+		// Make a joint array
+		double[] jointObservation = new double[observation1.length + observation2.length];
+		System.arraycopy(observation1, 0, jointObservation, 0, observation1.length);
+		System.arraycopy(observation2, 0, jointObservation, observation1.length, observation2.length);
+		return getProbability(jointObservation, timeStep);
+	}
+
+	/**
+	 * Return the kernel count (i think this is the correlation integral effectively?)
+	 *  for this vector without any dynamic correlation exlcusion
+	 * 
+	 * @param observation
+	 * @return
+	 */
+	public int getCount(double[] observation) {
+		KernelCount kernelCount;
+		if (useBins) {
+			kernelCount = getCountFromBins(observation, 0, false);
+		} else {
+			kernelCount = getCountComparingToAll(observation, 0, false);
+		}
+		return kernelCount.count;
+	}
+
+	/**
+	 * <p>Return the kernel count for the probability with dynamic
+	 *  correlation exlcusion if it has been switched on.</p>
+	 *  
+	 * <p>The user should be aware that a call to getProbability(double[], int)
+	 * will not return the same value as getCount(double[], int) / totalObservations,
+	 * since dynamic correlation exclusion could be swithced on. To achieve this, the
+	 * user could call getCompleteKernelCount(double[], int[], boolean) and use the
+	 * returned count and totalObservationsForCount in the returned KernelCount object.
+	 * </p>
+	 * 
+	 * @param observation
+	 * @param timeStep
+	 * @return
+	 */
+	public int getCount(double[] observation, int timeStep) {
+		KernelCount kernelCount;
+		if (useBins) {
+			kernelCount = getCountFromBins(observation, timeStep, excludeDynamicCorrelations);
+		} else {
+			kernelCount = getCountComparingToAll(observation, timeStep, excludeDynamicCorrelations);
+		}
+		return kernelCount.count;
+	}
+
+	/**
+	 * Return the kernel count for the probability with dynamic
+	 *  correlation exlcusion if it has been switched on
+	 * 
+	 * @param observation
+	 * @param timeStep
+	 * @param giveListOfCorrelatedPoints
+	 * @return
+	 */
+	public KernelCount getCompleteKernelCount(double[] observation, int timeStep,
+			boolean giveListOfCorrelatedPoints) {
+		KernelCount kernelCount;
+		if (useBins) {
+			kernelCount = getCountFromBins(observation, timeStep,
+					excludeDynamicCorrelations, giveListOfCorrelatedPoints);
+		} else {
+			kernelCount = getCountComparingToAll(observation, timeStep,
+					excludeDynamicCorrelations, giveListOfCorrelatedPoints);
+		}
+		return kernelCount;
+	}
+
+	/**
+	 * Return the kernel count for the joint vector without any dynamic
+	 *  correlation exlcusion
+	 *  
+	 * @param observation1
+	 * @param observation2
+	 * @return
+	 */
+	public int getCount(double[] observation1, double[] observation2) {
+		// Make a joint array
+		double[] jointObservation = new double[observation1.length + observation2.length];
+		System.arraycopy(observation1, 0, jointObservation, 0, observation1.length);
+		System.arraycopy(observation2, 0, jointObservation, observation1.length, observation2.length);
+		return getCount(jointObservation);
+	}
+
+	/**
+	 * Return the kernel estimate for the joint probability with dynamic
+	 *  correlation exlcusion if it has been swithced on
+	 *  
+	 * @param observation1
+	 * @param observation2
+	 * @param timeStep
+	 * @return
+	 */
+	public int getCount(double[] observation1, double[] observation2, int timeStep) {
+		// Make a joint array
+		double[] jointObservation = new double[observation1.length + observation2.length];
+		System.arraycopy(observation1, 0, jointObservation, 0, observation1.length);
+		System.arraycopy(observation2, 0, jointObservation, observation1.length, observation2.length);
+		return getCount(jointObservation, timeStep);
+	}
+
+	/**
+	 * Return the kernel count object for the joint probability with dynamic
+	 *  correlation exlcusion if it has been swithced on
+	 *  
+	 * @param observation1
+	 * @param observation2
+	 * @param timeStep
+	 * @return
+	 */
+	public KernelCount getCompleteKernelCount(double[] observation1, double[] observation2,
+			int timeStep, boolean giveListOfCorrelatedPoints) {
+		// Make a joint array
+		double[] jointObservation = new double[observation1.length + observation2.length];
+		System.arraycopy(observation1, 0, jointObservation, 0, observation1.length);
+		System.arraycopy(observation2, 0, jointObservation, observation1.length, observation2.length);
+		return getCompleteKernelCount(jointObservation, timeStep, giveListOfCorrelatedPoints);
+	}
+
+	/**
+	 * Give kernel estimated count for this observation.
+	 *  Do this using bins, so we only need to compare to neighbouring bins 
+	 *   and of course add in the observations in this bin.
+	 *  Achieves this by trawling through neighbouring bins in a recursive fashion.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @return KernelCount object with the count and total observations the count was taken from
+	 */
+	private KernelCount getCountFromBins(double[] observation, int timeStep, 
+			boolean dynCorrExclusion) {
+		return getCountFromBins(observation, timeStep, dynCorrExclusion, false);
+	}
+
+	/**
+	 * Give kernel estimated count for this observation.
+	 *  Do this using bins, so we only need to compare to neighbouring bins 
+	 *   and of course add in the observations in this bin.
+	 *  Achieves this by trawling through neighbouring bins in a recursive fashion.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @param giveListOfCorrelatedPoints include the list of time indices whose vectors
+	 *  were correlated with the given vector
+	 * @return KernelCount object with the count and total observations the count was taken from
+	 */
+	protected KernelCount getCountFromBins(double[] observation, int timeStep, 
+			boolean dynCorrExclusion, boolean giveListOfCorrelatedPoints) {
+		// First count the number of observations in the same bin
+		int count;
+		boolean[] isCorrelatedWithArgument = null;
+		if (giveListOfCorrelatedPoints) {
+			isCorrelatedWithArgument = new boolean[totalObservations];
+		}
+		int multiDimBin = 0;
+		int[] multiDimBinArray = null;
+		if (usingIntegerIndexBins) {
+			multiDimBin = getMultiDimBinIndex(observation);
+			count = observations[multiDimBin].size();
+			if (giveListOfCorrelatedPoints || makeCorrelatedPointAddedCallback) {
+				for (int i = 0; i < observations[multiDimBin].size(); i++) {
+					TimeStampedObservation tso = (TimeStampedObservation) observations[multiDimBin].elementAt(i);
+					if (giveListOfCorrelatedPoints) {
+						isCorrelatedWithArgument[tso.timeStep] = true;
+					}
+					if (makeCorrelatedPointAddedCallback) {
+						correlatedPointAddedCallback(tso.timeStep);
+					}
+				}
+			}
+		} else {
+			multiDimBinArray = getMultiDimBinArray(observation);
+			IntArray intArrayMultiDimBin = new IntArray(multiDimBinArray);
+			Vector<TimeStampedObservation> observationsInThisBin =
+				(Vector<TimeStampedObservation>) observationsByHash.get(intArrayMultiDimBin);
+			// Shouldn't need to check (observationsInThisBin != null) here since 
+			// at least this observation itself should have been added in here at some point.
+			// Will need to change this though if we ever start using the kernel estimator to check
+			// probabilities for new vectors that were not previously added to the observations.
+			count = observationsInThisBin.size();
+			if (giveListOfCorrelatedPoints || makeCorrelatedPointAddedCallback) {
+				for (int i = 0; i < observationsInThisBin.size(); i++) {
+					TimeStampedObservation tso = (TimeStampedObservation) observationsInThisBin.elementAt(i);
+					if (giveListOfCorrelatedPoints) {
+						isCorrelatedWithArgument[tso.timeStep] = true;
+					}
+					if (makeCorrelatedPointAddedCallback) {
+						correlatedPointAddedCallback(tso.timeStep);
+					}
+				}
+			}
+		}
+		
+		int totalTimePointsCompared = totalObservations;
+		
+		if (dynCorrExclusion) {
+			// Need to remove any observations that were *closer* than timeProximityForDynamicCorrelationExclusion
+			int closeTimePointsToCompare = (timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: timeStep;
+			closeTimePointsToCompare += (totalObservations - timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: totalObservations - timeStep - 1;
+			closeTimePointsToCompare++; // Add one for comparison to self
+			totalTimePointsCompared -= closeTimePointsToCompare;
+			// Use a heuristic to select best way to eliminate dynamic correlations here:
+			int countToRemove = 0;
+			if (closeTimePointsToCompare * dimensions < count) {
+				// Check all the neighbouring points in time to see if
+				//  they were in the same bin and have been counted
+				for (int t = Math.max(0,timeStep-timeProximityForDynamicCorrelationExclusion+1);
+						t < Math.min(totalObservations,timeStep+timeProximityForDynamicCorrelationExclusion); t++) {
+					int otherMultiDimBin = getMultiDimBinIndex(rawData[t]);
+					if (otherMultiDimBin == multiDimBin) {
+						countToRemove++;
+						if (giveListOfCorrelatedPoints) {
+							isCorrelatedWithArgument[t] = false;
+						}
+						if (makeCorrelatedPointAddedCallback) {
+							correlatedPointRemovedCallback(t);
+						}
+					}
+				}
+			} else {
+				// check all the points in this bin 
+				if (usingIntegerIndexBins) {
+					for (int i = 0; i < observations[multiDimBin].size(); i++) {
+						TimeStampedObservation tso = (TimeStampedObservation) observations[multiDimBin].elementAt(i);
+						if (Math.abs(tso.timeStep - timeStep) < timeProximityForDynamicCorrelationExclusion) {
+							countToRemove++;
+							if (giveListOfCorrelatedPoints) {
+								isCorrelatedWithArgument[tso.timeStep] = false;
+							}
+							if (makeCorrelatedPointAddedCallback) {
+								correlatedPointRemovedCallback(tso.timeStep);
+							}
+						}
+					}
+				} else {
+					Vector<TimeStampedObservation> observationsInThisBin =
+						(Vector<TimeStampedObservation>) observationsByHash.get(multiDimBinArray);
+					if (observationsInThisBin != null) {
+						for (int i = 0; i < observationsInThisBin.size(); i++) {
+							TimeStampedObservation tso = (TimeStampedObservation) observationsInThisBin.elementAt(i);
+							if (Math.abs(tso.timeStep - timeStep) < timeProximityForDynamicCorrelationExclusion) {
+								countToRemove++;
+								if (giveListOfCorrelatedPoints) {
+									isCorrelatedWithArgument[tso.timeStep] = false;
+								}
+								if (makeCorrelatedPointAddedCallback) {
+									correlatedPointRemovedCallback(tso.timeStep);
+								}
+							}
+						}
+					}
+				}
+			}
+			count -= countToRemove;
+		}
+
+		// Now look for correlated points in neighbouring bins
+		int[] currentNeighbourBinIndices = new int[dimensions];
+		// Run for one bin lower (assuming it exists)
+		count += addCount(observation, currentNeighbourBinIndices, 0,
+				getBinIndex(observation[0], 0) - 1, false,
+				dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+		// Run along this bin for the moment, changing one of the other indices
+		count += addCount(observation,currentNeighbourBinIndices, 0,
+				getBinIndex(observation[0], 0), true,
+				dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+		// Run for the next bin up (assuming it exists)
+		count += addCount(observation, currentNeighbourBinIndices, 0,
+				getBinIndex(observation[0], 0) + 1, false,
+				dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+		
+		KernelCount kernelCount = new KernelCount(count, totalTimePointsCompared, isCorrelatedWithArgument);
+		return kernelCount;
+	}
+
+	/**
+	 * Give kernel estimated probability for this observation.
+	 *  Do this using bins, so we only need to compare to neighbouring bins 
+	 *   and of course add in the observations in this bin.
+	 *  Achieves this by trawling through neighbouring bins in a recursive fashion.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @return
+	 */
+	private double getProbabilityFromBins(double[] observation, int timeStep, 
+			boolean dynCorrExclusion) {
+		
+		KernelCount kernelCount = getCountFromBins(observation, timeStep, dynCorrExclusion);
+		
+		return (double) kernelCount.count / (double) kernelCount.totalObservationsForCount;
+	}
+	
+	/**
+	 * Give kernel estimated count for this observation, by comparing to all
+	 *  given observations.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @return KernelCount object with the count and total observations the count was taken from
+	 */
+	private KernelCount getCountComparingToAll(double[] observation, int timeStep, 
+			boolean dynCorrExclusion) {
+		return getCountComparingToAll(observation, timeStep, dynCorrExclusion, false);
+	}
+
+
+	/**
+	 * Give kernel estimated count for this observation, by comparing to all
+	 *  given observations.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @return KernelCount object with the count and total observations the count was taken from
+	 */
+	protected KernelCount getCountComparingToAll(double[] observation, int timeStep, 
+			boolean dynCorrExclusion, boolean giveListOfCorrelatedPoints) {
+		int count = 0;
+		boolean[] isCorrelatedWithArgument = null;
+		if (giveListOfCorrelatedPoints) {
+			isCorrelatedWithArgument = new boolean[totalObservations];
+		}
+		for (int t = 0; t < totalObservations; t++) {
+			if (!dynCorrExclusion || 
+					(Math.abs(t - timeStep) >= timeProximityForDynamicCorrelationExclusion)) {
+				// Only add in the contribution from this point
+				//  when we're not doing dynamic correlation exclusion
+				//  or if it's outside the exclusion zone
+				int thisStepKernelValue = stepKernel(observation, rawData[t]);
+				count += thisStepKernelValue;
+				if (giveListOfCorrelatedPoints) {
+					isCorrelatedWithArgument[t] = (thisStepKernelValue > 0);
+				}
+				if (makeCorrelatedPointAddedCallback && (thisStepKernelValue > 0)) {
+					correlatedPointAddedCallback(t);
+				}
+			}
+		}
+		int totalTimePointsCompared = totalObservations;
+		if (dynCorrExclusion) {
+			// Needed to remove any observations that were *closer* than timeProximityForDynamicCorrelationExclusion
+			//  from the total of time points that were compared.
+			int closeTimePoints = (timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: timeStep;
+			closeTimePoints += (totalObservations - timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: totalObservations - timeStep - 1;
+			closeTimePoints++; // Add one for comparison to self
+			totalTimePointsCompared -= closeTimePoints;
+		}
+		KernelCount kernelCount = new KernelCount(count, totalTimePointsCompared, isCorrelatedWithArgument);
+		return kernelCount;
+	}
+
+	/**
+	 * Give kernel estimated probability for this observation, by comparing to all
+	 *  given observations.
+	 * 
+	 * @param observation
+	 * @param timeStep of the given observation (required for dynamic correlation exclusion)
+	 * @param dynCorrExclusion whether to use dynamic correlation exclusion
+	 * @return
+	 */
+	private double getProbabilityComparingToAll(double[] observation, int timeStep, 
+			boolean dynCorrExclusion) {
+		
+		KernelCount kernelCount = getCountComparingToAll(observation, timeStep, dynCorrExclusion);
+		return (double) kernelCount.count / (double) kernelCount.totalObservationsForCount;
+	}
+	
+	private int getBinIndex(double value, int dimension) {
+		int bin = (int) Math.floor((value - mins[dimension]) / epsilonInUse[dimension]);
+		// Check for any rounding errors on the bin assignment:
+		if (bin >= bins[dimension]) {
+			bin = bins[dimension] - 1;
+		}
+		if (bin < 0) {
+			bin = 0;
+		}
+		return bin;
+	}
+	
+	public int getMultiDimBinIndex(double[] value) {
+		int multiDimBin = 0;
+		for (int d = 0; d < dimensions; d++) {
+			multiDimBin += getBinIndex(value[d], d) * multipliers[d];
+		}
+		return multiDimBin;
+	}
+
+	public int[] getMultiDimBinArray(double[] value) {
+		int[] multiDimBinArray = new int[dimensions];
+		for (int d = 0; d < dimensions; d++) {
+			multiDimBinArray[d] = getBinIndex(value[d], d);
+		}
+		return multiDimBinArray;
+	}
+
+	public int getMultiDimBinIndexFromSingles(int[] singleBinIndices) {
+		int multiDimBin = 0;
+		for (int d = 0; d < dimensions; d++) {
+			multiDimBin += singleBinIndices[d] * multipliers[d];
+		}
+		return multiDimBin;
+	}
+
+	/**
+	 * Count the number of data points within epsilon, 
+	 * for multi dimensional bin indices as specified
+	 * 
+	 * @param neighbourBinIndices
+	 * @param currentIndex
+	 * @param currentIndexValue
+	 * @param possibleCentralBin
+	 * @param dynCorrExclusion
+	 * @param timeStep
+	 * @param isCorrelatedWithArgument set if the vector at each given time point is included 
+	 * @return
+	 */
+	private int addCount(double[] observation, int[] neighbourBinIndices,
+			int currentIndex, int currentIndexValue, boolean possibleCentralBin,
+			boolean dynCorrExclusion, int timeStep, boolean[] isCorrelatedWithArgument) {
+		
+		int count = 0;
+		
+		// Check that the current index is in range
+		if (currentIndexValue < 0) {
+			return 0;
+		}
+		if (currentIndexValue >= bins[currentIndex]) {
+			return 0;
+		}
+		neighbourBinIndices[currentIndex] = currentIndexValue;
+		
+		if (currentIndex == dimensions - 1) {
+			// this is the last dimension, need to check the neighbouring bin here
+			if (possibleCentralBin) {
+				// This is definitely the central bin, don't
+				//  count it, we've already done that outside
+				return 0;
+			}
+			if (usingIntegerIndexBins) {
+				int multiDimIndex = getMultiDimBinIndexFromSingles(neighbourBinIndices);
+				for (int i = 0; i < observations[multiDimIndex].size(); i++) {
+					TimeStampedObservation tso = (TimeStampedObservation) observations[multiDimIndex].elementAt(i);
+					if (!dynCorrExclusion || 
+							(Math.abs(tso.timeStep - timeStep) >= timeProximityForDynamicCorrelationExclusion)) {
+						// Only add in the contribution from this point
+						//  when we're not doing dynamic correlation exclusion
+						//  or if it's outside the exclusion zone
+						int thisStepKernelValue = stepKernel(observation, tso.observation);
+						count += thisStepKernelValue;
+						if (isCorrelatedWithArgument != null) {
+							isCorrelatedWithArgument[tso.timeStep] = (thisStepKernelValue > 0);
+						}
+						if (makeCorrelatedPointAddedCallback && (thisStepKernelValue > 0)) {
+							correlatedPointAddedCallback(tso.timeStep);
+						}
+					}
+				}
+			} else {
+				IntArray intArrayBinNeighbourIndices = new IntArray(neighbourBinIndices);
+				Vector<TimeStampedObservation> observationsInThisBin =
+					(Vector<TimeStampedObservation>) observationsByHash.get(intArrayBinNeighbourIndices);
+				if (observationsInThisBin != null) {
+					for (int i = 0; i < observationsInThisBin.size(); i++) {
+						TimeStampedObservation tso = (TimeStampedObservation) observationsInThisBin.elementAt(i);
+						if (!dynCorrExclusion || 
+								(Math.abs(tso.timeStep - timeStep) >= timeProximityForDynamicCorrelationExclusion)) {
+							// Only add in the contribution from this point
+							//  when we're not doing dynamic correlation exclusion
+							//  or if it's outside the exclusion zone
+							int thisStepKernelValue = stepKernel(observation, tso.observation);
+							count += thisStepKernelValue;
+							if (isCorrelatedWithArgument != null) {
+								isCorrelatedWithArgument[tso.timeStep] = (thisStepKernelValue > 0);
+							}
+							if (makeCorrelatedPointAddedCallback && (thisStepKernelValue > 0)) {
+								correlatedPointAddedCallback(tso.timeStep);
+							}
+						}
+					}
+				}
+			}
+		} else {
+			// pass on the responsibility to do the checks:
+			// Run for one index lower (assuming it exists)
+			count += addCount(observation, neighbourBinIndices, currentIndex+1,
+					getBinIndex(observation[currentIndex+1], currentIndex+1) - 1, false,
+					dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+			// Run for this index
+			count += addCount(observation,neighbourBinIndices, currentIndex+1,
+					getBinIndex(observation[currentIndex+1], currentIndex+1), possibleCentralBin,
+					dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+			// Run for the next index up (assuming it exists)
+			count += addCount(observation, neighbourBinIndices, currentIndex+1,
+					getBinIndex(observation[currentIndex+1], currentIndex+1) + 1, false,
+					dynCorrExclusion, timeStep, isCorrelatedWithArgument);
+		}
+		
+		return count;
+	}
+	
+	/**
+	 * Return the step kernel for the two data points
+	 * 
+	 * @param observation
+	 * @param candidate
+	 * @return
+	 */
+	public int stepKernel(double[] observation, double[] candidate) {
+		for (int d = 0; d < dimensions; d++) {
+			if (Math.abs(observation[d] - candidate[d]) > epsilonInUse[d]) {
+				return 0;
+			}
+		}
+		// The candidate is within epsilon of the observation
+		return 1;
+	}
+
+	public void setNormalise(boolean normalise) {
+		this.normalise = normalise;
+	}
+	
+	public boolean isNormalise() {
+		return normalise;
+	}
+
+	public void setDynamicCorrelationExclusion(int timeWindow) {
+		excludeDynamicCorrelations = true;
+		timeProximityForDynamicCorrelationExclusion = timeWindow;
+	}
+	
+	public void clearDynamicCorrelationExclusion() {
+		excludeDynamicCorrelations = false;
+	}
+	
+	public boolean isExcludeDynamicCorrelations() {
+		return excludeDynamicCorrelations;
+	}
+
+	public boolean isForceCompareToAll() {
+		return forceCompareToAll;
+	}
+
+	/**
+	 * Only recommended to set this for debugging purposes
+	 * 
+	 * @param forceCompareToAll
+	 */
+	public void setForceCompareToAll(boolean forceCompareToAll) {
+		this.forceCompareToAll = forceCompareToAll;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+	
+	/**
+	 * A callback for where a correlated point is found.
+	 * Used in child classes, but not here since we have set makeCorrelatedPointAddedCallback to false.
+	 *
+	 */
+	protected void correlatedPointAddedCallback(int correlatedTimeStep) {
+		
+	}
+
+	/**
+	 * A callback for where a correlated point is removed due to dynamic correlated exclusion.
+	 * Used in child classes, but not here since we have set makeCorrelatedPointAddedCallback to false.
+	 *
+	 */
+	protected void correlatedPointRemovedCallback(int removedCorrelatedTimeStep) {
+		
+	}
+}

java/source/infodynamics/measures/continuous/kernel/KernelEstimatorSingleVariate.java

@@ -0,0 +1,294 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+import java.util.Arrays;
+
+/**
+ * Class to maintain probability distribution function for
+ *  a single variable, using kernel estimates.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class KernelEstimatorSingleVariate {
+
+	private double epsilon = 0.1;
+	private double epsilonInUse;
+	private double min = 0;
+	private double max = 0;
+	private int bins = 0;
+	private int totalObservations = 0;
+	private TimeStampedObservation[][] sortedObservations = null;
+	private boolean debug = false;
+	
+	private boolean normalise = true;
+	
+	private boolean excludeDynamicCorrelations = false;
+	private int timeProximityForDynamicCorrelationExclusion = 100;
+
+	/**
+	 * 
+	 * Private class to store a time-stamped data point.
+	 * This allows us to eliminate dynamic correlations later.
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	private class TimeStampedObservation implements Comparable {
+		public int timeStep;
+		public double observation;
+		
+		TimeStampedObservation(int time, double dataPoint) {
+			timeStep = time;
+			observation = dataPoint;
+		}
+
+		/**
+		 * Compare the data values of the two time points
+		 * 
+		 * @param obj
+		 * @return
+		 */
+		public int compareTo(Object obj) {
+			TimeStampedObservation tso2 = (TimeStampedObservation) obj;
+			if (observation < tso2.observation) {
+				return -1;
+			} else if (observation > tso2.observation) {
+				return 1;
+			}
+			return 0;
+		}
+	}
+
+	public KernelEstimatorSingleVariate() {
+	}
+
+	/**
+	 * Initialise the estimator before passing any observations in.
+	 *
+	 * @param epsilon
+	 */
+	public void initialise(double epsilon) {
+		this.epsilon = epsilon;
+		sortedObservations = null;
+	}
+	
+	public void setObservations(double[] data) {
+		setObservations(data, 0);
+	}
+	
+	public void setObservations(double[] data, int startTime) {
+		min = MatrixUtils.minStartFromIndex(data, startTime);
+		max = MatrixUtils.maxStartFromIndex(data, startTime);
+		totalObservations = data.length - startTime;
+		
+		if (normalise) {
+			// Compute what the epsilonInUse should be here:
+			//  it should expand with the standard deviation.
+			// This saves us from normalising all of the incoming data points!
+			double std = MatrixUtils.stdDev(data);
+			epsilonInUse = epsilon * std;
+		} else {
+			epsilonInUse = epsilon;
+		}
+
+		// Create the bins
+		Vector<TimeStampedObservation>[] observations = null;
+		bins = (int) Math.ceil((max - min) / epsilonInUse);
+		if (bins == 0) {
+			// The max and min are the same.
+			// Should still have one bin here to put all the data in,
+			//  otherwise when we go to look up which bin an element
+			//  is in we would get an exception.
+			// Is mathematically akin to the spread being non-zero but within
+			//  epsilon anyway.
+			bins = 1;
+		}
+		
+		if (debug) {
+			System.out.println("Max: " + max + ", min: " + min +
+				", bins: " + bins);
+		}
+		
+		observations = new Vector[bins];
+		for (int v = 0; v < bins; v++) {
+			observations[v] = new Vector<TimeStampedObservation>();
+		}
+		
+		// Add each observation
+		for (int i = startTime; i < data.length; i++) {
+			int bin = getBinIndex(data[i]);
+			TimeStampedObservation tso = new TimeStampedObservation(i, data[i]);
+			// System.out.println(i + " " + observations.length +
+			//		" " + max + " " + min + " " + epsilon);
+			observations[bin].add(tso);
+		}
+		
+		// Now sort the bins, to allow faster counting later
+		sortedObservations = new TimeStampedObservation[bins][];
+		int total = 0;
+		for (int v = 0; v < bins; v++) {
+			// The class cast here causes a run time cast exception:
+			// sortedObservations[v] = (TimeStampedObservation[]) observations[v].toArray();
+			// It seems crazy, but to get around this we need to do
+			//  the following:
+			sortedObservations[v] = new TimeStampedObservation[observations[v].size()];
+			for (int o = 0; o < sortedObservations[v].length; o++) {
+				sortedObservations[v][o] = (TimeStampedObservation) observations[v].elementAt(o);
+			}
+			// Sort into ascending order
+			Arrays.sort(sortedObservations[v]);
+			total += sortedObservations[v].length;
+			if (debug) {
+				System.out.println("Num observations in bin " + v + ": " + sortedObservations[v].length);
+			}
+		}
+		if (total != totalObservations) {
+			throw new RuntimeException("We have not stored all observations");
+		}
+		
+	}
+	
+	/**
+	 * Get the probability of this observation without any dynamic correlation exclusion
+	 * 
+	 * @param observation
+	 * @return
+	 */
+	public double getProbability(double observation) {
+		return getProbability(observation, 0, false);
+	}
+	
+	/**
+	 * Get the probability of this observation using the existing settings for 
+	 *  dynamic correlation exclusion
+	 * 
+	 * @param observation
+	 * @param timeStep
+	 * @return
+	 */
+	public double getProbability(double observation, int timeStep) {
+		return getProbability(observation, timeStep, excludeDynamicCorrelations);
+	}
+	
+	private double getProbability(double observation, int timeStep, 
+			boolean dynCorrExclusion) {
+		int bin = getBinIndex(observation);
+		// First count the number of observations in the same bin
+		int count = sortedObservations[bin].length;
+		int totalTimePointsCompared = totalObservations;
+
+		// If required eliminate dynamic correlations
+		if (dynCorrExclusion) {
+			// Need to remove any observations that were *closer* than timeProximityForDynamicCorrelationExclusion
+			int closeTimePointsToCompare = (timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: timeStep;
+			closeTimePointsToCompare += (totalObservations - timeStep >= timeProximityForDynamicCorrelationExclusion) ?
+					timeProximityForDynamicCorrelationExclusion - 1: totalObservations - timeStep - 1;
+			closeTimePointsToCompare++; // Add one for comparison to self
+			totalTimePointsCompared -= closeTimePointsToCompare;
+			for (int t = 0; t < sortedObservations[bin].length; t++) {
+				if (Math.abs(sortedObservations[bin][t].timeStep - timeStep) < timeProximityForDynamicCorrelationExclusion) {
+					count--;
+				}
+			}
+		}
+		if (debug) {
+			System.out.println("Count from bin " + bin + " = " + count +
+					(dynCorrExclusion ? "" : " no") + " dynamic correlation exclusion.");
+		}
+		
+		// Now check the lower bin:
+		if (bin > 0) {
+			// Find the cut-off point where values in the lower bin
+			//  are no longer within epsilon of the given value.
+			int topIndex;
+			for (topIndex = sortedObservations[bin-1].length;
+					(topIndex > 0) && (sortedObservations[bin-1][topIndex-1].observation > observation - epsilonInUse);
+					topIndex--) {
+				// This observation is within epsilon.
+				// Before adding to the count just check if it's a dynamic correlation if required:
+				if (!dynCorrExclusion ||
+						(Math.abs(sortedObservations[bin-1][topIndex-1].timeStep - timeStep) < timeProximityForDynamicCorrelationExclusion)) {
+					count++;
+				}
+			}
+			// Don't need to do this addition anymore, it's incorporated above:
+			// Post-condition: 
+			// Lower bin has (sortedObservations[bin-1].length - topIndex)
+			//  values within epsilon of our observation;
+			// count += sortedObservations[bin-1].length - topIndex;
+		}
+		if (debug) {
+			System.out.println("Count after lower bin " + (bin - 1) + " = " + count);
+		}
+		
+		// Now check the upper bin:
+		if (bin < bins - 1) {
+			// Find the cut-off point where values in the upper bin
+			//  are no longer within epsilon of the given value
+			int bottomIndex;
+			for (bottomIndex = 0;
+					(bottomIndex < sortedObservations[bin+1].length) &&
+						(sortedObservations[bin+1][bottomIndex].observation < observation + epsilonInUse);
+					bottomIndex++) {
+				// This observation is within epsilon.
+				// Before adding to the count just check if it's a dynamic correlation if required:
+				if (!dynCorrExclusion ||
+						(Math.abs(sortedObservations[bin+1][bottomIndex].timeStep - timeStep) < timeProximityForDynamicCorrelationExclusion)) {
+					count++;
+				}
+			}
+			// Don't need to do this addition anymore, it's incorporated above:
+			// Post-condition: 
+			// Upper bin has bottomIndex
+			//  values within epsilon of our observation;
+			// count += bottomIndex;
+		}
+		if (debug) {
+			System.out.println("Count after upper bin " + (bin + 1) + " = " + count);
+		}
+
+		return (double) count / (double) totalTimePointsCompared;
+	}
+	
+	private int getBinIndex(double value) {
+		int bin = (int) Math.floor((value - min) / epsilonInUse);
+		// Check for any rounding errors on the bin assignment:
+		if (bin >= bins) {
+			bin = bins - 1;
+		}
+		if (bin < 0) {
+			bin = 0;
+		}
+		return bin;
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public boolean isNormalise() {
+		return normalise;
+	}
+
+	public void setNormalise(boolean normalise) {
+		this.normalise = normalise;
+	}
+
+	public void setDynamicCorrelationExclusion(int timeWindow) {
+		excludeDynamicCorrelations = true;
+		timeProximityForDynamicCorrelationExclusion = timeWindow;
+	}
+	
+	public void clearDynamicCorrelationExclusion() {
+		excludeDynamicCorrelations = false;
+	}
+	
+	public boolean isExcludeDynamicCorrelations() {
+		return excludeDynamicCorrelations;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/KernelEstimatorTransferEntropy.java

@@ -0,0 +1,155 @@
+/**
+ * 
+ */
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Kernel estimator for use with the transfer entropy.</p>
+ * 
+ * <p>Extends KernelEstimatorMultiVariate, using the super class to manage the history of the destination
+ * variable, and adds the next state and source on top of this. Any calls to the super class methods will only
+ * function on the joint history.
+ * </p> 
+ * 
+ * @author Joseph Lizier joseph.lizier at gmail.com
+ *
+ */
+public class KernelEstimatorTransferEntropy extends KernelEstimatorMultiVariate {
+
+	private double epsilonSource;
+	private double epsilonSourceInUse;
+	
+	private double[] destNext;
+	private double[] source;
+	
+	// Store the current observations passed in the synchronized method getCount
+	//  waiting for callbacks from the underlying kernel estimator
+	private double destNextObs;
+	private double sourceObs;
+	// Counts of destPastNext, destPastSource, destPastNextSource to be filled in 
+	//  with the callbacks
+	private int countNextPast;
+	private int countPastSource;
+	private int countNextPastSource;
+	
+	public KernelEstimatorTransferEntropy() {
+		super();
+		// Make sure when get a callbacl when correlated points are found
+		makeCorrelatedPointAddedCallback = true;
+	}
+
+	public void initialise(int dimensions, double epsilon) {
+		super.initialise(dimensions, epsilon);
+		this.epsilonSource = epsilon;
+	}
+	
+	public void initialise(int dimensions, double epsilonDest, 
+			double epsilonSource) {
+		super.initialise(dimensions, epsilonDest);
+		this.epsilonSource = epsilonSource;
+	}
+	
+	public void setObservations(double[][] destPastVectors,
+			double[] destNext, double[] source) {
+		setObservations(destPastVectors);
+		// epsilonInUse has been computed for the destination.
+		// TODO We could compute and set it directly here so
+		//  we don't have a mismatch between any of the vector
+		//  variables.
+		
+		if (normalise) {
+			double std = MatrixUtils.stdDev(source);
+			epsilonSourceInUse = epsilonSource * std;
+		} else {
+			epsilonSourceInUse = epsilonSource;
+		}
+
+		this.source = source;
+		this.destNext = destNext;
+	}
+	
+	/**
+	 * Compute the required counts for Transfer Entropy using kernel estimation on the destination's past.
+	 * Use callbacks to check if the joint counts need to be incremented.
+	 * 
+	 * If observationTimeStep &lt; 0, then no dynamic correlation exclusion will be attempted
+	 *
+	 * @param destPast
+	 * @param destNextObs
+	 * @param sourceObs
+	 * @param observationTimeStep
+	 * @return
+	 */
+	public synchronized TransferEntropyKernelCounts getCount(
+			double[] destPast, double destNextObs,
+			double sourceObs, int observationTimeStep) {
+		
+		// Prepare for any callbacks
+		countNextPast = 0;
+		countPastSource = 0;
+		countNextPastSource = 0;
+		this.destNextObs = destNextObs;
+		this.sourceObs = sourceObs;
+
+		// Get the count, and have the joint counts filled in via callbacks
+		int countPast;
+		if (observationTimeStep < 0) {
+			countPast = super.getCount(destPast);
+		} else {
+			countPast = super.getCount(destPast, observationTimeStep);
+		}
+		
+		TransferEntropyKernelCounts teKernelCount = 
+			new TransferEntropyKernelCounts(countPast, countNextPast, countPastSource, countNextPastSource);
+		return teKernelCount;
+	}
+
+	public void setEpsSource(double epsilonSource) {
+		this.epsilonSource = epsilonSource;
+	}
+
+	/**
+	 * A callback for where a correlated point is found at
+	 * correlatedTimeStep in the destination's past.
+	 * Now check whether we need to increment the joint counts.
+	 *
+	 */
+	protected void correlatedPointAddedCallback(int correlatedTimeStep) {
+		boolean sourceMatches = false;
+		if (Math.abs(sourceObs - source[correlatedTimeStep]) <= epsilonSourceInUse) {
+			countPastSource++;
+			sourceMatches = true;
+		}
+		// The epsilons across the destination variables should all be approximately 
+		//  equal, so just use the first one.
+		if (Math.abs(destNextObs - destNext[correlatedTimeStep]) <= epsilonInUse[0]) {
+			countNextPast++;
+			if (sourceMatches) {
+				countNextPastSource++;
+			}
+		}
+	}
+
+	/**
+	 * A callback for where a correlated point is removed due to dynamic correlated exclusion.
+	 * The removal is for the point at correlatedTimeStep in the destination's past.
+	 * Now check whether we need to decrement the joint counts.
+	 */
+	protected void correlatedPointRemovedCallback(int removedCorrelatedTimeStep) {
+		boolean sourceMatches = false;
+		if (Math.abs(sourceObs - source[removedCorrelatedTimeStep]) <= epsilonSourceInUse) {
+			countPastSource--;
+			sourceMatches = true;
+		}
+		// The epsilons across the destination variables should all be approximately 
+		//  equal, so just use the first one.
+		if (Math.abs(destNextObs - destNext[removedCorrelatedTimeStep]) <= epsilonInUse[0]) {
+			countNextPast--;
+			if (sourceMatches) {
+				countNextPastSource--;
+			}
+		}
+	}
+}

java/source/infodynamics/measures/continuous/kernel/KernelEstimatorTransferEntropyMultiVariate.java

@@ -0,0 +1,237 @@
+/**
+ * 
+ */
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Kernel estimator for use with the transfer entropy.</p>
+ * 
+ * <p>Extends KernelEstimatorMultiVariate, using the super class to manage the history of the destination
+ * variable, and adds the next state and source on top of this. Any calls to the super class methods will only
+ * function on the joint history.
+ * </p> 
+ * 
+ * @author Joseph Lizier joseph.lizier at gmail.com
+ *
+ */
+public class KernelEstimatorTransferEntropyMultiVariate extends KernelEstimatorMultiVariate {
+
+	private int sourceDimensions;
+	private double[] epsilonSource;
+	private double[] epsilonSourceInUse;
+
+	// Keep a separate epsilon for the destination next state, just in case
+	//  we're doing something funky and using different variables to track
+	//  the destination's past and next state (e.g. in swarm analysis).
+	private double epsilonDestNextFixed;
+	private double[] epsilonDestNext;
+	private double[] epsilonDestNextInUse;
+
+	private double[][] destNext;
+	private double[][] source;
+	
+	// Store the current observations passed in the synchronized method getCount
+	//  waiting for callbacks from the underlying kernel estimator
+	private double[] destNextObs;
+	private double[] sourceObs;
+	// Counts of destPastNext, destPastSource, destPastNextSource to be filled in 
+	//  with the callbacks
+	private int countNextPast;
+	private int countPastSource;
+	private int countNextPastSource;
+	
+	public KernelEstimatorTransferEntropyMultiVariate() {
+		super();
+		// Make sure when get a callbacl when correlated points are found
+		makeCorrelatedPointAddedCallback = true;
+	}
+
+	public void initialise(int dimensions, double epsilon) {
+		initialise(dimensions, dimensions, epsilon, epsilon);
+	}
+	
+	public void initialise(int destDimensionsWithPast, int sourceDimensions, 
+			double epsilonDest, double epsilonSource) {
+		super.initialise(destDimensionsWithPast, epsilonDest);
+		this.sourceDimensions = sourceDimensions;
+		this.epsilonSource = new double[sourceDimensions];
+		for (int d = 0; d < sourceDimensions; d++) {
+			this.epsilonSource[d] = epsilonSource;
+		}
+		epsilonSourceInUse = new double[sourceDimensions];
+		// Track that we're using a fixed epsilon for destination next
+		epsilonDestNext = null;
+		epsilonDestNextFixed = epsilonDest;
+	}
+	
+	public void initialise(double[] epsilonDest, 
+			double[] epsilonSource) {
+		super.initialise(epsilonDest);
+		this.epsilonSource = epsilonSource;
+		epsilonSourceInUse = new double[sourceDimensions];
+		// Assume that we are doing an ordinary TE computation (dest past
+		//  and next state are the same variable)
+		epsilonDestNext = epsilonDest;
+	}
+
+	/**
+	 * 
+	 * @param destPastVectors is the joint vector of the 
+	 *   past k states of the <dimensions> joint variables.
+	 *   Each vector has values x_c,t: [x_0,0; x_0,0; .. ; x_d,0; x_0,1 .. ] 
+	 *   i.e. it puts all values for a given time step in at once.
+	 * @param destNext
+	 * @param source
+	 */
+	public void setObservations(double[][] destPastVectors,
+			double[][] destNext, double[][] source) {
+		setObservations(destPastVectors);
+		// epsilonInUse has been computed for the destination.
+		// TODO We could compute and set it directly here so
+		//  we don't have a mismatch between any of the vector
+		//  variables.
+		
+		if (normalise) {
+			for (int d = 0; d < sourceDimensions; d++) {
+				double std = MatrixUtils.stdDev(source, d);
+				epsilonSourceInUse[d] = epsilonSource[d] * std;
+			}
+		} else {
+			for (int d = 0; d < sourceDimensions; d++) {
+				epsilonSourceInUse[d] = epsilonSource[d];
+			}
+		}
+
+		// Set epsilon for the destination next state here, now
+		//  that we can be sure of it's number of dimensions (in case
+		//  we're doing something funky with different dest past and next
+		//  state variables).
+		int destNextDimensions = destNext[0].length;
+		if (epsilonDestNext == null) {
+			// We must be using a fixed epsilon
+			epsilonDestNext = new double[destNextDimensions];
+			for (int d = 0; d < destNextDimensions; d++) {
+				epsilonDestNext[d] = epsilonDestNextFixed;
+			}
+		}
+		epsilonDestNextInUse = new double[destNextDimensions];
+		if (normalise) {
+			for (int d = 0; d < destNextDimensions; d++) {
+				double std = MatrixUtils.stdDev(destNext, d);
+				epsilonDestNextInUse[d] = epsilonDestNext[d] * std;
+			}
+		} else {
+			for (int d = 0; d < destNextDimensions; d++) {
+				epsilonDestNextInUse[d] = epsilonDestNext[d];
+			}
+		}
+
+		this.source = source;
+		this.destNext = destNext;
+	}
+	
+	/**
+	 * Compute the required counts for Transfer Entropy using kernel estimation on the destination's past.
+	 * Use callbacks to check if the joint counts need to be incremented.
+	 * 
+	 * If observationTimeStep &lt; 0, then no dynamic correlation exclusion will be attempted
+	 * 
+	 * @param destPast
+	 * @param destNextObs
+	 * @param sourceObs
+	 * @param observationTimeStep
+	 * @return
+	 */
+	public synchronized TransferEntropyKernelCounts getCount(
+			double[] destPast, double[] destNextObs,
+			double[] sourceObs, int observationTimeStep) {
+		
+		// Prepare for any callbacks
+		countNextPast = 0;
+		countPastSource = 0;
+		countNextPastSource = 0;
+		this.destNextObs = destNextObs;
+		this.sourceObs = sourceObs;
+
+		// Get the count, and have the joint counts filled in via callbacks
+		int countPast;
+		if (observationTimeStep < 0) {
+			countPast = super.getCount(destPast);
+		} else {
+			countPast = super.getCount(destPast, observationTimeStep);
+		}
+		
+		TransferEntropyKernelCounts teKernelCount = 
+			new TransferEntropyKernelCounts(countPast, countNextPast, countPastSource, countNextPastSource);
+		return teKernelCount;
+	}
+
+	public void setEpsSource(double[] epsilonSource) {
+		this.epsilonSource = epsilonSource;
+	}
+
+	/**
+	 * A callback for where a correlated point is found at
+	 * correlatedTimeStep in the destination's past.
+	 * Now check whether we need to increment the joint counts.
+	 *
+	 */
+	protected void correlatedPointAddedCallback(int correlatedTimeStep) {
+		boolean sourceMatches = false;
+		if (stepKernel(sourceObs, source[correlatedTimeStep], epsilonSourceInUse) > 0) {
+			countPastSource++;
+			sourceMatches = true;
+		}
+		// The epsilons across the history of each of the joint
+		//  destination variables should all be approximately 
+		//  equal, so just use the first ones for each dimension.
+		//  (i.e. use the first dimensions epsilons, which
+		//  is achieved simply by passing in epsilon, since
+		//  stepKernel just uses the first dimensions elements
+		//  of the widths argument).
+		if (stepKernel(destNextObs, destNext[correlatedTimeStep], epsilonDestNextInUse) > 0) {
+			countNextPast++;
+			if (sourceMatches) {
+				countNextPastSource++;
+			}
+		}
+	}
+
+	/**
+	 * A callback for where a correlated point is removed due to dynamic correlated exclusion.
+	 * The removal is for the point at correlatedTimeStep in the destination's past.
+	 * Now check whether we need to decrement the joint counts.
+	 */
+	protected void correlatedPointRemovedCallback(int removedCorrelatedTimeStep) {
+		boolean sourceMatches = false;
+		if (stepKernel(sourceObs, source[removedCorrelatedTimeStep], epsilonSourceInUse) > 0) {
+			countPastSource--;
+			sourceMatches = true;
+		}
+		// The epsilons across the history of each of the joint
+		//  destination variables should all be approximately 
+		//  equal, so just use the first ones for each dimension.
+		//  (i.e. use the first dimensions epsilons, which
+		//  is achieved simply by passing in epsilon, since
+		//  stepKernel just uses the first dimensions elements
+		//  of the widths argument).
+		if (stepKernel(destNextObs, destNext[removedCorrelatedTimeStep], epsilonDestNextInUse) > 0) {
+			countNextPast--;
+			if (sourceMatches) {
+				countNextPastSource--;
+			}
+		}
+	}
+	
+	protected int stepKernel(double[] vector1, double[] vector2, double[] widths) {
+		for (int d = 0; d < vector1.length; d++) {
+			if (Math.abs(vector1[d] - vector2[d]) > widths[d]) {
+				return 0;
+			}
+		}
+		// The candidate is within epsilon of the observation
+		return 1;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/MultiInfoCalculatorKernel.java

@@ -0,0 +1,527 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.MultiInfoCalculator;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+import java.util.Random;
+
+
+public class MultiInfoCalculatorKernel implements
+	MultiInfoCalculator {
+
+	KernelEstimatorSingleVariate[] svkeMarginals = null;
+	KernelEstimatorMultiVariate mvkeJoint = null;
+
+	private int dimensions = 0;
+	private int totalObservations = 0;
+	private boolean debug = false;
+	private double[][] observations;
+	private Vector<double[]> individualObservations;
+	private double lastAverage;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+
+	private boolean underSample = false;
+	private double samplingFactor = 0.1;
+	private Random rand;
+	public static final String SAMPLING_FACTOR_PROP_NAME = "SAMPLING_FACTOR";
+
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+
+	public MultiInfoCalculatorKernel() {
+		mvkeJoint = new KernelEstimatorMultiVariate();
+	}
+
+	/**
+	 * Initialise using a default epsilon
+	 * 
+	 * @param dimensions1
+	 */
+	public void initialise(int dimensions) {
+		initialise(dimensions, epsilon);
+	}
+
+	public void initialise(int dimensions, double epsilon) {
+		this.epsilon = epsilon;
+		observations = null;
+		if (this.dimensions != dimensions) {
+			// Need to create a new array of marginal kernel estimators 
+			this.dimensions = dimensions;
+			svkeMarginals = new KernelEstimatorSingleVariate[dimensions];
+			for (int i = 0; i < dimensions; i++) {
+				svkeMarginals[i] = new KernelEstimatorSingleVariate();
+				svkeMarginals[i].setNormalise(normalise);
+				if (dynCorrExcl) {
+					svkeMarginals[i].setDynamicCorrelationExclusion(dynCorrExclTime);
+				} else {
+					svkeMarginals[i].clearDynamicCorrelationExclusion();
+				}
+			}
+		}
+		// Initialise the marginal kernel estimators
+		for (int i = 0; i < dimensions; i++) {
+			svkeMarginals[i].initialise(epsilon);
+		}
+		// Initialise the joint kernel estimator
+		mvkeJoint.initialise(dimensions, epsilon);
+		lastAverage = 0.0;
+	}
+
+	/**
+	 * Set the observations for the PDFs.
+	 * Should only be called once, the last call contains the
+	 *  observations that are used (they are not accumulated). 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double observations[][]) throws Exception {
+		if (observations[0].length != dimensions) {
+			throw new Exception("Incorrect number of dimensions " + observations[0].length +
+					" in supplied observations (expected " + dimensions + ")");
+		}
+		for (int d = 0; d < dimensions; d++) {
+			svkeMarginals[d].setObservations(MatrixUtils.selectColumn(observations, d));
+		}
+		mvkeJoint.setObservations(observations);
+		totalObservations = observations.length;
+		this.observations = observations;
+	}
+	
+	/**
+	 * User elects to set observations one by one rather than in one go.
+	 * Will need to call endIndividualObservations before calling any of the
+	 *  compute functions, otherwise the previous observations will be used.
+	 */
+	public void startIndividualObservations() {
+		if (dynCorrExcl) {
+			// We have not properly implemented dynamic correlation exclusion for
+			//  multiple observation sets, so throw an error
+			throw new RuntimeException("Addition of multiple observation sets is not currently " +
+					"supported with property DYN_CORR_EXCL set");
+		}
+		individualObservations = new Vector<double[]>();
+	}
+	
+	public void addObservation(double observation[]) {
+		if (underSample && (rand.nextDouble() >= samplingFactor)) {
+			// Don't take this sample
+			return;
+		}
+		individualObservations.add(observation);
+	}
+	
+	public void endIndividualObservations() throws Exception {
+		double[][] data = new double[individualObservations.size()][];
+		for (int t = 0; t < data.length; t++) {
+			data[t] = individualObservations.elementAt(t);
+		}
+		// Allow vector to be reclaimed
+		individualObservations = null;
+		setObservations(data);
+	}
+
+	/**
+	 * Compute the MI from the observations we were given
+	 * 
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double mi = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			if (debug) {
+				System.out.print(b + ": ");
+			}
+			double marginalProbProducts = 1.0;
+			for (int d = 0; d < dimensions; d++) {
+				double marginalProb = svkeMarginals[d].getProbability(observations[b][d], b);
+				if (debug) {
+					System.out.print(observations[b][d] + " p=" + marginalProb + ", ");
+				}
+				marginalProbProducts *= marginalProb;
+			}
+			double probJoint = mvkeJoint.getCount(observations[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probJoint > 0.0) {
+				// TODO Should probably check that marginalProbProducts has not 
+				//  gone to zero (this is possible with several multiplications
+				//  of 1/N, though is unlikely). I'm not sure what we would do if
+				//  it had gone to zero though ... ignore this value?
+				logTerm = probJoint / marginalProbProducts;
+				cont = Math.log(logTerm);
+			}
+			mi += cont;
+			if (debug) {
+				System.out.println(", p(joint) = " + probJoint
+						+ " -> " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (mi/Math.log(2.0)));
+			}
+		}
+		lastAverage = mi / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * Extra utility method to return the joint entropy
+	 * 
+	 * @return
+	 */
+	public double computeAverageJointEntropy() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvkeJoint.getCount(observations[b], b);
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = - Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob
+						+ " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the entropy of the first set of joint variables
+	 * 
+	 * @return
+	 */
+	public double computeAverageMarginalEntropy(int variableIndex) {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = svkeMarginals[variableIndex].getProbability(observations[b][variableIndex], b);
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = -Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob
+						+ " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the information distance
+	 * 
+	 * @return
+	 */
+	public double computeAverageInfoDistanceOfObservations() {
+		double infoDistance = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double marginalProbProducts = 1.0;
+			for (int d = 0; d < dimensions; d++) {
+				marginalProbProducts *= svkeMarginals[d].getProbability(observations[b][d], b);
+			}
+			double probJoint = mvkeJoint.getProbability(observations[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probJoint > 0.0) {
+				// TODO Should probably check that marginalProbProducts has not 
+				//  gone to zero (this is possible with several multiplications
+				//  of 1/N, though is unlikely). I'm not sure what we would do if
+				//  it had gone to zero though ... ignore this value?
+				//  It's not easy to ignore since we can't say p log p -> 0 here
+				//  because marginalProbProducts is not multiplying out the front
+				logTerm = marginalProbProducts / (probJoint * probJoint);
+				cont = Math.log(logTerm);
+			}
+			infoDistance += cont;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) +
+						" -> sum: " + (infoDistance/Math.log(2.0)));
+			}
+		}
+		return infoDistance / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Compute the local MI values for the previous observations.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(observations, true);
+	}
+
+	/**
+	 * Compute the local MI values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalUsingPreviousObservations(double states[][]) {
+		return computeLocalUsingPreviousObservations(states, false);
+	}
+	
+	/**
+	 * Internal implementation
+	 */
+	protected double[] computeLocalUsingPreviousObservations(double states[][],
+			boolean isOurPreviousObservations) {
+		double mi = 0.0;
+		int timeSteps = states.length;
+		double[] localMi = new double[timeSteps];
+		double probJoint;
+		for (int b = 0; b < timeSteps; b++) {
+			double marginalProbProducts = 1.0;
+			for (int d = 0; d < dimensions; d++) {
+				if (isOurPreviousObservations) {
+					marginalProbProducts *= svkeMarginals[d].getProbability(states[b][d], b);
+				} else {
+					marginalProbProducts *= svkeMarginals[d].getProbability(states[b][d]);
+				}
+			}
+			if (isOurPreviousObservations) {
+				probJoint = mvkeJoint.getProbability(states[b], b);
+			} else {
+				probJoint = mvkeJoint.getProbability(states[b]);
+			}
+			double logTerm = 0.0;
+			localMi[b] = 0.0;
+			if (probJoint > 0.0) {
+				// TODO Should probably check that marginalProbProducts has not 
+				//  gone to zero (this is possible with several multiplications
+				//  of 1/N, though is unlikely). I'm not sure what we would do if
+				//  it had gone to zero though ... ignore this value?
+				logTerm = probJoint / marginalProbProducts;
+				localMi[b] = Math.log(logTerm) / Math.log(2.0);
+			}
+			mi += localMi[b];
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + localMi[b] + " -> sum: " + mi);
+			}
+		}
+		lastAverage = mi / (double) totalObservations;
+		return localMi;
+	}
+
+	
+	public double[] computeLocalJointEntropyOfPreviousObservations() throws Exception {
+		return computeLocalJointEntropyUsingPreviousObservations(observations, true);
+	}
+
+	/**
+	 * Compute the local joint entropy values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 *  
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalJointEntropyUsingPreviousObservations(double states[][]) {
+		return computeLocalJointEntropyUsingPreviousObservations(states, false);
+	}
+	
+	/**
+	 * Internal implementation
+	 * 
+	 * @param states
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	protected double[] computeLocalJointEntropyUsingPreviousObservations(
+			double states[][], boolean isOurPreviousObservations) {
+
+		int timeSteps = states.length;
+		double[] localJoint = new double[timeSteps];
+		double probJoint;
+		for (int b = 0; b < totalObservations; b++) {
+			if (isOurPreviousObservations) {
+				probJoint = mvkeJoint.getProbability(states[b], b);
+			} else {
+				probJoint = mvkeJoint.getProbability(states[b]);
+			}
+			localJoint[b] = 0.0;
+			if (probJoint > 0.0) {
+				localJoint[b] = - Math.log(probJoint) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + probJoint + " -> " + localJoint[b]);
+			}
+		}
+		return localJoint;
+	}
+
+	
+	public double[] computeLocalMarginalEntropyOfPreviousObservations(int variableIndex) {
+		return computeLocalMarginalEntropyUsingPreviousObservations(observations, variableIndex, true);
+	}
+
+	/**
+	 * 
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 *  
+	 * @param states
+	 * @param useProbsForWhichVar use 1 for variable 1, 2 for variable 2
+	 * @return
+	 */
+	public double[] computeLocalMarginalEntropyUsingPreviousObservations(double states[][], int variableIndex) {
+		return computeLocalMarginalEntropyUsingPreviousObservations(states, variableIndex, false);
+	}
+	
+	/**
+	 * Internal implementation
+	 * 
+	 * @param states
+	 * @param variableIndex
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	protected double[] computeLocalMarginalEntropyUsingPreviousObservations(
+			double states[][], int variableIndex, boolean isOurPreviousObservations) {
+		int timeSteps = states.length;
+		double[] localEntropy = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			if (isOurPreviousObservations) {
+				prob = svkeMarginals[variableIndex].getProbability(states[b][variableIndex], b);
+			} else {
+				prob = svkeMarginals[variableIndex].getProbability(states[b][variableIndex]);
+			}
+			localEntropy[b] = 0.0;
+			if (prob > 0.0) {
+				localEntropy[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localEntropy[b]);
+			}
+		}
+		return localEntropy;
+	}
+
+	/**
+	 * Compute the local Info distance values for the previously provided
+	 *  observations to compute the probabilities.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalInfoDistanceOfPreviousObservations() {
+		return computeLocalInfoDistanceUsingPreviousObservations(observations, true);
+	}
+
+	/**
+	 * Compute the local Info distance values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalInfoDistanceUsingPreviousObservations(double[][] states) {
+		return computeLocalInfoDistanceUsingPreviousObservations(states, false);
+	}
+
+	/**
+	 * Internal implementation
+	 * 
+	 * @param states
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	protected double[] computeLocalInfoDistanceUsingPreviousObservations(
+			double[][] states, boolean isOurPreviousObservations) {
+		
+		int timeSteps = states.length;
+		double[] localInfoDistance = new double[timeSteps];
+		double probJoint;
+		for (int b = 0; b < timeSteps; b++) {
+			double marginalProbProducts = 1.0;
+			for (int d = 0; d < dimensions; d++) {
+				if (isOurPreviousObservations) {
+					marginalProbProducts *= svkeMarginals[d].getProbability(states[b][d], b);
+				} else {
+					marginalProbProducts *= svkeMarginals[d].getProbability(states[b][d]);
+				}
+			}
+			if (isOurPreviousObservations) {
+				probJoint = mvkeJoint.getProbability(states[b], b);
+			} else {
+				probJoint = mvkeJoint.getProbability(states[b]);
+			}
+			double logTerm = 0.0;
+			localInfoDistance[b] = 0.0;
+			if (probJoint > 0.0) {
+				// TODO Should probably check that marginalProbProducts has not 
+				//  gone to zero (this is possible with several multiplications
+				//  of 1/N, though is unlikely). I'm not sure what we would do if
+				//  it had gone to zero though ... ignore this value?
+				//  It's not easy to ignore since we can't say p log p -> 0 here
+				//  because marginalProbProducts is not multiplying out the front
+				logTerm = marginalProbProducts / (probJoint * probJoint);
+				localInfoDistance[b] = Math.log(logTerm) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + localInfoDistance[b]);
+			}
+		}
+		return localInfoDistance;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			for (int d = 0; d < dimensions; d++) {
+				svkeMarginals[d].setNormalise(normalise);
+			}
+			mvkeJoint.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+			if (dynCorrExcl) {
+				for (int d = 0; d < dimensions; d++) {
+					svkeMarginals[d].setDynamicCorrelationExclusion(dynCorrExclTime);
+				}
+				mvkeJoint.setDynamicCorrelationExclusion(dynCorrExclTime);
+			} else {
+				for (int d = 0; d < dimensions; d++) {
+					svkeMarginals[d].clearDynamicCorrelationExclusion();
+				}
+				mvkeJoint.clearDynamicCorrelationExclusion();
+			}
+		} else if (propertyName.equalsIgnoreCase(SAMPLING_FACTOR_PROP_NAME)) {
+			// Use less than 100 % of samples in the addObservation method
+			samplingFactor = Double.parseDouble(propertyValue);
+			underSample = (samplingFactor < 1.0);
+			rand = new Random();
+		} else {
+			// Property not recognised
+			return;
+		}
+		if (debug) {
+			System.out.println("Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+}

java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateKernel.java

@@ -0,0 +1,689 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+public class MutualInfoCalculatorMultiVariateKernel implements
+	MutualInfoCalculatorMultiVariate {
+
+	KernelEstimatorMultiVariate mvke1 = null;
+	KernelEstimatorMultiVariate mvke2 = null;
+	KernelEstimatorMultiVariate mvkeJoint = null;
+
+	private int totalObservations = 0;
+	// private int dimensions1 = 0;
+	// private int dimensions2 = 0;
+	private boolean debug = false;
+	private double[][] observations1; 
+	private double[][] observations2; 
+	private double lastAverage;
+	private boolean miComputed;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	private boolean forceCompareToAll = false;
+	public static final String FORCE_KERNEL_COMPARE_TO_ALL = "FORCE_KERNEL_COMPARE_TO_ALL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	public MutualInfoCalculatorMultiVariateKernel() {
+		mvke1 = new KernelEstimatorMultiVariate();
+		mvke2 = new KernelEstimatorMultiVariate();
+		mvkeJoint = new KernelEstimatorMultiVariate();
+		mvke1.setNormalise(normalise);
+		mvke2.setNormalise(normalise);
+		mvkeJoint.setNormalise(normalise);
+	}
+
+	/**
+	 * Initialise using a default epsilon
+	 * 
+	 * @param dimensions1
+	 * @param dimensions2
+	 */
+	public void initialise(int dimensions1, int dimensions2) {
+		initialise(dimensions1, dimensions2, epsilon);
+	}
+
+	public void initialise(int dimensions1, int dimensions2, double epsilon) {
+		this.epsilon = epsilon;
+		mvke1.initialise(dimensions1, epsilon);
+		mvke2.initialise(dimensions2, epsilon);
+		mvkeJoint.initialise(dimensions1 + dimensions2, epsilon);
+		// this.dimensions1 = dimensions1;
+		// this.dimensions2 = dimensions2;
+		lastAverage = 0.0;
+		miComputed = false;
+	}
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		// TODO If we ever implement these (which will require changing the kernel
+		//  estimators) we will need to throw an exception if dynamic correlation
+		//  exclusion was set.
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	/**
+	 * Set the observations for the PDFs.
+	 * Should only be called once, the last call contains the
+	 *  observations that are used (they are not accumulated). 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double observations1[][], double observations2[][]) throws Exception {
+		mvke1.setObservations(observations1);
+		mvke2.setObservations(observations2);
+		// This call will throw an exception for us if the length of observations1 and 2 
+		//  are not the same
+		mvkeJoint.setObservations(observations1, observations2);
+		totalObservations = observations1.length;
+		this.observations1 = observations1;
+		this.observations2 = observations2;
+	}
+	
+	/**
+	 * Compute the MI from the observations we were given
+	 * 
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double mi = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob1 = mvke1.getProbability(observations1[b], b);
+			double prob2 = mvke2.getProbability(observations2[b], b);
+			double probJoint = mvkeJoint.getProbability(observations1[b], observations2[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probJoint > 0.0) {
+				// If we have counted joint correlations, we must have marginals for each
+				logTerm = probJoint / (prob1 * prob2);
+				cont = Math.log(logTerm);
+			}
+			mi += cont;
+			if (debug) {
+				System.out.printf("%d: (%.5f, %.5f, %.5f) %.5f -> %.5f -> %.5f\n",
+						b, prob1, prob2, probJoint, logTerm, cont, mi);
+			}
+		}
+		lastAverage = mi / (double) totalObservations / Math.log(2.0);
+		miComputed = true;
+		return lastAverage;
+	}
+
+	/**
+	 * Compute the MI if data were reordered.
+	 * 
+	 * @param newOrdering
+	 * @return MI under the reordering scheme
+	 */
+	public double computeAverageLocalOfObservations(int[] newOrdering) throws Exception {
+		// Store the real observations and their MI:
+		double actualMI = lastAverage;
+		double[][] originalData2 = observations2;
+		double[][] data2;
+		
+		// Generate a new re-ordered data2
+		data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, newOrdering);
+		observations2 = data2;
+		// Perform new initialisations
+		mvkeJoint.initialise(observations1[0].length + originalData2[0].length, epsilon);
+		// Set new observations
+		mvkeJoint.setObservations(observations1, data2);
+		// Compute the MI
+		double newMI = computeAverageLocalOfObservations();
+		
+		// Restore the actual MI and the observations
+		lastAverage = actualMI;
+		observations2 = originalData2;
+		mvkeJoint.initialise(observations1[0].length + originalData2[0].length, epsilon);
+		mvkeJoint.setObservations(observations1, originalData2);
+		return newMI;
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(observations1.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = lastAverage;
+		double[][] originalData1 = observations1;
+		double[][] originalData2 = observations2;
+		double[][] data2;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Generate a new re-ordered data2
+			data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, newOrderings[i]);
+			observations2 = data2;
+			// Perform new initialisations
+			mvkeJoint.initialise(originalData1[0].length + originalData2[0].length, epsilon);
+			// Set new observations
+			mvkeJoint.setObservations(originalData1, data2);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservations();
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		lastAverage = actualMI;
+		observations2 = originalData2;
+		mvkeJoint.initialise(originalData1[0].length + originalData2[0].length, epsilon);
+		mvkeJoint.setObservations(originalData1, originalData2);
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualMI;
+		
+		return measDistribution;
+	}
+
+	/**
+	 * Extra utility method to return the joint entropy
+	 * 
+	 * @return
+	 */
+	public double computeAverageJointEntropy() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvkeJoint.getProbability(observations1[b], observations2[b], b);
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = - Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the entropy of the first set of joint variables
+	 * 
+	 * @return
+	 */
+	public double computeAverageEntropyOfObservation1() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvke1.getProbability(observations1[b], b);
+			double cont = 0.0;
+			// Comparing the prob to 0.0 should be fine - it would have to be
+			//  an impossible number of samples for us to hit machine resolution here.
+			if (prob > 0.0) {
+				cont = -Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the entropy of the second set of joint variables
+	 * 
+	 * @return
+	 */
+	public double computeAverageEntropyOfObservation2() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvke2.getProbability(observations2[b], b);
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = -Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the information distance
+	 * 
+	 * @return
+	 */
+	public double computeAverageInfoDistanceOfObservations() {
+		double infoDistance = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob1 = mvke1.getProbability(observations1[b], b);
+			double prob2 = mvke2.getProbability(observations2[b], b);
+			double probJoint = mvkeJoint.getProbability(observations1[b], observations2[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probJoint > 0.0) {
+				logTerm = (prob1 * prob2) / (probJoint * probJoint);
+				cont = Math.log(logTerm);
+			}
+			infoDistance += cont;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (infoDistance/Math.log(2.0)));
+			}
+		}
+		return infoDistance / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Compute the local MI values for the previous observations.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(observations1, observations2, true);
+	}
+
+
+	/**
+	 * Compute the local MI values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalUsingPreviousObservations(double states1[][], double states2[][]) {
+		return computeLocalUsingPreviousObservations(states1, states2, false);
+	}
+	
+	/**
+	 * Internal method implementing local computation
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	protected double[] computeLocalUsingPreviousObservations(double states1[][],
+			double states2[][], boolean isOurPreviousObservations) {
+		
+		double mi = 0.0;
+		int timeSteps = states1.length;
+		double[] localMi = new double[timeSteps];
+		double prob1, prob2, probJoint;
+		for (int b = 0; b < timeSteps; b++) {
+			if (isOurPreviousObservations) {
+				// We've been called with our previous observations, so we
+				//  can pass the time step through for dynamic correlation exclusion
+				prob1 = mvke1.getProbability(states1[b], b);
+				prob2 = mvke2.getProbability(states2[b], b);
+				probJoint = mvkeJoint.getProbability(states1[b], states2[b], b);
+			} else {
+				// We don't know whether these were our previous observation or not
+				//  so we don't do dynamic correlation exclusion
+				prob1 = mvke1.getProbability(states1[b]);
+				prob2 = mvke2.getProbability(states2[b]);
+				probJoint = mvkeJoint.getProbability(states1[b], states2[b]);
+			}
+			double logTerm = 0.0;
+			localMi[b] = 0.0;
+			if (probJoint > 0.0) {
+				// By necessity prob1 and prob2 will be > 0.0
+				logTerm = probJoint / (prob1 * prob2);
+				localMi[b] = Math.log(logTerm) / Math.log(2.0);
+			}
+			mi += localMi[b];
+			if (debug) {
+				System.out.printf("%d: (%.5f, %.5f, %.5f) %.5f -> %.5f -> %.5f\n",
+						b, prob1, prob2, probJoint, logTerm, localMi[b], mi);
+			}
+		}
+		lastAverage = mi / (double) totalObservations;
+		miComputed = true;
+		return localMi;
+	}
+
+	
+	/**
+	 * Compute the local joint entropy values of the previously provided
+	 *  observations.
+	 *  
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalJointEntropyOfPreviousObservations() throws Exception {
+		return computeLocalJointEntropyUsingPreviousObservations(observations1,
+				observations2, true);
+	}
+
+	/**
+	 * Compute the local joint entropy values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 *  
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalJointEntropyUsingPreviousObservations(double states1[][], double states2[][]) {
+		return computeLocalJointEntropyUsingPreviousObservations(states1,
+				states2, false);
+	}
+	
+	/**
+	 * Internal implementation
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	private double[] computeLocalJointEntropyUsingPreviousObservations(
+			double states1[][], double states2[][], boolean isOurPreviousObservations) {
+		int timeSteps = states1.length;
+		double[] localJoint = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			if (isOurPreviousObservations) {
+				prob = mvkeJoint.getProbability(observations1[b], observations2[b], b);
+			} else {
+				prob = mvkeJoint.getProbability(observations1[b], observations2[b]);
+			}
+			localJoint[b] = 0.0;
+			if (prob > 0.0) {
+				localJoint[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localJoint[b]);
+			}
+		}
+		return localJoint;
+	}
+
+	/**
+	 * Compute the local entropy values for the previously provided
+	 *  observations for VARIABLE 1 to compute the probabilities.
+	 * 
+	 * @param states1
+	 *  
+	 * @return
+	 */
+	public double[] computeLocalEntropy1OfPreviousObservations() {
+		return computeLocalEntropyFromPreviousObservations(observations1, 1, true);
+	}
+
+	/**
+	 * Compute the local entropy values for these given values, using the previously provided
+	 *  observations for VARIABLE 1 to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 * 
+	 * @param states1
+	 *  
+	 * @return
+	 */
+	public double[] computeLocalEntropy1UsingPreviousObservations(double[][] states) {
+		return computeLocalEntropyFromPreviousObservations(states, 1, false);
+	}
+
+	/**
+	 * Compute the local entropy values for the previously provided
+	 *  observations for VARIABLE 1 to compute the probabilities.
+	 * 
+	 * @param states2
+	 *  
+	 * @return
+	 */
+	public double[] computeLocalEntropy2OfPreviousObservations() {
+		return computeLocalEntropyFromPreviousObservations(observations2, 2, true);
+	}
+
+	/**
+	 * Compute the local entropy values for these given values, using the previously provided
+	 *  observations for VARIABLE 2 to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 *  
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalEntropy2UsingPreviousObservations(double states[][]) {
+		return computeLocalEntropyFromPreviousObservations(states, 2, false);
+	}
+
+	/**
+	 * Utility function to implement computeLocalEntropy1FromPreviousObservations
+	 *  and computeLocalEntropy2FromPreviousObservations
+	 *  
+	 * @param states
+	 * @param useProbsForWhichVar use 1 for variable 1, 2 for variable 2
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	private double[] computeLocalEntropyFromPreviousObservations(
+			double states[][], int useProbsForWhichVar, boolean isOurPreviousObservations) {
+		int timeSteps = states.length;
+		double[] localEntropy = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			if (useProbsForWhichVar == 1) {
+				if (isOurPreviousObservations) {
+					prob = mvke1.getProbability(states[b], b);
+				} else {
+					prob = mvke1.getProbability(states[b]);
+				}
+			} else {
+				if (isOurPreviousObservations) {
+					prob = mvke2.getProbability(states[b], b);
+				} else {
+					prob = mvke2.getProbability(states[b]);
+				}
+			}
+			localEntropy[b] = 0.0;
+			if (prob > 0.0) {
+				localEntropy[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localEntropy[b]);
+			}
+		}
+		return localEntropy;
+	}
+
+	/**
+	 * Compute the local Info distance values for the previously provided
+	 *  observations to compute the probabilities.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalInfoDistanceOfPreviousObservations() {
+		return computeLocalInfoDistanceUsingPreviousObservations(observations1,
+				observations2, true);
+	}
+
+	/**
+	 * Compute the local Info distance values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * Calls to this method will not harness dynamic correlation exclusion (if set)
+	 *  since we don't know whether it's the same time set or not.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalInfoDistanceUsingPreviousObservations(double[][] states1, double[][] states2) {
+		return computeLocalInfoDistanceUsingPreviousObservations(states1,
+				states2, false);
+	}
+
+	
+	protected double[] computeLocalInfoDistanceUsingPreviousObservations(
+			double[][] states1, double[][] states2, boolean isOurPreviousObservations) {
+		int timeSteps = states1.length;
+		double[] localInfoDistance = new double[timeSteps];
+		double prob1, prob2, probJoint;
+		for (int b = 0; b < timeSteps; b++) {
+			if (isOurPreviousObservations) {
+				prob1 = mvke1.getProbability(states1[b], b);
+				prob2 = mvke2.getProbability(states2[b], b);
+				probJoint = mvkeJoint.getProbability(states1[b], states2[b], b);
+			} else {
+				prob1 = mvke1.getProbability(states1[b]);
+				prob2 = mvke2.getProbability(states2[b]);
+				probJoint = mvkeJoint.getProbability(states1[b], states2[b]);
+			}
+			double logTerm = 0.0;
+			localInfoDistance[b] = 0.0;
+			if (probJoint > 0.0) {
+				logTerm = (prob1 * prob2) / (probJoint * probJoint);
+				localInfoDistance[b] = Math.log(logTerm) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + localInfoDistance[b]);
+			}
+		}
+		return localInfoDistance;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
+	/**
+	 * Set properties for the mutual information calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>{@link #EPSILON_PROP_NAME}</li>
+	 * 		<li>{@link #NORMALISE_PROP_NAME}</li>
+	 * 		<li>{@link #DYN_CORR_EXCL_TIME_NAME}</li>
+	 * 		<li>{@link #FORCE_KERNEL_COMPARE_TO_ALL}</li>
+	 * </ul>
+	 * 
+	 * Note that dynamic correlation exclusion may have unexpected results if multiple
+	 *  observation sets have been added. This is because multiple observation sets
+	 *  are treated as though they are from a single time series, so observations from
+	 *  near the end of observation set i will be excluded from comparison to 
+	 *  observations near the beginning of observation set (i+1). 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			mvke1.setNormalise(normalise);
+			mvke2.setNormalise(normalise);
+			mvkeJoint.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+			if (dynCorrExcl) {
+				mvke1.setDynamicCorrelationExclusion(dynCorrExclTime);
+				mvke2.setDynamicCorrelationExclusion(dynCorrExclTime);
+				mvkeJoint.setDynamicCorrelationExclusion(dynCorrExclTime);
+			} else {
+				mvke1.clearDynamicCorrelationExclusion();
+				mvke2.clearDynamicCorrelationExclusion();
+				mvkeJoint.clearDynamicCorrelationExclusion();
+			}
+		} else if (propertyName.equalsIgnoreCase(FORCE_KERNEL_COMPARE_TO_ALL)) {
+			forceCompareToAll = Boolean.parseBoolean(propertyValue);
+			mvke1.setForceCompareToAll(forceCompareToAll);
+			mvke2.setForceCompareToAll(forceCompareToAll);
+			mvkeJoint.setForceCompareToAll(forceCompareToAll);
+		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
+			int diff = Integer.parseInt(propertyValue);
+			if (diff != 0) {
+				throw new RuntimeException(PROP_TIME_DIFF + " property != 0 not implemented yet");
+			}
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	public int getNumObservations() {
+		return totalObservations;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateWithDiscreteKernel.java

@@ -0,0 +1,604 @@
+package infodynamics.measures.continuous.kernel;
+
+import java.util.Arrays;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariateWithDiscrete;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+public class MutualInfoCalculatorMultiVariateWithDiscreteKernel implements
+	MutualInfoCalculatorMultiVariateWithDiscrete {
+
+	KernelEstimatorMultiVariate mvke = null;
+	KernelEstimatorMultiVariate[] mvkeForEachDiscrete = null;
+	int base = 0;
+
+	private int totalObservations = 0;
+	// private int dimensions1 = 0;
+	// private int dimensions2 = 0;
+	private boolean debug = false;
+	private double[][] contObservations; 
+	private int[] discObservations;
+	private int[] discCounts;
+	private double lastAverage;
+	private boolean miComputed;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+
+	// No dynamic correlation exclusion time, since we won't track time
+	//  for the conditional observations
+	
+	private boolean forceCompareToAll = false;
+	public static final String FORCE_KERNEL_COMPARE_TO_ALL = "FORCE_KERNEL_COMPARE_TO_ALL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	private boolean usingSingleKernelWidthValue = true;
+	private double[] epsilons = null;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	public MutualInfoCalculatorMultiVariateWithDiscreteKernel() {
+		mvke = new KernelEstimatorMultiVariate();
+		mvke.setNormalise(normalise);
+	}
+
+	/**
+	 * Initialise using a the current settings for the kernel width
+	 *  (which is the default kernel width {@link DEFAULT_EPSILON}
+	 *   if it has not yet been set)
+	 * 
+	 * @param dimensions for number of continuous variables
+	 * @param base for discrete variable
+	 */
+	public void initialise(int dimensions, int base) {
+		if (usingSingleKernelWidthValue) {
+			mvke.initialise(dimensions, epsilon);
+		} else {
+			// epsilons must have been initialised previously in this case
+			mvke.initialise(epsilons);
+		}
+		initialiseCommon(base);
+	}
+
+	/**
+	 * Initialise using the supplied kernel width for all continuous variables
+	 * 
+	 * @param dimensions for number of continuous variables
+	 * @param base for discrete variable
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int dimensions, int base, double epsilon) {
+		this.epsilon = epsilon;
+		usingSingleKernelWidthValue = true;
+		mvke.initialise(dimensions, epsilon);
+		initialiseCommon(base);
+	}
+
+	/**
+	 * Initialise using the supplied kernel width for all continuous variables
+	 * 
+	 * @param base for discrete variable
+	 * @param epsilons kernel width for each continuous variable
+	 */
+	public void initialise(int base, double epsilons[]) {
+		this.epsilons = epsilons;
+		usingSingleKernelWidthValue = false;
+		mvke.initialise(epsilons);
+		initialiseCommon(base);
+	}
+
+	protected void initialiseCommon(int base) {
+		this.base = base;
+		mvkeForEachDiscrete = new KernelEstimatorMultiVariate[base];
+		for (int i = 0; i < base; i++) {
+			mvkeForEachDiscrete[i] = new KernelEstimatorMultiVariate();
+			// We won't normalise the conditional calculators, but feed them
+			//  the same kernel width used by the full marginal one
+			mvkeForEachDiscrete[i].setNormalise(false);
+			mvkeForEachDiscrete[i].setForceCompareToAll(forceCompareToAll);
+			// Don't initialise these calculators yet - we'll wait until
+			//  we have the adjusted epsilon from the full marginal space
+		}
+		discCounts = new int[base];
+		// this.dimensions1 = dimensions1;
+		// this.dimensions2 = dimensions2;
+		lastAverage = 0.0;
+		miComputed = false;
+	}
+	
+	/**
+	 * Set the observations for the PDFs.
+	 * Should only be called once, the last call contains the
+	 *  observations that are used (they are not accumulated). 
+	 * 
+	 * @param observations
+	 */
+	public void setObservations(double continuousObservations[][], int discreteObservations[]) throws Exception {
+		if (continuousObservations.length != discreteObservations.length) {
+			throw new Exception("Observations are not of the same length");
+		}
+		this.contObservations = continuousObservations;
+		mvke.setObservations(continuousObservations);
+		setDiscreteData(continuousObservations, discreteObservations);
+		totalObservations = continuousObservations.length;
+	}
+	
+	protected void setDiscreteData(double continuousObservations[][], int discreteObservations[]) {
+		// Clear the discrete counts:
+		Arrays.fill(discCounts, 0);
+		// Compute the observation counts for the discrete state
+		for (int t = 0; t < discreteObservations.length; t++) {
+			discCounts[discreteObservations[t]]++;
+		}
+		for (int i = 0; i < base; i++) {
+			// Extract the observations for when this base value occurs:
+			double[][] obsForThisDiscValue = MatrixUtils.extractSelectedPointsMatchingCondition(
+					continuousObservations, discreteObservations, i, discCounts[i]);
+			// Set the kernel width for the relevant kernel estimator:
+			mvkeForEachDiscrete[i].initialise(mvke.epsilonInUse);
+			// Set these observations for the relevant kernel estimator:
+			mvkeForEachDiscrete[i].setObservations(obsForThisDiscValue);
+		}
+		this.discObservations = discreteObservations;
+	}
+	
+	/**
+	 * Compute the MI from the observations we were given.
+	 * 
+	 * @return MI in bits
+	 */
+	public double computeAverageLocalOfObservations() {
+		double mi = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double probCont = mvke.getProbability(contObservations[b]);
+			double condProbCont = mvkeForEachDiscrete[discObservations[b]].getProbability(contObservations[b]);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (condProbCont > 0.0) {
+				// If we have counted joint correlations, we must have marginals for each
+				logTerm = condProbCont / probCont;
+				cont = Math.log(logTerm);
+			}
+			mi += cont;
+			if (debug) {
+				System.out.printf("%d: %.3f, %d, (%.3f %d, %.5f %d) %.5f -> %.5f -> %.5f\n",
+						b, contObservations[b][0], discObservations[b],
+						condProbCont, mvkeForEachDiscrete[discObservations[b]].getCount(contObservations[b]),
+						probCont, mvke.getCount(contObservations[b]),
+						logTerm, cont, mi);
+			}
+		}
+		lastAverage = mi / (double) totalObservations / Math.log(2.0);
+		miComputed = true;
+		return lastAverage;
+	}
+
+	/**
+	 * Compute the MI if data were reordered.
+	 * 
+	 * @param newOrdering
+	 * @return MI under the reordering scheme
+	 */
+	public double computeAverageLocalOfObservations(int[] newOrdering) throws Exception {
+		// Store the real observations and their MI:
+		double actualMI = lastAverage;
+		int[] originalDiscrete = discObservations;
+		
+		// Generate a new re-ordered data2
+		int[] newDiscrete = MatrixUtils.extractSelectedTimePoints(originalDiscrete, newOrdering);
+		
+		// Perform new initialisations on the discrete pdfs
+		setDiscreteData(contObservations, newDiscrete);
+		// Compute the MI
+		double newMI = computeAverageLocalOfObservations();
+		
+		// Restore the actual MI and the observations
+		lastAverage = actualMI;
+		setDiscreteData(contObservations, originalDiscrete);
+
+		return newMI;
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(contObservations.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = lastAverage;
+		int[] originalDiscrete = discObservations;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Generate a new re-ordered data2
+			int[] newDiscrete = MatrixUtils.extractSelectedTimePoints(originalDiscrete, newOrderings[i]);
+			// Perform new initialisations on the discrete pdfs
+			setDiscreteData(contObservations, newDiscrete);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservations();
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		lastAverage = actualMI;
+		setDiscreteData(contObservations, originalDiscrete);
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualMI;
+		
+		return measDistribution;
+	}
+
+	/**
+	 * Extra utility method to return the joint entropy
+	 * 
+	 * @return
+	 */
+	public double computeAverageJointEntropy() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvkeForEachDiscrete[discObservations[b]].getProbability(contObservations[b])
+							* (double) discCounts[discObservations[b]] / (double) totalObservations;
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = - Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the entropy of the first set of joint variables
+	 * 
+	 * @return
+	 */
+	public double computeAverageEntropyOfObservation1() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = mvke.getProbability(contObservations[b]);
+			double cont = 0.0;
+			// Comparing the prob to 0.0 should be fine - it would have to be
+			//  an impossible number of samples for us to hit machine resolution here.
+			if (prob > 0.0) {
+				cont = -Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the entropy of the second set of joint variables
+	 * 
+	 * @return
+	 */
+	public double computeAverageEntropyOfObservation2() {
+		double entropy = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob = (double) discCounts[discObservations[b]] / (double) totalObservations;
+			double cont = 0.0;
+			if (prob > 0.0) {
+				cont = -Math.log(prob);
+			}
+			entropy += cont;
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + cont/Math.log(2.0) + " -> sum: " + (entropy/Math.log(2.0)));
+			}
+		}
+		return entropy / (double) totalObservations / Math.log(2.0);
+	}
+
+	/**
+	 * Extra utility method to return the information distance
+	 * 
+	 * @return
+	 */
+	public double computeAverageInfoDistanceOfObservations() {
+		throw new RuntimeException("Not implemented yet");
+		/*
+		double infoDistance = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			double prob1 = mvke.getProbability(observations1[b], b);
+			double prob2 = mvke2.getProbability(observations2[b], b);
+			double probJoint = mvkeJoint.getProbability(observations1[b], observations2[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probJoint > 0.0) {
+				logTerm = (prob1 * prob2) / (probJoint * probJoint);
+				cont = Math.log(logTerm);
+			}
+			infoDistance += cont;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (infoDistance/Math.log(2.0)));
+			}
+		}
+		return infoDistance / (double) totalObservations / Math.log(2.0);
+		*/
+	}
+
+	/**
+	 * Compute the local MI values for the previous observations.
+	 * 
+	 * @return
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(contObservations, discObservations);
+	}
+
+	/**
+	 * Compute the local MI values for these given values, using the previously provided
+	 *  observations to compute the probabilities.
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalUsingPreviousObservations(double states1[][],
+			int[] states2) {
+		
+		double mi = 0.0;
+		int timeSteps = states1.length;
+		double[] localMi = new double[timeSteps];
+		double condProbCont, probCont;
+		for (int b = 0; b < timeSteps; b++) {
+			probCont = mvke.getProbability(states1[b]);
+			condProbCont = mvkeForEachDiscrete[states2[b]].getProbability(states1[b]);
+			double logTerm = 0.0;
+			localMi[b] = 0.0;
+			if (condProbCont > 0.0) {
+				// By necessity prob1 and prob2 will be > 0.0
+				logTerm = condProbCont / probCont;
+				localMi[b] = Math.log(logTerm) / Math.log(2.0);
+			}
+			mi += localMi[b];
+			if (debug) {
+				System.out.printf("%d: (%.5f, %.5f) %.5f -> %.5f -> %.5f\n",
+						b, condProbCont, probCont, logTerm, localMi[b], mi);
+			}
+		}
+		lastAverage = mi / (double) totalObservations;
+		miComputed = true;
+		return localMi;
+	}
+
+	
+	/**
+	 * Compute the local joint entropy values of the previously provided
+	 *  observations.
+	 *  
+	 * @param states1
+	 * @param states2
+	 * @return
+	 */
+	public double[] computeLocalJointEntropyOfPreviousObservations() throws Exception {
+		return computeLocalJointEntropyUsingPreviousObservations(contObservations,
+				discObservations);
+	}
+
+	/**
+	/**
+	 * Internal implementation
+	 * 
+	 * @param states1
+	 * @param states2
+	 * @param isOurPreviousObservations
+	 * @return
+	 */
+	public double[] computeLocalJointEntropyUsingPreviousObservations(
+			double states1[][], int states2[]) {
+		int timeSteps = states1.length;
+		double[] localJoint = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			prob = mvkeForEachDiscrete[states2[b]].getProbability(states1[b]) *
+				(double) discCounts[states2[b]] / (double) totalObservations;
+			localJoint[b] = 0.0;
+			if (prob > 0.0) {
+				localJoint[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localJoint[b]);
+			}
+		}
+		return localJoint;
+	}
+
+	/**
+	 * Compute the local entropy values for the previously provided
+	 *  observations for VARIABLE 1 to compute the probabilities.
+	 * 
+	 * @param states1
+	 *  
+	 * @return
+	 */
+	public double[] computeLocalEntropy1OfPreviousObservations() {
+		return computeLocalEntropyFromPreviousObservations(contObservations);
+	}
+
+
+	/**
+	 * Compute the local entropy values for the previously provided
+	 *  observations for VARIABLE 2 to compute the probabilities.
+	 * 
+	 * @param states2
+	 *  
+	 * @return
+	 */
+	public double[] computeLocalEntropy2OfPreviousObservations() {
+		return computeLocalEntropyFromPreviousObservations(discObservations);
+	}
+
+	/**
+	 * Utility function to implement computeLocalEntropy1FromPreviousObservations
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public double[] computeLocalEntropyFromPreviousObservations(
+			double states[][]) {
+		int timeSteps = states.length;
+		double[] localEntropy = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			prob = mvke.getProbability(states[b]);
+			localEntropy[b] = 0.0;
+			if (prob > 0.0) {
+				localEntropy[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localEntropy[b]);
+			}
+		}
+		return localEntropy;
+	}
+
+	/**
+	 * Utility function to implement computeLocalEntropy1FromPreviousObservations
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public double[] computeLocalEntropyFromPreviousObservations(
+			int states[]) {
+		int timeSteps = states.length;
+		double[] localEntropy = new double[timeSteps];
+		double prob;
+		for (int b = 0; b < totalObservations; b++) {
+			prob = (double) discCounts[states[b]] / (double) totalObservations;
+			localEntropy[b] = 0.0;
+			if (prob > 0.0) {
+				localEntropy[b] = - Math.log(prob) / Math.log(2.0);
+			}
+			if (debug) {
+				System.out.println(b + ": " + prob + " -> " + localEntropy[b]);
+			}
+		}
+		return localEntropy;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
+	/**
+	 * Set properties for the mutual information calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>{@link #EPSILON_PROP_NAME} - applies to full marginal space of continuous</li>
+	 * 		<li>{@link #NORMALISE_PROP_NAME}</li>
+	 * 		<li>{@link #FORCE_KERNEL_COMPARE_TO_ALL}</li>
+	 * </ul>
+	 * 
+	 * Note that dynamic correlation exclusion may have unexpected results if multiple
+	 *  observation sets have been added. This is because multiple observation sets
+	 *  are treated as though they are from a single time series, so observations from
+	 *  near the end of observation set i will be excluded from comparison to 
+	 *  observations near the beginning of observation set (i+1). 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			usingSingleKernelWidthValue = true;
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			mvke.setNormalise(normalise);
+			// Don't set the normalise property on the conditional
+			//  kernel estimation calculators - we'll set these directly
+			//  from the joint space
+		} else if (propertyName.equalsIgnoreCase(FORCE_KERNEL_COMPARE_TO_ALL)) {
+			forceCompareToAll = Boolean.parseBoolean(propertyValue);
+			mvke.setForceCompareToAll(forceCompareToAll);
+			for (int i = 0; i < base; i++) {
+				mvkeForEachDiscrete[i].setForceCompareToAll(forceCompareToAll);
+			}
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	public int getNumObservations() {
+		return totalObservations;
+	}
+	
+	/**
+	 * Return the kernel widths used by the MI calculator for the
+	 *  continuous variables here.
+	 * This should not be called until the observations have been set.
+	 * These are the kernel widths actually applied to the data (not the 
+	 *  number of standard deviations, if we are using normalisation).
+	 * 
+	 * @return an array of doubles with the kernel widths.
+	 */
+	public double[] getKernelWidthsInUse() {
+		// Return a copy so that the user can't mess with it
+		return Arrays.copyOf(mvke.epsilonInUse, mvke.epsilonInUse.length);
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorKernel.java

@@ -0,0 +1,446 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.measures.continuous.kernel.TransferEntropyKernelCounts;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Iterator;
+
+
+/**
+ * 
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>
+ * TODO Implement dynamic correlation exclusion with multiple observation sets. (see the
+ *  way this is done in Plain calculator).
+ * </p>
+ * 
+ * @author Joseph Lizier
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorKernel
+	extends TransferEntropyCommon implements TransferEntropyCalculator {
+
+	protected KernelEstimatorTransferEntropy teKernelEstimator = null;
+
+	// Keep joint vectors so we don't need to regenerate them
+	protected double[][] destPastVectors;
+	protected double[]   destNextValues;
+	protected double[]   sourceValues;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	private boolean forceCompareToAll = false;
+	public static final String FORCE_KERNEL_COMPARE_TO_ALL = "FORCE_KERNEL_COMPARE_TO_ALL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public TransferEntropyCalculatorKernel() {
+		super();
+		teKernelEstimator = new KernelEstimatorTransferEntropy();
+		teKernelEstimator.setNormalise(normalise);
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		super.initialise(k); // calls initialise();
+	}
+
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() {
+		teKernelEstimator.initialise(k, epsilon);
+		destPastVectors = null;
+		destNextValues = null;
+		sourceValues = null;
+	}
+	
+	/**
+	 * Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>K_PROP_NAME</li>
+	 * 		<li>EPSILON_PROP_NAME</li>
+	 * 		<li>NORMALISE_PROP_NAME</li>
+	 * 		<li>DYN_CORR_EXCL_TIME_NAME</li>
+	 * 		<li>FORCE_KERNEL_COMPARE_TO_ALL</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			teKernelEstimator.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+			if (dynCorrExcl) {
+				teKernelEstimator.setDynamicCorrelationExclusion(dynCorrExclTime);
+			} else {
+				teKernelEstimator.clearDynamicCorrelationExclusion();
+			}
+		} else if (propertyName.equalsIgnoreCase(FORCE_KERNEL_COMPARE_TO_ALL)) {
+			forceCompareToAll = Boolean.parseBoolean(propertyValue);
+			teKernelEstimator.setForceCompareToAll(forceCompareToAll);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		destPastVectors = new double[totalObservations][k];
+		destNextValues = new double[totalObservations];
+		sourceValues = new double[totalObservations];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[]> iterator = vectorOfDestinationObservations.iterator();
+		for (double[] source : vectorOfSourceObservations) {
+			double[] destination = iterator.next();
+			double[][] currentDestPastVectors = makeJointVectorForPast(destination);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					destPastVectors, startObservation, 0, currentDestPastVectors.length, k);
+			System.arraycopy(destination, k, destNextValues, startObservation, destination.length - k);
+			System.arraycopy(source, k - 1, sourceValues, startObservation, source.length - k);
+			startObservation += destination.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		teKernelEstimator.setObservations(destPastVectors, destNextValues, sourceValues);
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfDestinationObservations.size() > 1;
+		if (addedMoreThanOneObservationSet && dynCorrExcl) {
+			// We have not properly implemented dynamic correlation exclusion for
+			//  multiple observation sets, so throw an error
+			throw new RuntimeException("Addition of multiple observation sets is not currently " +
+					"supported with property DYN_CORR_EXCL set");
+		}
+
+		// And clear the vector of observations
+		vectorOfSourceObservations = null;
+		vectorOfDestinationObservations = null;
+	}
+	
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destPastVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			TransferEntropyKernelCounts kernelCounts = teKernelEstimator.getCount(destPastVectors[b],
+					destNextValues[b], sourceValues[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (kernelCounts.countNextPastSource > 0) {
+				logTerm = ((double) kernelCounts.countNextPastSource / (double) kernelCounts.countPastSource) /
+							((double) kernelCounts.countNextPast / (double) kernelCounts.countPast);
+				cont = Math.log(logTerm);
+			}
+			te += cont;
+			if (debug) {
+				System.out.println(b + ": " + destPastVectors[b][0] + " (" +
+						kernelCounts.countNextPastSource + " / " + kernelCounts.countPastSource + ") / (" +
+						kernelCounts.countNextPast + " / " + kernelCounts.countPast + ") = " + 
+						logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations,
+	 *  using the Grassberger correction for the point count k: log_e(k) ~= digamma(k).</p>
+	 * <p>Kaiser and Schreiber, Physica D 166 (2002) pp. 43-62 suggest (on p. 57) that for the TE
+	 *   though the adverse correction of the bias correction is worse than the correction
+	 *   itself (because the probabilities being multiplied/divided are not independent), 
+	 *   so recommend not to use this method.
+	 * </p>
+	 * <p>It is implemented here for testing purposes only.</p>
+	 * 
+	 */
+	public double computeAverageLocalOfObservationsWithCorrection() throws Exception {
+		
+		double te = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			TransferEntropyKernelCounts kernelCounts = teKernelEstimator.getCount(destPastVectors[b],
+					destNextValues[b], sourceValues[b], b);
+			double cont = 0.0;
+			if (kernelCounts.countNextPastSource > 0) {
+				cont = MathsUtils.digamma(kernelCounts.countNextPastSource) - 
+						MathsUtils.digamma(kernelCounts.countPastSource) -
+						MathsUtils.digamma(kernelCounts.countNextPast) +
+						MathsUtils.digamma(kernelCounts.countPast);
+			}
+			te += cont;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		// Average it, and convert results to bytes
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * Computes the local transfer entropies for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(null, null, true);
+	}
+
+	/**
+	 * Comptues local transfer entropies for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalUsingPreviousObservations(double[] source, double[] destination) throws Exception {
+		return computeLocalUsingPreviousObservations(source, destination, false);
+	}
+	
+	/**
+	 * Returns the local TE at every time point.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalUsingPreviousObservations(double[] source, double[] destination, boolean isPreviousObservations) throws Exception {
+		
+		double[][] newDestPastVectors;
+		double[] newDestNextValues;
+		double[] newSourceValues;
+
+		if (isPreviousObservations) {
+			// We've already computed the joint vectors for these observations
+			newDestPastVectors = destPastVectors;
+			newDestNextValues = destNextValues;
+			newSourceValues = sourceValues;
+		} else {
+			// We need to compute a new set of joint vectors
+			newDestPastVectors = makeJointVectorForPast(destination);
+			newDestNextValues = MatrixUtils.select(destination, k, destination.length - k);
+			newSourceValues = MatrixUtils.select(source, k - 1, source.length - k);
+		}
+
+		double te = 0.0;
+		int numLocalObservations = newDestPastVectors.length;
+		double[] localTE;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localTE = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localTE = new double[numLocalObservations + k];
+			offset = k;
+		}
+		TransferEntropyKernelCounts kernelCounts;
+		for (int b = 0; b < numLocalObservations; b++) {
+			if (isPreviousObservations) {
+				kernelCounts = teKernelEstimator.getCount(newDestPastVectors[b],
+						newDestNextValues[b], newSourceValues[b], b);
+			} else {
+				kernelCounts = teKernelEstimator.getCount(newDestPastVectors[b],
+						newDestNextValues[b], newSourceValues[b], -1);
+			}
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (kernelCounts.countNextPastSource > 0) {
+				logTerm = ((double) kernelCounts.countNextPastSource / (double) kernelCounts.countPastSource) /
+							((double) kernelCounts.countNextPast / (double) kernelCounts.countPast);
+				local = Math.log(logTerm);
+			}
+			localTE[offset + b] = local;
+			te += local;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (local/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) numLocalObservations / Math.log(2.0);
+		return localTE;
+	}
+
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *
+	 * Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(totalObservations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 *  
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		
+		double actualTE = computeAverageLocalOfObservations();
+		
+		// Space for the source observations:
+		double[] oldSourceValues = sourceValues;
+		
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the source in the destPastSourceVectors 
+			//  and destNextPastSourceVectors vectors
+			sourceValues = MatrixUtils.extractSelectedTimePoints(oldSourceValues, newOrderings[p]);
+			
+			// Make the equivalent operations of intialise
+			teKernelEstimator.initialise(k, epsilon);
+			// Make the equivalent operations of setObservations:
+			teKernelEstimator.setObservations(destPastVectors, destNextValues, sourceValues);
+			// And get a TE value for this realisation:
+			double newTe = computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the local variables:
+		lastAverage = actualTE;
+		sourceValues = oldSourceValues;
+		// And set the kernel estimator back to their previous state
+		teKernelEstimator.initialise(k, epsilon);
+		teKernelEstimator.setObservations(destPastVectors, destNextValues, sourceValues);
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualTE;
+		return measDistribution;
+	}
+
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		teKernelEstimator.setDebug(debug);
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorKernelPlain.java

@@ -0,0 +1,462 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+
+import java.util.Iterator;
+
+
+/**
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()].
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>This implementation is simple, involving O(n^2) comparisons.
+ * It incurs a larger memory cost than TransferEntropyCalculatorKernelSimple since
+ * it can accomodate multiple time series samples being added. This is a larger memory
+ * cost also than TransferEntropyCalculatorKernelPlainIterators but has a significantly
+ * lower cost in time than it.
+ * </p>
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorKernelPlain
+	extends TransferEntropyCommon implements TransferEntropyCalculator {
+
+	// Keep joint vectors so we don't need to regenerate them
+	protected double[][] destNextPastSourceVectors;
+	
+	// Indices into array of count results
+	private static final int NEXT_PAST_SOURCE = 0;
+	private static final int PAST_SOURCE = 1;
+	private static final int NEXT_PAST = 2;
+	private static final int PAST = 3;	
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	private double[] epsilons;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public TransferEntropyCalculatorKernelPlain() {
+		super();
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		super.initialise(k); // calls initialise()
+	}
+	
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() {
+		destNextPastSourceVectors = null;
+		epsilons = new double[k + 2];		
+	}
+
+	/**
+	 * Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>EPSILON_PROP_NAME</li>
+	 * 		<li>NORMALISE_PROP_NAME</li>
+	 * 		<li>DYN_CORR_EXCL_TIME_NAME</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+		}
+		if (debug) {
+			System.out.println("Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		destNextPastSourceVectors = new double[totalObservations][k + 2];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[]> iterator = vectorOfDestinationObservations.iterator();
+		for (double[] source : vectorOfSourceObservations) {
+			double[] destination = iterator.next();
+			double[][] currentDestNextPastSourceVectors = makeJointVectorForNextPastSource(destination, source);
+			MatrixUtils.arrayCopy(currentDestNextPastSourceVectors, 0, 0,
+					destNextPastSourceVectors, startObservation, 0, currentDestNextPastSourceVectors.length, k + 2);
+			startObservation += destination.length - k;
+		}
+		
+		if (normalise) {
+			// Adjust epsilon for each dimension
+			for (int c = 0; c < k + 2; c++) {
+				double std = MatrixUtils.stdDev(destNextPastSourceVectors, c);
+				epsilons[c] = epsilon * std;
+			}
+		} else {
+			for (int c = 0; c < k + 2; c++) {
+				epsilons[c] = epsilon;
+			}
+		}
+		
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfDestinationObservations.size() > 1;
+		if (addedMoreThanOneObservationSet && dynCorrExcl) {
+			// We have not properly implemented dynamic correlation exclusion for
+			//  multiple observation sets, so throw an error
+			throw new RuntimeException("Addition of multiple observation sets is not currently " +
+					"supported with property DYN_CORR_EXCL set");
+		}
+
+		// And clear the vector of observations
+		vectorOfSourceObservations = null;
+		vectorOfDestinationObservations = null;
+	}
+	
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destNextPastSourceVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			// Get the probability counts for the bth sample tuple
+			int[] counts = getCounts(destNextPastSourceVectors[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				logTerm = ((double) counts[NEXT_PAST_SOURCE] / (double) counts[PAST_SOURCE]) /
+							((double) counts[NEXT_PAST] / (double) counts[PAST]);
+				cont = Math.log(logTerm);
+			}
+			te += cont;
+			if (debug) {
+				System.out.println(b + ": " + destNextPastSourceVectors[b][0] + " (" + counts[NEXT_PAST_SOURCE] + " / " +
+						counts[PAST_SOURCE] + ") / (" +
+						counts[NEXT_PAST] + " / " + counts[PAST] + ") = " + 
+						logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations,
+	 *  using the Grassberger correction for the point count k: log_e(k) ~= digamma(k).</p>
+	 * <p>Kaiser and Schreiber suggest (on p. 57) that for the TE
+	 *   though the adverse correction of the bias correction is worse than the correction
+	 *   itself (because the probabilities being multiplied/divided are not independent), 
+	 *   so recommend not to use this method.
+	 * </p>
+	 * <p>It is implemented here for testing purposes only.</p>
+	 * 
+	 * @see Kaiser and Schreiber, Physica D 166 (2002) pp. 43-62
+	 */
+	public double computeAverageLocalOfObservationsWithCorrection() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destNextPastSourceVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			int[] counts = getCounts(destNextPastSourceVectors[b], b);
+			double cont = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				cont = MathsUtils.digamma(counts[NEXT_PAST_SOURCE]) -
+						MathsUtils.digamma(counts[PAST_SOURCE]) -
+						MathsUtils.digamma(counts[NEXT_PAST]) +
+						MathsUtils.digamma(counts[PAST]);
+			}
+			te += cont;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		// Average it, and convert results to bytes
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * Computes the local transfer entropies for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(null, null, true);
+	}
+
+	/**
+	 * Comptues local transfer entropies for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalUsingPreviousObservations(double[] source, double[] destination) throws Exception {
+		return computeLocalUsingPreviousObservations(source, destination, false);
+	}
+	
+	/**
+	 * Returns the local TE at every time point.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalUsingPreviousObservations(double[] source, double[] destination, boolean isPreviousObservations) throws Exception {
+		
+		double[][] newDestNextPastSourceVectors;
+
+		if (isPreviousObservations) {
+			// We've already computed the joint vectors for these observations
+			newDestNextPastSourceVectors = destNextPastSourceVectors;
+		} else {
+			// We need to compute a new set of joint vectors
+			newDestNextPastSourceVectors = makeJointVectorForNextPastSource(destination, source);
+		}
+
+		double te = 0.0;
+		int numLocalObservations = newDestNextPastSourceVectors.length;
+		double[] localTE;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localTE = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localTE = new double[numLocalObservations + k];
+			offset = k;
+		}
+		for (int b = 0; b < numLocalObservations; b++) {
+			int[] counts = getCounts(newDestNextPastSourceVectors[b], isPreviousObservations ? b : -1);
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				logTerm = ((double) counts[NEXT_PAST_SOURCE] / (double) counts[PAST_SOURCE]) /
+							((double) counts[NEXT_PAST] / (double) counts[PAST]);
+				local = Math.log(logTerm);
+			}
+			localTE[offset + b] = local;
+			te += local;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (local/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) numLocalObservations / Math.log(2.0);
+		return localTE;
+	}
+
+	/**
+	 * Compute the next, past and source values into a joint vector
+	 * 
+	 * @param destination
+	 * @param source
+	 * @return
+	 */
+	private double[][] makeJointVectorForNextPastSource(double[] destination, double[] source) {
+		double[][] destNextPastSourceVectors = new double[destination.length - k][k + 2];
+		for (int t = k; t < destination.length; t++) {
+			for (int i = 0; i < k + 1; i++) {
+				destNextPastSourceVectors[t - k][i] = destination[t - i];
+			}
+			destNextPastSourceVectors[t - k][k + 1] = source[t - 1];
+		}
+		return destNextPastSourceVectors;
+	}
+	
+	/**
+	 * Return the set of probability counts for the observation at time step 
+	 * observationNumber checked against those stored in
+	 * destNextPastSourceVectors.
+	 * 
+	 * @param observation
+	 * @param observationNumber if &lt; 0 we don't try dynamic correlation exclusion
+	 * @return
+	 */
+	protected int[] getCounts(double[] observation, int observationIndex) {
+		int[] counts = new int[4];
+		for (int b = 0; b < totalObservations; b++) {
+			// If this is one of our samples, and we're doing dynamic correlation exclusion
+			//  and it's within the window, don't count it
+			if ((observationIndex >= 0) && dynCorrExcl &&  
+					(Math.abs(b - observationIndex) < dynCorrExclTime)) {
+				// This sample is within the dyn corr excl window,
+				//  so don't count it
+				continue;
+			}
+
+			if (!compareStateVectors(observation, b, 1, k)) {
+				// Past vectors are different, no point continuing the comparison
+				continue;
+			}
+			counts[PAST]++;
+			boolean nextMatches = Math.abs(observation[0] - 
+					destNextPastSourceVectors[b][0]) <= epsilons[0];
+			boolean sourceMatches = Math.abs(observation[k + 1] - 
+					destNextPastSourceVectors[b][k + 1]) <= epsilons[k + 1];
+			counts[NEXT_PAST] += nextMatches ? 1 : 0;
+			counts[PAST_SOURCE] += sourceMatches ? 1 : 0;
+			counts[NEXT_PAST_SOURCE] += (nextMatches && sourceMatches) ? 1 : 0;
+		}
+		return counts;
+	}
+	
+	/**
+	 * Return whether the vectors at destNextPastSourceVectors[sampleIndex] and
+	 * the observation are the same for length values from
+	 * fromOffset
+	 * 
+	 * @param observation
+	 * @param sampleIndex
+	 * @param fromOffset
+	 * @param length
+	 * @return
+	 */
+	protected boolean compareStateVectors(double[] observation,
+			int sampleIndex, int fromOffset, int length) {
+		int i;
+		for (i = 0; i < length; i++) {
+			if (Math.abs(observation[fromOffset + i] - 
+					destNextPastSourceVectors[sampleIndex][fromOffset + i]) > epsilons[fromOffset + i]) {
+				// This pair can't be within the kernel
+				return false;
+			}
+		}
+		return true;
+	}
+
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		throw new RuntimeException("Not implemented in this calculator");
+	}
+	
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		throw new RuntimeException("Not implemented in this calculator");
+	}
+
+	/**
+	 * <p>Computes the probability counts for each previously supplied observations</p>
+	 * <p>Implemented primarily for debug purposes</p>
+	 * 
+	 */
+	public KernelCount[][] computeMatchesForEachObservations(boolean giveListOfCorrelatedPoints) throws Exception {
+		KernelCount[][] counts = new KernelCount[totalObservations][4];
+		for (int b = 0; b < totalObservations; b++) {
+			int[] intCounts = getCounts(destNextPastSourceVectors[b], b);
+			counts[b] = new KernelCount[4];
+			for (int i = 0; i < 4; i++) {
+				int totalObsCount = 0;
+				if (dynCorrExcl) {
+					// Need to remove any observations that were *closer* than timeProximityForDynamicCorrelationExclusion
+					int closeTimePointsToCompare = (b >= dynCorrExclTime) ?
+							dynCorrExclTime - 1: b;
+					closeTimePointsToCompare += (totalObservations - b >= dynCorrExclTime) ?
+							dynCorrExclTime - 1: totalObservations - b - 1;
+					closeTimePointsToCompare++; // Add one for comparison to self
+					totalObsCount = totalObservations - closeTimePointsToCompare;
+				} else {
+					totalObsCount = totalObservations;
+				}
+				counts[b][i] = new KernelCount(intCounts[i], totalObsCount);
+				// And ignore giveListOfCorrelatedPoints for now.
+			}
+		}
+		return counts;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorKernelPlainIterators.java

@@ -0,0 +1,474 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MeasurementDistribution;
+
+import java.util.Iterator;
+
+
+/**
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>This implementation is simple, involving O(n^2) comparisons.
+ * It wraps the original observations in an iterator, to save on memory cost
+ * of constructing joint vectors, but this results in a significant time cost
+ * over TransferEntropyCalculatorKernelPlain.
+ * </p>
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorKernelPlainIterators
+	extends TransferEntropyCommon implements TransferEntropyCalculator {
+
+	// Keep a record of which time series and time index in it
+	//  each observation number corresponds to
+	protected int[] timeSeriesIndex;
+	protected int[] timeStepIndex;
+	
+	// Indices into array of count results
+	private static final int NEXT_PAST_SOURCE = 0;
+	private static final int PAST_SOURCE = 1;
+	private static final int NEXT_PAST = 2;
+	private static final int PAST = 3;	
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	private double epsDest;
+	private double epsSource;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public TransferEntropyCalculatorKernelPlainIterators() {
+		super();
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		super.initialise(k); // calls initialise();
+	}
+
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() {
+		timeSeriesIndex = null;
+		timeStepIndex = null;
+	}
+	
+	/**
+	 * Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>EPSILON_PROP_NAME</li>
+	 * 		<li>NORMALISE_PROP_NAME</li>
+	 * 		<li>DYN_CORR_EXCL_TIME_NAME</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+		}
+		if (debug) {
+			System.out.println("Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+
+		// Now track which time series and step in it each observation comes from
+		timeSeriesIndex = new int[totalObservations];
+		timeStepIndex = new int[totalObservations];
+		int obs = 0;
+		int currentTimeSeriesIndex = 0;
+		double sumDest = 0.0, sumSqDest = 0.0, sumSource = 0.0, sumSqSource = 0.0;
+		Iterator<double[]> iterator = vectorOfSourceObservations.iterator();
+		for (double[] destination : vectorOfDestinationObservations) {
+			for (int t = k; t < destination.length; t++) {
+				timeSeriesIndex[obs] = currentTimeSeriesIndex;
+				timeStepIndex[obs++] = t;
+			}
+			currentTimeSeriesIndex++;
+			if (normalise) {
+				double[] source = iterator.next();
+				// Track the sum and sum of squares of the dest and source
+				// First add in the first k - 1 steps of history (dest only)
+				for (int t = 0; t < k - 1; t++) {
+					sumDest += destination[t];
+					sumSqDest += destination[t] * destination[t];
+				}
+				// Now do all of the previous steps
+				for (int t = k - 1; t < destination.length - 1; t++) {
+					sumDest += destination[t];
+					sumSqDest += destination[t] * destination[t];
+					sumSource += source[t];
+					sumSqSource += source[t] * source[t];
+				}
+				// Finally add the last destination step (dest only)
+				sumDest += destination[destination.length - 1];
+				sumSqDest += destination[destination.length - 1] * destination[destination.length - 1];
+			}
+		}
+		
+		if (normalise) {
+			double meanDest = sumDest / (double) totalObservations;
+			double meanSource = sumSource / (double) totalObservations;
+			double meanSqDest = sumSqDest / (double) totalObservations;
+			double meanSqSource = sumSqSource / (double) totalObservations;
+			double stdDest = Math.sqrt(meanSqDest - meanDest*meanDest);
+			double stdSource = Math.sqrt(meanSqSource - meanSource*meanSource);
+			// Adjust epsilon for the dest and source
+			epsDest = epsilon * stdDest;
+			epsSource = epsilon * stdSource;
+		} else {
+			epsDest = epsilon;
+			epsSource = epsilon;
+		}
+		
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfDestinationObservations.size() > 1;
+		if (addedMoreThanOneObservationSet && dynCorrExcl) {
+			// This is fine for this calculator - see getCounts
+		}
+	}
+	
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		
+		double te = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			int timeSeries = timeSeriesIndex[b];
+			double[] source = vectorOfSourceObservations.elementAt(timeSeries);
+			double[] dest = vectorOfDestinationObservations.elementAt(timeSeries);
+			int[] counts = getCounts(source, dest, timeStepIndex[b], timeSeries);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				logTerm = ((double) counts[NEXT_PAST_SOURCE] / (double) counts[PAST_SOURCE]) /
+							((double) counts[NEXT_PAST] / (double) counts[PAST]);
+				cont = Math.log(logTerm);
+			}
+			te += cont;
+			if (debug) {
+				System.out.println(b + ": " + dest[timeStepIndex[b]] + " (" + counts[NEXT_PAST_SOURCE] + " / " +
+						counts[PAST_SOURCE] + ") / (" +
+						counts[NEXT_PAST] + " / " + counts[PAST] + ") = " + 
+						logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations,
+	 *  using the Grassberger correction for the point count k: log_e(k) ~= digamma(k).</p>
+	 * <p>Kaiser and Schreiber, Physica D 166 (2002) pp. 43-62 suggest (on p. 57) that for the TE
+	 *   though the adverse correction of the bias correction is worse than the correction
+	 *   itself (because the probabilities being multiplied/divided are not independent), 
+	 *   so recommend not to use this method.
+	 * </p>
+	 * <p>It is implemented here for testing purposes only.</p>
+	 * 
+	 */
+	public double computeAverageLocalOfObservationsWithCorrection() throws Exception {
+		
+		double te = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			int timeSeries = timeSeriesIndex[b];
+			double[] source = vectorOfSourceObservations.elementAt(timeSeries);
+			double[] dest = vectorOfDestinationObservations.elementAt(timeSeries);
+			int[] counts = getCounts(source, dest, timeStepIndex[b], timeSeries);
+			double cont = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				cont = MathsUtils.digamma(counts[NEXT_PAST_SOURCE]) -
+						MathsUtils.digamma(counts[PAST_SOURCE]) -
+						MathsUtils.digamma(counts[NEXT_PAST]) +
+						MathsUtils.digamma(counts[PAST]);
+			}
+			te += cont;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		// Average it, and convert results to bytes
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * Computes the local transfer entropies for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(null, null, true);
+	}
+
+	/**
+	 * Comptues local transfer entropies for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalUsingPreviousObservations(double[] source, double[] destination) throws Exception {
+		return computeLocalUsingPreviousObservations(source, destination, false);
+	}
+	
+	/**
+	 * Returns the local TE at every time point.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalUsingPreviousObservations(double[] source, double[] destination, boolean isPreviousObservations) throws Exception {
+		
+		int numLocalObservations;
+		if (isPreviousObservations) {
+			numLocalObservations = totalObservations;
+		} else {
+			numLocalObservations = destination.length - k;
+		}
+
+		double te = 0.0;
+		double[] localTE;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localTE = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localTE = new double[numLocalObservations + k];
+			offset = k;
+		}
+		for (int b = 0; b < numLocalObservations; b++) {
+			int timeSeries = -1;
+			int timeStep = b;
+			if (isPreviousObservations) {
+				timeSeries = timeSeriesIndex[b];
+				timeStep = timeStepIndex[b];
+				source = vectorOfSourceObservations.elementAt(timeSeries);
+				destination = vectorOfDestinationObservations.elementAt(timeSeries);
+			}
+			int[] counts = getCounts(source, destination, timeStep, timeSeries);
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (counts[NEXT_PAST_SOURCE] > 0) {
+				logTerm = ((double) counts[NEXT_PAST_SOURCE] / (double) counts[PAST_SOURCE]) /
+							((double) counts[NEXT_PAST] / (double) counts[PAST]);
+				local = Math.log(logTerm);
+			}
+			localTE[offset + b] = local;
+			te += local;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (local/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) numLocalObservations / Math.log(2.0);
+		return localTE;
+	}
+
+	/**
+	 * Return the set of probability counts for the observation at time step 
+	 * timeStep of the given source and destination checked against those stored in
+	 * the supplied samples.
+	 * timeSeries tells us which sample these were from, in case we want to do
+	 * dynamic correlation exclusion. If it is -1, then they are not from one of the samples.
+	 * 
+	 * @param observation
+	 * @param observationNumber
+	 * @return
+	 */
+	protected int[] getCounts(double[] source, double[] dest, int timeStep, int timeSeries) {
+		int[] counts = new int[4];
+		for (int b = 0; b < totalObservations; b++) {
+			// Pull out the current source and dest samples to compare to
+			double[] sampleSource = vectorOfSourceObservations.elementAt(timeSeriesIndex[b]);
+			double[] sampleDest = vectorOfDestinationObservations.elementAt(timeSeriesIndex[b]);
+			int t = timeStepIndex[b];
+			// If this is one of our samples, and we're doing dynamic correlation exclusion,
+			//  and they're from the same sample time series,
+			//  and it's within the window ==> don't count it
+			if ((timeSeries >= 0) && dynCorrExcl &&
+					(timeSeriesIndex[b] == timeSeries) &&
+					(Math.abs(t - timeStep) < dynCorrExclTime)) {
+				// This sample is within the dyn corr excl window,
+				//  so don't count it
+				continue;
+			}
+
+			if (!compareHistory(dest, timeStep, sampleDest, t)) {
+				// Past vectors are different, no point continuing the comparison
+				continue;
+			}
+			counts[PAST]++;
+			boolean nextMatches = Math.abs(dest[timeStep] - sampleDest[t]) <= epsDest;
+			boolean sourceMatches = Math.abs(source[timeStep - 1] - sampleSource[t - 1])
+										<= epsSource;
+			counts[NEXT_PAST] += nextMatches ? 1 : 0;
+			counts[PAST_SOURCE] += sourceMatches ? 1 : 0;
+			counts[NEXT_PAST_SOURCE] += (nextMatches && sourceMatches) ? 1 : 0;
+		}
+		return counts;
+	}
+	
+	/**
+	 * Return whether the past k time steps (prior to t1 and t2) of the
+	 *  two time series series1 and series2 match within epsDest at each point
+	 * 
+	 * @param series1
+	 * @param t1
+	 * @param series2
+	 * @param t2
+	 * @return
+	 */
+	protected boolean compareHistory(double[] series1, int t1, double[] series2, int t2) {
+		for (int i = 1; i <= k; i++) {
+			if (Math.abs(series1[t1 - i] - series2[t2 - i]) > epsDest) {
+				// This pair can't be within the kernel
+				return false;
+			}
+		}
+		return true;
+	}
+	
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		throw new RuntimeException("Not implemented in this calculator");
+	}
+	
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		throw new RuntimeException("Not implemented in this calculator");
+	}
+
+	/**
+	 * <p>Computes the probability counts for each previously supplied observations</p>
+	 * <p>Implemented primarily for debug purposes</p>
+	 * 
+	 */
+	public KernelCount[][] computeMatchesForEachObservations(boolean giveListOfCorrelatedPoints) throws Exception {
+		KernelCount[][] counts = new KernelCount[totalObservations][4];
+		for (int b = 0; b < totalObservations; b++) {
+			int timeSeries = timeSeriesIndex[b];
+			double[] source = vectorOfSourceObservations.elementAt(timeSeries);
+			double[] dest = vectorOfDestinationObservations.elementAt(timeSeries);
+			int[] intCounts = getCounts(source, dest, timeStepIndex[b], timeSeries);
+			counts[b] = new KernelCount[4];
+			for (int i = 0; i < 4; i++) {
+				int totalObsCount = 0;
+				if (dynCorrExcl) {
+					// Need to remove any observations that were in the same
+					//  sample time series and *closer* than dynCorrExclTime
+					int observationsInSameTimeSeries = dest.length;
+					int obsIndexInItsTimeSeries = timeStepIndex[b] - k;
+					int closeTimePointsToCompare = (obsIndexInItsTimeSeries >= dynCorrExclTime) ?
+							dynCorrExclTime - 1: obsIndexInItsTimeSeries;
+					closeTimePointsToCompare += (observationsInSameTimeSeries - obsIndexInItsTimeSeries >= dynCorrExclTime) ?
+							dynCorrExclTime - 1: observationsInSameTimeSeries - obsIndexInItsTimeSeries - 1;
+					closeTimePointsToCompare++; // Add one for comparison to self
+					totalObsCount = totalObservations - closeTimePointsToCompare;
+				} else {
+					totalObsCount = totalObservations;
+				}
+				counts[b][i] = new KernelCount(intCounts[i], totalObsCount);
+				// And ignore giveListOfCorrelatedPoints for now.
+			}
+		}
+		return counts;
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorKernelSeparate.java

@@ -0,0 +1,638 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.measures.continuous.kernel.KernelEstimatorMultiVariate;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Iterator;
+
+
+/**
+ * 
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * Uses separate kernel estimators for each probability calculation required.
+ * </p>
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>
+ * TODO Implement dynamic correlation exclusion with multiple observation sets. (see the
+ *  way this is done in Plain calculator).
+ * </p>
+ * 
+ * @author Joseph Lizier
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorKernelSeparate
+	extends TransferEntropyCommon implements TransferEntropyCalculator {
+
+	protected KernelEstimatorMultiVariate mvkeDestinationPast = null;
+	protected KernelEstimatorMultiVariate mvkeDestinationNextPast = null;
+	protected KernelEstimatorMultiVariate mvkeDestinationPastSource = null;
+	protected KernelEstimatorMultiVariate mvkeDestinationNextPastSource = null;
+
+	// Keep joint vectors so we don't need to regenerate them
+	protected double[][] destPastVectors;
+	protected double[][] destNextPastVectors;
+	protected double[][] destPastSourceVectors;
+	protected double[][] destNextPastSourceVectors;
+	
+	private boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	private boolean dynCorrExcl = false;
+	private int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	private boolean forceCompareToAll = false;
+	public static final String FORCE_KERNEL_COMPARE_TO_ALL = "FORCE_KERNEL_COMPARE_TO_ALL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	private double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public TransferEntropyCalculatorKernelSeparate() {
+		super();
+		mvkeDestinationPast = new KernelEstimatorMultiVariate();
+		mvkeDestinationNextPast = new KernelEstimatorMultiVariate();
+		mvkeDestinationPastSource = new KernelEstimatorMultiVariate();
+		mvkeDestinationNextPastSource = new KernelEstimatorMultiVariate();
+		mvkeDestinationPast.setNormalise(normalise);
+		mvkeDestinationNextPast.setNormalise(normalise);
+		mvkeDestinationPastSource.setNormalise(normalise);
+		mvkeDestinationNextPastSource.setNormalise(normalise);
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		super.initialise(k); // calls initialise()
+	}
+
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() throws Exception {
+		mvkeDestinationPast.initialise(k, epsilon);
+		mvkeDestinationNextPast.initialise(k+1, epsilon);
+		mvkeDestinationPastSource.initialise(k+1, epsilon);
+		mvkeDestinationNextPastSource.initialise(k+2, epsilon);
+		destPastVectors = null;
+		destNextPastVectors = null;
+		destPastSourceVectors = null;
+		destNextPastSourceVectors = null;
+	}
+	
+	/**
+	 * Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>K_PROP_NAME</li>
+	 * 		<li>EPSILON_PROP_NAME</li>
+	 * 		<li>NORMALISE_PROP_NAME</li>
+	 * 		<li>DYN_CORR_EXCL_TIME_NAME</li>
+	 * 		<li>FORCE_KERNEL_COMPARE_TO_ALL</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			mvkeDestinationPast.setNormalise(normalise);
+			mvkeDestinationNextPast.setNormalise(normalise);
+			mvkeDestinationPastSource.setNormalise(normalise);
+			mvkeDestinationNextPastSource.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+			if (dynCorrExcl) {
+				mvkeDestinationPast.setDynamicCorrelationExclusion(dynCorrExclTime);
+				mvkeDestinationNextPast.setDynamicCorrelationExclusion(dynCorrExclTime);
+				mvkeDestinationPastSource.setDynamicCorrelationExclusion(dynCorrExclTime);
+				mvkeDestinationNextPastSource.setDynamicCorrelationExclusion(dynCorrExclTime);
+			} else {
+				mvkeDestinationPast.clearDynamicCorrelationExclusion();
+				mvkeDestinationNextPast.clearDynamicCorrelationExclusion();
+				mvkeDestinationPastSource.clearDynamicCorrelationExclusion();
+				mvkeDestinationNextPastSource.clearDynamicCorrelationExclusion();
+			}
+		} else if (propertyName.equalsIgnoreCase(FORCE_KERNEL_COMPARE_TO_ALL)) {
+			forceCompareToAll = Boolean.parseBoolean(propertyValue);
+			mvkeDestinationPast.setForceCompareToAll(forceCompareToAll);
+			mvkeDestinationNextPast.setForceCompareToAll(forceCompareToAll);
+			mvkeDestinationPastSource.setForceCompareToAll(forceCompareToAll);
+			mvkeDestinationNextPastSource.setForceCompareToAll(forceCompareToAll);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println("Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	/*
+	 * Old implementation of set observations - now we defer to the super class
+	 *  and let it call startAdd, Add and FinaliseAdd.
+	 * 
+	 * ------------------------
+	 * 
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * 
+	public void setObservations(double[] source, double[] destination) {
+		totalObservations = destination.length - k;
+		// We're just going to be wasteful with memory for the moment since it's easier to
+		//  implement for now:
+		// Construct joint vectors past
+		destPastVectors = makeJointVectorForPast(destination);
+		mvkeDestinationPast.setObservations(destPastVectors);
+		// Construct joint vectors for next and past
+		destNextPastVectors = makeJointVectorForNextPast(destination);
+		mvkeDestinationNextPast.setObservations(destNextPastVectors);
+		// Construct joint vectors for past and source
+		destPastSourceVectors = makeJointVectorForPastSource(destination, source);
+		mvkeDestinationPastSource.setObservations(destPastSourceVectors);
+		// Construct joint vectors for next and past and source
+		destNextPastSourceVectors = makeJointVectorForNextPastSource(destination, source);
+		mvkeDestinationNextPastSource.setObservations(destNextPastSourceVectors);
+	}
+	*/
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		destPastVectors = new double[totalObservations][k];
+		destNextPastVectors = new double[totalObservations][k + 1];
+		destPastSourceVectors = new double[totalObservations][k + 1];
+		destNextPastSourceVectors = new double[totalObservations][k + 2];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[]> iterator = vectorOfDestinationObservations.iterator();
+		for (double[] source : vectorOfSourceObservations) {
+			double[] destination = iterator.next();
+			double[][] currentDestPastVectors = makeJointVectorForPast(destination);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					destPastVectors, startObservation, 0, currentDestPastVectors.length, k);
+			double[][] currentDestNextPastVectors = makeJointVectorForNextPast(destination);
+			MatrixUtils.arrayCopy(currentDestNextPastVectors, 0, 0,
+					destNextPastVectors, startObservation, 0, currentDestNextPastVectors.length, k + 1);
+			double[][] currentDestPastSourceVectors = makeJointVectorForPastSource(destination, source);
+			MatrixUtils.arrayCopy(currentDestPastSourceVectors, 0, 0,
+					destPastSourceVectors, startObservation, 0, currentDestPastSourceVectors.length, k + 1);
+			double[][] currentDestNextPastSourceVectors = makeJointVectorForNextPastSource(destination, source);
+			MatrixUtils.arrayCopy(currentDestNextPastSourceVectors, 0, 0,
+					destNextPastSourceVectors, startObservation, 0, currentDestNextPastSourceVectors.length, k + 2);
+			startObservation += destination.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		mvkeDestinationPast.setObservations(destPastVectors);
+		mvkeDestinationNextPast.setObservations(destNextPastVectors);
+		mvkeDestinationPastSource.setObservations(destPastSourceVectors);
+		mvkeDestinationNextPastSource.setObservations(destNextPastSourceVectors);
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfDestinationObservations.size() > 1;
+		if (addedMoreThanOneObservationSet && dynCorrExcl) {
+			// We have not properly implemented dynamic correlation exclusion for
+			//  multiple observation sets, so throw an error
+			throw new RuntimeException("Addition of multiple observation sets is not currently " +
+					"supported with property DYN_CORR_EXCL set");
+		}
+
+		// And clear the vector of observations
+		vectorOfSourceObservations = null;
+		vectorOfDestinationObservations = null;
+	}
+	
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destNextPastSourceVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			double countPast = mvkeDestinationPast.getCount(destPastVectors[b], b);
+			double countNextPast = mvkeDestinationNextPast.getCount(destNextPastVectors[b], b);
+			double countPastSource = mvkeDestinationPastSource.getCount(destPastSourceVectors[b], b);
+			double countNextPastSource = mvkeDestinationNextPastSource.getCount(destNextPastSourceVectors[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (countNextPastSource > 0) {
+				logTerm = (countNextPastSource / countPastSource) / (countNextPast / countPast);
+				cont = Math.log(logTerm);
+			}
+			te += cont;
+			if (debug) {
+				System.out.println(b + ": " + destPastVectors[b][0] + " (" + countNextPastSource + " / " + countPastSource + ") / (" +
+						countNextPast + " / " + countPast + ") = " + 
+						logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations,
+	 *  using the Grassberger correction for the point count k: log_e(k) ~= digamma(k).</p>
+	 * <p>Kaiser and Schreiber, Physica D 166 (2002) pp. 43-62 suggest (on p. 57) that for the TE
+	 *   though the adverse correction of the bias correction is worse than the correction
+	 *   itself (because the probabilities being multiplied/divided are not independent), 
+	 *   so recommend not to use this method.
+	 * </p>
+	 * <p>It is implemented here for testing purposes only.</p>
+	 * 
+	 */
+	public double computeAverageLocalOfObservationsWithCorrection() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destNextPastSourceVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			int countPast = mvkeDestinationPast.getCount(destPastVectors[b], b);
+			int countNextPast = mvkeDestinationNextPast.getCount(destNextPastVectors[b], b);
+			int countPastSource = mvkeDestinationPastSource.getCount(destPastSourceVectors[b], b);
+			int countNextPastSource = mvkeDestinationNextPastSource.getCount(destNextPastSourceVectors[b], b);
+			double cont = 0.0;
+			if (countNextPastSource > 0) {
+				cont = MathsUtils.digamma(countNextPastSource) - 
+						MathsUtils.digamma(countPastSource) -
+						MathsUtils.digamma(countNextPast) +
+						MathsUtils.digamma(countPast);
+			}
+			te += cont;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		// Average it, and convert results to bytes
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/*
+	 * TEST ONLY - THIS SHOULD not be used:
+	 * 
+	 * This is the way I previsouly computed TE in octave code.
+	 * This method of estimating p(i_n, i_n+1, j_n) should not be correct, because
+	 *  there is no need to multiply by the prob of observation.
+	 * I'm just writing it in here to test .. 
+	 * 
+	 * @return
+	 * @throws Exception
+	private double computeAverageLocalOfObservationsWithMultiplier() throws Exception {
+
+		double te = 0.0;
+		double sumJointProb = 0;
+		for (int b = 0; b < totalObservations; b++) {
+			double probPast = mvkeDestinationPast.getProbability(destPastVectors[b], b);
+			double probNextPast = mvkeDestinationNextPast.getProbability(destNextPastVectors[b], b);
+			double probPastSource = mvkeDestinationPastSource.getProbability(destPastSourceVectors[b], b);
+			double probNextPastSource = mvkeDestinationNextPastSource.getProbability(destNextPastSourceVectors[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (probNextPastSource > ZERO_COMPARATOR) {
+				logTerm = (probNextPastSource / probPastSource) / (probNextPast / probPast);
+				cont = Math.log(logTerm);
+			}
+			sumJointProb += probNextPastSource;
+			te += cont * probNextPastSource;
+		}
+		lastAverage = te / sumJointProb / Math.log(2.0);
+		return lastAverage;
+	}
+	*/
+	
+	/**
+	 * Computes the local transfer entropies for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(null, null, true);
+	}
+
+	/**
+	 * Comptues local transfer entropies for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalUsingPreviousObservations(double[] source, double[] destination) throws Exception {
+		return computeLocalUsingPreviousObservations(source, destination, false);
+	}
+	
+	/**
+	 * Returns the local TE at every time point.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalUsingPreviousObservations(double[] source, double[] destination, boolean isPreviousObservations) throws Exception {
+		
+		double[][] newDestPastVectors;
+		double[][] newDestNextPastVectors;
+		double[][] newDestPastSourceVectors;
+		double[][] newDestNextPastSourceVectors;
+
+		if (isPreviousObservations) {
+			// We've already computed the joint vectors for these observations
+			newDestPastVectors = destPastVectors;
+			newDestNextPastVectors = destNextPastVectors;
+			newDestPastSourceVectors = destPastSourceVectors;
+			newDestNextPastSourceVectors = destNextPastSourceVectors;
+		} else {
+			// We need to compute a new set of joint vectors
+			newDestPastVectors = makeJointVectorForPast(destination);
+			newDestNextPastVectors = makeJointVectorForNextPast(destination);
+			newDestPastSourceVectors = makeJointVectorForPastSource(destination, source);
+			newDestNextPastSourceVectors = makeJointVectorForNextPastSource(destination, source);
+		}
+
+		double te = 0.0;
+		int numLocalObservations = newDestPastVectors.length;
+		double[] localTE;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localTE = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localTE = new double[numLocalObservations + k];
+			offset = k;
+		}
+		double countPast, countNextPast, countPastSource, countNextPastSource;
+		for (int b = 0; b < numLocalObservations; b++) {
+			if (isPreviousObservations) {
+				countPast = mvkeDestinationPast.getCount(newDestPastVectors[b], b);
+				countNextPast = mvkeDestinationNextPast.getCount(newDestNextPastVectors[b], b);
+				countPastSource = mvkeDestinationPastSource.getCount(newDestPastSourceVectors[b], b);
+				countNextPastSource = mvkeDestinationNextPastSource.getCount(newDestNextPastSourceVectors[b], b);
+			} else {
+				countPast = mvkeDestinationPast.getCount(newDestPastVectors[b]);
+				countNextPast = mvkeDestinationNextPast.getCount(newDestNextPastVectors[b]);
+				countPastSource = mvkeDestinationPastSource.getCount(newDestPastSourceVectors[b]);
+				countNextPastSource = mvkeDestinationNextPastSource.getCount(newDestNextPastSourceVectors[b]);
+			}
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (countNextPastSource > 0) {
+				logTerm = (countNextPastSource / countPastSource) / (countNextPast / countPast);
+				local = Math.log(logTerm) / Math.log(2.0);
+			}
+			localTE[offset + b] = local;
+			te += local;
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + local + " -> sum: " + te);
+			}
+		}
+		lastAverage = te / (double) numLocalObservations;
+		return localTE;
+	}
+
+	/**
+	 * Combine the past and source values into a joint vector
+	 * 
+	 * @param destination
+	 * @param source
+	 * @return
+	 */
+	private double[][] makeJointVectorForPastSource(double[] destination, double[] source) {
+		double[][] destPastSourceVectors = new double[destination.length - k][k + 1];
+		for (int t = k; t < destination.length; t++) {
+			for (int i = 0; i < k; i++) {
+				destPastSourceVectors[t - k][i] = destination[t - i - 1];
+			}
+			destPastSourceVectors[t - k][k] = source[t - 1];
+		}
+		return destPastSourceVectors;
+	}
+
+	/**
+	 * Compute the next, past and source values into a joint vector
+	 * 
+	 * @param destination
+	 * @param source
+	 * @return
+	 */
+	private double[][] makeJointVectorForNextPastSource(double[] destination, double[] source) {
+		double[][] destNextPastSourceVectors = new double[destination.length - k][k + 2];
+		for (int t = k; t < destination.length; t++) {
+			for (int i = 0; i < k + 1; i++) {
+				destNextPastSourceVectors[t - k][i] = destination[t - i];
+			}
+			destNextPastSourceVectors[t - k][k + 1] = source[t - 1];
+		}
+		return destNextPastSourceVectors;
+	}
+	
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *
+	 * Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(totalObservations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 *  
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+				
+		double actualTE = computeAverageLocalOfObservations();
+		
+		// Save the relevant source observations here:
+		double[] originalSourceValuesInJoint = MatrixUtils.selectColumn(destPastSourceVectors, k);
+		
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Check that the length of the reorderings is OK
+			if (newOrderings[p].length != totalObservations) {
+				throw new Exception("Length " + newOrderings[p].length +
+						" of reordering " + p + " in newOrderings does not " +
+						" match the number of observations " + totalObservations);
+			}
+
+			// Generate a new re-ordered data set for the source in the destPastSourceVectors 
+			//  and destNextPastSourceVectors vectors
+			MatrixUtils.reorderVectorIntoMatrix(originalSourceValuesInJoint, newOrderings[p],
+					destPastSourceVectors, k);
+			MatrixUtils.reorderVectorIntoMatrix(originalSourceValuesInJoint, newOrderings[p],
+					destNextPastSourceVectors, k+1);
+			
+			// Make the equivalent operations of intialise
+			mvkeDestinationPastSource.initialise(k+1, epsilon);
+			mvkeDestinationNextPastSource.initialise(k+2, epsilon);
+			// Make the equivalent operations of setObservations:
+			mvkeDestinationPastSource.setObservations(destPastSourceVectors);
+			mvkeDestinationNextPastSource.setObservations(destNextPastSourceVectors);
+			// And get a TE value for this realisation:
+			double newTe = computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the local variables:
+		lastAverage = actualTE;
+		// Restore the source observations in the joint vectors
+		MatrixUtils.copyIntoColumn(destPastSourceVectors, k, originalSourceValuesInJoint);
+		MatrixUtils.copyIntoColumn(destNextPastSourceVectors, k+1, originalSourceValuesInJoint);
+		// And set the mulit-variate kernel estimators back to their previous state
+		mvkeDestinationPastSource.initialise(k+1, epsilon);
+		mvkeDestinationNextPastSource.initialise(k+2, epsilon);
+		mvkeDestinationPastSource.setObservations(destPastSourceVectors);
+		mvkeDestinationNextPastSource.setObservations(destNextPastSourceVectors);
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualTE;
+		return measDistribution;
+	}
+
+	/**
+	 * <p>Computes the probability counts for each previously supplied observations</p>
+	 * <p>Implemented primarily for debug purposes</p>
+	 * 
+	 */
+	public KernelCount[][] computeMatchesForEachObservations(boolean giveListOfCorrelatedPoints) throws Exception {
+		
+		KernelCount[][] counts = new KernelCount[totalObservations][4];
+		for (int b = 0; b < totalObservations; b++) {
+			counts[b][0] = mvkeDestinationPast.
+				getCompleteKernelCount(destPastVectors[b], b, giveListOfCorrelatedPoints);
+			counts[b][1] = mvkeDestinationNextPast.
+				getCompleteKernelCount(destNextPastVectors[b], b, giveListOfCorrelatedPoints);
+			counts[b][2] = mvkeDestinationPastSource.
+				getCompleteKernelCount(destPastSourceVectors[b], b, giveListOfCorrelatedPoints);
+			counts[b][3] = mvkeDestinationNextPastSource.
+				getCompleteKernelCount(destNextPastSourceVectors[b], b, giveListOfCorrelatedPoints);
+		}
+		return counts;
+	}
+
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		mvkeDestinationPast.setDebug(debug);
+		mvkeDestinationNextPast.setDebug(debug);
+		mvkeDestinationPastSource.setDebug(debug);
+		mvkeDestinationNextPastSource.setDebug(debug);
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorMultiVariateKernel.java

@@ -0,0 +1,783 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.measures.continuous.TransferEntropyCalculatorMultiVariate;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.measures.continuous.kernel.TransferEntropyKernelCounts;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Iterator;
+import java.util.Vector;
+
+
+/**
+ * 
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * This calculator handles multi-variate source and destination variables.
+ * </p>
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>initialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>
+ * TODO Implement dynamic correlation exclusion with multiple observation sets. (see the
+ *  way this is done in Plain calculator).
+ * </p>
+ * 
+ * @author Joseph Lizier
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorMultiVariateKernel
+	extends TransferEntropyCommon implements TransferEntropyCalculatorMultiVariate {
+
+	protected KernelEstimatorTransferEntropyMultiVariate teKernelEstimator = null;
+	// Keep a kernel estimator for the next state, in case we wish to compute
+	//  Active info storage also:
+	protected KernelEstimatorMultiVariate nextStateKernelEstimator = null;
+
+	/**
+	 * Storage for source observations for addObservsations
+	 */
+	protected Vector<double[][]> vectorOfJointSourceObservations;
+	/**
+	 * Storage for destination observations for addObservsations
+	 */
+	protected Vector<double[][]> vectorOfJointDestinationObservations;
+
+	// Keep joint vectors so we don't need to regenerate them
+	protected double[][] destPastVectors;
+	protected double[][] destNextVectors;
+	protected double[][] sourceVectors;
+	
+	protected int destDimensions = 1;
+	protected int sourceDimensions = 1;
+	
+	// Store the local conditional probability of next on past state as 
+	//  computed during a local TE computation, in case the caller wants to
+	//  compute the local active info storage next.
+	protected double[] localProbNextCondPast;
+	
+	protected boolean normalise = true;
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	
+	protected boolean dynCorrExcl = false;
+	protected int dynCorrExclTime = 100;
+	public static final String DYN_CORR_EXCL_TIME_NAME = "DYN_CORR_EXCL";
+	
+	protected boolean forceCompareToAll = false;
+	public static final String FORCE_KERNEL_COMPARE_TO_ALL = "FORCE_KERNEL_COMPARE_TO_ALL";
+	
+	/**
+	 * Default value for epsilon
+	 */
+	public static final double DEFAULT_EPSILON = 0.25;
+	/**
+	 * Kernel width
+	 */
+	protected double epsilon = DEFAULT_EPSILON;
+	public static final String EPSILON_PROP_NAME = "EPSILON";
+	
+	/**
+	 * Creates a new instance of the kernel-estimate style transfer entropy calculator
+	 *
+	 */
+	public TransferEntropyCalculatorMultiVariateKernel() {
+		super();
+		teKernelEstimator = new KernelEstimatorTransferEntropyMultiVariate();
+		teKernelEstimator.setNormalise(normalise);
+		nextStateKernelEstimator = new KernelEstimatorMultiVariate();
+		nextStateKernelEstimator.setNormalise(normalise);
+	}
+
+	/**
+	 * Initialises the calculator with the existing value for epsilon
+	 * 
+	 * @param k history length
+	 */
+	public void initialise(int k) throws Exception {
+		initialise(k, epsilon);
+	}
+	
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param k history length
+	 * @param epsilon kernel width
+	 */
+	public void initialise(int k, double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		initialise(k, 1, 1); // assume 1 dimension in source and dest
+	}
+
+	public void initialise(int k, int destDimensions, int sourceDimensions) throws Exception {
+		this.destDimensions = destDimensions;
+		this.sourceDimensions = sourceDimensions;
+		super.initialise(k); // calls initialise();
+	}
+	
+	public void initialise(int destDimensions, int sourceDimensions) throws Exception {
+		this.destDimensions = destDimensions;
+		this.sourceDimensions = sourceDimensions;
+		super.initialise(k); // calls initialise();
+	}
+
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() {
+		teKernelEstimator.initialise(k * destDimensions,
+				sourceDimensions, epsilon, epsilon);
+		nextStateKernelEstimator.initialise(destDimensions, epsilon);
+		destPastVectors = null;
+		destNextVectors = null;
+		sourceVectors = null;
+		localProbNextCondPast = null;
+	}
+	
+	/**
+	 * Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>K_PROP_NAME</li>
+	 * 		<li>EPSILON_PROP_NAME</li>
+	 * 		<li>NORMALISE_PROP_NAME</li>
+	 * 		<li>DYN_CORR_EXCL_TIME_NAME</li>
+	 * 		<li>FORCE_KERNEL_COMPARE_TO_ALL</li>
+	 * </ul> 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			epsilon = Double.parseDouble(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+			teKernelEstimator.setNormalise(normalise);
+			nextStateKernelEstimator.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			dynCorrExclTime = Integer.parseInt(propertyValue);
+			dynCorrExcl = (dynCorrExclTime > 0);
+			if (dynCorrExcl) {
+				teKernelEstimator.setDynamicCorrelationExclusion(dynCorrExclTime);
+				nextStateKernelEstimator.setDynamicCorrelationExclusion(dynCorrExclTime);
+			} else {
+				teKernelEstimator.clearDynamicCorrelationExclusion();
+				nextStateKernelEstimator.clearDynamicCorrelationExclusion();
+			}
+		} else if (propertyName.equalsIgnoreCase(FORCE_KERNEL_COMPARE_TO_ALL)) {
+			forceCompareToAll = Boolean.parseBoolean(propertyValue);
+			teKernelEstimator.setForceCompareToAll(forceCompareToAll);
+			nextStateKernelEstimator.setForceCompareToAll(forceCompareToAll);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	/**
+	 * Set the observations to compute the probabilities from 
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void setObservations(double[][] source, double[][] destination) throws Exception {
+		startAddObservations();
+		addObservations(source, destination);
+		finaliseAddObservations();
+	}
+	
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+		
+		// We've found the set of start and end times for this pair
+		startAddObservations();
+		for (int[] timePair : startAndEndTimePairs) {
+			int startTime = timePair[0];
+			int endTime = timePair[1];
+			addObservations(source, destination, startTime, endTime - startTime + 1);
+		}
+		finaliseAddObservations();
+	}
+
+	public void setObservations(double[][] source, double[][] destination,
+			boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		
+		boolean[] jointSourceValid = MatrixUtils.andRows(sourceValid);
+		boolean[] jointDestValid = MatrixUtils.andRows(destValid);
+		setObservations(source, destination, jointSourceValid, jointDestValid);
+	}
+
+	@Override
+	public void startAddObservations() {
+		vectorOfJointSourceObservations = new Vector<double[][]>();
+		vectorOfJointDestinationObservations = new Vector<double[][]>();
+	}
+
+	/**
+	 * Add observations of a single-dimensional source and destination pair.
+	 * 
+	 * Only allow this call if source and destination dimenions were 1.
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	@Override
+	public void addObservations(double[] source, double[] destination) throws Exception {
+		double[][] sourceMatrix = new double[source.length][1];
+		MatrixUtils.copyIntoColumn(sourceMatrix, 0, source);
+		double[][] destMatrix = new double[destination.length][1];
+		MatrixUtils.copyIntoColumn(destMatrix, 0, destination);
+		addObservations(sourceMatrix, destMatrix);
+	}
+
+	/**
+	 * Add observations of a single-dimensional source and destination pair.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	@Override
+	public void addObservations(double[] source, double[] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		double[][] sourceMatrix = new double[numTimeSteps][1];
+		MatrixUtils.copyIntoColumn(sourceMatrix, 0, 0, source, startTime, numTimeSteps);
+		double[][] destMatrix = new double[destination.length][1];
+		MatrixUtils.copyIntoColumn(destMatrix, 0, 0, destination, startTime, numTimeSteps);
+		addObservations(sourceMatrix, destMatrix);
+	}
+
+	/**
+	 * Add observations of the joint source and destinations
+	 * 
+	 * @param source
+	 * @param destination
+	 * @throws Exception
+	 */
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		if (source.length != destination.length) {
+			throw new Exception(String.format("Source and destination lengths (%d and %d) must match!",
+					source.length, destination.length));
+		}
+		int thisSourceDimensions = source[0].length;
+		int thisDestDimensions = destination[0].length;
+		if ((thisDestDimensions != destDimensions) || (thisSourceDimensions != sourceDimensions)) {
+			throw new Exception("Cannot add observsations for source and destination variables " +
+					" of " + thisSourceDimensions + " and " + thisDestDimensions +
+					" dimensions respectively for TE calculator set up for " + sourceDimensions + " " +
+					destDimensions + " source and destination dimensions respectively");
+		}
+		if (vectorOfJointSourceObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		vectorOfJointSourceObservations.add(source);
+		vectorOfJointDestinationObservations.add(destination);
+	}
+	
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[][] source, double[][] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		double[][] sourceToAdd = new double[numTimeSteps][source[0].length];
+		System.arraycopy(source, startTime, sourceToAdd, 0, numTimeSteps);
+		double[][] destToAdd = new double[numTimeSteps][destination[0].length];
+		System.arraycopy(destination, startTime, destToAdd, 0, numTimeSteps);
+		addObservations(sourceToAdd, destToAdd);
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[][] destination : vectorOfJointDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		destPastVectors = new double[totalObservations][k * destDimensions];
+		destNextVectors = new double[totalObservations][destDimensions];
+		sourceVectors = new double[totalObservations][sourceDimensions];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[][]> iterator = vectorOfJointDestinationObservations.iterator();
+		for (double[][] source : vectorOfJointSourceObservations) {
+			double[][] destination = iterator.next();
+			double[][] currentDestPastVectors = makeJointVectorForPast(destination);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					destPastVectors, startObservation, 0, currentDestPastVectors.length,
+					k * destDimensions);
+			MatrixUtils.arrayCopy(destination, k, 0,
+					destNextVectors, startObservation, 0,
+					destination.length - k, destDimensions);
+			MatrixUtils.arrayCopy(source, k - 1, 0,
+					sourceVectors, startObservation, 0,
+					source.length - k, sourceDimensions);
+			startObservation += destination.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		teKernelEstimator.setObservations(destPastVectors, destNextVectors, sourceVectors);
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfJointDestinationObservations.size() > 1;
+		
+		// And clear the vector of observations
+		vectorOfJointSourceObservations = null;
+		vectorOfJointDestinationObservations = null;
+	}
+	
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations</p> 
+	 * 
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		
+		double te = 0.0;
+		if (debug) {
+			MatrixUtils.printMatrix(System.out, destPastVectors);
+		}
+		for (int b = 0; b < totalObservations; b++) {
+			TransferEntropyKernelCounts kernelCounts = teKernelEstimator.getCount(destPastVectors[b],
+					destNextVectors[b], sourceVectors[b], b);
+			double logTerm = 0.0;
+			double cont = 0.0;
+			if (kernelCounts.countNextPastSource > 0) {
+				logTerm = ((double) kernelCounts.countNextPastSource / (double) kernelCounts.countPastSource) /
+							((double) kernelCounts.countNextPast / (double) kernelCounts.countPast);
+				cont = Math.log(logTerm);
+			}
+			te += cont;
+			if (debug) {
+				System.out.println(b + ": " + destPastVectors[b][0] + " (" +
+						kernelCounts.countNextPastSource + " / " + kernelCounts.countPastSource + ") / (" +
+						kernelCounts.countNextPast + " / " + kernelCounts.countPast + ") = " + 
+						logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+		}
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * <p>Computes the average Transfer Entropy for the previously supplied observations,
+	 *  using the Grassberger correction for the point count k: log_e(k) ~= digamma(k).</p>
+	 * <p>Kaiser and Schreiber, Physica D 166 (2002) pp. 43-62 suggest (on p. 57) that for the TE
+	 *   though the adverse correction of the bias correction is worse than the correction
+	 *   itself (because the probabilities being multiplied/divided are not independent), 
+	 *   so recommend not to use this method.
+	 * </p>
+	 * <p>It is implemented here for testing purposes only.</p>
+	 * 
+	 */
+	public double computeAverageLocalOfObservationsWithCorrection() throws Exception {
+		
+		double te = 0.0;
+		for (int b = 0; b < totalObservations; b++) {
+			TransferEntropyKernelCounts kernelCounts = teKernelEstimator.getCount(destPastVectors[b],
+					destNextVectors[b], sourceVectors[b], b);
+			double cont = 0.0;
+			if (kernelCounts.countNextPastSource > 0) {
+				cont = MathsUtils.digamma(kernelCounts.countNextPastSource) - 
+						MathsUtils.digamma(kernelCounts.countPastSource) -
+						MathsUtils.digamma(kernelCounts.countNextPast) +
+						MathsUtils.digamma(kernelCounts.countPast);
+			}
+			te += cont;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		// Average it, and convert results to bytes
+		lastAverage = te / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
+	}
+
+	/**
+	 * Computes the local transfer entropies for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		return computeLocalUsingPreviousObservations(null, null, true);
+	}
+
+	/**
+	 * Comptues local transfer entropies for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalUsingPreviousObservations(double[][] source, double[][] destination) throws Exception {
+		return computeLocalUsingPreviousObservations(source, destination, false);
+	}
+	
+	/**
+	 * Returns the local TE at every time point.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalUsingPreviousObservations(double[][] source,
+			double[][] destination, boolean isPreviousObservations) throws Exception {
+		
+		double[][] newDestPastVectors;
+		double[][] newDestNextValues;
+		double[][] newSourceValues;
+
+		if (isPreviousObservations) {
+			// We've already computed the joint vectors for these observations
+			newDestPastVectors = destPastVectors;
+			newDestNextValues = destNextVectors;
+			newSourceValues = sourceVectors;
+		} else {
+			// We need to compute a new set of joint vectors
+			newDestPastVectors = makeJointVectorForPast(destination);
+			newDestNextValues = new double[destination.length - k][destDimensions];
+			MatrixUtils.arrayCopy(destination, k, 0,
+					newDestNextValues, 0, 0,
+					destination.length - k, destDimensions);
+			newSourceValues = new double[source.length - k][sourceDimensions];
+			MatrixUtils.arrayCopy(source, k - 1, 0,
+					newSourceValues, 0, 0,
+					source.length - k, sourceDimensions);
+		}
+
+		double te = 0.0;
+		int numLocalObservations = newDestPastVectors.length;
+		double[] localTE;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localTE = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localTE = new double[numLocalObservations + k];
+			offset = k;
+		}
+		localProbNextCondPast = new double[numLocalObservations];
+		double avKernelCount = 0;
+		TransferEntropyKernelCounts kernelCounts;
+		for (int b = 0; b < numLocalObservations; b++) {
+			// System.out.print("Observation number " + String.valueOf(b) + "\n");
+			if (isPreviousObservations) {
+				kernelCounts = teKernelEstimator.getCount(
+						newDestPastVectors[b],
+						newDestNextValues[b], newSourceValues[b], b);
+			} else {
+				kernelCounts = teKernelEstimator.getCount(
+						newDestPastVectors[b],
+						newDestNextValues[b], newSourceValues[b], -1);
+			}
+			avKernelCount += kernelCounts.countNextPastSource;
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (kernelCounts.countPast > 0) {
+				// Store this ratio for a potential active info calculation later
+				localProbNextCondPast[b] = (double) kernelCounts.countNextPast / (double) kernelCounts.countPast;
+			}
+			if (kernelCounts.countNextPastSource > 0) {
+				logTerm = ((double) kernelCounts.countNextPastSource / (double) kernelCounts.countPastSource) /
+							localProbNextCondPast[b];
+				local = Math.log(logTerm);
+			}
+			localTE[offset + b] = local;
+			te += local;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (local/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		avKernelCount = avKernelCount / (double) numLocalObservations;
+		if (debug) {
+			System.out.printf("Average kernel count was %.3f\n", avKernelCount);
+		}
+		lastAverage = te / (double) numLocalObservations / Math.log(2.0);
+		return localTE;
+	}
+
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *
+	 * Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(totalObservations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 *  
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		
+		double actualTE = computeAverageLocalOfObservations();
+		
+		// Space for the source observations:
+		double[][] oldSourceValues = sourceVectors;
+		
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the source in the destPastSourceVectors 
+			//  and destNextPastSourceVectors vectors
+			sourceVectors = MatrixUtils.extractSelectedTimePoints(oldSourceValues, newOrderings[p]);
+			
+			// Make the equivalent operations of intialise
+			teKernelEstimator.initialise(k * destDimensions,
+					sourceDimensions, epsilon, epsilon);
+			// Make the equivalent operations of setObservations:
+			teKernelEstimator.setObservations(destPastVectors, destNextVectors, sourceVectors);
+			// And get a TE value for this realisation:
+			double newTe = computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the local variables:
+		lastAverage = actualTE;
+		sourceVectors = oldSourceValues;
+		// And set the kernel estimator back to their previous state
+		teKernelEstimator.initialise(k * destDimensions,
+				sourceDimensions, epsilon, epsilon);
+		teKernelEstimator.setObservations(destPastVectors, destNextVectors, sourceVectors);
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualTE;
+		return measDistribution;
+	}
+
+	/**
+	 * Computes the local active info storage for the previous supplied observations.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 *  
+	 * Precondition: The user must have computed the local TEs first for these
+	 *  vectors.
+	 * 
+	 */
+	public double[] computeLocalActiveOfPreviousObservations() throws Exception {
+		return computeLocalActiveUsingPreviousObservations(null, true);
+	}
+
+	/**
+	 * Comptues local active info storage for the given observations, using the previously supplied
+	 * observations to compute the PDFs.
+	 * I don't think it's such a good idea to do this for continuous variables (e.g. where
+	 * one can get kernel estimates for probabilities of zero now) but I've implemented
+	 * it anyway. I guess getting kernel estimates of zero here is no different than what
+	 * can occur with dynamic correlation exclusion.
+	 * 
+	 * Precondition: The user must have computed the local TEs first for these
+	 *  vectors
+	 * 
+	 * @param source
+	 * @param destination
+	 * @return
+	 * @throws Exception
+	 */
+	public double[] computeLocalActiveUsingPreviousObservations(double[][] destination) throws Exception {
+		return computeLocalActiveUsingPreviousObservations(destination, false);
+	}
+	
+	/**
+	 * Returns the local active info at every time point.
+	 * 
+	 * Precondition: The user must have computed the local TEs first for these
+	 *   vectors.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param isPreviousObservations
+	 * @return
+	 * @throws Exception
+	 */
+	private double[] computeLocalActiveUsingPreviousObservations(
+			double[][] destination, boolean isPreviousObservations) throws Exception {
+
+		// Precondition: the local TE must have already been computed
+		if (localProbNextCondPast == null) {
+			throw new RuntimeException("A local TE must have been computed before " +
+					"the local active info storage can be computed by TransferEntropyCalculatorMultiVariateKernel");
+		}
+		
+		double[][] newDestNextValues;
+
+		// Set the observations on the kernel estimator:
+		nextStateKernelEstimator.setObservations(destNextVectors);
+		
+		// Now set which observations we're going to compute the local 
+		//  active info of:
+		if (isPreviousObservations) {
+			// We've already computed the joint vectors for these observations
+			newDestNextValues = destNextVectors;
+		} else {
+			// We need to compute a new set of joint vectors
+			newDestNextValues = new double[destination.length - k][destDimensions];
+			MatrixUtils.arrayCopy(destination, k, 0,
+					newDestNextValues, 0, 0,
+					destination.length - k, destDimensions);
+		}
+
+		int numLocalObservations = newDestNextValues.length;
+		double[] localActive;
+		int offset = 0;
+		if (isPreviousObservations && addedMoreThanOneObservationSet) {
+			// We're returning the local values for a set of disjoint
+			//  observations. So we don't add k zeros to the start
+			localActive = new double[numLocalObservations];
+			offset = 0;
+		} else {
+			localActive = new double[numLocalObservations + k];
+			offset = k;
+		}
+		double nextStateProb;
+		for (int b = 0; b < numLocalObservations; b++) {
+			if (isPreviousObservations) {
+				nextStateProb = nextStateKernelEstimator.getProbability(
+						newDestNextValues[b], b);
+			} else {
+				nextStateProb = nextStateKernelEstimator.getProbability(
+						newDestNextValues[b], -1);
+			}
+			double logTerm = 0.0;
+			double local = 0.0;
+			if (localProbNextCondPast[b] > 0) {
+				logTerm = localProbNextCondPast[b] / nextStateProb;
+				local = Math.log(logTerm);
+			}
+			localActive[offset + b] = local;
+			/*
+			if (debug) {
+				System.out.println(b + ": " + logTerm + " -> " + (local/Math.log(2.0)) + " -> sum: " + (te/Math.log(2.0)));
+			}
+			*/
+		}
+		return localActive;
+	}
+	
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		teKernelEstimator.setDebug(debug);
+	}
+
+	/**
+	 * Generate a vector for each time step, containing the past k states of the destination.
+	 * Note that each state of the destination is a joint vector of destDimensions variables.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return array of vectors for each time step
+	 */
+	private double[][] makeJointVectorForPast(double[][] destination) {
+		try {
+			// We want one less delay vector here - we don't need the last k point,
+			//  because there is no next state for these.
+			return MatrixUtils.makeDelayEmbeddingVector(destination, k, k-1, destination.length - k);
+		} catch (Exception e) {
+			// The parameters for the above call should be fine, so we don't expect to
+			//  throw an Exception here - embed in a RuntimeException if it occurs 
+			throw new RuntimeException(e);
+		}
+	}
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyCalculatorMultiVariateSingleObservationsKernel.java

@@ -0,0 +1,227 @@
+package infodynamics.measures.continuous.kernel;
+
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Iterator;
+import java.util.Vector;
+
+/**
+ * 
+ * <p>
+ * Implements a transfer entropy calculator using kernel estimation.
+ * (see Schreiber, PRL 85 (2) pp.461-464, 2000)</p> 
+ * 
+ * <p>
+ * This calculator handles multi-variate source and destination variables,
+ *  and should only be used to add observation tuples, i.e.
+ *  (source, destination next state, destination past)
+ *  one at a time. This allows the user to specify the variable that
+ *  should be used as the destination past, for advanced applications.
+ * </p>
+ * 
+ * <p>
+ * Usage:
+ * 	<ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ *		<li>initialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(),
+ * 			(addObservations|addSingleObservation)*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not guaranteed to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * 	</ol>
+ * </p>
+ * 
+ * <p>
+ * TODO Implement dynamic correlation exclusion with multiple observation sets. (see the
+ *  way this is done in Plain calculator).
+ * TODO Think about added error-trapping code to make sure the user only makes one type of addObservations call.
+ * </p>
+ * 
+ * @author Joseph Lizier
+ * @see For transfer entropy: Schreiber, PRL 85 (2) pp.461-464, 2000; http://dx.doi.org/10.1103/PhysRevLett.85.461
+ * @see For local transfer entropy: Lizier et al, PRE 77, 026110, 2008; http://dx.doi.org/10.1103/PhysRevE.77.026110
+ *
+ */
+public class TransferEntropyCalculatorMultiVariateSingleObservationsKernel
+		extends TransferEntropyCalculatorMultiVariateKernel {
+
+	/**
+	 * Storage for destination history observations for addObservsations
+	 */
+	protected Vector<double[][]> vectorOfJointDestinationPastObservations;
+
+	protected int destPastDimensions = 1;
+
+	public TransferEntropyCalculatorMultiVariateSingleObservationsKernel() {
+		super();
+	}
+
+	/**
+	 * Initialises the calculator
+	 * 
+	 * @param epsilon kernel width
+	 */
+	public void initialise(double epsilon) throws Exception {
+		this.epsilon = epsilon;
+		initialise(1, 1); // assume 1 dimension in source and dest
+	}
+
+	public void initialise(int destDimensions, int sourceDimensions) throws Exception {
+		this.destDimensions = destDimensions;
+		this.sourceDimensions = sourceDimensions;
+		this.destPastDimensions = destDimensions; // assume same
+		super.initialise(1); // Feeds k=1 to super and calls initialise();
+	}
+
+	public void initialiseAllDimensions(int destDimensions, int destPastDimensions,
+			int sourceDimensions) throws Exception {
+		this.destDimensions = destDimensions;
+		this.sourceDimensions = sourceDimensions;
+		this.destPastDimensions = destPastDimensions;
+		
+		// Mimic super.initialise(1) (it would replace dest and source
+		//  dimensions if we're not careful)
+		addedMoreThanOneObservationSet = false;
+		k = 1;
+		initialise();
+	}
+
+	/**
+	 * Set the observations to compute the probabilities from 
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void setObservations(double[][] source, double[][] destination,
+			double[][] destinationPast) throws Exception {
+		startAddObservations();
+		addObservations(source, destination, destinationPast);
+		finaliseAddObservations();
+	}
+	
+	@Override
+	public void startAddObservations() {
+		vectorOfJointDestinationPastObservations = new Vector<double[][]>();
+		super.startAddObservations();
+	}
+
+	/**
+	 * Add observations of the joint source and destinations
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param destinationPast
+	 * @throws Exception
+	 */
+	public void addObservations(double[][] source, double[][] destination,
+			double[][] destinationPast) throws Exception {
+		if (destinationPast.length != destination.length) {
+			throw new Exception(String.format("Destination past and destination lengths (%d and %d) must match!",
+					destinationPast.length, destination.length));
+		}
+		int thisDestPastDimensions = destinationPast[0].length;
+		if ((thisDestPastDimensions != destPastDimensions)) {
+			throw new Exception("Cannot add observsations for destination past variables " +
+					" of " + thisDestPastDimensions +
+					" dimensions for TE calculator set up for " + destPastDimensions +
+					" destination past dimensions");
+		}
+		if (vectorOfJointDestinationPastObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		vectorOfJointDestinationPastObservations.add(destinationPast);
+		super.addObservations(source, destination);
+	}
+
+	/**
+	 * Add a single observation of the joint source, destinations and
+	 * destination past
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param destinationPast
+	 * @throws Exception
+	 */
+	public void addSingleObservation(double[] source, double[] destination,
+			double[] destinationPast) throws Exception {
+		int thisSourceDimensions = source.length;
+		int thisDestDimensions = destination.length;
+		int thisDestPastDimensions = destinationPast.length;
+		if ((thisDestDimensions != destDimensions) ||
+				(thisSourceDimensions != sourceDimensions) ||
+				(thisDestPastDimensions != destPastDimensions)) {
+			throw new Exception("Cannot add observsations for source, destination and destPast variables " +
+					" of " + thisSourceDimensions + " and " + thisDestDimensions + " and " +
+					thisDestPastDimensions +
+					" dimensions respectively for TE calculator set up for " + sourceDimensions + ", " +
+					destDimensions + " and " + destPastDimensions +
+					" source, destination and destPast dimensions respectively");
+		}
+		if (vectorOfJointDestinationPastObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		// Now make the multidimensional arrays to add in
+		double[][] sourceContainer = new double[1][];
+		double[][] destContainer = new double[1][];
+		double[][] destPastContainer = new double[1][];
+		sourceContainer[0] = source;
+		destContainer[0] = destination;
+		destPastContainer[0] = destinationPast;
+		vectorOfJointDestinationPastObservations.add(destPastContainer);
+		super.addObservations(sourceContainer, destContainer);
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[][] destination : vectorOfJointDestinationObservations) {
+			// No need t jump k values in, since we've got the destination
+			//  past values held separately
+			totalObservations += destination.length;
+		}
+		destPastVectors = new double[totalObservations][destPastDimensions];
+		destNextVectors = new double[totalObservations][destDimensions];
+		sourceVectors = new double[totalObservations][sourceDimensions];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[][]> iterator = vectorOfJointDestinationObservations.iterator();
+		Iterator<double[][]> iteratorDestPast = vectorOfJointDestinationPastObservations.iterator();
+		for (double[][] source : vectorOfJointSourceObservations) {
+			double[][] destination = iterator.next();
+			double[][] destinationPast = iteratorDestPast.next();
+			// Add in all observations - no need to offset by k since
+			//  we've got the destination past held separately.
+			MatrixUtils.arrayCopy(destinationPast, 0, 0,
+					destPastVectors, startObservation, 0, destinationPast.length,
+					destPastDimensions);
+			MatrixUtils.arrayCopy(destination, 0, 0,
+					destNextVectors, startObservation, 0,
+					destination.length, destDimensions);
+			MatrixUtils.arrayCopy(source, 0, 0,
+					sourceVectors, startObservation, 0,
+					source.length, sourceDimensions);
+			startObservation += destination.length;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		teKernelEstimator.setObservations(destPastVectors, destNextVectors, sourceVectors);
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfJointDestinationObservations.size() > 1;
+		
+		// And clear the vector of observations
+		vectorOfJointSourceObservations = null;
+		vectorOfJointDestinationObservations = null;
+		vectorOfJointDestinationPastObservations = null;
+	}
+
+}

java/source/infodynamics/measures/continuous/kernel/TransferEntropyKernelCounts.java

@@ -0,0 +1,33 @@
+package infodynamics.measures.continuous.kernel;
+
+/**
+ * Structure to hold the results of the kernel
+ *  estimation for one time point
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ *
+ */
+public class TransferEntropyKernelCounts {
+	public int countPast;
+	public int countNextPast;
+	public int countPastSource;
+	public int countNextPastSource;
+	
+	/**
+	 * Create a new TransferEntropyKernelCounts object
+	 * 
+	 * @param countPast
+	 * @param countNextPast
+	 * @param countPastSource
+	 * @param countNextPastSource
+	 */
+	public TransferEntropyKernelCounts(int countPast,
+			int countNextPast, int countPastSource,
+			int countNextPastSource) {
+		this.countPast = countPast;
+		this.countNextPast = countNextPast;
+		this.countPastSource = countPastSource;
+		this.countNextPastSource = countNextPastSource;
+	}
+}
+

java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java

@@ -0,0 +1,259 @@
+package infodynamics.measures.continuous.kozachenko;
+
+import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+
+/**
+ * Compute the entropy using the Kozachenko estimation method.
+ * See:
+ * - "A statistical estimate for the entropy of a random vector"
+ * 		Kozachenko, L., Leonenko, N.,
+ * 		Problems of Information Transmission, 23 (1987) 9–16
+ * - "Estimating mutual information"
+ * 		Kraskov, A., Stogbauer, H., Grassberger, P.,
+ * 		Physical Review E 69, (2004) 066138
+ * - "Measuring Global Behaviour of Multi-Agent Systems from
+ *  	Pair-Wise Mutual Information",
+ *  	George Mathews, Hugh Durrant-Whyte, and Mikhail Prokopenko
+ * 
+ * This class computes it exactly as in "Estimating mutual information", i.e. using natural
+ *  units and twice the minimum distance.
+ *  Implementing this to check if our other implementation was correct.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class EntropyCalculatorMultiVariateKozachenko  
+	implements EntropyCalculatorMultiVariate {
+
+	protected boolean debug = false;
+	private int totalObservations;
+	private int dimensions;
+	protected double[][] rawData;
+	private double lastEntropy;
+	private double[] lastLocalEntropy;
+	private boolean isComputed;
+	
+	public static final double EULER_MASCHERONI_CONSTANT = 0.5772156;
+	
+		
+	public EntropyCalculatorMultiVariateKozachenko() {
+		totalObservations = 0;
+		dimensions = 0;
+		isComputed = false;
+		lastLocalEntropy = null;
+	}
+
+	public void initialise(int dimensions) {
+		this.dimensions = dimensions;
+		rawData = null;
+		totalObservations = 0;
+		isComputed = false;
+		lastLocalEntropy = null;
+	}
+
+	public void setObservations(double[][] observations) {
+		rawData = observations;
+		totalObservations = observations.length;
+		isComputed = false;
+		lastLocalEntropy = null;
+	}
+
+	/**
+	 * Each row of the data is an observation; each column of
+	 *  the row is a new variable in the multivariate observation.
+	 * This method signature allows the user to call setObservations for
+	 *  joint time series without combining them into a single joint time
+	 *  series (we do the combining for them).
+	 * 
+	 * @param data1
+	 * @param data2
+	 * @throws Exception When the length of the two arrays of observations do not match.
+	 */
+	public void setObservations(double[][] data1,
+			double[][] data2) throws Exception {
+		int timeSteps = data1.length;
+		if ((data1 == null) || (data2 == null)) {
+			throw new Exception("Cannot have null data arguments");
+		}
+		if (data1.length != data2.length) {
+			throw new Exception("Length of data1 (" + data1.length + ") is not equal to the length of data2 (" +
+					data2.length + ")");
+		}
+		int data1Variables = data1[0].length;
+		int data2Variables = data2[0].length;
+		double[][] data = new double[timeSteps][data1Variables + data2Variables];
+		for (int t = 0; t < timeSteps; t++) {
+			System.arraycopy(data1[t], 0, data[t], 0, data1Variables);
+			System.arraycopy(data2[t], 0, data[t], data1Variables, data2Variables);
+		}
+		// Now defer to the normal setObservations method
+		setObservations(data);
+	}
+
+	/**
+	 * Computes average entropy of previously provided observations.
+	 * 
+	 * @return entropy in natural units
+	 */
+	public double computeAverageLocalOfObservations() {
+		if (isComputed) {
+			return lastEntropy;
+		}
+		double sdTermHere = sdTerm(totalObservations, dimensions);
+		double emConstHere = eulerMacheroniTerm(totalObservations);
+		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData);
+		double entropy = 0.0;
+		if (debug) {
+			System.out.println("t,\tminDist,\tlogMinDist,\tsum");
+		}
+		for (int t = 0; t < rawData.length; t++) {
+			entropy += Math.log(2.0 * minDistance[t]);
+			if (debug) {
+				System.out.println(t + ",\t" + 
+						minDistance[t] + ",\t" +
+						Math.log(minDistance[t]) + ",\t" +
+						entropy);
+			}
+		}
+		// Using natural units
+		// entropy /= Math.log(2);
+		entropy *= (double) dimensions / (double) totalObservations;
+		if (debug) {
+			System.out.println("Sum part:   " + entropy);
+			System.out.println("Euler part: " + emConstHere);
+			System.out.println("Sd term:    " + sdTermHere);
+		}
+		entropy += emConstHere;
+		entropy += sdTermHere;
+		lastEntropy = entropy;
+		isComputed = true;
+		return entropy;
+	}
+
+	/**
+	 * Computes local entropies of given values, using previously provided observations.
+	 * 
+	 * @return local entropies in natural units
+	 */
+	public double[] computeLocalOfPreviousObservations() {
+		if (lastLocalEntropy != null) {
+			return lastLocalEntropy;
+		}
+
+		double sdTermHere = sdTerm(totalObservations, dimensions);
+		double emConstHere = eulerMacheroniTerm(totalObservations);
+		double constantToAddIn = sdTermHere + emConstHere;
+		
+		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData);
+		double entropy = 0.0;
+		double[] localEntropy = new double[rawData.length];
+		if (debug) {
+			System.out.println("t,\tminDist,\tlogMinDist,\tlocal,\tsum");
+		}
+		for (int t = 0; t < rawData.length; t++) {
+			localEntropy[t] = Math.log(2.0 * minDistance[t]) * (double) dimensions;
+			// using natural units
+			// localEntropy[t] /= Math.log(2);
+			localEntropy[t] += constantToAddIn;
+			entropy += localEntropy[t];
+			if (debug) {
+				System.out.println(t + ",\t" + 
+						minDistance[t] + ",\t" +
+						Math.log(minDistance[t]) + ",\t" +
+						localEntropy[t] + ",\t" +
+						entropy);
+			}
+		}
+		entropy /= (double) totalObservations;
+		lastEntropy = entropy;
+		lastLocalEntropy = localEntropy;
+		return localEntropy;
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states) throws Exception {
+		throw new Exception("Local method for other data not implemented");
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states1, double[][] states2) throws Exception {
+		throw new Exception("Local method for other data not implemented");
+	}
+
+	/**
+	 * Returns the value of the Euler-Mascheroni term.
+	 * Public for debugging purposes
+	 * 
+	 * @return
+	 */
+	public double eulerMacheroniTerm(int N) {
+		// Using natural units
+		// return EULER_MASCHERONI_CONSTANT / Math.log(2);
+		try {
+			return -MathsUtils.digamma(1) + MathsUtils.digamma(N);
+		} catch (Exception e) {
+			// Exception will only be thrown if N < 0
+			return 0;
+		}
+	}
+	
+	/**
+	 * Returns the value of the Sd term
+	 * Public for debugging purposes
+	 * 
+	 * @param numObservations
+	 * @param dimensions
+	 * @return
+	 */
+	public double sdTerm(int numObservations, int dimensions) {
+		// To compute directly:
+		// double unLoggedSdTerm = 
+		//	Math.pow(Math.PI/4.0, ((double) dimensions) / 2.0) /	// Brought 2^d term from denominator into Pi term
+		//	MathsUtils.gammaOfArgOn2Plus1(dimensions);
+		
+		// But we need to compute it carefully, to allow the maximum range of dimensions
+		// double unLoggedSdTerm = 
+		//	1.0 / MathsUtils.gammaOfArgOn2Plus1IncludeDivisor(dimensions, 
+		//			Math.pow(Math.PI, ((double) dimensions) / 2.0));
+		// Don't include the 2^d in the above divisor, since that makes the divisor < 1, which 
+		//  doesn't help at all.
+		// unLoggedSdTerm /= Math.pow(2, dimensions);
+		// return Math.log(unLoggedSdTerm) / Math.log(2);
+		// Using natural units
+		// return Math.log(unLoggedSdTerm);
+		
+		// But even that method falls over by about d = 340.
+		// Break down the log into the log of a factorial term and the log of a constant term
+		double constantTerm = Math.pow(Math.PI / 4.0, (double) dimensions / 2.0);
+		double result = 0.0;
+		if (dimensions % 2 == 0) {
+			// d even
+			// Now take log (1/(d/2)!) = -log (1/(d/2)!) = -sum(d/2 --) {log d/2}
+			for (int d = dimensions/2; d > 1; d--) {
+				result -= Math.log(d);
+			}
+		} else {
+			// d odd
+			constantTerm *= Math.pow(2.0, (double) (dimensions + 1) / 2.0);
+			constantTerm /= Math.sqrt(Math.PI);
+			// Now take log (1/d!!) = - log (d!!) = - sum(d -= 2) {log d}
+			for (int d = dimensions; d > 1; d -= 2) {
+				result -= Math.log(d);
+			}
+		}
+		result += Math.log(constantTerm);
+		return result;
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return lastEntropy;
+	}
+
+	public int getNumObservations() {
+		return totalObservations;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/ConditionalMutualInfoCalculatorMultiVariateKraskov.java

@@ -0,0 +1,248 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+/**
+ * <p>Compute the Conditional Mutual Information between two vectors,
+ * conditioned on a third, using the Kraskov estimation method,
+ * as extended by Frenzel and Pompe.</p>
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * Two child classes actually implement the two algorithms in the Kraskov paper.</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * @see "Partial Mutual Information for Coupling Analysis of Multivariate Time Series", Frenzel and Pompe, 2007
+ * 
+ * TODO Finish writing this class - changing it from original Kraskov one
+ * 
+ * @author Joseph Lizier
+ */
+public abstract class ConditionalMutualInfoCalculatorMultiVariateKraskov {
+
+	/**
+	 * we compute distances to the kth neighbour in the joint space
+	 */
+	protected int k;
+	protected double[][] data1;
+	protected double[][] data2;
+	protected double[][] dataCond;
+	protected boolean debug;
+	protected double condMi;
+	protected boolean condMiComputed;
+	
+	// Storage for the norms from each observation to each other one
+	protected double[][] xNorms;
+	protected double[][] yNorms;
+	protected double[][] zNorms;
+	// Keep the norms each time (making reordering very quick)
+	//  (Should only be set to false for testing)
+	public static boolean tryKeepAllPairsNorms = true;
+	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 2000;
+	
+	public final static String PROP_K = "k";
+	public final static String PROP_NORM_TYPE = "NORM_TYPE";
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+
+	public ConditionalMutualInfoCalculatorMultiVariateKraskov() {
+		super();
+		k = 1; // by default
+	}
+
+	public void initialise(int dimensions1, int dimensions2, int dimensions3) {
+		condMi = 0.0;
+		condMiComputed = false;
+		xNorms = null;
+		yNorms = null;
+		zNorms = null;
+		data1 = null;
+		data2 = null;
+		// No need to keep the dimensions here
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			EuclideanUtils.setNormToUse(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void addObservations(double[][] var1, double[][] var2,
+			double[][] conditionedVar) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] var1, double[][] var2,
+			double[][] conditionedVar, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] var1, double[][] var2,
+			double[][] conditionedVar, boolean[] var1Valid,
+			boolean[] var2Valid, boolean[] conditionedValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] var1, double[][] var2,
+			double[][] conditionedVar, boolean[][] var1Valid,
+			boolean[][] var2Valid, boolean[][] conditionedValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] observations1,
+			double[][] observations2, double[][] obsConditioned) throws Exception {
+		if (observations1.length != observations2.length) {
+			throw new Exception("Time steps for observations2 " +
+					observations2.length + " does not match the length " +
+					"of observations1 " + observations1.length);
+		}
+		if ((observations1[0].length == 0) || (observations2[0].length == 0)) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		// Normalise it if required
+		if (normalise) {
+			// Take a copy since we're going to normalise it
+			data1 = MatrixUtils.normaliseIntoNewArray(observations1);
+			data2 = MatrixUtils.normaliseIntoNewArray(observations2);
+			dataCond = MatrixUtils.normaliseIntoNewArray(obsConditioned);
+		} else {
+			data1 = observations1;
+			data2 = observations2;
+			dataCond = obsConditioned;
+		}
+	}
+
+	/**
+	 * Compute the norms for each time series
+	 *
+	 */
+	protected void computeNorms() {
+		int N = data1.length; // number of observations
+		
+		xNorms = new double[N][N];
+		yNorms = new double[N][N];
+		zNorms = new double[N][N];
+		for (int t = 0; t < N; t++) {
+			// Compute the norms from t to all other time points
+			double[][] xyzNormsForT = EuclideanUtils.computeNorms(data1,
+					data2, dataCond, t);
+			for (int t2 = 0; t2 < N; t2++) {
+				xNorms[t][t2] = xyzNormsForT[t2][0];
+				yNorms[t][t2] = xyzNormsForT[t2][1];
+				zNorms[t][t2] = xyzNormsForT[t2][2];
+			}
+		}
+	}
+	
+	public abstract double computeAverageLocalOfObservations() throws Exception;
+
+	/**
+	 * Compute what the average conditional MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here.
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public abstract double computeAverageLocalOfObservations(int[] reordering) throws Exception;
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y,z) correlations, while retaining the p(x,z), p(y) marginals, to check how
+	 *  significant this conditional mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(data1.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y,z) correlations, while retaining the p(x,z), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of conditional MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!condMiComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = condMi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		condMi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = condMi;
+		return measDistribution;
+	}
+
+	public abstract double[] computeLocalOfPreviousObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states1, double[][] states2) throws Exception {
+		// If implemented, will need to incorporate any time difference here.
+		throw new Exception("Local method not implemented yet");
+	}
+	
+	public abstract String printConstants(int N) throws Exception ;
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return condMi;
+	}
+	
+	public int getNumObservations() {
+		return data1.length;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/ConditionalMutualInfoCalculatorMultiVariateKraskov1.java

@@ -0,0 +1,309 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Conditional Mutual Info using the Kraskov estimation method,
+ * as extended by Frenzel and Pompe.
+ * Uses the first algorithm (defined at end of p.2 of the Kraskov paper)</p>
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * </p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * @see "Partial Mutual Information for Coupling Analysis of Multivariate Time Series", Frenzel and Pompe, 2007
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class ConditionalMutualInfoCalculatorMultiVariateKraskov1
+	extends ConditionalMutualInfoCalculatorMultiVariateKraskov {
+	
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+
+	/**
+	 * Compute the average conditional MI from the previously set observations
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		return computeAverageLocalOfObservations(null);
+	}
+
+	/**
+	 * Compute what the average conditional MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * If reordering is null, it is assumed there is no reordering of
+	 *  the y variable.
+	 * 
+	 * @param reordering the reordered time steps of the y variable
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData2 = data2;
+			if (reordering != null) {
+				// Generate a new re-ordered data2
+				data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, reordering);
+			}
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			data2 = originalData2;
+			return newMI;
+		}
+		
+		// Else we'll use the arrays of marginal distances
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		
+		// Count the average number of points within eps_xz and eps_yz and eps_z
+		double averageDiGammas = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  using x, y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			int tForY = (reordering == null) ? t : reordering[t];
+
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				int t2ForY = (reordering == null) ? t2 : reordering[t2];
+				// Joint norm is the max of all three marginals
+				jointNorm[t2] = Math.max(xNorms[t][t2], Math.max(yNorms[tForY][t2ForY], zNorms[t][t2]));
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose (x,z) distance is less
+			//  than eps, and whose (y,z) distance is less than eps, and
+			//  whose z distance is less than eps.
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (zNorms[t][t2] < epsilon) {
+					n_z++;
+					if (xNorms[t][t2] < epsilon) {
+						n_xz++;
+					}
+					int t2ForY = (reordering == null) ? t2 : reordering[t2];
+					if (yNorms[tForY][t2ForY] < epsilon) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			// Note: we're using digamma function which has opposite sign to the harmonic
+			//  number used by Frenzel and Pompe, and is also offset by a constant (though
+			//  this cancels out)
+			averageDiGammas += MathsUtils.digamma(n_z+1) - MathsUtils.digamma(n_xz+1)
+							- MathsUtils.digamma(n_yz+1);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f", avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) + averageDiGammas;
+		condMiComputed = true;
+		return condMi;
+	}
+	
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x, y and z 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		
+		// Count the average number of points within eps_xz, eps_yz and eps_z
+		double averageDiGammas = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyzNorms = EuclideanUtils.computeNorms(data1, data2, dataCond, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = Math.max(xyzNorms[t2][0], Math.max(xyzNorms[t2][1], xyzNorms[t2][2]));
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose (x,z) distance is less
+			//  than eps, whose (y,z) distance is less than eps, and whose
+			//  z distance is less than eps
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyzNorms[t2][2] < epsilon) {
+					n_z++;
+					if (xyzNorms[t2][0] < epsilon) {
+						n_xz++;
+					}
+					if (xyzNorms[t2][1] < epsilon) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_z+1) - MathsUtils.digamma(n_xz+1)
+						- MathsUtils.digamma(n_yz+1);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+					avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) + averageDiGammas;
+		condMiComputed = true;
+		return condMi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		double[] localCondMi = new double[N];
+		
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		
+		// Count the average number of points within eps_xz and eps_yz and eps_z
+		double averageDiGammas = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get x and y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity.
+			double[][] xyzNorms = EuclideanUtils.computeNorms(data1, data2, dataCond, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = Math.max(xyzNorms[t2][0], Math.max(xyzNorms[t2][1], xyzNorms[t2][2]));
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than eps, and whose y distance is less than eps
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyzNorms[t2][2] < epsilon) {
+					n_z++;
+					if (xyzNorms[t2][0] < epsilon) {
+						n_xz++;
+					}
+					if (xyzNorms[t2][1] < epsilon) {
+						n_yz++;
+					}
+				}
+			}
+			// And take the digamma:
+			double digammaNxzPlusOne = MathsUtils.digamma(n_xz+1);
+			double digammaNyzPlusOne = MathsUtils.digamma(n_yz+1);
+			double digammaNzPlusOne = MathsUtils.digamma(n_z+1);
+			
+			localCondMi[t] = digammaK - digammaNxzPlusOne - digammaNyzPlusOne + digammaNzPlusOne;
+			
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And keep track of the average
+			averageDiGammas += digammaNzPlusOne - digammaNxzPlusOne - digammaNyzPlusOne;
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+					avNxz, avNyz, avNz));
+		}
+		
+		condMi = digammaK + averageDiGammas;
+		condMiComputed = true;
+		return localCondMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e",
+				k, MathsUtils.digamma(k));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/ConditionalMutualInfoCalculatorMultiVariateKraskov2.java

@@ -0,0 +1,497 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Conditional Mutual Info using the Kraskov estimation method,
+ * as extended by Frenzel and Pompe.</p>
+ * <p>
+ * Uses the second algorithm (defined at start of p.3 of the Kraskov paper) - 
+ *  note that Frenzel and Pompe only extended the technique directly for the
+ *  first algorithm, though here we define it for the second.
+ *  It is unclear exactly how to do this, since we need to account for
+ *  the 1/k factor, which changes in the space of the second
+ *  MI. I've taken a guess, though use of 
+ *  {@link ConditionalMutualInfoCalculatorMultiVariateKraskov1}
+ *  is perhaps recommended instead of this class.</p>
+ *  
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * </p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * 		http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * @see "Partial Mutual Information for Coupling Analysis of Multivariate Time Series", Frenzel and Pompe, 2007
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class ConditionalMutualInfoCalculatorMultiVariateKraskov2
+	extends ConditionalMutualInfoCalculatorMultiVariateKraskov {
+
+	protected static final int JOINT_NORM_VAL_COLUMN = 0;
+	protected static final int JOINT_NORM_TIMESTEP_COLUMN = 1;
+
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+	
+	/**
+	 * Compute what the average conditional MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData2 = data2;
+			// Generate a new re-ordered data2
+			data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, reordering);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			data2 = originalData2;
+			return newMI;
+		}
+		
+		// Otherwise we will use the norms we've already computed, and use a "virtual"
+		//  reordered data2.
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_xz and eps_yz and eps_z
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_xz and eps_yz ad eps_z for this time step:
+			//  First get x and y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+
+			int tForY = reordering[t];
+
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				int t2ForY = reordering[t2];
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xNorms[t][t2],
+							Math.max(yNorms[tForY][t2ForY], zNorms[t][t2]));
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y,z} as the maximum x and y and z norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (yNorms[tForY][reordering[timeStepOfJthPoint]] > eps_y) {
+					eps_y = yNorms[tForY][reordering[timeStepOfJthPoint]];
+				}
+				if (zNorms[t][timeStepOfJthPoint] > eps_z) {
+					eps_z = zNorms[t][timeStepOfJthPoint];
+				}
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y, and whose z distance is less
+			//  than or equal to eps_z
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (zNorms[t][t2] <= eps_z) {
+					n_z++;
+					if (xNorms[t][t2] <= eps_x) {
+						n_xz++;
+					}
+					if (yNorms[tForY][reordering[t2]] <= eps_y) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_z) - MathsUtils.digamma(n_xz)
+								- MathsUtils.digamma(n_yz);
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+						avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0 / (double) k + 
+			+ averageInverseCountInJointYZ + averageDiGammas;
+		condMiComputed = true;
+		return condMi;
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			return computeAverageLocalOfObservationsWhileComputingDistances();
+		}
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y and eps_z for this time step:
+			//  using x and y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xNorms[t][t2],
+							Math.max(yNorms[t][t2], zNorms[t][t2]));
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y,z} as the maximum x and y and z norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (yNorms[t][timeStepOfJthPoint] > eps_y) {
+					eps_y = yNorms[t][timeStepOfJthPoint];
+				}
+				if (zNorms[t][timeStepOfJthPoint] > eps_z) {
+					eps_z = zNorms[t][timeStepOfJthPoint];
+				}
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y, and whose z distance is less
+			//  than or equal to eps_z
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (zNorms[t][t2] <= eps_z) {
+					n_z++;
+					if (xNorms[t][t2] <= eps_x) {
+						n_xz++;
+					}
+					if (yNorms[t][t2] <= eps_y) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_z) - MathsUtils.digamma(n_xz)
+								- MathsUtils.digamma(n_yz);
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f, Average n_y=%.3f",
+					avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0 / (double) k +
+			averageInverseCountInJointYZ + averageDiGammas;
+		condMiComputed = true;
+		return condMi;
+	}
+
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y and z
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y and eps_z for this time step:
+			//  First get x and y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyzNorms = EuclideanUtils.computeNorms(data1, data2, dataCond, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xyzNorms[t2][0], 
+						Math.max(xyzNorms[t2][1], xyzNorms[t2][2]));
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j];
+				if (xyzNorms[timeStepOfJthPoint][0] > eps_x) {
+					eps_x = xyzNorms[timeStepOfJthPoint][0];
+				}
+				if (xyzNorms[timeStepOfJthPoint][1] > eps_y) {
+					eps_y = xyzNorms[timeStepOfJthPoint][1];
+				}
+				if (xyzNorms[timeStepOfJthPoint][2] > eps_z) {
+					eps_z = xyzNorms[timeStepOfJthPoint][2];
+				}
+			}
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y, and whose z distance is less
+			//  than or equal to eps_z
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyzNorms[t2][2] <= eps_z) {
+					n_z++;
+					if (xyzNorms[t2][0] <= eps_x) {
+						n_xz++;
+					}
+					if (xyzNorms[t2][1] <= eps_y) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_z) - MathsUtils.digamma(n_xz)
+								- MathsUtils.digamma(n_yz);
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_zx=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+						avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0 / (double) k +
+				averageInverseCountInJointYZ + averageDiGammas;
+		condMiComputed = true;
+		return condMi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		double[] localCondMi = new double[N];
+		
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y and eps_z for this time step:
+			//  First get x and y and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyzNorms = EuclideanUtils.computeNorms(data1, data2, dataCond, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xyzNorms[t2][0],
+							Math.max(xyzNorms[t2][1], xyzNorms[t2][2]));
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y,z} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xyzNorms[timeStepOfJthPoint][0] > eps_x) {
+					eps_x = xyzNorms[timeStepOfJthPoint][0];
+				}
+				if (xyzNorms[timeStepOfJthPoint][1] > eps_y) {
+					eps_y = xyzNorms[timeStepOfJthPoint][1];
+				}
+				if (xyzNorms[timeStepOfJthPoint][0] > eps_z) {
+					eps_z = xyzNorms[timeStepOfJthPoint][2];
+				}
+			}
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y, and whose z distance is less
+			//  than or equal to eps_z
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyzNorms[t2][2] <= eps_z) {
+					n_z++;
+					if (xyzNorms[t2][0] <= eps_x) {
+						n_xz++;
+					}
+					if (xyzNorms[t2][1] <= eps_y) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma:
+			double digammaNxz = MathsUtils.digamma(n_xz);
+			double digammaNyz = MathsUtils.digamma(n_yz);
+			double digammaNz = MathsUtils.digamma(n_z);
+			
+			localCondMi[t] = digammaK - digammaNxz - digammaNyz + digammaNz
+						- 1.0 / (double) k + 1.0/(double) n_yz ;
+
+			averageDiGammas += digammaNz - digammaNxz - digammaNyz;
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+					avNxz, avNyz, avNz));
+		}
+		
+		condMi = digammaK + averageDiGammas - 1.0 / (double) k +
+			averageInverseCountInJointYZ;
+		condMiComputed = true;
+		return localCondMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e",
+				k, MathsUtils.digamma(k));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/ConditionalMutualInfoCalculatorMultiVariateWithDiscreteKraskov.java

@@ -0,0 +1,565 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSource;
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+/**
+ * <p>Compute the Conditional Mutual Information between a discrete variable and a 
+ *  vector of continuous variables, conditioned on another vector of continuous variables
+ *  using the Kraskov estimation method.</p>
+ * <p>Uses Kraskov method type 2, since type 1 only looks at points with
+ * distances strictly less than the kth variable, which won't work for one marginal
+ * being discrete.</p>
+ * 
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public class ConditionalMutualInfoCalculatorMultiVariateWithDiscreteKraskov implements ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSource {
+
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+	
+	/**
+	 * we compute distances to the kth neighbour
+	 */
+	protected int k;
+	protected double[][] continuousDataX;
+	protected double[][] conditionedDataZ;
+	protected int[] discreteDataY;
+	protected int[] counts;
+	protected int base;
+	protected boolean debug;
+	protected double condMi;
+	protected boolean miComputed;
+	
+	// Storage for the norms from each observation to each other one
+	protected double[][] xNorms;
+	protected double[][] zNorms;
+	protected double[][] xzNorms;
+	// Keep the norms each time (making reordering very quick)
+	//  (Should only be set to false for testing)
+	public static boolean tryKeepAllPairsNorms = true;
+	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 2000;
+	
+	public final static String PROP_K = "k";
+	public final static String PROP_NORM_TYPE = "NORM_TYPE";	
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+
+	public ConditionalMutualInfoCalculatorMultiVariateWithDiscreteKraskov() {
+		super();
+		k = 1; // by default
+	}
+
+	/**
+	 * Initialise the calculator.
+	 * 
+	 * @param dimensions number of joint continuous variables
+	 * @param base number of discrete states
+	 * @param dimensionsCond the number of joint continuous variables
+	 *     to condition on
+	 */
+	public void initialise(int dimensions, int base, int dimensionsCond) {
+		condMi = 0.0;
+		miComputed = false;
+		xNorms = null;
+		continuousDataX = null;
+		discreteDataY = null;
+		// No need to keep the dimenions for the conditional variables here
+		this.base = base;
+	}
+
+	/**
+	 * 
+	 * @param propertyName name of the property to set
+	 * @param propertyValue value to set on that property
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			EuclideanUtils.setNormToUse(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations, double[][] conditionedObservations)
+			throws Exception {
+		
+		if ((continuousObservations.length != discreteObservations.length) ||
+			(continuousObservations.length != conditionedObservations.length)) {
+			throw new Exception("Time steps for observations2 " +
+					discreteObservations.length + " does not match the length " +
+					"of observations1 " + continuousObservations.length +
+					" and of conditionedObservations " + conditionedObservations.length);
+		}
+		if (continuousObservations[0].length == 0) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		if (conditionedObservations[0].length == 0) {
+			throw new Exception("Computing MI with a null set of conditioned data");
+		}
+		continuousDataX = continuousObservations;
+		discreteDataY = discreteObservations;
+		conditionedDataZ = conditionedObservations;
+		if (normalise) {
+			// Take a copy since we're going to normalise it
+			continuousDataX = MatrixUtils.normaliseIntoNewArray(continuousObservations);
+			conditionedDataZ = MatrixUtils.normaliseIntoNewArray(conditionedObservations);
+		}
+		// count the discrete states:
+		counts = new int[base];
+		for (int t = 0; t < discreteDataY.length; t++) {
+			counts[discreteDataY[t]]++;
+		}
+		for (int b = 0; b < counts.length; b++) {
+			if (counts[b] < k) {
+				throw new RuntimeException("This implementation assumes there are at least k items in each discrete bin");
+			}
+		}
+	}
+
+	/**
+	 * Compute the norms for each marginal time series
+	 *
+	 */
+	protected void computeNorms() {
+		int N = continuousDataX.length; // number of observations
+		
+		xNorms = new double[N][N];
+		zNorms = new double[N][N];
+		xzNorms = new double[N][N];
+		for (int t = 0; t < N; t++) {
+			// Compute the norms from t to all other time points
+			for (int t2 = 0; t2 < N; t2++) {
+				if (t2 == t) {
+					xNorms[t][t2] = Double.POSITIVE_INFINITY;
+					zNorms[t][t2] = Double.POSITIVE_INFINITY;
+					xzNorms[t][t2] = Double.POSITIVE_INFINITY;
+					continue;
+				}
+				// Compute norm in the continuous space
+				xNorms[t][t2] = EuclideanUtils.norm(continuousDataX[t], continuousDataX[t2]);
+				zNorms[t][t2] = EuclideanUtils.norm(conditionedDataZ[t], conditionedDataZ[t2]);
+				xzNorms[t][t2] = Math.max(xNorms[t][t2], zNorms[t][t2]);
+			}
+		}
+	}
+	
+	/**
+	 * Compute what the average conditional MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		int N = continuousDataX.length; // number of observations
+		if (!tryKeepAllPairsNorms || (N > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			// Generate a new re-ordered set of discrete data
+			int[] originalDiscreteData = discreteDataY;
+			discreteDataY = MatrixUtils.extractSelectedTimePoints(discreteDataY, reordering);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			discreteDataY = originalDiscreteData;
+			return newMI;
+		}
+		
+		// Otherwise we will use the norms we've already computed, and use a "virtual"
+		//  reordered data2.
+		int[] reorderedDiscreteData = MatrixUtils.extractSelectedTimePoints(discreteDataY, reordering);
+
+		if (xNorms == null) {
+			computeNorms();
+		}
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_z for this time step:
+			//  using max of x and z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][0] = Math.max(xNorms[t][t2], zNorms[t][t2]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][1] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			// just do a linear search for the minimum epsilon value
+			timeStepsOfKthMins = MatrixUtils.kMinIndicesSubjectTo(
+					jointNorm, 0, k, reorderedDiscreteData, reorderedDiscreteData[t]);
+			// and now we have the closest k points.
+			// Find eps_{x,y,z} as the maximum x and y and z norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (zNorms[t][timeStepOfJthPoint] > eps_z) {
+					eps_z = zNorms[t][timeStepOfJthPoint];
+				}
+			}
+
+			// Count the number of points whose distances are less
+			//  than or equal to eps in each required joint space
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (zNorms[t][t2] <= eps_z) {
+					n_z++;
+					if (xNorms[t][t2] <= eps_x) {
+						n_xz++;
+					}
+					if (reorderedDiscreteData[t] == reorderedDiscreteData[t2]) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_xz) + MathsUtils.digamma(n_yz)
+					- MathsUtils.digamma(n_z);
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.println(String.format("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f",
+						avNxz, avNyz, avNz));
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0/(double)k +
+			averageInverseCountInJointYZ - averageDiGammas;
+		miComputed = true;
+		return condMi;
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (!tryKeepAllPairsNorms || (continuousDataX.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			return computeAverageLocalOfObservationsWhileComputingDistances();
+		}
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = continuousDataX.length; // number of observations
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_z for this time step:
+			//  using x,z norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][0] = Math.max(xNorms[t][t2], zNorms[t][t2]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][1] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			// just do a linear search for the minimum epsilon value
+			timeStepsOfKthMins = MatrixUtils.kMinIndicesSubjectTo(
+					jointNorm, 0, k, discreteDataY, discreteDataY[t]);
+			// and now we have the closest k points.
+			// Find eps_{x,y,z} as the maximum x and y and z norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (zNorms[t][timeStepOfJthPoint] > eps_z) {
+					eps_z = zNorms[t][timeStepOfJthPoint];
+				}
+			}
+
+			// Count the number of points whose x,y,z distances are less
+			//  than or equal to eps (not including this point)
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (zNorms[t][t2] <= eps_z) {
+					n_z++;
+					if (xNorms[t][t2] <= eps_x) {
+						n_xz++;
+					}
+					if (discreteDataY[t] == discreteDataY[t2]) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_xz) + MathsUtils.digamma(n_yz)
+						- MathsUtils.digamma(n_z);
+			averageInverseCountInJointYZ += 1.0 / (double) n_yz;
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double) N;
+			System.out.printf("Average n_xz=%.3f (-> digam=%.3f %.3f), Average n_yz=%.3f (-> digam=%.3f)",
+					avNxz, MathsUtils.digamma((int) avNxz), MathsUtils.digamma((int) avNxz - 1), avNyz, MathsUtils.digamma((int) avNyz));
+			System.out.printf(", Average n_z=%.3f (-> digam=%.3f)\n", avNz, MathsUtils.digamma((int) avNz));
+			System.out.printf("Independent average num in joint box is %.3f\n", (avNxz * avNyz / (double) N));
+			System.out.println(String.format("digamma(k)=%.3f - 1/k=%.3f + <1/n_yz>=%.3f - averageDiGammas=%.3f\n",
+					MathsUtils.digamma(k), 1.0/(double)k, averageInverseCountInJointYZ,
+					averageDiGammas));
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0/(double)k +
+			averageInverseCountInJointYZ - averageDiGammas;
+		miComputed = true;
+		return condMi;
+	}
+
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = continuousDataX.length; // number of observations
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double averageInverseCountInJointYZ = 0;
+		double avNxz = 0;
+		double avNyz = 0;
+		double avNz = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_* for this time step:
+			//  First get xz norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xzNorms = EuclideanUtils.computeNorms(continuousDataX, conditionedDataZ, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][0] = Math.max(xzNorms[t2][0], 
+						xzNorms[t2][1]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][1] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_z = 0.0;
+			int[] timeStepsOfKthMins = null;
+			// just do a linear search for the minimum epsilon value
+			//  subject to the discrete variable value
+			timeStepsOfKthMins = MatrixUtils.kMinIndicesSubjectTo(
+					jointNorm, 0, k, discreteDataY, discreteDataY[t]);
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j];
+				if (xzNorms[timeStepOfJthPoint][0] > eps_x) {
+					eps_x = xzNorms[timeStepOfJthPoint][0];
+				}
+				if (xzNorms[timeStepOfJthPoint][1] > eps_z) {
+					eps_z = xzNorms[timeStepOfJthPoint][1];
+				}
+			}
+
+			// Count the number of points whose distances is less
+			//  than or equal to eps
+			int n_xz = 0;
+			int n_yz = 0;
+			int n_z = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xzNorms[t2][1] <= eps_z) {
+					n_z++;
+					if (xzNorms[t2][0] <= eps_x) {
+						n_xz++;
+					}
+					if (discreteDataY[t] == discreteDataY[t2]) {
+						n_yz++;
+					}
+				}
+			}
+			avNxz += n_xz;
+			avNyz += n_yz;
+			avNz += n_z;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_xz) + MathsUtils.digamma(n_yz)
+				- MathsUtils.digamma(n_z);
+		}
+		averageDiGammas /= (double) N;
+		averageInverseCountInJointYZ /= (double) N;
+		if (debug) {
+			avNxz /= (double)N;
+			avNyz /= (double)N;
+			avNz /= (double)N;
+			System.out.printf("Average n_xz=%.3f, Average n_yz=%.3f, Average n_z=%.3f\n",
+					avNxz, avNyz, avNz);
+		}
+		
+		condMi = MathsUtils.digamma(k) - 1.0/(double)k +
+			averageInverseCountInJointYZ - averageDiGammas;
+		miComputed = true;
+		return condMi;
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(continuousDataX.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = condMi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		condMi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = condMi;
+		return measDistribution;
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] continuousStates,
+			int[] discreteStates) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return condMi;
+	}
+	
+	public int getNumObservations() {
+		return continuousDataX.length;
+	}
+
+	public double[] computeLocalUsingPreviousObservations(
+			double[][] contStates, int[] discreteStates,
+			double[][] conditionedStates) throws Exception {
+		// TODO Auto-generated method stub
+		throw new Exception("Not implemented yet");
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov.java

@@ -0,0 +1,296 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.MultiInfoCalculator;
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Vector;
+
+
+/**
+ * <p>Compute the Multi-Information (or integration) using the Kraskov estimation method.
+ * Two child classes actually implement the two algorithms in the Kraskov paper.</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public abstract class MultiInfoCalculatorKraskov implements
+		MultiInfoCalculator {
+
+	/**
+	 * we compute distances to the kth neighbour
+	 */
+	protected int k;
+	protected double[][] data;
+	protected boolean debug;
+	protected double mi;
+	protected boolean miComputed;
+	private Vector<double[]> individualObservations;
+
+	protected int N; // number of observations
+	protected int V; // number of variables
+	
+	// Storage for the norms for each marginal variable from each observation to each other one
+	protected double[][][] norms;
+	// Keep the norms each time (making reordering very quick)
+	//  (Should only be set to false for testing)
+	protected boolean tryKeepAllPairsNorms = true;
+	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 4000;
+	
+	public final static String PROP_K = "k";
+	public final static String PROP_NORM_TYPE = "NORM_TYPE";
+	public final static String PROP_TRY_TO_KEEP_ALL_PAIRS_NORM = "TRY_KEEP_ALL_PAIRS_NORM";
+	
+	public MultiInfoCalculatorKraskov() {
+		super();
+		k = 1; // by default
+	}
+
+	public void initialise(int dimensions) {
+		V = dimensions;
+		mi = 0.0;
+		miComputed = false;
+		norms = null;
+		data = null;
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			EuclideanUtils.setNormToUse(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_TRY_TO_KEEP_ALL_PAIRS_NORM)) {
+			tryKeepAllPairsNorms = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void setObservations(double[][] observations) throws Exception {
+		if ((observations == null) || (observations[0].length == 0)) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		if (observations[0].length != V) {
+			throw new Exception("Incorrect number of dimensions " + observations[0].length +
+					" in supplied observations (expected " + V + ")");
+		}
+		data = observations;
+		N = data.length;
+	}
+
+	/**
+	 * Set observations from two separate time series (join the rows at each time step
+	 *  together to make a joint vector)
+	 * 
+	 * @param observations1
+	 * @param observations2
+	 */
+	public void setObservations(double[][] observations1, double[][] observations2) throws Exception {
+		if ((observations1 == null) || (observations1[0].length == 0) ||
+				(observations2 == null) || (observations2[0].length == 0)) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		if (observations1.length != observations2.length) {
+			throw new Exception("Length of the time series to be joined to not match");
+		}
+		if (observations1[0].length + observations2[0].length != V) {
+			throw new Exception("Incorrect number of dimensions " + 
+					(observations1[0].length + observations2[0].length) +
+					" in supplied observations (expected " + V + ")");
+		}
+		N = observations1.length;
+		data = new double[N][V];
+		for (int t = 0; t < N; t++) {
+			int v = 0;
+			for (int i = 0; i < observations1[t].length; i++) {
+				data[t][v++] = observations1[t][i];
+			}
+			for (int i = 0; i < observations2[t].length; i++) {
+				data[t][v++] = observations2[t][i];
+			}
+		}
+		return;
+	}
+	
+	/**
+	 * User elects to set observations one by one rather than in one go.
+	 * Will need to call endIndividualObservations before calling any of the
+	 *  compute functions, otherwise the previous observations will be used.
+	 */
+	public void startIndividualObservations() {
+		individualObservations = new Vector<double[]>();
+	}
+	
+	public void addObservation(double observation[]) {
+		individualObservations.add(observation);
+	}
+	
+	public void endIndividualObservations() throws Exception {
+		double[][] data = new double[individualObservations.size()][];
+		for (int t = 0; t < data.length; t++) {
+			data[t] = individualObservations.elementAt(t);
+		}
+		// Allow vector to be reclaimed
+		individualObservations = null;
+		setObservations(data);
+	}
+
+
+	/**
+	 * Compute the norms for each marginal time series
+	 *
+	 */
+	protected void computeNorms() {
+		
+		norms = new double[V][N][N];
+		for (int t = 0; t < N; t++) {
+			// Compute the norms from t to all other time points
+			double[][] normsForT = EuclideanUtils.computeNorms(data, t);
+			for (int t2 = 0; t2 < N; t2++) {
+				for (int v = 0; v < V; v++) {
+					norms[v][t][t2] = normsForT[t2][v];
+				}
+			}
+		}
+	}
+	
+	public abstract double computeAverageLocalOfObservations() throws Exception;
+
+	/**
+	 * Compute what the average MI would look like were all time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The reordering array contains the reordering for each marginal variable (first index).
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here.
+	 * 
+	 * @param reordering the specific new orderings to use. First index is the variable number
+	 *  (minus 1, since we don't reorder the first variable),
+	 *  second index is the time step, the value is the reordered time step to use
+	 *  for that variable at the given time step.
+	 * @return
+	 * @throws Exception
+	 */
+	public abstract double computeAverageLocalOfObservations(int[][] reordering) throws Exception;
+
+	/**
+	 * Compute the significance of the multi-information of the previously supplied observations.
+	 * We destroy the p(x,y,z,..) correlations, while retaining the p(x), p(y),.. marginals, to check how
+	 *  significant this multi-information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][][] newOrderings = new int[numPermutationsToCheck][][];
+		// Generate numPermutationsToCheck * V permutations of 0 .. data.length-1
+		for (int n = 0; n < numPermutationsToCheck; n++) {
+			newOrderings[n] = rg.generateDistinctRandomPerturbations(data.length, V-1);
+		}
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y,z,..) correlations, while retaining the p(x), p(y),.. marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use. First index is the reordering index,
+	 *  second index is the variable number (minus 1, since we don't reorder the first variable),
+	 *  third index is the reordered variable number for that position.
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = mi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		mi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualMI;
+		return measDistribution;
+	}
+
+	public abstract double[] computeLocalOfPreviousObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return mi;
+	}
+	
+	public abstract String printConstants(int N) throws Exception;
+
+	/**
+	 * Utility to take a reordering matrix and return the array of reordered time indices from
+	 *  which to find the reordered data to be inserted at timeStep.
+	 * 
+	 * @param reordering the specific new orderings to use. First index is the variable number
+	 *  (can be for all variables, or one less than all if the first is not to be reordered),
+	 *  second index is the time step, the value is the reordered time step to use
+	 *  for that variable at the given time step.
+	 *  If null, no reordering is performed.
+	 * @param timeStep
+	 * @return
+	 */
+	protected int[] reorderedTimeStepsForEachMarginal(int[][] reordering, int timeStep) {
+		// Create storage for the reordered time steps for the variables
+		int[] tForEachMarginal = new int[V];
+		if (reordering == null) {
+			// We're not reordering
+			for (int v = 0; v < V; v++) {
+				tForEachMarginal[v] = timeStep;
+			}
+		} else {
+			boolean reorderingFirstColumn = (reordering.length == V);
+			int reorderIndex = 0;
+			// Handle the first column
+			if (reorderingFirstColumn) {
+				tForEachMarginal[0] = reordering[reorderIndex++][timeStep];
+			} else {
+				tForEachMarginal[0] = timeStep;
+			}
+			// Handle subsequent columns
+			for (int v = 1; v < V; v++) {
+				tForEachMarginal[v] = reordering[reorderIndex++][timeStep];
+			}
+		}
+		return tForEachMarginal;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov1.java

@@ -0,0 +1,297 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Multi Info using the Kraskov estimation method.
+ * Uses the first algorithm (defined on p.4/5 of the paper)</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MultiInfoCalculatorKraskov1
+	extends MultiInfoCalculatorKraskov {
+	
+	/**
+	 * Compute the average MI from the previously set observations
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (miComputed) {
+			return mi;
+		}
+		return computeAverageLocalOfObservations(null);
+	}
+
+	/**
+	 * Compute what the average MI would look like were the 2nd, 3rd, etc time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * If reordering is null, it is assumed there is no reordering of
+	 *  the y variable.
+	 * 
+	 * @param reordering the specific new orderings to use. First index is the variable number
+	 *  (can be for all variables, or one less than all if the first is not to be reordered),
+	 *  second index is the time step, the value is the reordered time step to use
+	 *  for that variable at the given time step.
+	 *  If null, no reordering is performed.
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[][] reordering) throws Exception {
+		if (V == 1) {
+			miComputed = true;
+			return 0.0;
+		}
+		if (!tryKeepAllPairsNorms || (data.length * V > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData = data;
+			if (reordering != null) {
+				// Generate a new re-ordered data
+				data = MatrixUtils.reorderDataForVariables(originalData, reordering);
+			}
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data
+			data = originalData;
+			return newMI;
+		}
+		
+		if (norms == null) {
+			computeNorms();
+		}
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+		
+		int cutoffForKthMinLinear = (int) (Math.log(N) / Math.log(2.0));
+
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First grab marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).			
+			
+			// Create storage for the reordered time steps for the variables
+			int[] tForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t);
+
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				int[] t2ForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t2);
+				// Find the max marginal norm between the vector at t and the reordered vector
+				//  at t2:
+				jointNorm[t2] = 0;
+				for (int v = 0; v < V; v++) {
+					double normForThisVar = norms[v][tForEachMarginal[v]][t2ForEachMarginal[v]];
+					if (normForThisVar > jointNorm[t2]) {
+						jointNorm[t2] = normForThisVar;
+					}
+				}
+			}
+			// Then find the kth closest neighbour:
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				int[] t2ForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t2);
+				for (int v = 0; v < V; v++) {
+					if (norms[v][tForEachMarginal[v]][t2ForEachMarginal[v]] < epsilon) {
+						n_x[v]++;
+					}
+				}
+			}
+			// Track the averages, and take the digamma before adding into the 
+			//  average:
+			for (int v = 0; v < V; v++) {
+				avNx[v] += n_x[v];
+				averageDiGammas += MathsUtils.digamma(n_x[v]+1);
+			}
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = MathsUtils.digamma(k) - averageDiGammas + (double) (V - 1) * MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+	
+	/**
+	 * This method correctly computes the average local MI, but recomputes the 
+	 *  distances for each marginal variable between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		
+		if (V == 1) {
+			miComputed = true;
+			return 0.0;
+		}
+		// Count the average number of points within eps for each marginal variable
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+		int cutoffForKthMinLinear = (int) (Math.log(N) / Math.log(2.0));
+
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] normsForT = EuclideanUtils.computeNorms(data, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = MatrixUtils.max(normsForT[t2]);
+			}
+			// Then find the kth closest neighbour:
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				for (int v = 0; v < V; v++) {
+					if (normsForT[t2][v] < epsilon) {
+						n_x[v]++;
+					}
+				}
+			}
+			for (int v = 0; v < V; v++) {
+				avNx[v] += n_x[v];
+			}
+			// And take the digamma before adding into the 
+			//  average:
+			for (int v = 0; v < V; v++) {
+				averageDiGammas += MathsUtils.digamma(n_x[v]+1);
+			}
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = MathsUtils.digamma(k) - averageDiGammas + (double) (V - 1) * MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		double[] localMi = new double[N];
+		int cutoffForKthMinLinear = (int) (Math.log(N) / Math.log(2.0));
+
+		if (V == 1) {
+			miComputed = true;
+			return localMi;
+		}
+
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		double Vminus1TimesdigammaN = (double) (V - 1) * MathsUtils.digamma(N);
+		
+		// Count the average number of points within eps_x[v] for each marginal v
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] norms = EuclideanUtils.computeNorms(data, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = MatrixUtils.max(norms[t2]);
+			}
+			
+			// Then find the kth closest neighbour:
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+						
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				for (int v = 0; v < V; v++) {
+					if (norms[t2][v] < epsilon) {
+						n_x[v]++;
+					}
+				}
+			}
+
+			// And take the digammas, and add into the local
+			localMi[t] = digammaK + Vminus1TimesdigammaN;
+			for (int v = 0; v < V; v++) {
+				double digammaNxPlusOne = MathsUtils.digamma(n_x[v]+1);
+				localMi[t] -= digammaNxPlusOne;
+				// And keep track of the averages
+				averageDiGammas += digammaNxPlusOne;
+				avNx[v] += n_x[v];
+			}
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = digammaK - averageDiGammas + Vminus1TimesdigammaN;
+		miComputed = true;
+		return localMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e + digamma(N=%d)=%.3e => %.3e",
+				k, MathsUtils.digamma(k), N, MathsUtils.digamma(N),
+				(MathsUtils.digamma(k) + MathsUtils.digamma(N)));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov2.java

@@ -0,0 +1,345 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Multi Info using the Kraskov estimation method.
+ * Uses the second algorithm (defined on p.5 of the paper)</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * TODO should use the kthMinIndices code from MatrixUtils as per MutualInfoCalculatorMultiVariateKraskov2
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MultiInfoCalculatorKraskov2
+	extends MultiInfoCalculatorKraskov {
+
+	protected static final int JOINT_NORM_VAL_COLUMN = 0;
+	protected static final int JOINT_NORM_TIMESTEP_COLUMN = 1;
+
+	/**
+	 * Compute the average MI from the previously set observations
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (miComputed) {
+			return mi;
+		}
+		return computeAverageLocalOfObservations(null);
+	}
+
+	/**
+	 * Compute what the average MI would look like were the 2nd, 3rd, etc time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * 
+	 * @param reordering the specific new orderings to use. First index is the variable number
+	 *  (can be for all variables, or one less than all if the first is not to be reordered),
+	 *  second index is the time step, the value is the reordered time step to use
+	 *  for that variable at the given time step.
+	 *  If null, no reordering is performed.
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[][] reordering) throws Exception {
+		if (V == 1) {
+			miComputed = true;
+			return 0.0;
+		}
+		if (!tryKeepAllPairsNorms || (data.length * V > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData = data;
+			// Generate a new re-ordered data
+			if (reordering != null) {
+				// Generate a new re-ordered data
+				data = MatrixUtils.reorderDataForVariables(originalData, reordering);
+			}
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data
+			data = originalData;
+			return newMI;
+		}
+		
+		// Otherwise we will use the norms we've already computed, and use a "virtual"
+		//  reordered data2.
+		
+		if (norms == null) {
+			computeNorms();
+		}
+		
+		// Count the average number of points within eps_x[v]
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for each marginal for this time step:
+			//  First grab marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+
+			// Get the reordered time steps for the variables
+			int[] tForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t);
+
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				int[] t2ForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t2);
+				// Find the max marginal norm between the vector at t and the reordered vector
+				//  at t2:
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = 0;
+				for (int v = 0; v < V; v++) {
+					double normForThisVar = norms[v][tForEachMarginal[v]][t2ForEachMarginal[v]];
+					if (normForThisVar > jointNorm[t2][JOINT_NORM_VAL_COLUMN]) {
+						jointNorm[t2][JOINT_NORM_VAL_COLUMN] = normForThisVar;
+					}
+				}
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double[] eps_x = new double[V];
+			if (k == 1) {
+				// just do a linear search for the minimum epsilon value
+				int timeStepOfMin = MatrixUtils.minIndex(jointNorm, JOINT_NORM_VAL_COLUMN);
+				int[] timeStepOfMinForEachMarginal =
+					reorderedTimeStepsForEachMarginal(reordering, timeStepOfMin);
+				for (int v = 0; v < V; v++) {
+					eps_x[v] = norms[v][tForEachMarginal[v]][timeStepOfMinForEachMarginal[v]];
+				}
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				// Find eps_{x,y} as the maximum x and y norms amongst this set:
+				for (int j = 0; j < k; j++) {
+					int timeStepOfJthPoint = (int)jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+					int[] timeStepOfJthPointForEachMarginal =
+						reorderedTimeStepsForEachMarginal(reordering, timeStepOfJthPoint);
+					for (int v = 0; v < V; v++) {
+						if (norms[v][tForEachMarginal[v]][timeStepOfJthPointForEachMarginal[v]] > eps_x[v]) {
+							eps_x[v] = norms[v][tForEachMarginal[v]][timeStepOfJthPointForEachMarginal[v]];
+						}
+					}
+				}
+			}
+			
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps in that marginal dimension
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				int[] t2ForEachMarginal = reorderedTimeStepsForEachMarginal(reordering, t2);
+				for (int v = 0; v < V; v++) {
+					if (norms[v][tForEachMarginal[v]][t2ForEachMarginal[v]] <= eps_x[v]) {
+						n_x[v]++;
+					}
+				}
+			}
+			// Track the averages, and take the digamma before adding into the 
+			//  average:
+			for (int v = 0; v < V; v++) {
+				avNx[v] += n_x[v];
+				averageDiGammas += MathsUtils.digamma(n_x[v]);
+			}
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = MathsUtils.digamma(k) - (double) (V - 1) /(double)k - averageDiGammas +
+				(double) (V - 1) * MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+
+		if (V == 1) {
+			miComputed = true;
+			return 0.0;
+		}
+		// Count the average number of points within eps for each marginal variable
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+	
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x (for each marginal) for this time step:
+			//  First get the marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] normsForT = EuclideanUtils.computeNorms(data, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = MatrixUtils.max(normsForT[t2]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			
+			// Then find the k closest neighbours:
+			double[] eps_x = new double[V];
+			if (k == 1) {
+				// just do a linear search for the minimum epsilon value
+				int timeStepOfMin = MatrixUtils.minIndex(jointNorm, JOINT_NORM_VAL_COLUMN);
+				for (int v = 0; v < V; v++) {
+					eps_x[v] = normsForT[timeStepOfMin][v];
+				}
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				// Find eps_{x,y} as the maximum x and y norms amongst this set:
+				for (int j = 0; j < k; j++) {
+					int timeStepOfJthPoint = (int)jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+					for (int v = 0; v < V; v++) {
+						if (normsForT[timeStepOfJthPoint][v] > eps_x[v]) {
+							eps_x[v] = normsForT[timeStepOfJthPoint][v];
+						}
+					}
+				}
+			}
+			
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps in that marginal dimension
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				for (int v = 0; v < V; v++) {
+					if (normsForT[t2][v] <= eps_x[v]) {
+						n_x[v]++;
+					}
+				}
+			}
+			// Track the averages, and take the digamma before adding into the 
+			//  average:
+			for (int v = 0; v < V; v++) {
+				avNx[v] += n_x[v];
+				averageDiGammas += MathsUtils.digamma(n_x[v]);
+			}
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = MathsUtils.digamma(k) - (double) (V - 1) /(double)k - averageDiGammas +
+				(double) (V - 1) * MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		double[] localMi = new double[N];
+		if (V == 1) {
+			miComputed = true;
+			return localMi;
+		}
+
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		double Vminus1TimesDigammaN = (double) (V - 1) * MathsUtils.digamma(N);
+		double Vminus1TimesInvK = (double) (V - 1) / (double)k;
+		
+		// Count the average number of points within eps_x[v] for each marginal v
+		double averageDiGammas = 0;
+		double[] avNx = new double[V];
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x (for each marginal) for this time step:
+			//  First get the marginal norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] normsForT = EuclideanUtils.computeNorms(data, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = MatrixUtils.max(normsForT[t2]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+
+			// Then find the k closest neighbours:
+			double[] eps_x = new double[V];
+			if (k == 1) {
+				// just do a linear search for the minimum epsilon value
+				int timeStepOfMin = MatrixUtils.minIndex(jointNorm, JOINT_NORM_VAL_COLUMN);
+				for (int v = 0; v < V; v++) {
+					eps_x[v] = normsForT[timeStepOfMin][v];
+				}
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				// Find eps_{x,y} as the maximum x and y norms amongst this set:
+				for (int j = 0; j < k; j++) {
+					int timeStepOfJthPoint = (int)jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+					for (int v = 0; v < V; v++) {
+						if (normsForT[timeStepOfJthPoint][v] > eps_x[v]) {
+							eps_x[v] = normsForT[timeStepOfJthPoint][v];
+						}
+					}
+				}
+			}
+			
+			// Count the number of points (in each marginal variable) 
+			//  whose marginal distance is less than eps in that marginal dimension
+			int[] n_x = new int[V];
+			for (int t2 = 0; t2 < N; t2++) {
+				for (int v = 0; v < V; v++) {
+					if (normsForT[t2][v] <= eps_x[v]) {
+						n_x[v]++;
+					}
+				}
+			}
+
+			// Track the averages, and take the digamma before adding into the 
+			//  local:
+			localMi[t] = digammaK - Vminus1TimesInvK + Vminus1TimesDigammaN;
+			for (int v = 0; v < V; v++) {
+				double digammaNx = MathsUtils.digamma(n_x[v]);
+				localMi[t] -= digammaNx;
+				// And keep track of the averages
+				avNx[v] += n_x[v];
+				averageDiGammas += digammaNx;
+			}
+		}
+		if (debug) {
+			for (int v = 0; v < V; v++) {
+				avNx[v] /= (double)N;
+				System.out.print(String.format("Average n_x[%d]=%.3f, ", v, avNx[v]));
+			}
+			System.out.println();
+		}
+		
+		mi = digammaK - Vminus1TimesInvK - averageDiGammas + Vminus1TimesDigammaN;
+		miComputed = true;
+		return localMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e - 1/k=%.3e + digamma(N=%d)=%.3e => %.3e",
+				k, MathsUtils.digamma(k), 1.0/(double)k, N, MathsUtils.digamma(N),
+				(MathsUtils.digamma(k) - 1.0/(double)k + MathsUtils.digamma(N)));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov.java

@@ -0,0 +1,249 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+/**
+ * <p>Compute the Mutual Information between two vectors using the Kraskov estimation method.</p>
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * Two child classes actually implement the two algorithms in the Kraskov paper.</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public abstract class MutualInfoCalculatorMultiVariateKraskov implements
+		MutualInfoCalculatorMultiVariate {
+
+	/**
+	 * we compute distances to the kth neighbour
+	 */
+	protected int k;
+	protected double[][] data1;
+	protected double[][] data2;
+	protected boolean debug;
+	protected double mi;
+	protected boolean miComputed;
+	
+	// Storage for the norms from each observation to each other one
+	protected double[][] xNorms;
+	protected double[][] yNorms;
+	// Keep the norms each time (making reordering very quick)
+	//  (Should only be set to false for testing)
+	public static boolean tryKeepAllPairsNorms = true;
+	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 2000;
+	
+	public final static String PROP_K = "k";
+	public final static String PROP_NORM_TYPE = "NORM_TYPE";
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+	private int timeDiff = 0;
+
+	public MutualInfoCalculatorMultiVariateKraskov() {
+		super();
+		k = 1; // by default
+	}
+
+	public void initialise(int dimensions1, int dimensions2) {
+		mi = 0.0;
+		miComputed = false;
+		xNorms = null;
+		yNorms = null;
+		data1 = null;
+		data2 = null;
+		// No need to keep the dimenions here
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			EuclideanUtils.setNormToUse(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
+			timeDiff = Integer.parseInt(propertyValue);
+			if (timeDiff < 0) {
+				throw new RuntimeException("Time difference must be >= 0. Flip data1 and data2 around if required.");
+			}
+		}
+	}
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		// Will need to handle timeDiff here
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		// Will need to handle timeDiff here
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] observations1,
+			double[][] observations2) throws Exception {
+		if (observations1.length != observations2.length) {
+			throw new Exception("Time steps for observations2 " +
+					observations2.length + " does not match the length " +
+					"of observations1 " + observations1.length);
+		}
+		if ((observations1[0].length == 0) || (observations2[0].length == 0)) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		// Grab the data
+		if (timeDiff == 0) {
+			data1 = observations1;
+			data2 = observations2;
+		} else {
+			// MI with time difference: I(x_{1,n}; x_{2,n+timeDiff})
+			if (observations1.length < timeDiff) {
+				throw new Exception("Computing MI with too few observations " +
+						observations1.length + " given time diff " + timeDiff);
+			}
+			data1 = MatrixUtils.selectRows(observations1, 0, observations1.length - timeDiff);
+			data2 = MatrixUtils.selectRows(observations2, timeDiff, observations2.length - timeDiff);
+		}
+		// Normalise it if required
+		if (normalise) {
+			// Take a copy since we're going to normalise it
+			data1 = MatrixUtils.normaliseIntoNewArray(data1);
+			data2 = MatrixUtils.normaliseIntoNewArray(data2);
+		}
+	}
+
+	/**
+	 * Compute the norms for each marginal time series
+	 *
+	 */
+	protected void computeNorms() {
+		int N = data1.length; // number of observations
+		
+		xNorms = new double[N][N];
+		yNorms = new double[N][N];
+		for (int t = 0; t < N; t++) {
+			// Compute the norms from t to all other time points
+			double[][] xyNormsForT = EuclideanUtils.computeNorms(data1, data2, t);
+			for (int t2 = 0; t2 < N; t2++) {
+				xNorms[t][t2] = xyNormsForT[t2][0];
+				yNorms[t][t2] = xyNormsForT[t2][1];
+			}
+		}
+	}
+	
+	public abstract double computeAverageLocalOfObservations() throws Exception;
+
+	/**
+	 * Compute what the average MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here.
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public abstract double computeAverageLocalOfObservations(int[] reordering) throws Exception;
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(data1.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = mi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		mi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = mi;
+		return measDistribution;
+	}
+
+	public abstract double[] computeLocalOfPreviousObservations() throws Exception;
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states1, double[][] states2) throws Exception {
+		// If implemented, will need to incorporate any time difference here.
+		throw new Exception("Local method not implemented yet");
+	}
+	
+	public abstract String printConstants(int N) throws Exception ;
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return mi;
+	}
+	
+	public int getNumObservations() {
+		return data1.length;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java

@@ -0,0 +1,276 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Mutual Info using the Kraskov estimation method.
+ * Uses the first algorithm (defined at end of p.2 of the paper)</p>
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * </p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MutualInfoCalculatorMultiVariateKraskov1
+	extends MutualInfoCalculatorMultiVariateKraskov {
+	
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+
+	/**
+	 * Compute the average MI from the previously set observations
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		return computeAverageLocalOfObservations(null);
+	}
+
+	/**
+	 * Compute what the average MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * If reordering is null, it is assumed there is no reordering of
+	 *  the y variable.
+	 * 
+	 * @param reordering the reordered time steps of the y variable
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData2 = data2;
+			if (reordering != null) {
+				// Generate a new re-ordered data2
+				data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, reordering);
+			}
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			data2 = originalData2;
+			return newMI;
+		}
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  using x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			int tForY = (reordering == null) ? t : reordering[t];
+
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				int t2ForY = (reordering == null) ? t2 : reordering[t2];
+				jointNorm[t2] = Math.max(xNorms[t][t2], yNorms[tForY][t2ForY]);
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than eps, and whose y distance is less than eps
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xNorms[t][t2] < epsilon) {
+					n_x++;
+				}
+				int t2ForY = (reordering == null) ? t2 : reordering[t2];
+				if (yNorms[tForY][t2ForY] < epsilon) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x+1) + MathsUtils.digamma(n_y+1);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+	
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyNorms = EuclideanUtils.computeNorms(data1, data2, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = Math.max(xyNorms[t2][0], xyNorms[t2][1]);
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than eps, and whose y distance is less than eps
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyNorms[t2][0] < epsilon) {
+					n_x++;
+				}
+				if (xyNorms[t2][1] < epsilon) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x+1) + MathsUtils.digamma(n_y+1);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		double[] localMi = new double[N];
+		
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		double digammaN = MathsUtils.digamma(N);
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity.
+			double[][] xyNorms = EuclideanUtils.computeNorms(data1, data2, t);
+			double[] jointNorm = new double[N];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2] = Math.max(xyNorms[t2][0], xyNorms[t2][1]);
+			}
+			// Then find the kth closest neighbour, using a heuristic to 
+			// select whether to keep the k mins only or to do a sort.
+			double epsilon = 0.0;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum
+				epsilon = MatrixUtils.kthMin(jointNorm, k);
+			} else {
+				// Sort the array of joint norms first
+				java.util.Arrays.sort(jointNorm);
+				// And find the distance to it's kth closest neighbour
+				// (we subtract one since the array is indexed from zero)
+				epsilon = jointNorm[k-1];
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than eps, and whose y distance is less than eps
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyNorms[t2][0] < epsilon) {
+					n_x++;
+				}
+				if (xyNorms[t2][1] < epsilon) {
+					n_y++;
+				}
+			}
+			// And take the digamma:
+			double digammaNxPlusOne = MathsUtils.digamma(n_x+1);
+			double digammaNyPlusOne = MathsUtils.digamma(n_y+1);
+			
+			localMi[t] = digammaK - digammaNxPlusOne - digammaNyPlusOne + digammaN;
+			
+			avNx += n_x;
+			avNy += n_y;
+			// And keep track of the average
+			averageDiGammas += digammaNxPlusOne + digammaNyPlusOne;
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = digammaK - averageDiGammas + digammaN;
+		miComputed = true;
+		return localMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e + digamma(N=%d)=%.3e => %.3e",
+				k, MathsUtils.digamma(k), N, MathsUtils.digamma(N),
+				(MathsUtils.digamma(k) + MathsUtils.digamma(N)));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov2.java

@@ -0,0 +1,412 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Compute the Mutual Info using the Kraskov estimation method.
+ * Uses the second algorithm (defined at start of p.3 of the paper)</p>
+ * <p>Computes this directly looking at the marginal space for each variable, rather than
+ * using the multi-info (or integration) in the marginal spaces.
+ * </p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MutualInfoCalculatorMultiVariateKraskov2
+	extends MutualInfoCalculatorMultiVariateKraskov {
+
+	protected static final int JOINT_NORM_VAL_COLUMN = 0;
+	protected static final int JOINT_NORM_TIMESTEP_COLUMN = 1;
+
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+	
+	/**
+	 * Compute what the average MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			double[][] originalData2 = data2;
+			// Generate a new re-ordered data2
+			data2 = MatrixUtils.extractSelectedTimePointsReusingArrays(originalData2, reordering);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			data2 = originalData2;
+			return newMI;
+		}
+		
+		// Otherwise we will use the norms we've already computed, and use a "virtual"
+		//  reordered data2.
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+
+			int tForY = reordering[t];
+
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				int t2ForY = reordering[t2];
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xNorms[t][t2], yNorms[tForY][t2ForY]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (yNorms[tForY][reordering[timeStepOfJthPoint]] > eps_y) {
+					eps_y = yNorms[tForY][reordering[timeStepOfJthPoint]];
+				}
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xNorms[t][t2] <= eps_x) {
+					n_x++;
+				}
+				if (yNorms[tForY][reordering[t2]] <= eps_y) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (!tryKeepAllPairsNorms || (data1.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			return computeAverageLocalOfObservationsWhileComputingDistances();
+		}
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  using x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xNorms[t][t2], yNorms[t][t2]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xNorms[t][timeStepOfJthPoint] > eps_x) {
+					eps_x = xNorms[t][timeStepOfJthPoint];
+				}
+				if (yNorms[t][timeStepOfJthPoint] > eps_y) {
+					eps_y = yNorms[t][timeStepOfJthPoint];
+				}
+			}
+			
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xNorms[t][t2] <= eps_x) {
+					n_x++;
+				}
+				if (yNorms[t][t2] <= eps_y) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyNorms = EuclideanUtils.computeNorms(data1, data2, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xyNorms[t2][0], xyNorms[t2][1]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xyNorms[timeStepOfJthPoint][0] > eps_x) {
+					eps_x = xyNorms[timeStepOfJthPoint][0];
+				}
+				if (xyNorms[timeStepOfJthPoint][1] > eps_y) {
+					eps_y = xyNorms[timeStepOfJthPoint][1];
+				}
+			}
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyNorms[t2][0] <= eps_x) {
+					n_x++;
+				}
+				if (xyNorms[t2][1] <= eps_y) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		int N = data1.length; // number of observations
+		int cutoffForKthMinLinear = (int) (CUTOFF_MULTIPLIER * Math.log(N) / Math.log(2.0));
+		double[] localMi = new double[N];
+		
+		// Constants:
+		double digammaK = MathsUtils.digamma(k);
+		double invK = 1.0 / (double)k;
+		double digammaN = MathsUtils.digamma(N);
+		
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  First get x and y norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[][] xyNorms = EuclideanUtils.computeNorms(data1, data2, t);
+			double[][] jointNorm = new double[N][2];
+			for (int t2 = 0; t2 < N; t2++) {
+				jointNorm[t2][JOINT_NORM_VAL_COLUMN] = Math.max(xyNorms[t2][0], xyNorms[t2][1]);
+				// And store the time step for back reference after the 
+				//  array is sorted.
+				jointNorm[t2][JOINT_NORM_TIMESTEP_COLUMN] = t2;
+			}
+			// Then find the k closest neighbours:
+			double eps_x = 0.0;
+			double eps_y = 0.0;
+			int[] timeStepsOfKthMins = null;
+			if (k <= cutoffForKthMinLinear) {
+				// just do a linear search for the minimum epsilon value
+				timeStepsOfKthMins = MatrixUtils.kMinIndices(jointNorm, JOINT_NORM_VAL_COLUMN, k);
+			} else {
+				// Sort the array of joint norms
+				java.util.Arrays.sort(jointNorm, FirstIndexComparatorDouble.getInstance());
+				// and now we have the closest k points.
+				timeStepsOfKthMins = new int[k];
+				for (int j = 0; j < k; j++) {
+					timeStepsOfKthMins[j] = (int) jointNorm[j][JOINT_NORM_TIMESTEP_COLUMN];
+				}
+			}
+			// and now we have the closest k points.
+			// Find eps_{x,y} as the maximum x and y norms amongst this set:
+			for (int j = 0; j < k; j++) {
+				int timeStepOfJthPoint = timeStepsOfKthMins[j]; 
+				if (xyNorms[timeStepOfJthPoint][0] > eps_x) {
+					eps_x = xyNorms[timeStepOfJthPoint][0];
+				}
+				if (xyNorms[timeStepOfJthPoint][1] > eps_y) {
+					eps_y = xyNorms[timeStepOfJthPoint][1];
+				}
+			}
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x, and whose y distance is less
+			//  than or equal to eps_y
+			int n_x = 0;
+			int n_y = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xyNorms[t2][0] <= eps_x) {
+					n_x++;
+				}
+				if (xyNorms[t2][1] <= eps_y) {
+					n_y++;
+				}
+			}
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma:
+			double digammaNx = MathsUtils.digamma(n_x);
+			double digammaNy = MathsUtils.digamma(n_y);
+			
+			localMi[t] = digammaK - invK - digammaNx - digammaNy + digammaN;
+
+			averageDiGammas += digammaNx + digammaNy;
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = digammaK - invK - averageDiGammas + digammaN;
+		miComputed = true;
+		return localMi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		String constants = String.format("digamma(k=%d)=%.3e - 1/k=%.3e + digamma(N=%d)=%.3e => %.3e",
+				k, MathsUtils.digamma(k), 1.0/(double)k, N, MathsUtils.digamma(N),
+				(MathsUtils.digamma(k) - 1.0/(double)k + MathsUtils.digamma(N)));
+		return constants;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskovByMulti.java

@@ -0,0 +1,352 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Hashtable;
+
+
+/**
+ * <p>Compute the Mutual Information between two vectors using the Kraskov estimation method.
+ * Computes this using the multi-info (or integration) in the marginal spaces.
+ * Two child classes actually implement the two algorithms in the Kraskov paper.</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public abstract class MutualInfoCalculatorMultiVariateKraskovByMulti implements
+		MutualInfoCalculatorMultiVariate {
+
+	/**
+	 * Storage for the properties ready to pass onto the underlying MI calculators
+	 */
+	private Hashtable<String,String> props;
+	
+	/**
+	 * Properties for the underlying MultiInfoCalculatorKraskov.
+	 * Added here so they can be accessed externally and the accessor doesn't need
+	 *  to know that they're really part of the underlying multi-info calculators.
+	 */
+	public final static String PROP_K = MultiInfoCalculatorKraskov.PROP_K;
+	public final static String PROP_NORM_TYPE = MultiInfoCalculatorKraskov.PROP_NORM_TYPE;
+	public final static String PROP_TRY_TO_KEEP_ALL_PAIRS_NORM = MultiInfoCalculatorKraskov.PROP_TRY_TO_KEEP_ALL_PAIRS_NORM;
+
+	/**
+	 * MultiInfo calculator for the joint space
+	 */
+	protected MultiInfoCalculatorKraskov multiInfoJoint;
+	/**
+	 * MultiInfo calculator for marginal space 1
+	 */
+	protected MultiInfoCalculatorKraskov multiInfo1;
+	/**
+	 * MultiInfo calculator for marginal space 2
+	 */
+	protected MultiInfoCalculatorKraskov multiInfo2;
+
+	private double[][] data1;
+	private double[][] data2;
+	private int dimensions1;
+	private int dimensions2;
+	private int numObservations;
+	
+	protected boolean debug;
+	protected double mi;
+	protected boolean miComputed;
+	
+	public MutualInfoCalculatorMultiVariateKraskovByMulti() {
+		super();
+		props = new Hashtable<String,String>();
+		createMultiInfoCalculators();
+	}
+
+	/**
+	 * Create the underlying Kraskov multi info calculators
+	 *
+	 */
+	protected abstract void createMultiInfoCalculators();
+	
+	public void initialise(int dimensions1, int dimensions2) {
+		mi = 0.0;
+		miComputed = false;
+		numObservations = 0;
+		data1 = null;
+		data2 = null;
+		// Set the properties for the Kraskov multi info calculators
+		for (String key : props.keySet()) {
+			multiInfoJoint.setProperty(key, props.get(key));
+			multiInfo1.setProperty(key, props.get(key));
+			multiInfo2.setProperty(key, props.get(key));
+		}
+		// Initialise the Kraskov multi info calculators
+		multiInfoJoint.initialise(dimensions1 + dimensions2);
+		multiInfo1.initialise(dimensions1);
+		multiInfo2.initialise(dimensions2);
+		this.dimensions1 = dimensions1;
+		this.dimensions2 = dimensions2;
+	}
+
+	/**
+	 * Sets properties for the calculator.
+	 * Valid properties include:
+	 * <ul>
+	 * 	<li>Any valid properties for MultiInfoCalculatorKraskov.setProperty</li>
+	 * </ul>
+	 * One should set MultiInfoCalculatorKraskov.PROP_K here, the number
+	 *  of neighbouring points one should count up to in determining the joint kernel size. 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
+			int diff = Integer.parseInt(propertyValue);
+			if (diff != 0) {
+				throw new RuntimeException(PROP_TIME_DIFF + " property != 0 not implemented yet");
+			}
+		}
+		// No other local properties here, so 
+		// assume it was a property for the MI calculator
+		props.put(propertyName, propertyValue);
+	}
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	/**
+	 * Set the observations from which to compute the mutual information
+	 * 
+	 * @param observations1
+	 * @param observations2
+	 */
+	public void setObservations(double[][] observations1,
+			double[][] observations2) throws Exception {
+		if (observations1.length != observations2.length) {
+			throw new Exception("Time steps for observations2 " +
+					observations2.length + " does not match the length " +
+					"of observations1 " + observations1.length);
+		}
+		if ((observations1[0].length == 0) || (observations2[0].length == 0)) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		data1 = observations1;
+		data2 = observations2;
+		multiInfoJoint.setObservations(data1, data2);
+		multiInfo1.setObservations(data1);
+		multiInfo2.setObservations(data2);
+		numObservations = data1.length;
+	}
+
+	/**
+	 * 
+	 * @return the average mutual information
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		double jointMultiInfo = multiInfoJoint.computeAverageLocalOfObservations();
+		shareNormsIfPossible();
+		// Now compute the marginal multi-infos
+		double marginal1MultiInfo = multiInfo1.computeAverageLocalOfObservations();
+		double marginal2MultiInfo = multiInfo2.computeAverageLocalOfObservations();
+		// And return the mutual info
+		mi = jointMultiInfo - marginal1MultiInfo - marginal2MultiInfo;
+		if (debug) {
+			System.out.println("jointMultiInfo=" + jointMultiInfo + " - marginal1MultiInfo=" +
+					marginal1MultiInfo + " - marginal2MultiInfo=" + marginal2MultiInfo + 
+					" = " + mi);
+		}
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * Compute what the average MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here.
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		int[][] reorderingForJointSpace = null;
+		int[][] reorderingFor2Space = null;
+		if (reordering != null) {
+			// We need to make the reordering for the second marginal data set apply 
+			//  to each variable within that data set, and keep all variables in the first data
+			//  set unchanged
+			reorderingForJointSpace = new int[dimensions1 + dimensions2][reordering.length];
+			reorderingFor2Space = new int[dimensions2][];
+			for (int t = 0; t < numObservations; t++) {
+				// Keep the first marginal space not reordered
+				for (int v = 0; v < dimensions1; v++) {
+					reorderingForJointSpace[v][t] = t;
+				}
+				// Reorder the second marginal space to match the requested reordering
+				for (int v = 0; v < dimensions2; v++) {
+					reorderingForJointSpace[v + dimensions1][t] = reordering[t];
+				}
+			}
+			for (int v = 0; v < dimensions2; v++) {
+				reorderingFor2Space[v] = reorderingForJointSpace[v + dimensions1];
+			}
+		}
+		double jointMultiInfo = multiInfoJoint.computeAverageLocalOfObservations(reorderingForJointSpace);
+		shareNormsIfPossible();
+		// Now compute the marginal multi-info in space 1 without reordering
+		double marginal1MultiInfo = multiInfo1.computeAverageLocalOfObservations();
+		// Now compute the marginal multi-info in space 2 with the reordering applied
+		double marginal2MultiInfo = multiInfo2.computeAverageLocalOfObservations(reorderingFor2Space);
+		// And return the mutual info
+		mi = jointMultiInfo - marginal1MultiInfo - marginal2MultiInfo;
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * If the underlying joint space calculator has computed the norms, share them
+	 *  with the marginal calculators 
+	 *
+	 */
+	protected void shareNormsIfPossible() {
+		if (multiInfoJoint.norms != null) {
+			// Share the norms already computed for the joint space:
+			if (multiInfo1.norms == null) {
+				multiInfo1.norms = new double[dimensions1][][];
+				for (int v = 0; v < dimensions1; v++) {
+					multiInfo1.norms[v] = multiInfoJoint.norms[v];
+				}
+			}
+			if (multiInfo2.norms == null) {
+				multiInfo2.norms = new double[dimensions2][][];
+				for (int v = 0; v < dimensions2; v++) {
+					multiInfo2.norms[v] = multiInfoJoint.norms[dimensions1 + v];
+				}
+			}
+		}
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(data1.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = mi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		mi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = mi;
+		return measDistribution;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		double[] localJointMultiInfo = multiInfoJoint.computeLocalOfPreviousObservations();
+		shareNormsIfPossible();
+		// Now compute the marginal multi-infos
+		double[] localMarginal1MultiInfo = multiInfo1.computeLocalOfPreviousObservations();
+		MatrixUtils.subtractInPlace(localJointMultiInfo, localMarginal1MultiInfo);
+		double[] localMarginal2MultiInfo = multiInfo2.computeLocalOfPreviousObservations();
+		MatrixUtils.subtractInPlace(localJointMultiInfo, localMarginal2MultiInfo);
+		// And return the mutual info
+		mi = multiInfoJoint.getLastAverage() - multiInfo1.getLastAverage() - multiInfo2.getLastAverage();
+		miComputed = true;
+		return localJointMultiInfo;
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] states1, double[][] states2) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+		multiInfoJoint.debug = debug;
+		multiInfo1.debug = debug;
+		multiInfo2.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return mi;
+	}
+
+	public String printConstants(int N) throws Exception {
+		return multiInfoJoint.printConstants(N);
+	}
+
+	public int getNumObservations() {
+		return numObservations;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskovByMulti1.java

@@ -0,0 +1,29 @@
+package infodynamics.measures.continuous.kraskov;
+
+/**
+ * <p>Compute the Mutual Information between two vectors using the Kraskov estimation method.
+ * Computes this using the multi-info (or integration) in the marginal spaces, using algorithm 1</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public class MutualInfoCalculatorMultiVariateKraskovByMulti1 extends
+		MutualInfoCalculatorMultiVariateKraskovByMulti {
+
+	/**
+	 * 
+	 */
+	public MutualInfoCalculatorMultiVariateKraskovByMulti1() {
+		super();
+	}
+
+	@Override
+	protected void createMultiInfoCalculators() {
+		multiInfoJoint = new MultiInfoCalculatorKraskov1();
+		multiInfo1 = new MultiInfoCalculatorKraskov1();
+		multiInfo2 =  new MultiInfoCalculatorKraskov1();
+	
+	}
+	
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskovByMulti2.java

@@ -0,0 +1,29 @@
+package infodynamics.measures.continuous.kraskov;
+
+/**
+ * <p>Compute the Mutual Information between two vectors using the Kraskov estimation method.
+ * Computes this using the multi-info (or integration) in the marginal spaces, using algorithm 2</p>
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public class MutualInfoCalculatorMultiVariateKraskovByMulti2 extends
+		MutualInfoCalculatorMultiVariateKraskovByMulti {
+
+	/**
+	 * 
+	 */
+	public MutualInfoCalculatorMultiVariateKraskovByMulti2() {
+		super();
+	}
+
+	@Override
+	protected void createMultiInfoCalculators() {
+		multiInfoJoint = new MultiInfoCalculatorKraskov2();
+		multiInfo1 = new MultiInfoCalculatorKraskov2();
+		multiInfo2 =  new MultiInfoCalculatorKraskov2();
+	
+	}
+	
+}

java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java

@@ -0,0 +1,432 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariateWithDiscrete;
+import infodynamics.utils.EuclideanUtils;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+/**
+ * <p>Compute the Mutual Information between a vector of continuous variables and discrete
+ *  variable using the Kraskov estimation method.</p>
+ * <p>Uses Kraskov method type 2, since type 1 only looks at points with
+ * distances strictly less than the kth variable, which won't work for one marginal
+ * being discrete.</p>
+ * <p>I have noticed that there are quite large bias negative values here
+ * where small K is used (e.g. for binary data splitting continuous into
+ * two distinct groups, 1400 observations, K=4 has bias ~ -.35)</p>
+ * 
+ * @see "Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P., Physical Review E 69, (2004) 066138
+ * @see http://dx.doi.org/10.1103/PhysRevE.69.066138
+ * 
+ * @author Joseph Lizier
+ */
+public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements MutualInfoCalculatorMultiVariateWithDiscrete {
+
+	// Multiplier used in hueristic for determining whether to use a linear search
+	//  for min kth element or a binary search.
+	protected static final double CUTOFF_MULTIPLIER = 1.5;
+	
+	/**
+	 * we compute distances to the kth neighbour
+	 */
+	protected int k;
+	protected double[][] continuousData;
+	protected int[] discreteData;
+	protected int[] counts;
+	protected int base;
+	protected boolean debug;
+	protected double mi;
+	protected boolean miComputed;
+	
+	// Storage for the norms from each observation to each other one
+	protected double[][] xNorms;
+	// Keep the norms each time (making reordering very quick)
+	//  (Should only be set to false for testing)
+	public static boolean tryKeepAllPairsNorms = true;
+	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 2000;
+	
+	public final static String PROP_K = "k";
+	public final static String PROP_NORM_TYPE = "NORM_TYPE";	
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+
+	public MutualInfoCalculatorMultiVariateWithDiscreteKraskov() {
+		super();
+		k = 1; // by default
+	}
+
+	/**
+	 * Initialise the calculator.
+	 * 
+	 * @param dimensions number of joint continuous variables
+	 * @param base number of discrete states
+	 */
+	public void initialise(int dimensions, int base) {
+		mi = 0.0;
+		miComputed = false;
+		xNorms = null;
+		continuousData = null;
+		discreteData = null;
+		// No need to keep the dimenions here
+		this.base = base;
+	}
+
+	/**
+	 * 
+	 * @param propertyName name of the property to set
+	 * @param propertyValue value to set on that property
+	 */
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			EuclideanUtils.setNormToUse(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[] sourceValid, boolean[] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] source, double[][] destination, boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void startAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void finaliseAddObservations() {
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations) throws Exception {
+		if (continuousObservations.length != discreteObservations.length) {
+			throw new Exception("Time steps for observations2 " +
+					discreteObservations.length + " does not match the length " +
+					"of observations1 " + continuousObservations.length);
+		}
+		if (continuousObservations[0].length == 0) {
+			throw new Exception("Computing MI with a null set of data");
+		}
+		continuousData = continuousObservations;
+		discreteData = discreteObservations;
+		if (normalise) {
+			// Take a copy since we're going to normalise it
+			continuousData = MatrixUtils.normaliseIntoNewArray(continuousObservations);
+		}
+		// count the discrete states:
+		counts = new int[base];
+		for (int t = 0; t < discreteData.length; t++) {
+			counts[discreteData[t]]++;
+		}
+		for (int b = 0; b < counts.length; b++) {
+			if (counts[b] < k) {
+				throw new RuntimeException("This implementation assumes there are at least k items in each discrete bin");
+			}
+		}
+	}
+
+	/**
+	 * Compute the norms for each marginal time series
+	 *
+	 */
+	protected void computeNorms() {
+		int N = continuousData.length; // number of observations
+		
+		xNorms = new double[N][N];
+		for (int t = 0; t < N; t++) {
+			// Compute the norms from t to all other time points
+			for (int t2 = 0; t2 < N; t2++) {
+				if (t2 == t) {
+					xNorms[t][t2] = Double.POSITIVE_INFINITY;
+					continue;
+				}
+				// Compute norm in the continuous space
+				xNorms[t][t2] = EuclideanUtils.norm(continuousData[t], continuousData[t2]);
+			}
+		}
+	}
+	
+	/**
+	 * Compute what the average MI would look like were the second time series reordered
+	 *  as per the array of time indices in reordering.
+	 * The user should ensure that all values 0..N-1 are represented exactly once in the
+	 *  array reordering and that no other values are included here. 
+	 * 
+	 * @param reordering
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+		int N = continuousData.length; // number of observations
+		if (!tryKeepAllPairsNorms || (N > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			// Generate a new re-ordered set of discrete data
+			int[] originalDiscreteData = discreteData;
+			discreteData = MatrixUtils.extractSelectedTimePoints(discreteData, reordering);
+			// Compute the MI
+			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
+			// restore data2
+			discreteData = originalDiscreteData;
+			return newMI;
+		}
+		
+		// Otherwise we will use the norms we've already computed, and use a "virtual"
+		//  reordered data2.
+		int[] reorderedDiscreteData = MatrixUtils.extractSelectedTimePoints(discreteData, reordering);
+
+		if (xNorms == null) {
+			computeNorms();
+		}
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  using x norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			// Then find the k closest neighbours in the same discrete bin
+			double eps_x = MatrixUtils.kthMinSubjectTo(xNorms[t], k, reorderedDiscreteData, reorderedDiscreteData[t]);			
+			
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x
+			int n_x = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (xNorms[t][t2] <= eps_x) {
+					n_x++;
+				}
+			}
+			// n_y is number of points in that bin minus that point
+			int n_y = counts[reorderedDiscreteData[t]] - 1;
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		if (!tryKeepAllPairsNorms || (continuousData.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
+			return computeAverageLocalOfObservationsWhileComputingDistances();
+		}
+		
+		if (xNorms == null) {
+			computeNorms();
+		}
+		int N = continuousData.length; // number of observations
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  using x norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			// Then find the k closest neighbours in the same discrete bin
+			double eps_x = MatrixUtils.kthMinSubjectTo(xNorms[t], k, discreteData, discreteData[t]);			
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x (not including this point)
+			int n_x = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if ((t2 != t) && (xNorms[t][t2] <= eps_x)) {
+					// xNorms has infinity for t == t2 anyway ...
+					n_x++;
+				}
+			}
+			// n_y is number of points in that bin, minus this point
+			int n_y = counts[discreteData[t]] - 1;
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);			
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f (-> digam=%.3f %.3f), Average n_y=%.3f (-> digam=%.3f)",
+					avNx, MathsUtils.digamma((int) avNx), MathsUtils.digamma((int) avNx - 1), avNy, MathsUtils.digamma((int) avNy)));
+			System.out.printf("Independent average num in joint box is %.3f\n", (avNx * avNy / (double) N));
+			System.out.println(String.format("digamma(k)=%.3f - 1/k=%.3f - averageDiGammas=%.3f + digamma(N)=%.3f\n",
+					MathsUtils.digamma(k), 1.0/(double)k, averageDiGammas, MathsUtils.digamma(N)));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * This method correctly computes the average local MI, but recomputes the x and y 
+	 *  distances between all tuples in time.
+	 * Kept here for cases where we have too many observations
+	 *  to keep the norm between all pairs, and for testing purposes.
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
+		int N = continuousData.length; // number of observations
+
+		// Count the average number of points within eps_x and eps_y
+		double averageDiGammas = 0;
+		double avNx = 0;
+		double avNy = 0;
+		
+		for (int t = 0; t < N; t++) {
+			// Compute eps_x and eps_y for this time step:
+			//  First get x norms to all neighbours
+			//  (note that norm of point t to itself will be set to infinity).
+			double[] norms = new double[N]; 
+			for (int t2 = 0; t2 < N; t2++) {
+				if (t2 == t) {
+					norms[t2] = Double.POSITIVE_INFINITY;
+					continue;
+				}
+				// Compute norm in the continuous space
+				norms[t2] = EuclideanUtils.norm(continuousData[t], continuousData[t2]);
+			}
+
+			// Then find the k closest neighbours in the same discrete bin
+			double eps_x = MatrixUtils.kthMinSubjectTo(norms, k, discreteData, discreteData[t]);			
+
+			// Count the number of points whose x distance is less
+			//  than or equal to eps_x
+			int n_x = 0;
+			for (int t2 = 0; t2 < N; t2++) {
+				if (norms[t2] <= eps_x) {
+					n_x++;
+				}
+			}
+			// n_y is number of points in that bin, minus that point
+			int n_y = counts[discreteData[t]] - 1;
+			avNx += n_x;
+			avNy += n_y;
+			// And take the digamma before adding into the 
+			//  average:
+			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
+		}
+		averageDiGammas /= (double) N;
+		if (debug) {
+			avNx /= (double)N;
+			avNy /= (double)N;
+			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
+		}
+		
+		mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
+		miComputed = true;
+		return mi;
+	}
+
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public synchronized MeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(continuousData.length, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of the mutual information of the previously supplied observations.
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
+	 *  significant this mutual information actually was.
+	 *  
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
+	 *  which was performed for Transfer entropy.
+	 * 
+	 * @param newOrderings the specific new orderings to use
+	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		if (!miComputed) {
+			computeAverageLocalOfObservations();
+		}
+		// Store the real observations and their MI:
+		double actualMI = mi;
+		
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		
+		int countWhereMiIsMoreSignificantThanOriginal = 0;
+		for (int i = 0; i < numPermutationsToCheck; i++) {
+			// Compute the MI under this reordering
+			double newMI = computeAverageLocalOfObservations(newOrderings[i]);
+			measDistribution.distribution[i] = newMI;
+			if (debug){
+				System.out.println("New MI was " + newMI);
+			}
+			if (newMI >= actualMI) {
+				countWhereMiIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the actual MI and the observations
+		mi = actualMI;
+
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMiIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = mi;
+		return measDistribution;
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] continuousStates,
+			int[] discreteStates) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public double getLastAverage() {
+		return mi;
+	}
+	
+	public int getNumObservations() {
+		return continuousData.length;
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorKraskov.java

@@ -0,0 +1,36 @@
+package infodynamics.measures.continuous.kraskov;
+
+/**
+ * <p>Compute the Transfer Entropy using the Kraskov estimation method for underlying
+ * mutual information calculation, using direct calculation of the mutual information
+ * rather than breaking it down into component multi-informations.<p>
+ * 
+ * @see TransferEntropyCalculatorKraskovByMulti
+ * @see MutualInfoCalculatorMultiVariateDirect
+ * @see MutualInfoCalculatorMultiVariateDirect1
+ * @see MutualInfoCalculatorMultiVariateDirect2
+ * @author Joseph Lizier joseph.lizier at gmail.com
+ *
+ */
+public class TransferEntropyCalculatorKraskov
+	extends TransferEntropyCalculatorKraskovByMulti {
+
+	/**
+	 * Create the underlying Kraskov MI calculators using direct multi-variate MI calculators
+	 *
+	 */
+	protected void createKraskovMiCalculators() {
+		if (kraskovAlgorithmNumber == 1) {
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskov1();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskov1();
+		} else {
+			// Algorithm 2
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskov2();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskov2();
+		}
+	}
+	
+	protected void shareDataBetweenUnderlyingCalculators() {
+		// Nothing to do here
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorKraskovByMulti.java

@@ -0,0 +1,497 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Hashtable;
+import java.util.Iterator;
+
+
+/**
+ * <p>Compute the Transfer Entropy using the Kraskov estimation method.<p>
+ * <p>Transfer Entropy is defined in:
+ * <ul>
+ * 		<li>"Measuring information transfer", T. Schreiber, Physical Review E 85, 2,
+ * 		(2000), pp. 461-464</li>
+ * 		 <li>"Information transfer in continuous processes", A. Kaiser, T. Schreiber,
+ * 		Physica D, 166 (2002), pp. 43-62</li>
+ * </ul>
+ * Interestingly, in the Physica D paper, the authors begin discussing in section 5.2.3.2 how
+ *  to use "fixed mass" kernel estimators across all of the underlying entropies, but
+ *  state they cannot work out how to correct for the correlations between the counts.
+ * </p>
+ * 
+ * <p>This correction is effectively what was done in Kraskov et al's method for computing the
+ *  Mutual information:
+ * <ul>
+ * 		<li>"Estimating mutual information", Kraskov, A., Stogbauer, H., Grassberger, P.,
+ * 		Physical Review E 69, (2004) 066138</li>
+ * 		<li>"Synchronization and Interdependence Measures and their Applications to the
+ * 		Electroencephalogram of Epilepsy Patients and Clustering of Data",
+ *		Alexander Kraskov, Publication Series of the John von Neumann Institute for Computing,
+ *		Volume 24, John von Neumann Institute for Computing, J\"{u}lich, Germany, 2004 (PhD Thesis)
+ *		</li>
+ * </ul>
+ * </p>
+ * 
+ * <p>Now, in Kraskov "Synchronization ..." the author presents a method for estimating
+ * Transfer Entropy as a sum of two mutual informations (each estimated by his technique).
+ * We implement this estimation of the transfer entropy here.
+ * We compute <code>I[ (X',X^k); Y ] - I [X^k;Y ]</code> rather than
+ * <code>I[ (Y,X^k); X' ] - I [X^k;X' ]</code> as 
+ *  Kraskov found this to be the more accurate option.
+ * </p>
+ * 
+ * <p>
+ * Usage:
+ * <ol>
+ * 		<li>Construct</li>
+ * 		<li>SetProperty() for each property</li>
+ * 		<li>intialise()</li>
+ * 		<li>setObservations(), or [startAddObservations(), addObservations()*, finaliseAddObservations()]
+ *   Note: If not using setObservations(), the results from computeLocal or getSignificance
+ *    are not likely to be particularly sensible.</li> 
+ * 		<li>computeAverageLocalOfObservations() or ComputeLocalOfPreviousObservations()</li>
+ * </ol>
+ * </p>
+ * 
+ * @author Joseph Lizier joseph.lizier at gmail.com
+ *
+ */
+public class TransferEntropyCalculatorKraskovByMulti
+	extends TransferEntropyCommon implements TransferEntropyCalculator {
+
+	/**
+	 * Which Kraskov MI algorithm number to use (1 or 2)
+	 */
+	protected int kraskovAlgorithmNumber = 2;
+	private boolean algChanged = false;
+	/**
+	 * Storage for the properties ready to pass onto the underlying MI calculators
+	 */
+	private Hashtable<String,String> props;
+	/**
+	 * MI calculator for (Past,Next) to Source
+	 */
+	protected MutualInfoCalculatorMultiVariate mickNextPastToSource;
+	/**
+	 * MI calculator for Past to Source
+	 */
+	protected MutualInfoCalculatorMultiVariate mickPastToSource;
+
+	public final static String PROP_KRASKOV_ALG_NUM = "ALG_NUM";
+
+	protected double[][] jointPastVectors;
+	protected double[][] jointNextAndPastVectors;
+	protected double[][] sourceVectors;
+
+	/**
+	 * Construct the calculator
+	 *
+	 */
+	public TransferEntropyCalculatorKraskovByMulti() {
+		super();
+		props = new Hashtable<String,String>();
+		createKraskovMiCalculators();
+	}
+
+	public void initialise(int k) throws Exception {
+		super.initialise(k); // calls initialise();
+	}
+
+	/**
+	 * Initialise using default or existing values for k and epsilon
+	 */
+	public void initialise() throws Exception {
+		if (algChanged) {
+			// Create the Kraskov MI calculators for the new algorithm number.
+			createKraskovMiCalculators();
+			algChanged = false;
+		}
+		// Set the properties for the Kraskov MI calculators
+		for (String key : props.keySet()) {
+			mickPastToSource.setProperty(key, props.get(key));
+			mickNextPastToSource.setProperty(key, props.get(key));
+		}
+		// Initilalise the Kraskov MI calculators
+		initialiseKraskovCalculators();
+		addedMoreThanOneObservationSet = false;
+	}
+	
+	/**
+	 * Create the underlying Kraskov MI calculators 
+	 *
+	 */
+	protected void createKraskovMiCalculators() {
+		if (kraskovAlgorithmNumber == 1) {
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskovByMulti1();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskovByMulti1();
+		} else {
+			// Algorithm 2
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskovByMulti2();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskovByMulti2();
+		}
+	}
+	
+	protected void initialiseKraskovCalculators() throws Exception {
+		mickPastToSource.initialise(k, 1);
+		mickNextPastToSource.initialise(k+1, 1);
+	}
+	
+	/**
+	 * Sets properties for the calculator.
+	 * Valid properties include:
+	 * <ul>
+	 *  <li>K_PROP_NAME</li>
+	 * 	<li>PROP_KRASKOV_ALG_NUM</li>
+	 * 	<li>Any valid properties for MutualInfoCalculatorMultiVariateKraskov.setProperty except
+	 * 			for MutualInfoCalculatorMultiVariate.PROP_TIME_DIFF</li>
+	 * </ul>
+	 * One should set MutualInfoCalculatorMultiVariateKraskov.PROP_K here, the number
+	 *  of neighbouring points one should count up to in determining the joint kernel size. 
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 */
+	public void setProperty(String propertyName, String propertyValue)
+			throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		if (propertyName.equalsIgnoreCase(K_PROP_NAME)) {
+			k = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_KRASKOV_ALG_NUM)) {
+			int previousAlgNumber = kraskovAlgorithmNumber;
+			kraskovAlgorithmNumber = Integer.parseInt(propertyValue);
+			if ((kraskovAlgorithmNumber != 1) && (kraskovAlgorithmNumber != 2)) {
+				throw new Exception("Kraskov algorithm number (" + kraskovAlgorithmNumber
+						+ ") must be either 1 or 2");
+			}
+			if (kraskovAlgorithmNumber != previousAlgNumber) {
+				algChanged = true;
+			}
+		} else if (propertyName.equalsIgnoreCase(MutualInfoCalculatorMultiVariate.PROP_TIME_DIFF)) {
+			// Do nothing, simply prevent this property being set on the 
+			//  mutual info calculator.
+		} else {
+			// Assume it was a property for the MI calculator
+			props.put(propertyName, propertyValue);
+		}
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		jointPastVectors = new double[totalObservations][k];
+		jointNextAndPastVectors = new double[totalObservations][k+1];
+		sourceVectors = new double[totalObservations][1];
+
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[]> iterator = vectorOfDestinationObservations.iterator();
+		for (double[] source : vectorOfSourceObservations) {
+			double[] destination = iterator.next();
+			double[][] currentDestPastVectors = makeJointVectorForPast(destination);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					jointPastVectors, startObservation, 0, currentDestPastVectors.length, k);
+			double[][] currentDestNextPastVectors = makeJointVectorForNextPast(destination);
+			MatrixUtils.arrayCopy(currentDestNextPastVectors, 0, 0,
+					jointNextAndPastVectors, startObservation, 0, currentDestNextPastVectors.length, k + 1);
+			try {
+				MatrixUtils.copyIntoColumn(sourceVectors, 0, startObservation,
+					source, k - 1, source.length - k);
+			} catch (Exception e) {
+				// The above params should not throw an Exception, so
+				//  wrap in a runtime exception
+				throw new RuntimeException(e);
+			}
+			startObservation += destination.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		try {
+			mickPastToSource.setObservations(jointPastVectors, sourceVectors);
+			mickNextPastToSource.setObservations(jointNextAndPastVectors, sourceVectors);
+		} catch (Exception e) {
+			// The above should not throw an exception since they were constructed here
+			//  of the same time length, so wrap in a runtime exception
+			throw new RuntimeException(e);
+		}
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfDestinationObservations.size() > 1;
+		
+		// And clear the vector of observations
+		vectorOfSourceObservations = null;
+		vectorOfDestinationObservations = null;
+	}
+
+	/**
+	 * Returns the transfer entropy calculated as I[ (X',X^k); Y ] - I [X^k;Y ]
+	 * using the underlying Kraskov MI calculators 
+	 * 
+	 * @return the transfer entropy
+	 */
+	public double computeAverageLocalOfObservations() throws Exception {
+		double miPastNextToSource = mickNextPastToSource.computeAverageLocalOfObservations();
+		shareDataBetweenUnderlyingCalculators();
+		double miPastToSource = mickPastToSource.computeAverageLocalOfObservations();
+		lastAverage = miPastNextToSource - miPastToSource;
+		if (debug) {
+			System.out.println("miPastNextToSource=" + miPastNextToSource + ", miPastToSource=" +
+					miPastToSource + " = " + lastAverage);
+		}
+		return lastAverage;
+	}
+
+	/**
+	 * If mickPastToSource can utilise anything from mickPastNextToSource after the latter
+	 * has run computeAverageLocalOfObservations, arrange that here 
+	 *
+	 */
+	protected void shareDataBetweenUnderlyingCalculators() {
+		if (! MutualInfoCalculatorMultiVariateKraskovByMulti.class.isInstance(mickNextPastToSource)) {
+			// We don't know of what to share for other calculator types.
+			// Subclasses may know and can over-ride this method.
+			return;
+		}
+		MutualInfoCalculatorMultiVariateKraskovByMulti micmvkNextPastToSource =
+			(MutualInfoCalculatorMultiVariateKraskovByMulti) mickNextPastToSource;
+		MutualInfoCalculatorMultiVariateKraskovByMulti micmvkPastToSource =
+			(MutualInfoCalculatorMultiVariateKraskovByMulti) mickPastToSource;
+		if (micmvkNextPastToSource.multiInfoJoint.norms != null) {
+			// Share the norms already computed with the other mutual info calculator.
+			// Just assign the joint norms for it, it will filter them through to the
+			//  marginals itself.
+			if (micmvkPastToSource.multiInfoJoint.norms == null) {
+				micmvkPastToSource.multiInfoJoint.norms = new double[k + 1][][];
+				// Point to the norms for the past variables
+				for (int v = 0; v < k; v++) {
+					micmvkPastToSource.multiInfoJoint.norms[v] =
+						micmvkNextPastToSource.multiInfoJoint.norms[1+v];
+				}
+				// Point to the norms for the source variable
+				micmvkPastToSource.multiInfoJoint.norms[k] =
+					micmvkNextPastToSource.multiInfoJoint.norms[k + 1];
+			}
+		}
+	}
+	
+	/**
+	 * Returns the local transfer entropy values.
+	 * 
+	 * Where more than one time series has been added, the array
+	 *  contains the local values for each tuple in the order in
+	 *  which they were added.
+	 * 
+	 * If there was only a single time series added, the array
+	 *  contains k zero values before the local values.
+	 *  (This means the length of the return array is the same
+	 *  as the length of the input time series).
+	 * 
+	 * @return an array of the local transfer entropy values.
+	 */
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		// Initialise localTE with local MI of PastNextToSource
+		double[] localMiPastNextToSource = mickNextPastToSource.computeLocalOfPreviousObservations();
+		shareDataBetweenUnderlyingCalculators();
+		double[] localMiPastToSource = mickPastToSource.computeLocalOfPreviousObservations();
+		MatrixUtils.subtractInPlace(localMiPastNextToSource, localMiPastToSource);
+		lastAverage = MatrixUtils.mean(localMiPastNextToSource);
+		
+		if (debug) {
+			if (!addedMoreThanOneObservationSet) {
+				for (int i = 0; i < k; k++) {
+					System.out.println(0.0);
+				}
+			}
+			for (int t = 0; t < localMiPastNextToSource.length; t++) {
+				System.out.println(sourceVectors[t][0] + " -> " + 
+					jointNextAndPastVectors[t][0] + ", " +
+					jointNextAndPastVectors[t][1] + ": " +
+					(localMiPastNextToSource[t]+localMiPastToSource[t]) +
+					" - " + localMiPastToSource[t] +
+				    " = " + localMiPastNextToSource[t]);
+			}
+		}
+		
+		if (!addedMoreThanOneObservationSet) {
+			// We need to allow k zeros at the beginning
+			int offset = k;
+			double[] localTE = new double[localMiPastNextToSource.length + offset];
+			System.arraycopy(localMiPastNextToSource, 0, localTE, offset, localMiPastNextToSource.length);
+			return localTE;
+		} else {
+			return localMiPastNextToSource;
+		}
+	}
+
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *
+	 * Basically, we shuffle the source observations against the destination tuples.
+	 * This keeps the marginal PDFs the same (including the entropy rate of the destination)
+	 *  but destroys any correlation between the source and state change of the destination.
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(totalObservations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 * 
+	 * Implemented using "virtual" reorderings of the source variable (assuming there
+	 *  are a small enough number of time points).
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * 		If the length of each permutation in newOrderings
+	 * 		is not equal to numObservations, an Exception is thrown. 
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		
+		double actualTE = computeAverageLocalOfObservations();
+		
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the source in the destPastSourceVectors 
+			//  and destNextPastSourceVectors vectors
+			if (newOrderings[p].length != totalObservations) {
+				throw new Exception("permutation " + p +
+						" in newOrderings argument was not equal to totalObservations");
+			}
+			
+			// Compute the new component MIs under the "virtual" reordering of the source
+			//  variable (it is the second variable for each of these MI calculators, so it
+			//  is the one which will be reordered.
+			double newMiPastNextToSource = mickNextPastToSource.computeAverageLocalOfObservations(newOrderings[p]);
+			double newMiPastToSource = mickPastToSource.computeAverageLocalOfObservations(newOrderings[p]);
+			double newTe = newMiPastNextToSource - newMiPastToSource;
+			
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+		}
+				
+		// Return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualTE;
+		return measDistribution;
+	}
+
+	/**
+	 * As per {@link computeSignificance(int) computeSignificance()} but supplies
+	 *  the re-orderings of the observations of the source variables.
+	 * Explicitly reorders the source variable instead of allowing the underling MI
+	 *  calculators to do it virtually.
+	 *  
+	 * 
+	 * @param newOrderings first index is permutation number, i.e. newOrderings[i]
+	 * 		is an array of 1 permutation of 0..n-1, where there were n observations.
+	 * 		If the length of each permutation in newOrderings
+	 * 		is not equal to numObservations, an Exception is thrown. 
+	 * @return
+	 * @throws Exception
+	 */
+	public MeasurementDistribution computeSignificanceExplicitlyReordering(
+			int[][] newOrderings) throws Exception {
+		
+		int numPermutationsToCheck = newOrderings.length;
+		
+		double actualTE = computeAverageLocalOfObservations();
+		
+		// Save the relevant source observations here:
+		double[][] originalSourceValues = new double[totalObservations][];
+		for (int t = 0; t < totalObservations; t++) {
+			originalSourceValues[t] = sourceVectors[t];
+		}
+		
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the source in the destPastSourceVectors 
+			//  and destNextPastSourceVectors vectors
+			if (newOrderings[p].length != totalObservations) {
+				throw new Exception("permutation " + p +
+						" in newOrderings argument was not equal to totalObservations");
+			}
+			sourceVectors = MatrixUtils.extractSelectedTimePointsReusingArrays(
+					originalSourceValues, newOrderings[p]);
+			
+			// Make the equivalent operations of intialise
+			mickPastToSource.initialise(k, 1);
+			mickNextPastToSource.initialise(k+1, 1);
+			// Make the equivalent operations of setObservations:
+			try {
+				mickPastToSource.setObservations(jointPastVectors, sourceVectors);
+				mickNextPastToSource.setObservations(jointNextAndPastVectors, sourceVectors);
+			} catch (Exception e) {
+				// The above should not throw an exception since they were constructed here
+				//  of the same time length, so wrap in a runtime exception
+				throw new RuntimeException(e);
+			}
+			// And get a TE value for this realisation:
+			double newTe = computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+		}
+		
+		// Restore the local variables:
+		lastAverage = actualTE;
+		// Restore the source observations in the joint vectors
+		for (int t = 0; t < sourceVectors.length; t++) {
+			sourceVectors[t] = originalSourceValues[t];
+		}
+		// And set the MI estimators back to their previous state
+		mickPastToSource.initialise(k, 1);
+		mickNextPastToSource.initialise(k+1, 1);
+		try {
+			mickPastToSource.setObservations(jointPastVectors, sourceVectors);
+			mickNextPastToSource.setObservations(jointNextAndPastVectors, sourceVectors);
+		} catch (Exception e) {
+			// The above should not throw an exception since they were constructed here
+			//  of the same time length, so wrap in a runtime exception
+			throw new RuntimeException(e);
+		}
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = lastAverage;
+		return measDistribution;
+	}
+
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		mickNextPastToSource.setDebug(debug);
+		mickPastToSource.setDebug(debug);
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorMultiVariateKraskov.java

@@ -0,0 +1,37 @@
+package infodynamics.measures.continuous.kraskov;
+
+/**
+ * <p>Compute the Transfer Entropy for multi-variates using the Kraskov estimation method.<p>
+ * <p>This calculator extends the {@link TransferEntropyCalculatorKraskovByMulti TransferEntropyCalculatorKraskov}
+ *  by allowing multivariate sources and destinations. See 
+ *  {@link TransferEntropyCalculatorKraskovByMulti TransferEntropyCalculatorKraskov}
+ *  for further comments on the implementation of Transfer entropy via the Kraskov 
+ *  Mutual Information estimation.</p>
+ * </p>
+ * 
+ * 
+ * @author Joseph Lizier; joseph.lizier at gmail.com
+ *
+ */
+public class TransferEntropyCalculatorMultiVariateKraskov
+	extends TransferEntropyCalculatorMultiVariateKraskovByMulti {
+
+	/**
+	 * Create the underlying Kraskov MI calculators using direct multi-variate MI calculators
+	 *
+	 */
+	protected void createKraskovMiCalculators() {
+		if (kraskovAlgorithmNumber == 1) {
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskov1();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskov1();
+		} else {
+			// Algorithm 2
+			mickPastToSource = new MutualInfoCalculatorMultiVariateKraskov2();
+			mickNextPastToSource = new MutualInfoCalculatorMultiVariateKraskov2();
+		}
+	}
+	
+	protected void shareDataBetweenUnderlyingCalculators() {
+		// Nothing to do here
+	}
+}

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorMultiVariateKraskovByMulti.java

@@ -0,0 +1,321 @@
+package infodynamics.measures.continuous.kraskov;
+
+import infodynamics.measures.continuous.TransferEntropyCalculatorMultiVariate;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Iterator;
+import java.util.Vector;
+
+
+/**
+ * <p>Compute the Transfer Entropy using the Kraskov estimation method.<p>
+ * <p>This calculator extends the {@link TransferEntropyCalculatorKraskovByMulti TransferEntropyCalculatorKraskov}
+ *  by allowing multivariate sources and destinations. See 
+ *  {@link TransferEntropyCalculatorKraskovByMulti TransferEntropyCalculatorKraskov}
+ *  for further comments on the implementation of Transfer entropy via the Kraskov 
+ *  Mutual Information estimation.</p>
+ * </p>
+ * 
+ * 
+ * @author Joseph Lizier; joseph.lizier at gmail.com
+ *
+ */
+public class TransferEntropyCalculatorMultiVariateKraskovByMulti
+	extends TransferEntropyCalculatorKraskovByMulti implements TransferEntropyCalculatorMultiVariate {
+
+	private int destDimensions;
+	private int sourceDimensions;
+
+	/**
+	 * Storage for source observations for addObservsations
+	 */
+	private Vector<double[][]> vectorOfJointSourceObservations;
+	/**
+	 * Storage for destination observations for addObservsations
+	 */
+	private Vector<double[][]> vectorOfJointDestinationObservations;
+
+	public void initialise(int k) throws Exception {
+		// Assume the user only wants 1 source and 1 destination dimension
+		initialise(k, 1, 1);
+	}
+
+	public void initialise(int k, int destDimensions, int sourceDimensions) throws Exception {
+		super.initialise(k);
+		initialise(sourceDimensions, destDimensions);
+	}
+	
+	/**
+	 * Initialise using existing or default value of k
+	 */
+	public void initialise(int sourceDimensions, int destDimensions) throws Exception {
+		this.destDimensions = destDimensions;
+		this.sourceDimensions = sourceDimensions;
+	}
+
+	@Override
+	protected void initialiseKraskovCalculators() throws Exception {
+		mickPastToSource.initialise(k * destDimensions, sourceDimensions);
+		mickNextPastToSource.initialise((k+1) * destDimensions, sourceDimensions);
+	}
+
+	/**
+	 * Set the observations to compute the probabilities from 
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	public void setObservations(double[][] source, double[][] destination) throws Exception {
+		startAddObservations();
+		addObservations(source, destination);
+		finaliseAddObservations();
+	}
+	
+	@Override
+	public void startAddObservations() {
+		vectorOfJointSourceObservations = new Vector<double[][]>();
+		vectorOfJointDestinationObservations = new Vector<double[][]>();
+	}
+
+	/**
+	 * Add observations of a single-dimensional source and destination pair.
+	 * 
+	 * Only allow this call if source and destination dimenions were 1.
+	 * 
+	 * @param source
+	 * @param destination
+	 */
+	@Override
+	public void addObservations(double[] source, double[] destination) throws Exception {
+		double[][] sourceMatrix = new double[source.length][1];
+		MatrixUtils.copyIntoColumn(sourceMatrix, 0, source);
+		double[][] destMatrix = new double[destination.length][1];
+		MatrixUtils.copyIntoColumn(destMatrix, 0, destination);
+		addObservations(sourceMatrix, destMatrix);
+	}
+
+	/**
+	 * Add observations of a single-dimensional source and destination pair.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	@Override
+	public void addObservations(double[] source, double[] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		double[][] sourceMatrix = new double[numTimeSteps][1];
+		MatrixUtils.copyIntoColumn(sourceMatrix, 0, 0, source, startTime, numTimeSteps);
+		double[][] destMatrix = new double[destination.length][1];
+		MatrixUtils.copyIntoColumn(destMatrix, 0, 0, destination, startTime, numTimeSteps);
+		addObservations(sourceMatrix, destMatrix);
+	}
+
+	/**
+	 * Add observations of the joint source and destinations
+	 * 
+	 * @param source
+	 * @param destination
+	 * @throws Exception
+	 */
+	public void addObservations(double[][] source, double[][] destination) throws Exception {
+		if (source.length != destination.length) {
+			throw new Exception(String.format("Source and destination lengths (%d and %d) must match!",
+					source.length, destination.length));
+		}
+		int thisSourceDimensions = source[0].length;
+		int thisDestDimensions = destination[0].length;
+		if ((thisDestDimensions != destDimensions) || (thisSourceDimensions != sourceDimensions)) {
+			throw new Exception("Cannot add observsations for source and destination variables " +
+					" of " + thisSourceDimensions + " and " + thisDestDimensions +
+					" dimensions respectively for TE calculator set up for " + sourceDimensions + " " +
+					destDimensions + " source and destination dimensions respectively");
+		}
+		if (vectorOfJointSourceObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		vectorOfJointSourceObservations.add(source);
+		vectorOfJointDestinationObservations.add(destination);
+	}
+	
+	/**
+	 * Add some more observations.
+	 * 
+	 * @param source
+	 * @param destination
+	 * @param startTime first time index to take observations on
+	 * @param numTimeSteps number of time steps to use
+	 */
+	public void addObservations(double[][] source, double[][] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		double[][] sourceToAdd = new double[numTimeSteps][source[0].length];
+		System.arraycopy(source, startTime, sourceToAdd, 0, numTimeSteps);
+		double[][] destToAdd = new double[numTimeSteps][destination[0].length];
+		System.arraycopy(destination, startTime, destToAdd, 0, numTimeSteps);
+		addObservations(sourceToAdd, destToAdd);
+	}
+
+	/**
+	 * Flag that the observations are complete, probability distribution functions can now be built.
+	 *
+	 */
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[][] destination : vectorOfJointDestinationObservations) {
+			totalObservations += destination.length - k;
+		}
+		jointPastVectors = new double[totalObservations][k * destDimensions];
+		jointNextAndPastVectors = new double[totalObservations][(k+1) * destDimensions];
+		sourceVectors = new double[totalObservations][sourceDimensions];
+
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		Iterator<double[][]> iterator = vectorOfJointDestinationObservations.iterator();
+		for (double[][] source : vectorOfJointSourceObservations) {
+			double[][] destination = iterator.next();
+			double[][] currentDestPastVectors = makeJointVectorForPast(destination);
+			MatrixUtils.arrayCopy(currentDestPastVectors, 0, 0,
+					jointPastVectors, startObservation, 0, currentDestPastVectors.length,
+					k * destDimensions);
+			double[][] currentDestNextPastVectors = makeJointVectorForNextPast(destination);
+			MatrixUtils.arrayCopy(currentDestNextPastVectors, 0, 0,
+					jointNextAndPastVectors, startObservation, 0,
+					currentDestNextPastVectors.length, (k + 1) * destDimensions);
+			MatrixUtils.arrayCopy(source, k-1, 0, sourceVectors, startObservation, 0,
+					source.length - k, sourceDimensions);
+			startObservation += destination.length - k;
+		}
+		
+		// Now set the joint vectors in the kernel estimators
+		try {
+			mickPastToSource.setObservations(jointPastVectors, sourceVectors);
+			mickNextPastToSource.setObservations(jointNextAndPastVectors, sourceVectors);
+		} catch (Exception e) {
+			// The above should not throw an exception since they were constructed here
+			//  of the same time length, so wrap in a runtime exception
+			throw new RuntimeException(e);
+		}
+
+		// Store whether there was more than one observation set:
+		addedMoreThanOneObservationSet = vectorOfJointDestinationObservations.size() > 1;
+		
+		// And clear the vector of observations
+		vectorOfJointSourceObservations = null;
+		vectorOfJointDestinationObservations = null;
+	}
+
+	/**
+	 * If mickPastToSource can utilise anything from mickPastNextToSource after the latter
+	 * has run computeAverageLocalOfObservations, arrange that here 
+	 *
+	 */
+	protected void shareDataBetweenUnderlyingCalculators() {
+		if (! MutualInfoCalculatorMultiVariateKraskovByMulti.class.isInstance(mickNextPastToSource)) {
+			// We don't know of what to share for other calculator types.
+			// Subclasses may know and can over-ride this method.
+			return;
+		}
+		MutualInfoCalculatorMultiVariateKraskovByMulti micmvkNextPastToSource =
+			(MutualInfoCalculatorMultiVariateKraskovByMulti) mickNextPastToSource;
+		MutualInfoCalculatorMultiVariateKraskovByMulti micmvkPastToSource =
+			(MutualInfoCalculatorMultiVariateKraskovByMulti) mickPastToSource;
+		if (micmvkNextPastToSource.multiInfoJoint.norms != null) {
+			// Share the norms already computed with the other mutual info calculator.
+			// Just assign the joint norms for it, it will filter them through to the
+			//  marginals itself.
+			if (micmvkPastToSource.multiInfoJoint.norms == null) {
+				micmvkPastToSource.multiInfoJoint.norms =
+					new double[destDimensions * k + sourceDimensions][][];
+				// Point to the norms for the past variables
+				for (int t = 0; t < k; t++) {
+					for (int d = 0; d < destDimensions; d++) {
+						micmvkPastToSource.multiInfoJoint.norms[t*destDimensions + d] =
+							micmvkNextPastToSource.multiInfoJoint.norms[(1+t)*destDimensions + d];
+					}
+				}
+				// Point to the norms for the source variable
+				for (int s = 0; s < sourceDimensions; s++) {
+					micmvkPastToSource.multiInfoJoint.norms[k*destDimensions + s] =
+						micmvkNextPastToSource.multiInfoJoint.norms[(k + 1)*destDimensions + s];
+				}
+			}
+		}
+	}
+
+	/**
+	 * Generate a vector for each time step, containing the past k states of the destination.
+	 * Note that each state of the destination is a joint vector of destDimensions variables.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return array of vectors for each time step
+	 */
+	private double[][] makeJointVectorForPast(double[][] destination) {
+		try {
+			// We want one less delay vector here - we don't need the last k point,
+			//  because there is no next state for these.
+			return MatrixUtils.makeDelayEmbeddingVector(destination, k, k-1, destination.length - k);
+		} catch (Exception e) {
+			// The parameters for the above call should be fine, so we don't expect to
+			//  throw an Exception here - embed in a RuntimeException if it occurs 
+			throw new RuntimeException(e);
+		}
+	}
+
+	/**
+	 * Sets the observations to compute the PDFs from.
+	 * Cannot be called in conjunction with start/add/finaliseAddObservations.
+	 * destValid is a time series (with time indices the same as destination)
+	 *  indicating whether the destination at that point is valid.
+	 * sourceValid is the same for the source
+	 * 
+	 * @param source observations for the source variable
+	 * @param destination observations for the destination variable
+	 * @param sourceValid
+	 * @param destValid
+	 */
+	public void setObservations(double[][] source, double[][] destination,
+			boolean[] sourceValid, boolean[] destValid) throws Exception {
+		
+		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+		
+		// We've found the set of start and end times for this pair
+		startAddObservations();
+		for (int[] timePair : startAndEndTimePairs) {
+			int startTime = timePair[0];
+			int endTime = timePair[1];
+			addObservations(source, destination, startTime, endTime - startTime + 1);
+		}
+		finaliseAddObservations();
+	}
+
+	public void setObservations(double[][] source, double[][] destination,
+			boolean[][] sourceValid, boolean[][] destValid) throws Exception {
+		
+		boolean[] jointSourceValid = MatrixUtils.andRows(sourceValid);
+		boolean[] jointDestValid = MatrixUtils.andRows(destValid);
+		setObservations(source, destination, jointSourceValid, jointDestValid);
+	}
+
+	/**
+	 * Generate a vector for each time step, containing the past k states of
+	 *  the destination, and the current state.
+	 * Does not include a vector for the first k time steps.
+	 * 
+	 * @param destination
+	 * @return
+	 */
+	protected double[][] makeJointVectorForNextPast(double[][] destination) {
+		// We want all delay vectors here
+		return MatrixUtils.makeDelayEmbeddingVector(destination, k+1);
+	}
+
+	public void setDebug(boolean debug) {
+		super.setDebug(debug);
+		mickNextPastToSource.setDebug(debug);
+		mickPastToSource.setDebug(debug);
+	}
+}

java/source/infodynamics/measures/continuous/symbolic/ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSymbolic.java

@@ -0,0 +1,156 @@
+package infodynamics.measures.continuous.symbolic;
+
+import infodynamics.measures.continuous.ConditionalMutualInfoCalculatorMultiVariateWithDiscrete;
+import infodynamics.measures.discrete.ConditionalMutualInformationCalculator;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+public class ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSymbolic implements
+		ConditionalMutualInfoCalculatorMultiVariateWithDiscrete {
+
+	// The calculator used to do the grunt work
+	protected ConditionalMutualInformationCalculator condMiCalc;
+	
+	protected int dimensions;
+	
+	protected int[][] permutations;
+	// For each permutation index, holds the unique permutation id
+	protected int[] permutationIds;
+	// For each possible permutation id, holds the permutation index 
+	protected int[] idToPermutationIndex;
+	
+	// Array indices for the 2D array sorted by the first index
+	protected static final int VAL_COLUMN = 0;
+	protected static final int VAR_NUM_COLUMN = 1;
+
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+
+	public ConditionalMutualInfoCalculatorMultiVariateWithDiscreteSymbolic() {
+		// Nothing to do
+	}
+
+	public void initialise(int dimensions, int base, int condBase) throws Exception {
+		this.dimensions = dimensions;
+
+		// First work out how many permutations of orderings we could have
+		RandomGenerator rg = new RandomGenerator();
+		permutations = rg.generateAllDistinctPerturbations(dimensions);
+		// Now generate an int signature for each permutation:
+		permutationIds = new int[permutations.length];
+		for (int r = 0; r < permutations.length; r++) {
+			permutationIds[r] = generatePermutationId(permutations[r]);
+		}
+		// Now we have a list of permutations, each with an ID (which will be dimensions^dimensions)
+		// Generate a reverse mapping from permutation identifier to permutation id
+		idToPermutationIndex = new int[MathsUtils.power(dimensions, dimensions)];
+		// First initialise all mappings to -1 : this will force an Array lookup error if 
+		//  an identifier is not calculated correctly (most of the time)
+		for (int i = 0; i < idToPermutationIndex.length; i++) {
+			idToPermutationIndex[i] = -1;
+		}
+		for (int idIndex = 0; idIndex < permutationIds.length; idIndex++) {
+			idToPermutationIndex[permutationIds[idIndex]] = idIndex;
+		}
+		
+		condMiCalc = ConditionalMutualInformationCalculator.newInstance(permutationIds.length, base, condBase);
+		condMiCalc.initialise();
+	}
+
+	/**
+	 * Generate the unique permutation id for this permutation.
+	 * 
+	 * @param data
+	 * @return
+	 */
+	private int generatePermutationId(int[] data) {
+		int permutationId = 0;
+		for (int c = 0; c < dimensions; c++) {
+			permutationId *= dimensions;
+			permutationId +=  data[c];
+		}
+		return permutationId;
+	}
+
+	/**
+	 * Generate the unique permutation id for this permutation.
+	 * Convert the floating point variable numbers into ints first
+	 * 
+	 * @param data
+	 * @return
+	 */
+	private int generatePermutationId(double[] data) {
+		int permutationId = 0;
+		for (int c = 0; c < dimensions; c++) {
+			permutationId *= dimensions;
+			permutationId +=  (int) data[c];
+		}
+		return permutationId;
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equals(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations, int[] conditionalObservations) throws Exception {
+		if (normalise) {
+			// Normalise the continuous observations first
+			continuousObservations = MatrixUtils.normaliseIntoNewArray(continuousObservations);
+		}
+		// Construct the orderings for the continuous observations
+		int[] mappedPermutationIds = new int[continuousObservations.length];
+		for (int t = 0; t < continuousObservations.length; t++) {
+			// Work out what the order of the continuous variables was here:
+			double[][] variablesAndIndices = new double[dimensions][2];
+			for (int v = 0; v < dimensions; v++) {
+				variablesAndIndices[v][VAL_COLUMN] = continuousObservations[t][v];
+				variablesAndIndices[v][VAR_NUM_COLUMN] = v;
+			}
+			java.util.Arrays.sort(variablesAndIndices, FirstIndexComparatorDouble.getInstance());
+			// Now the second column contains the order of values here
+			double[] permutation = MatrixUtils.selectColumn(variablesAndIndices, VAR_NUM_COLUMN);
+			int permutationId = generatePermutationId(permutation);
+			mappedPermutationIds[t] = idToPermutationIndex[permutationId];
+		}
+		// Now we can set the observations
+		condMiCalc.addObservations(mappedPermutationIds, discreteObservations, conditionalObservations);
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		return condMiCalc.computeAverageLocalOfObservations();
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] contStates,
+			int[] discreteStates, int[] conditionedStates) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		return condMiCalc.computeSignificance(numPermutationsToCheck);
+	}
+	
+	public MeasurementDistribution computeSignificance(int[][] newOrderings)
+		throws Exception {
+		return condMiCalc.computeSignificance(newOrderings);
+	}
+
+	public void setDebug(boolean debug) {
+		// condMiCalc.setDebug(debug);
+	}
+
+	public double getLastAverage() {
+		return condMiCalc.getLastAverage();
+	}
+
+	public int getNumObservations() {
+		return condMiCalc.getNumObservations();
+	}
+
+}

java/source/infodynamics/measures/continuous/symbolic/MutualInfoCalculatorMultiVariateWithDiscreteSymbolic.java

@@ -0,0 +1,164 @@
+package infodynamics.measures.continuous.symbolic;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariateWithDiscrete;
+import infodynamics.measures.discrete.MutualInformationCalculator;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+public class MutualInfoCalculatorMultiVariateWithDiscreteSymbolic implements
+		MutualInfoCalculatorMultiVariateWithDiscrete {
+
+	// The calculator used to do the grunt work
+	protected MutualInformationCalculator miCalc;
+	
+	protected int dimensions;
+	
+	protected int[][] permutations;
+	// For each permutation index, holds the unique permutation id
+	protected int[] permutationIds;
+	// For each possible permutation id, holds the permutation index 
+	protected int[] idToPermutationIndex;
+	
+	// Array indices for the 2D array sorted by the first index
+	protected static final int VAL_COLUMN = 0;
+	protected static final int VAR_NUM_COLUMN = 1;
+
+	public static final String PROP_NORMALISE = "NORMALISE";
+	private boolean normalise = true;
+
+	public MutualInfoCalculatorMultiVariateWithDiscreteSymbolic() {
+		// Nothing to do
+	}
+
+	public void initialise(int dimensions, int base) throws Exception {
+		this.dimensions = dimensions;
+
+		// First work out how many permutations of orderings we could have
+		RandomGenerator rg = new RandomGenerator();
+		permutations = rg.generateAllDistinctPerturbations(dimensions);
+		// Now generate an int signature for each permutation:
+		permutationIds = new int[permutations.length];
+		for (int r = 0; r < permutations.length; r++) {
+			permutationIds[r] = generatePermutationId(permutations[r]);
+		}
+		// Now we have a list of permutations, each with an ID (which will be dimensions^dimensions)
+		// Generate a reverse mapping from permutation identifier to permutation id
+		idToPermutationIndex = new int[MathsUtils.power(dimensions, dimensions)];
+		// First initialise all mappings to -1 : this will force an Array lookup error if 
+		//  an identifier is not calculated correctly (most of the time)
+		for (int i = 0; i < idToPermutationIndex.length; i++) {
+			idToPermutationIndex[i] = -1;
+		}
+		for (int idIndex = 0; idIndex < permutationIds.length; idIndex++) {
+			idToPermutationIndex[permutationIds[idIndex]] = idIndex;
+		}
+		
+		// so we have permutationIds.length permutations of order of the continuous variables
+		//  and base possiblities of the discrete variable.
+		// Select the base for MI as the max between these (since our only MI calc at the
+		//  moment only handles one base)
+		int baseToUse = Math.max(permutationIds.length, base);
+		
+		// Make the base the maximum of the number of combinations of orderings of the
+		//  continuous variables and the discrete base.
+		miCalc = new MutualInformationCalculator(baseToUse,0);
+		miCalc.initialise();
+	}
+
+	/**
+	 * Generate the unique permutation id for this permutation.
+	 * 
+	 * @param data
+	 * @return
+	 */
+	private int generatePermutationId(int[] data) {
+		int permutationId = 0;
+		for (int c = 0; c < dimensions; c++) {
+			permutationId *= dimensions;
+			permutationId +=  data[c];
+		}
+		return permutationId;
+	}
+
+	/**
+	 * Generate the unique permutation id for this permutation.
+	 * Convert the floating point variable numbers into ints first
+	 * 
+	 * @param data
+	 * @return
+	 */
+	private int generatePermutationId(double[] data) {
+		int permutationId = 0;
+		for (int c = 0; c < dimensions; c++) {
+			permutationId *= dimensions;
+			permutationId +=  (int) data[c];
+		}
+		return permutationId;
+	}
+
+	public void setProperty(String propertyName, String propertyValue) {
+		if (propertyName.equals(PROP_NORMALISE)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		}
+	}
+
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations) throws Exception {
+		if (normalise) {
+			// Normalise the continuous observations first
+			continuousObservations = MatrixUtils.normaliseIntoNewArray(continuousObservations);
+		}
+		// Construct the orderings for the continuous observations
+		int[] mappedPermutationIds = new int[continuousObservations.length];
+		for (int t = 0; t < continuousObservations.length; t++) {
+			// Work out what the order of the continuous variables was here:
+			double[][] variablesAndIndices = new double[dimensions][2];
+			for (int v = 0; v < dimensions; v++) {
+				variablesAndIndices[v][VAL_COLUMN] = continuousObservations[t][v];
+				variablesAndIndices[v][VAR_NUM_COLUMN] = v;
+			}
+			java.util.Arrays.sort(variablesAndIndices, FirstIndexComparatorDouble.getInstance());
+			// Now the second column contains the order of values here
+			double[] permutation = MatrixUtils.selectColumn(variablesAndIndices, VAR_NUM_COLUMN);
+			int permutationId = generatePermutationId(permutation);
+			mappedPermutationIds[t] = idToPermutationIndex[permutationId];
+		}
+		// Now we can set the observations
+		miCalc.addObservations(mappedPermutationIds, discreteObservations);
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		return miCalc.computeAverageLocalOfObservations();
+	}
+
+	public double[] computeLocalUsingPreviousObservations(double[][] contStates,
+			int[] discreteStates) throws Exception {
+		throw new Exception("Local method not implemented yet");
+	}
+
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		return miCalc.computeSignificance(numPermutationsToCheck);
+	}
+
+	public MeasurementDistribution computeSignificance(int[][] newOrderings)
+			throws Exception {
+		return miCalc.computeSignificance(newOrderings);
+	}
+
+	public void setDebug(boolean debug) {
+		// miCalc.setDebug(debug);
+	}
+
+	public double getLastAverage() {
+		return miCalc.getLastAverage();
+	}
+
+	public int getNumObservations() {
+		return miCalc.getNumObservations();
+	}
+
+}

java/source/infodynamics/measures/continuous/symbolic/TransferEntropyCalculatorSymbolic.java

@@ -0,0 +1,318 @@
+package infodynamics.measures.continuous.symbolic;
+
+import infodynamics.measures.continuous.TransferEntropyCalculator;
+import infodynamics.measures.continuous.TransferEntropyCommon;
+import infodynamics.measures.discrete.ApparentTransferEntropyCalculator;
+import infodynamics.utils.FirstIndexComparatorDouble;
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+import java.util.Iterator;
+import java.util.Vector;
+
+
+/**
+ * Computing the transfer entropy symbollically
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com
+ * @see Mattha\"{u}s Staniek, Klaus Lehnertz, "Symbolic Transfer Entropy", PRL 100, 158101 (2008)
+ *
+ */
+public class TransferEntropyCalculatorSymbolic
+		extends TransferEntropyCommon
+		implements TransferEntropyCalculator {
+
+	// The calculator used to do the grunt work
+	protected ApparentTransferEntropyCalculator teCalc;
+
+	protected int maxEmbeddingLength; // = Max(l,k)
+	// l will default to the value of k unless l gets explicitly set
+	protected int l = 1;
+	protected boolean lWasSet = false;
+	
+	private int[][] destPermutations;
+	// For each permutation index, holds the unique permutation id
+	private int[] destPermutationIds;
+	// For each possible permutation id, holds the permutation index 
+	private int[] idToDestPermutationIndex;
+	
+	private int[][] sourcePermutations;
+	// For each permutation index, holds the unique permutation id
+	private int[] sourcePermutationIds;
+	// For each possible permutation id, holds the permutation index 
+	private int[] idToSourcePermutationIndex;
+
+	// Storage for the computation symbols
+	protected Vector<int[]> destSymbolsVector;
+	protected Vector<int[]> sourceSymbolsVector;
+	
+	// Array indices for the 2D array sorted by the first index
+	protected static final int VAL_COLUMN = 0;
+	protected static final int VAR_NUM_COLUMN = 1;
+
+	public static final String PROP_L = "L";
+	
+	public TransferEntropyCalculatorSymbolic() {
+		// Nothing to do
+	}
+
+	public void initialise(int k) throws Exception {
+		super.initialise(k); // calls initialise();
+	}
+
+	public void initialise() throws Exception {
+		if (!lWasSet) {
+			l = k;
+		}
+		
+		// The discrete TE calculator will be run with a base of the number of
+		//  permutations of the maximum embedding length (out of l,k), and history 1
+		//  since the symbols incorporate the past k states.
+		maxEmbeddingLength = Math.max(k, l);
+		
+		// First work out how many permutations of orderings we could have
+		RandomGenerator rg = new RandomGenerator();
+		destPermutations = rg.generateAllDistinctPerturbations(k);
+		// Now generate an int signature for each permutation:
+		destPermutationIds = new int[destPermutations.length];
+		for (int r = 0; r < destPermutations.length; r++) {
+			destPermutationIds[r] = generatePermutationId(destPermutations[r], k);
+		}
+		// Now we have a list of permutations, each with an ID (which will be up to k^k)
+		// Generate a reverse mapping from permutation identifier to permutation id
+		idToDestPermutationIndex = new int[MathsUtils.power(k, k)];
+		// First initialise all mappings to -1 : this will force an Array lookup error if 
+		//  an identifier is not calculated correctly (most of the time)
+		for (int i = 0; i < idToDestPermutationIndex.length; i++) {
+			idToDestPermutationIndex[i] = -1;
+		}
+		for (int idIndex = 0; idIndex < destPermutationIds.length; idIndex++) {
+			idToDestPermutationIndex[destPermutationIds[idIndex]] = idIndex;
+			/* System.out.print("Permutation "+ idIndex + ": ");
+			for (int j = 0; j < permutations[idIndex].length; j++) {
+				System.out.print(permutations[idIndex][j] + " ");
+			}
+			System.out.println(" -> id " + permutationIds[idIndex]);
+			*/
+		}
+
+		// Now generate the permutations for the source
+		sourcePermutations = rg.generateAllDistinctPerturbations(l);
+		// Now generate an int signature for each permutation:
+		sourcePermutationIds = new int[sourcePermutations.length];
+		for (int r = 0; r < sourcePermutations.length; r++) {
+			sourcePermutationIds[r] = generatePermutationId(sourcePermutations[r], l);
+		}
+		// Now we have a list of permutations, each with an ID (which will be up to l^l)
+		// Generate a reverse mapping from permutation identifier to permutation id
+		idToSourcePermutationIndex = new int[MathsUtils.power(l, l)];
+		// First initialise all mappings to -1 : this will force an Array lookup error if 
+		//  an identifier is not calculated correctly (most of the time)
+		for (int i = 0; i < idToSourcePermutationIndex.length; i++) {
+			idToSourcePermutationIndex[i] = -1;
+		}
+		for (int idIndex = 0; idIndex < sourcePermutationIds.length; idIndex++) {
+			idToSourcePermutationIndex[sourcePermutationIds[idIndex]] = idIndex;
+			/* System.out.print("Permutation "+ idIndex + ": ");
+			for (int j = 0; j < permutations[idIndex].length; j++) {
+				System.out.print(permutations[idIndex][j] + " ");
+			}
+			System.out.println(" -> id " + permutationIds[idIndex]);
+			*/
+		}
+
+		// The discrete calculator only uses a history of 1 here - k is built into
+		//  the permutations
+		int base = Math.max(destPermutationIds.length, sourcePermutationIds.length);
+		teCalc = ApparentTransferEntropyCalculator.newInstance(base, 1);
+		teCalc.initialise();
+	}
+
+	/**
+	 * <p>Set properties for the transfer entropy calculator.
+	 * These can include:
+	 * <ul>
+	 * 		<li>PROP_L</li>
+	 * </ul> 
+	 * and those for {@link TransferEntropyCommon#setProperty(String,String)}
+	 * </p>
+	 * 
+	 * @param propertyName
+	 * @param propertyValue
+	 * @throws Exception 
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+		super.setProperty(propertyName, propertyValue);
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(PROP_L)) {
+			l = Integer.parseInt(propertyValue);
+			lWasSet = true;
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println("Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+
+	public void finaliseAddObservations() {
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[] destination : vectorOfDestinationObservations) {
+			totalObservations += destination.length - maxEmbeddingLength;
+		}
+
+		destSymbolsVector = new Vector<int[]>();
+		sourceSymbolsVector = new Vector<int[]>();
+		
+		// Construct the symbols from the given observations
+		int startObservation = 0;
+		Iterator<double[]> iterator = vectorOfDestinationObservations.iterator();
+		for (double[] source : vectorOfSourceObservations) {
+			double[] destination = iterator.next();
+			// compute the embedding vectors for the destination and source
+			double[][] currentDestPastVectors = null;
+			double[][] sourceVectors = null;
+			try {
+				// we don't use the pre-defined makeDestPastVectors method because it skips off 
+				//  the last embedding vector
+				currentDestPastVectors = MatrixUtils.makeDelayEmbeddingVector(destination, k, k-1, destination.length - k + 1);
+				sourceVectors = MatrixUtils.makeDelayEmbeddingVector(source, l, l-1, source.length - l + 1);
+			} catch (Exception e) {
+				// The parameters for the above call should be fine, so we don't expect to
+				//  throw an Exception here - embed in a RuntimeException if it occurs 
+				throw new RuntimeException(e);
+			}
+			
+			// Now compute the permutation values for the dest
+			double[][] destVariablesAndIndices = new double[k][2];
+			int[] destSymbols = new int[currentDestPastVectors.length];
+			for (int t = 0; t < currentDestPastVectors.length; t++) {
+				// Work out what the order of the embedded variables was here:
+				for (int v = 0; v < k; v++) {
+					destVariablesAndIndices[v][VAL_COLUMN] = currentDestPastVectors[t][v];
+					destVariablesAndIndices[v][VAR_NUM_COLUMN] = v;
+				}
+				java.util.Arrays.sort(destVariablesAndIndices, FirstIndexComparatorDouble.getInstance());
+				// Now the second column contains the order of values here
+				double[] permutation = MatrixUtils.selectColumn(destVariablesAndIndices, VAR_NUM_COLUMN);
+				int permutationId = generatePermutationId(permutation, k);
+				destSymbols[t] = idToDestPermutationIndex[permutationId];
+			}
+
+			// Compute the permutation values for the source
+			double[][] sourceVariablesAndIndices = new double[l][2];
+			int[] sourceSymbols = new int[sourceVectors.length];
+			for (int t = 0; t < sourceVectors.length; t++) {
+				// Work out what the order of the embedded variables was here:
+				for (int v = 0; v < l; v++) {
+					sourceVariablesAndIndices[v][VAL_COLUMN] = sourceVectors[t][v];
+					sourceVariablesAndIndices[v][VAR_NUM_COLUMN] = v;
+				}
+				java.util.Arrays.sort(sourceVariablesAndIndices, FirstIndexComparatorDouble.getInstance());
+				// Now the second column contains the order of values here
+				double[] permutation = MatrixUtils.selectColumn(sourceVariablesAndIndices, VAR_NUM_COLUMN);
+				int permutationId = generatePermutationId(permutation, l);
+				sourceSymbols[t] = idToSourcePermutationIndex[permutationId];
+			}
+			
+			if (k > l) {
+				// l is smaller, so we will have more source embeddings than destination ones.
+				// Get rid of the first k - l source embeddings
+				sourceSymbols = MatrixUtils.select(sourceSymbols, k - l,
+						sourceSymbols.length - (k - l));
+			} else if (l > k) {
+				// k is smaller, so we will have more dest embeddings than source ones.
+				// Get rid of the first l - k source embeddings
+				destSymbols = MatrixUtils.select(destSymbols, l - k,
+						destSymbols.length - (l - k));
+			}
+			
+			// Add these observations in, and keep them locally
+			destSymbolsVector.add(destSymbols);
+			sourceSymbolsVector.add(sourceSymbols);
+			teCalc.addObservations(destSymbols, sourceSymbols);
+			
+			if (destination.length - maxEmbeddingLength != destSymbols.length - 1) {
+				throw new RuntimeException(
+					String.format("Number of observations %d doesn't match what's expected %d",
+						destSymbols.length - 1, destination.length - maxEmbeddingLength));
+			}
+			
+			startObservation += destSymbols.length - 1;
+		}
+		
+	}
+
+	/**
+	 * Generate the unique permutation id for this permutation.
+	 * 
+	 * @param data
+	 * @param base the number of elements being permuted
+	 * @return
+	 */
+	private int generatePermutationId(int[] data, int base) {
+		int permutationId = 0;
+		for (int c = 0; c < data.length; c++) {
+			permutationId *= base;
+			permutationId +=  data[c];
+		}
+		return permutationId;
+	}
+
+	/**
+	 * Generate the unique permutation id for this ordering of variable ids.
+	 * Convert the floating point variable ids into ints first
+	 * 
+	 * @param data
+	 * @param base the number of elements being permuted
+	 * @return
+	 */
+	private int generatePermutationId(double[] ids, int base) {
+		int permutationId = 0;
+		for (int c = 0; c < ids.length; c++) {
+			permutationId *= base;
+			permutationId +=  (int) ids[c];
+		}
+		return permutationId;
+	}
+
+	public double computeAverageLocalOfObservations() throws Exception {
+		lastAverage = teCalc.computeAverageLocalOfObservations();
+		return lastAverage;
+	}
+
+	public double[] computeLocalOfPreviousObservations() throws Exception {
+		double[] locals = new double[totalObservations];
+		int currentIndexInLocals = 0;
+		Iterator<int[]> iterator = destSymbolsVector.iterator();
+		for (int[] sourceSymbols : sourceSymbolsVector) {
+			int[] destSymbols = iterator.next();
+			double[] theseLocals = teCalc.computeLocalFromPreviousObservations(destSymbols, sourceSymbols);
+			System.arraycopy(theseLocals, 1, locals, currentIndexInLocals, theseLocals.length - 1);
+			currentIndexInLocals += theseLocals.length - 1;
+		}
+		lastAverage = MatrixUtils.mean(locals);
+		return locals;
+	}
+
+	public MeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		return teCalc.computeSignificance(numPermutationsToCheck);
+	}
+
+	/**
+	 * This method does not actually use the newOrderings supplied,
+	 *  which is not strictly what this method is meant to do.
+	 * 
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings)
+			throws Exception {
+		System.out.println("TESymbolic.computeSignificance(): Not using the new orderings supplied");
+		return teCalc.computeSignificance(newOrderings.length);
+	}
+}

java/source/infodynamics/measures/discrete/ActiveInformationCalculator.java

@@ -0,0 +1,969 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class ActiveInformationCalculator {
+
+	private double average = 0.0;
+	private double max = 0.0;
+	private double min = 0.0;
+	private int observations = 0;
+	private int k = 0; // history length k. Need initialised to 0 for changedSizes
+	private int base = 0; // number of individual states. Need initialised to 0 for changedSizes
+	private int[][]	jointCount = null; // Count for (i[t+1], i[t]) tuples
+	private int[] prevCount = null; // Count for i[t]		
+	private int[] nextCount = null; // Count for i[t+1]
+	private int[] maxShiftedValue = null; // states * (base^(history-1))
+	
+	private int base_power_k = 0;
+	private double log_base = 0;
+	
+	/**
+	 * User to create new instances through this factory method.
+	 * This allows us to return an efficient calculator for
+	 * base 2, for example, without the user needing to have 
+	 * knowledge of this.
+	 * @param base
+	 * @param history
+	 * 
+	 * @return
+	 */
+	public static ActiveInformationCalculator newInstance(int base, int history) {
+		return new ActiveInformationCalculator(base, history);
+	}
+	
+	protected ActiveInformationCalculator(int base, int history) {
+		super();
+		
+		this.base = base;
+		k = history;
+		base_power_k = MathsUtils.power(base, k);
+		log_base = Math.log(base);
+		
+		if (history < 1) {
+			throw new RuntimeException("History k " + history + " is not >= 1 for Entropy rate Calculator");
+		}
+		if (k > Math.log(Integer.MAX_VALUE) / log_base) {
+			throw new RuntimeException("Base and history combination too large");
+		}
+
+		// Create storage for counts of observations
+		jointCount = new int[base][base_power_k];
+		prevCount = new int[base_power_k];
+		nextCount = new int[base];
+
+		// Create constants for tracking prevValues
+		maxShiftedValue = new int[base];
+		for (int v = 0; v < base; v++) {
+			maxShiftedValue[v] = v * MathsUtils.power(base, k-1);
+		}
+	}
+
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 *
+	 */
+	public void initialise(){
+		average = 0.0;
+		max = 0.0;
+		min = 0.0;
+		observations = 0;
+		
+		MatrixUtils.fill(jointCount, 0);
+		MatrixUtils.fill(prevCount, 0);
+		MatrixUtils.fill(nextCount, 0);
+	}
+		
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+	 *
+	 * @param states time series of agent states
+	 */
+	public void addObservations(int states[]) {
+		int timeSteps = states.length;
+		// increment the count of observations:
+		observations += (timeSteps - k); 
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p];
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int t = k; t < timeSteps; t++) {
+			// Add to the count for this particular transition:
+			nextVal = states[t];
+			jointCount[nextVal][prevVal]++;
+			prevCount[prevVal]++;
+			nextCount[nextVal]++;					
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[t-k]];
+			prevVal *= base;
+			prevVal += states[t];
+		}		
+	}
+
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][]) {
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - k)*columns; 
+		
+		// Initialise and store the current previous value for each column
+		int[] prevVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			prevVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				prevVal[c] *= base;
+				prevVal[c] += states[p][c];
+			}
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add to the count for this particular transition:
+				// (cell's assigned as above)
+				nextVal = states[r][c];
+				jointCount[nextVal][prevVal[c]]++;
+				prevCount[prevVal[c]]++;
+				nextCount[nextVal]++;					
+				// Update the previous value:
+				prevVal[c] -= maxShiftedValue[states[r-k][c]];
+				prevVal[c] *= base;
+				prevVal[c] += states[r][c];
+			}
+		}		
+	}
+
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 */
+	public void addObservations(int states[][][]) {
+		int timeSteps = states.length;
+		if (timeSteps == 0) {
+			return;
+		}
+		int agentRows = states[0].length;
+		if (agentRows == 0) {
+			return;
+		}
+		int agentColumns = states[0][0].length;
+		// increment the count of observations:
+		observations += (timeSteps - k) * agentRows * agentColumns; 
+
+		// Initialise and store the current previous value for each column
+		int[][] prevVal = new int[agentRows][agentColumns];
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				prevVal[r][c] = 0;
+				for (int p = 0; p < k; p++) {
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[p][r][c];
+				}
+			}
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					// Add to the count for this particular transition:
+					// (cell's assigned as above)
+					nextVal = states[t][r][c];
+					jointCount[nextVal][prevVal[r][c]]++;
+					prevCount[prevVal[r][c]]++;
+					nextCount[nextVal]++;					
+					// Update the previous value:
+					prevVal[r][c] -= maxShiftedValue[states[t-k][r][c]];
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[t][r][c];
+				}
+			}
+		}		
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][], int col) {
+		int rows = states.length;
+		// increment the count of observations:
+		observations += (rows - k); 
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][col];
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int r = k; r < rows; r++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			nextVal = states[r][col];
+			jointCount[nextVal][prevVal]++;
+			prevCount[prevVal]++;
+			nextCount[nextVal]++;					
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[r-k][col]];
+			prevVal *= base;
+			prevVal += states[r][col];
+		}
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 */
+	public void addObservations(int states[][][], int agentIndex1, int agentIndex2) {
+		int timeSteps = states.length;
+		// increment the count of observations:
+		observations += (timeSteps - k); 
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][agentIndex1][agentIndex2];
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int t = k; t < timeSteps; t++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			nextVal = states[t][agentIndex1][agentIndex2];
+			jointCount[nextVal][prevVal]++;
+			prevCount[prevVal]++;
+			nextCount[nextVal]++;					
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[t-k][agentIndex1][agentIndex2]];
+			prevVal *= base;
+			prevVal += states[t][agentIndex1][agentIndex2];
+		}
+	}
+
+	/**
+	 * Returns the average local active information storage from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double mi = 0.0;
+		double miCont = 0.0;
+
+		max = 0;
+		min = 0;
+		for (int nextVal = 0; nextVal < base; nextVal++) {
+			// compute p_next
+			double p_next = (double) nextCount[nextVal] / (double) observations;
+			for (int prevVal = 0; prevVal < base_power_k; prevVal++) {
+				// compute p_prev
+				double p_prev = (double) prevCount[prevVal] / (double) observations;
+				// compute p(prev, next)
+				double p_joint = (double) jointCount[nextVal][prevVal] / (double) observations;
+				// Compute MI contribution:
+				if (p_joint * p_next * p_prev > 0.0) {
+					double logTerm = p_joint / (p_next * p_prev);
+					double localValue = Math.log(logTerm) / log_base;
+					miCont = p_joint * localValue;
+					if (localValue > max) {
+						max = localValue;
+					} else if (localValue < min) {
+						min = localValue;
+					}
+				} else {
+					miCont = 0.0;
+				}
+				mi += miCont;
+			}
+		}
+		
+		average = mi;
+		return mi;
+	}
+	
+	/**
+	 * Returns the average local entropy rate from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalEntropyRateOfObservations() {
+		double entRate = 0.0;
+		double entRateCont = 0.0;
+
+		for (int nextVal = 0; nextVal < base; nextVal++) {
+			for (int prevVal = 0; prevVal < base_power_k; prevVal++) {
+				// compute p_prev
+				double p_prev = (double) prevCount[prevVal] / (double) observations;
+				// compute p(prev, next)
+				double p_joint = (double) jointCount[nextVal][prevVal] / (double) observations;
+				// Compute entropy rate contribution:
+				if (p_joint > 0.0) {
+					double logTerm = p_joint / p_prev;
+					// Entropy rate takes the negative log:
+					double localValue = - Math.log(logTerm) / log_base;
+					entRateCont = p_joint * localValue;
+				} else {
+					entRateCont = 0.0;
+				}
+				entRate += entRateCont;
+			}
+		}
+		
+		return entRate;
+	}
+	
+	/**
+	 * Computes local active info storage for the given values
+	 *  
+	 * @param destNext
+	 * @param destPast
+	 * @param sourceCurrent
+	 * @return
+	 */
+	public double computeLocalFromPreviousObservations(int next, int past){
+		double logTerm = ( (double) jointCount[next][past] ) /
+		  ( (double) nextCount[next] *
+			(double) prevCount[past] );
+		logTerm *= (double) observations;
+		return Math.log(logTerm) / log_base;
+	}
+
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 *  
+	 * @param states time series of states
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[]){
+		int timeSteps = states.length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localActive = new double[timeSteps];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p];
+		}
+
+		int nextVal;
+		double logTerm = 0.0;
+		for (int t = k; t < timeSteps; t++) {
+			nextVal = states[t];
+			logTerm = ( (double) jointCount[nextVal][prevVal] ) /
+			  		  ( (double) nextCount[nextVal] *
+			  			(double) prevCount[prevVal] );
+			// Now account for the fact that we've
+			//  just used counts rather than probabilities,
+			//  and we've got two counts on the bottom
+			//  but one count on the top:
+			logTerm *= (double) observations;
+			localActive[t] = Math.log(logTerm) / log_base;
+			average += localActive[t];
+			if (localActive[t] > max) {
+				max = localActive[t];
+			} else if (localActive[t] < min) {
+				min = localActive[t];
+			}
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[t-k]];
+			prevVal *= base;
+			prevVal += states[t];
+		}
+		average = average/(double) (timeSteps - k);
+		
+		return localActive;
+		
+	}
+
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations(int states[][]){
+		int rows = states.length;
+		int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[][] localActive = new double[rows][columns];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int[] prevVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			prevVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				prevVal[c] *= base;
+				prevVal[c] += states[p][c];
+			}
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				nextVal = states[r][c];
+				logTerm = ( (double) jointCount[nextVal][prevVal[c]] ) /
+				  		  ( (double) nextCount[nextVal] *
+				  			(double) prevCount[prevVal[c]] );
+				// Now account for the fact that we've
+				//  just used counts rather than probabilities,
+				//  and we've got two counts on the bottom
+				//  but one count on the top:
+				logTerm *= (double) observations;
+				localActive[r][c] = Math.log(logTerm) / log_base;
+				average += localActive[r][c];
+				if (localActive[r][c] > max) {
+					max = localActive[r][c];
+				} else if (localActive[r][c] < min) {
+					min = localActive[r][c];
+				}
+				// Update the previous value:
+				prevVal[c] -= maxShiftedValue[states[r-k][c]];
+				prevVal[c] *= base;
+				prevVal[c] += states[r][c];
+			}
+		}
+		average = average/(double) (columns * (rows - k));
+		
+		return localActive;
+		
+	}
+	
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 * @return
+	 */
+	public double[][][] computeLocalFromPreviousObservations(int states[][][]){
+		int timeSteps = states.length;
+		int agentRows = states[0].length;
+		int agentColumns = states[0][0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[][][] localActive = new double[timeSteps][agentRows][agentColumns];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int[][] prevVal = new int[agentRows][agentColumns]; 
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				prevVal[r][c] = 0;
+				for (int p = 0; p < k; p++) {
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[p][r][c];
+				}
+			}
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					nextVal = states[t][r][c];
+					logTerm = ( (double) jointCount[nextVal][prevVal[r][c]] ) /
+					  		  ( (double) nextCount[nextVal] *
+					  			(double) prevCount[prevVal[r][c]] );
+					// Now account for the fact that we've
+					//  just used counts rather than probabilities,
+					//  and we've got two counts on the bottom
+					//  but one count on the top:
+					logTerm *= (double) observations;
+					localActive[t][r][c] = Math.log(logTerm) / log_base;
+					average += localActive[t][r][c];
+					if (localActive[t][r][c] > max) {
+						max = localActive[t][r][c];
+					} else if (localActive[t][r][c] < min) {
+						min = localActive[t][r][c];
+					}
+					// Update the previous value:
+					prevVal[r][c] -= maxShiftedValue[states[t-k][r][c]];
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[t][r][c];
+				}
+			}
+		}
+		average = average/(double) (agentRows * agentColumns * (timeSteps - k));
+		
+		return localActive;
+	}
+
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][], int col){
+		int rows = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localActive = new double[rows];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][col];
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int r = k; r < rows; r++) {
+			nextVal = states[r][col];
+			logTerm = ( (double) jointCount[nextVal][prevVal] ) /
+			  		  ( (double) nextCount[nextVal] *
+			  			(double) prevCount[prevVal] );
+			// Now account for the fact that we've
+			//  just used counts rather than probabilities,
+			//  and we've got two counts on the bottom
+			//  but one count on the top:
+			logTerm *= (double) observations;
+			localActive[r] = Math.log(logTerm) / log_base;
+			average += localActive[r];
+			if (localActive[r] > max) {
+				max = localActive[r];
+			} else if (localActive[r] < min) {
+				min = localActive[r];
+			}
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[r-k][col]];
+			prevVal *= base;
+			prevVal += states[r][col];
+		}
+		average = average/(double) (rows - k);
+		
+		return localActive;
+		
+	}
+	
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][][], int agentIndex1, int agentIndex2){
+		int timeSteps = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localActive = new double[timeSteps];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][agentIndex1][agentIndex2];
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int t = k; t < timeSteps; t++) {
+			nextVal = states[t][agentIndex1][agentIndex2];
+			logTerm = ( (double) jointCount[nextVal][prevVal] ) /
+			  		  ( (double) nextCount[nextVal] *
+			  			(double) prevCount[prevVal] );
+			// Now account for the fact that we've
+			//  just used counts rather than probabilities,
+			//  and we've got two counts on the bottom
+			//  but one count on the top:
+			logTerm *= (double) observations;
+			localActive[t] = Math.log(logTerm) / log_base;
+			average += localActive[t];
+			if (localActive[t] > max) {
+				max = localActive[t];
+			} else if (localActive[t] < min) {
+				min = localActive[t];
+			}
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[t-k][agentIndex1][agentIndex2]];
+			prevVal *= base;
+			prevVal += states[t][agentIndex1][agentIndex2];
+		}
+		average = average/(double) (timeSteps - k);
+		
+		return localActive;
+		
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage across an
+	 *  array of the states of homogeneous agents
+	 * Return an array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - time series array of states
+	 * @return
+	 */
+	public double[] computeLocal(int states[]) {
+		
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public double[][] computeLocal(int states[][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - array of states - 1st dimension is time, 2nd and 3rd are agent indices
+	 * @return
+	 */
+	public double[][][] computeLocal(int states[][][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local active information storage across an
+	 *  array of the states of homogeneous agents
+	 * Return the averagen
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - narray of states
+	 * @return
+	 */
+	public double computeAverageLocal(int states[]) {
+		
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local active information storage across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local active information storage across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - array of states - 1st dimension is time, 2nd and 3rd are agent indices
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public double[] computeLocal(int states[][], int col) {
+		
+		initialise();
+		addObservations(states, col);
+		return computeLocalFromPreviousObservations(states, col);
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - array of states - 1st dimension is time, 2nd and 3rd are agent indices
+	 * @param agentIndex1 row index of agent
+	 * @param agentIndex2 column index of agent
+	 * @return
+	 */
+	public double[] computeLocal(int states[][][], int agentIndex1, int agentIndex2) {
+		
+		initialise();
+		addObservations(states, agentIndex1, agentIndex2);
+		return computeLocalFromPreviousObservations(states, agentIndex1, agentIndex2);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local active information storage 
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][], int col) {
+		
+		initialise();
+		addObservations(states, col);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local active information storage 
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param maxEmbeddingLength - base of the states
+	 * @param states - array of states - 1st dimension is time, 2nd and 3rd are agent indices
+	 * @param agentIndex1 row index of agent
+	 * @param agentIndex2 column index of agent
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][][], int agentIndex1, int agentIndex2) {
+		
+		initialise();
+		addObservations(states, agentIndex1, agentIndex2);
+		return computeAverageLocalOfObservations();
+	}
+
+	public double getLastAverage() {
+		return average;
+	}
+
+	public double getLastMax() {
+		return max;
+	}
+
+	public double getLastMin() {
+		return min;
+	}
+	
+	/**
+	 * Writes the current probability distribution functions 
+	 *  
+	 * @return
+	 */
+	public void writePdfs() {
+		double mi = 0.0;
+		double miCont = 0.0;
+
+		System.out.println("nextVal p(next) prevVal p(prev) p(joint) logTerm localVal");
+		for (int nextVal = 0; nextVal < base; nextVal++) {
+			// compute p_next
+			double p_next = (double) nextCount[nextVal] / (double) observations;
+			for (int prevVal = 0; prevVal < base_power_k; prevVal++) {
+				// compute p_prev
+				double p_prev = (double) prevCount[prevVal] / (double) observations;
+				// compute p(prev, next)
+				double p_joint = (double) jointCount[nextVal][prevVal] / (double) observations;
+				// Compute MI contribution:
+				if (p_joint * p_next * p_prev > 0.0) {
+					double logTerm = p_joint / (p_next * p_prev);
+					double localValue = Math.log(logTerm) / log_base;
+					miCont = p_joint * localValue;
+					System.out.println(String.format("%7d    %.2f %7d    %.2f     %.2f    %.2f     %.2f",
+							nextVal, p_next, prevVal, p_prev, p_joint, logTerm, localValue));
+				} else {
+					miCont = 0.0;
+					System.out.println(String.format("%7d    %.2f %7d    %.2f     %.2f    %.2f     %.2f",
+							nextVal, p_next, prevVal, p_prev, p_joint, 0.0, 0.0));
+				}
+				mi += miCont;
+			}
+		}
+		System.out.println("Average is " + mi);
+		
+		return;
+	}
+
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[] x, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p];
+		}
+		return pastVal;
+	}
+
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param agentNumber
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[][] x, int agentNumber, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p][agentNumber];
+		}
+		return pastVal;
+	}
+
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param agentNumber
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[][][] x, int agentRow, int agentColumn, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p][agentRow][agentColumn];
+		}
+		return pastVal;
+	}
+}

java/source/infodynamics/measures/discrete/BlockEntropyCalculator.java

@@ -0,0 +1,517 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * Block entropy calculator
+ * Implemented separately from the single state entropy 
+ *  calculator to allow the single state calculator 
+ *  to have optimal performance.
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class BlockEntropyCalculator extends EntropyCalculator {
+
+	protected int blocksize = 0; // temporal blocksize to compute entropy over. Need initialised to 0 for changedSizes
+	protected int[] maxShiftedValue = null; // states * (base^(blocksize-1))
+
+	protected int base_power_blocksize = 0;
+
+	/**
+	 * User to create new instances through this factory method.
+	 * This allows us to return an efficient calculator for
+	 * base 2, for example, without the user needing to have 
+	 * knowledge of this. (This functionality is now obselete though)
+	 * 
+	 * @param blocksize
+	 * @param base
+	 * @return
+	 */
+	public static EntropyCalculator newInstance(int blocksize, int base) {
+		return new BlockEntropyCalculator(blocksize, base);
+	}
+	
+	/**
+	 * 
+	 * @param blocksize
+	 * @param base
+	 */
+	public BlockEntropyCalculator(int blocksize, int base) {
+
+		super(base);
+		
+		this.blocksize = blocksize;
+		base_power_blocksize = MathsUtils.power(base, blocksize);
+
+		if (blocksize <= 1) {
+			throw new RuntimeException("Blocksize " + blocksize + " is not > 1 for Block Entropy Calculator");
+		}
+		if (blocksize > Math.log(Integer.MAX_VALUE) / log_base) {
+			throw new RuntimeException("Base and blocksize combination too large");
+		}
+
+		// Create storage for counts of observations.
+		// We're recreating stateCount, which was created in super()
+		//  however the super() didn't create it for blocksize > 1
+		stateCount = new int[MathsUtils.power(base, blocksize)];
+
+		// Create constants for tracking stateValues
+		maxShiftedValue = new int[base];
+		for (int v = 0; v < base; v++) {
+			maxShiftedValue[v] = v * MathsUtils.power(base, blocksize-1);
+		}
+	}
+	
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 */
+	public void initialise(){
+		super.initialise();
+		MatrixUtils.fill(stateCount, 0);
+	}
+		
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][]) {
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - blocksize + 1)*columns; 
+		
+		// Initialise and store the current previous value for each column
+		int[] stateVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			stateVal[c] = 0;
+			for (int p = 0; p < blocksize - 1; p++) {
+				// Add the contribution from this observation
+				stateVal[c] += states[p][c];
+				// And shift up
+				stateVal[c] *= base;
+			}
+		}
+
+		// 1. Now count the tuples observed from the next row onwards
+		for (int r = blocksize - 1; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add the contribution from this observation
+				stateVal[c] += states[r][c];
+
+				// Add to the count for this particular state:
+				stateCount[stateVal[c]]++;
+				
+				// Remove the oldest observation from the state value
+				stateVal[c] -= maxShiftedValue[states[r-blocksize+1][c]];
+				stateVal[c] *= base;
+			}
+		}		
+	}
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 */
+	public void addObservations(int states[][][]) {
+		int timeSteps = states.length;
+		if (timeSteps == 0) {
+			return;
+		}
+		int agentRows = states[0].length;
+		if (agentRows == 0) {
+			return;
+		}
+		int agentColumns = states[0][0].length;
+		// increment the count of observations:
+		observations += (timeSteps - blocksize + 1) * agentRows * agentColumns; 
+		
+		// Initialise and store the current previous value for each column
+		int[][] stateVal = new int[agentRows][agentColumns];
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				stateVal[r][c] = 0;
+				for (int p = 0; p < blocksize - 1; p++) {
+					// Add the contribution from this observation
+					stateVal[r][c] += states[p][r][c];
+					// And shift up
+					stateVal[r][c] *= base;
+				}
+			}
+		}
+
+		// 1. Now count the tuples observed from the next row onwards
+		for (int t = blocksize - 1; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					// Add the contribution from this observation
+					stateVal[r][c] += states[t][r][c];
+	
+					// Add to the count for this particular state:
+					stateCount[stateVal[r][c]]++;
+					
+					// Remove the oldest observation from the state value
+					stateVal[r][c] -= maxShiftedValue[states[t-blocksize+1][r][c]];
+					stateVal[r][c] *= base;
+				}
+			}
+		}		
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][], int col) {
+		int rows = states.length;
+		// increment the count of observations:
+		observations += rows - blocksize + 1; 
+		
+		// Initialise and store the current previous value for each column
+		int stateVal = 0;
+		for (int p = 0; p < blocksize - 1; p++) {
+			// Add the contribution from this observation
+			stateVal += states[p][col];
+			// And shift up
+			stateVal *= base;
+		}
+
+		// 1. Count the tuples observed
+		for (int r = blocksize - 1; r < rows; r++) {
+			// Add the contribution from this observation
+			stateVal += states[r][col];
+
+			// Add to the count for this particular state:
+			stateCount[stateVal]++;
+			
+			// Remove the oldest observation from the state value
+			stateVal -= maxShiftedValue[states[r-blocksize+1][col]];
+			stateVal *= base;
+		}
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 */
+	public void addObservations(int states[][][], int agentIndex1, int agentIndex2) {
+		int timeSteps = states.length;
+		// increment the count of observations:
+		observations += timeSteps - blocksize + 1; 
+		
+		// Initialise and store the current previous value for each column
+		int stateVal = 0;
+		for (int p = 0; p < blocksize - 1; p++) {
+			// Add the contribution from this observation
+			stateVal += states[p][agentIndex1][agentIndex2];
+			// And shift up
+			stateVal *= base;
+		}
+
+		// 1. Count the tuples observed
+		for (int t = blocksize - 1; t < timeSteps; t++) {
+			// Add the contribution from this observation
+			stateVal += states[t][agentIndex1][agentIndex2];
+
+			// Add to the count for this particular state:
+			stateCount[stateVal]++;
+			
+			// Remove the oldest observation from the state value
+			stateVal -= maxShiftedValue[states[t-blocksize+1][agentIndex1][agentIndex2]];
+			stateVal *= base;
+		}
+	}
+
+	/**
+	 * Returns the average local entropy from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double ent = 0.0;
+		double entCont = 0.0;
+
+		max = 0;
+		min = 0;
+		for (int stateVal = 0; stateVal < base_power_blocksize; stateVal++) {
+			// compute p_state
+			double p_state = (double) stateCount[stateVal] / (double) observations;
+			if (p_state > 0.0) {
+				// Entropy takes the negative log:
+				double localValue = - Math.log(p_state) / log_2;
+				entCont = p_state * localValue;
+				if (localValue > max) {
+					max = localValue;
+				} else if (localValue < min) {
+					min = localValue;
+				}
+			} else {
+				entCont = 0.0;
+			}
+			ent += entCont;
+		}
+		
+		average = ent;
+		return ent;
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations(int states[][]){
+		int rows = states.length;
+		int columns = states[0].length;
+		
+		// Allocate for all rows even though we may not be assigning all of them
+		double[][] localEntropy = new double[rows][columns];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int[] stateVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			stateVal[c] = 0;
+			for (int p = 0; p < blocksize - 1; p++) {
+				stateVal[c] += states[p][c];
+				stateVal[c] *= base;
+			}
+		}
+		// StateVal just needs the next value put in before processing
+
+		for (int r = blocksize - 1; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add the most recent observation to the state count
+				stateVal[c] += states[r][c];
+				// Process the stateVal:
+				double p_state = (double) stateCount[stateVal[c]] / (double) observations;
+				// Entropy takes the negative log:
+				localEntropy[r][c] = - Math.log(p_state) / log_2;
+				average += localEntropy[r][c];
+				if (localEntropy[r][c] > max) {
+					max = localEntropy[r][c];
+				} else if (localEntropy[r][c] < min) {
+					min = localEntropy[r][c];
+				}
+				// Subtract out the oldest part of the state value:
+				stateVal[c] -= maxShiftedValue[states[r-blocksize+1][c]];
+				// And shift all upwards
+				stateVal[c] *= base;
+			}
+		}
+		average = average/(double) (columns * (rows - blocksize + 1));
+		
+		return localEntropy;
+		
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[][][] computeLocalFromPreviousObservations(int states[][][]){
+		int timeSteps = states.length;
+		int agentRows, agentColumns;
+		if (timeSteps == 0) {
+			agentRows = 0;
+			agentColumns = 0;
+		} else {
+			agentRows = states[0].length;
+			if (agentRows == 0) {
+				agentColumns = 0;
+			} else {
+				agentColumns = states[0][0].length;
+			}
+		}
+		
+		// Allocate for all rows even though we may not be assigning all of them
+		double[][][] localEntropy = new double[timeSteps][agentRows][agentColumns];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int[][] stateVal = new int[agentRows][agentColumns]; 
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				stateVal[r][c] = 0;
+				for (int p = 0; p < blocksize - 1; p++) {
+					stateVal[r][c] += states[p][r][c];
+					stateVal[r][c] *= base;
+				}
+			}
+		}
+		// StateVal just needs the next value put in before processing
+
+		for (int t = blocksize - 1; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					// Add the most recent observation to the state count
+					stateVal[r][c] += states[t][r][c];
+					// Process the stateVal:
+					double p_state = (double) stateCount[stateVal[r][c]] / (double) observations;
+					// Entropy takes the negative log:
+					localEntropy[t][r][c] = - Math.log(p_state) / log_2;
+					average += localEntropy[t][r][c];
+					if (localEntropy[t][r][c] > max) {
+						max = localEntropy[t][r][c];
+					} else if (localEntropy[t][r][c] < min) {
+						min = localEntropy[t][r][c];
+					}
+					// Subtract out the oldest part of the state value:
+					stateVal[r][c] -= maxShiftedValue[states[t-blocksize+1][r][c]];
+					// And shift all upwards
+					stateVal[r][c] *= base;
+				}
+			}
+		}
+		average = average/(double) (agentRows * agentColumns * (timeSteps - blocksize + 1));
+		
+		return localEntropy;
+		
+	}
+
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][], int col){
+		int rows = states.length;
+		//int columns = states[0].length;
+		// First row to consider the destination cell at:
+		int startRow = blocksize - 1;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntropy = new double[rows];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int stateVal = 0; 
+		for (int p = 0; p < blocksize - 1; p++) {
+			stateVal += states[p][col];
+			stateVal *= base;
+		}
+		// StateVal just needs the next value put in before processing
+
+		for (int r = startRow; r < rows; r++) {
+			// Add the most recent observation to the state count
+			stateVal += states[r][col];
+			// Process the stateVal:
+			double p_state = (double) stateCount[stateVal] / (double) observations;
+			// Entropy takes the negative log:
+			localEntropy[r] = - Math.log(p_state) / log_2;
+			average += localEntropy[r];
+			if (localEntropy[r] > max) {
+				max = localEntropy[r];
+			} else if (localEntropy[r] < min) {
+				min = localEntropy[r];
+			}
+			// Subtract out the oldest part of the state value:
+			stateVal -= maxShiftedValue[states[r-blocksize+1][col]];
+			// And shift all upwards
+			stateVal *= base;
+		}
+		average = average/(double) (rows - blocksize + 1);
+		
+		return localEntropy;
+		
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][][], int agentIndex1, int agentIndex2){
+		int timeSteps = states.length;
+		//int columns = states[0].length;
+		// First time to consider the destination cell at:
+		int startTime = blocksize - 1;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntropy = new double[timeSteps];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int stateVal = 0; 
+		for (int p = 0; p < blocksize - 1; p++) {
+			stateVal += states[p][agentIndex1][agentIndex2];
+			stateVal *= base;
+		}
+		// StateVal just needs the next value put in before processing
+
+		for (int t = startTime; t < timeSteps; t++) {
+			// Add the most recent observation to the state count
+			stateVal += states[t][agentIndex1][agentIndex2];
+			// Process the stateVal:
+			double p_state = (double) stateCount[stateVal] / (double) observations;
+			// Entropy takes the negative log:
+			localEntropy[t] = - Math.log(p_state) / log_2;
+			average += localEntropy[t];
+			if (localEntropy[t] > max) {
+				max = localEntropy[t];
+			} else if (localEntropy[t] < min) {
+				min = localEntropy[t];
+			}
+			// Subtract out the oldest part of the state value:
+			stateVal -= maxShiftedValue[states[t-blocksize+1][agentIndex1][agentIndex2]];
+			// And shift all upwards
+			stateVal *= base;
+		}
+		average = average/(double) (timeSteps - blocksize + 1);
+		
+		return localEntropy;
+		
+	}
+}

java/source/infodynamics/measures/discrete/ChannelCalculator.java

@@ -0,0 +1,51 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * An interface for calculators computing measures from a source to a destination.
+ * 
+ * 
+ * @author Joseph Lizier, jlizier at gmail.com
+ *
+ */
+public interface ChannelCalculator {
+
+	/**
+	 * Initialise the calculator
+	 *
+	 */
+	public void initialise();
+	
+	/**
+	 * Add observations for the source and destination
+	 * 
+	 * @param states
+	 * @param destIndex
+	 * @param sourceIndex
+	 */
+	public void addObservations(int states[][], int destIndex, int sourceIndex);
+
+	/**
+	 * Add observations for the source and destination
+	 * 
+	 * @param dest
+	 * @param source
+	 */
+	public void addObservations(int[] dest, int[] source);
+	
+	/**
+	 * Compute the value of the measure
+	 * 
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations();
+	
+	/**
+	 * Compute the significance of the average value for the channel measure here
+	 * 
+	 * @param numPermutationsToCheck
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck);
+}

java/source/infodynamics/measures/discrete/CombinedActiveEntRateCalculator.java

@@ -0,0 +1,516 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * A combined calculator for the active information, entropy rate and entropy.
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class CombinedActiveEntRateCalculator {
+
+	private double averageActive = 0.0;
+	private double maxActive = 0.0;
+	private double minActive = 0.0;
+	private double averageEntRate = 0.0;
+	private double maxEntRate = 0.0;
+	private double minEntRate = 0.0;
+	private double averageEntropy = 0.0;
+	private double maxEntropy = 0.0;
+	private double minEntropy = 0.0;
+	
+	private int observations = 0;
+	private int k = 0; // history length k. Need initialised to 0 for changedSizes
+	private int base = 0; // number of individual states. Need initialised to 0 for changedSizes
+	private int[][]	jointCount = null; // Count for (i[t+1], i[t]) tuples
+	private int[] prevCount = null; // Count for i[t]		
+	private int[] nextCount = null; // Count for i[t+1]
+
+	// Space-time results (ST)
+	// - for a homogeneous multi-agent system
+	public class CombinedActiveEntRateLocalSTResults {
+		public double[][] localActiveInfo;
+		public double[][] localEntropyRate;
+		public double[][] localEntropy;
+	}
+	
+	public class CombinedActiveEntRateLocalResults {
+		public double[] localActiveInfo;
+		public double[] localEntropyRate;
+		public double[] localEntropy;
+	}
+
+	public CombinedActiveEntRateCalculator() {
+		super();
+	}
+
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 *
+	 */
+	public void initialise(int history, int base){
+		averageActive = 0.0;
+		maxActive = 0.0;
+		minActive = 0.0;
+		observations = 0;
+		
+		boolean changedSizes = true;
+		if ((this.base == base) && (this.k == history)) {
+			changedSizes = false;
+		}
+		this.base = base;
+		k = history;
+		
+		if (history < 1) {
+			throw new RuntimeException("History k " + history + " is not >= 1 for ActiveInfo Calculator");
+		}
+		
+		if (changedSizes) {
+			// Create storage for counts of observations
+			jointCount = new int[base][MathsUtils.power(base, history)];
+			prevCount = new int[MathsUtils.power(base, history)];
+			nextCount = new int[base];
+		} else {
+			// Just set counts to zeros without recreating the space
+			MatrixUtils.fill(jointCount, 0);
+			MatrixUtils.fill(prevCount, 0);
+			MatrixUtils.fill(nextCount, 0);
+		}
+	}
+		
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states
+	 */
+	public void addObservations(int states[][]) {
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - k)*columns; 
+		
+		// 1. Count the tuples observed
+		int prevVal, nextVal;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add to the count for this particular transition:
+				// (cell's assigned as above)
+				nextVal = states[r][c];
+				prevVal = 0;
+				int multiplier = 1;
+				for (int p = 1; p <= k; p++) {
+					prevVal += states[r-p][c] * multiplier;			
+					multiplier *= base;
+				}
+				jointCount[nextVal][prevVal]++;
+				prevCount[prevVal]++;
+				nextCount[nextVal]++;					
+			}
+		}		
+	}
+	
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states
+	 */
+	public void addObservations(int states[][], int col) {
+		int rows = states.length;
+		// increment the count of observations:
+		observations += (rows - k); 
+		
+		// 1. Count the tuples observed
+		int prevVal, nextVal;
+		for (int r = k; r < rows; r++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			nextVal = states[r][col];
+			prevVal = 0;
+			int multiplier = 1;
+			for (int p = 1; p <= k; p++) {
+				prevVal += states[r-p][col] * multiplier;			
+				multiplier *= base;
+			}
+			jointCount[nextVal][prevVal]++;
+			prevCount[prevVal]++;
+			nextCount[nextVal]++;					
+		}
+	}
+
+	/**
+	 * Computes the average local values from
+	 *  the observed values which have been passed in previously. 
+	 * Access the averages, mins and maxes from the accessor methods.
+	 *  
+	 * @return
+	 */
+	public void computeAverageLocalOfObservations() {
+		double activeCont, entropyCont;
+
+		resetOverallStats();
+		double localActiveValue, localEntropyValue, localEntRateValue, logTerm;
+		
+		for (int nextVal = 0; nextVal < base; nextVal++) {
+			// compute p_next
+			double p_next = (double) nextCount[nextVal] / (double) observations;
+			// ** ENTROPY **
+			if (p_next > 0.0) {
+				// Entropy takes the negative log:
+				localEntropyValue = - Math.log(p_next) / Math.log(base);
+				entropyCont = p_next * localEntropyValue;
+				if (localEntropyValue > maxEntropy) {
+					maxEntropy = localEntropyValue;
+				} else if (localEntropyValue < minEntropy) {
+					minEntropy = localEntropyValue;
+				}
+			} else {
+				localEntropyValue = 0.0;
+				entropyCont = 0.0;
+				continue; // no point computing ent rate and active info, will be zeros
+			}
+			averageEntropy += entropyCont;
+			for (int prevVal = 0; prevVal < MathsUtils.power(base, k); prevVal++) {
+				// compute p_prev
+				double p_prev = (double) prevCount[prevVal] / (double) observations;
+				// compute p(prev, next)
+				double p_joint = (double) jointCount[nextVal][prevVal] / (double) observations;
+				if (p_joint > 0.0) {
+					// ** ACTIVE INFO **
+					logTerm = p_joint / (p_next * p_prev);
+					localActiveValue = Math.log(logTerm) / Math.log(base);
+					activeCont = p_joint * localActiveValue;
+					if (localActiveValue > maxActive) {
+						maxActive = localActiveValue;
+					} else if (localActiveValue < minActive) {
+						minActive = localActiveValue;
+					}
+				} else {
+					localActiveValue = 0.0;
+					activeCont = 0.0;
+				}
+				averageActive += activeCont;
+				// ** ENTROPY RATE ** = ENTROPY - ACTIVE
+				localEntRateValue = localEntropyValue - localActiveValue;
+				if (localEntRateValue > maxEntRate) {
+					maxEntRate = localEntRateValue;
+				} else if (localEntRateValue < maxEntRate) {
+					maxEntRate = localEntRateValue;
+				}
+			}
+		}
+		averageEntRate = averageEntropy - averageActive;
+		return;
+	}
+	
+	/**
+	 * Computes local values for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public CombinedActiveEntRateLocalSTResults computeLocalFromPreviousObservations(int states[][]){
+		int rows = states.length;
+		int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[][] localActive = new double[rows][columns];
+		double[][] localEntRate = new double[rows][columns];
+		double[][] localEntropy = new double[rows][columns];
+		
+		resetOverallStats();
+		
+		int prevVal, nextVal;
+		double logTerm;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				nextVal = states[r][c];
+				prevVal = 0;
+				int multiplier = 1;
+				for (int p = 1; p <= k; p++) {
+					prevVal += states[r-p][c] * multiplier;			
+					multiplier *= base;
+				}
+				// ** ENTROPY **
+				double p_next = (double) nextCount[nextVal] / (double) observations;
+				// Entropy takes the negative log:
+				localEntropy[r][c] = - Math.log(p_next) / Math.log(base);
+				averageEntropy += localEntropy[r][c];
+				if (localEntropy[r][c] > maxEntropy) {
+					maxEntropy = localEntropy[r][c];
+				} else if (localEntropy[r][c] < minEntropy) {
+					minEntropy = localEntropy[r][c];
+				}
+				// ** ACTIVE INFO **
+				logTerm = ( (double) jointCount[nextVal][prevVal] ) /
+				  		  ( (double) nextCount[nextVal] *
+				  			(double) prevCount[prevVal] );
+				// Now account for the fact that we've
+				//  just used counts rather than probabilities,
+				//  and we've got two counts on the bottom
+				//  but one count on the top:
+				logTerm *= (double) observations;
+				localActive[r][c] = Math.log(logTerm) / Math.log(base);
+				averageActive += localActive[r][c];
+				if (localActive[r][c] > maxActive) {
+					maxActive = localActive[r][c];
+				} else if (localActive[r][c] < minActive) {
+					minActive = localActive[r][c];
+				}
+				// ** ENTROPY RATE **
+				localEntRate[r][c] = localEntropy[r][c] - localActive[r][c];
+				if (localEntRate[r][c] > maxEntRate) {
+					maxEntRate = localEntRate[r][c];
+				} else if (localEntRate[r][c] < minEntRate) {
+					minEntRate = localEntRate[r][c];
+				}
+			}
+		}
+		averageActive = averageActive/(double) (columns * (rows - k));
+		averageEntropy = averageEntropy/(double) (columns * (rows - k));
+		averageEntRate = averageEntropy - averageActive;
+		
+		// Package results ready for return
+		CombinedActiveEntRateLocalSTResults results = new CombinedActiveEntRateLocalSTResults();
+		results.localActiveInfo = localActive;
+		results.localEntropyRate = localEntRate;
+		results.localEntropy = localEntropy;
+		
+		return results;
+	}
+	
+	/**
+	 * Computes local values for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public CombinedActiveEntRateLocalResults computeLocalFromPreviousObservations(int states[][], int col){
+		int rows = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localActive = new double[rows];
+		double[] localEntRate = new double[rows];
+		double[] localEntropy = new double[rows];
+
+		resetOverallStats();
+		
+		int prevVal, nextVal;
+		double logTerm = 0.0;
+		for (int r = k; r < rows; r++) {
+			nextVal = states[r][col];
+			prevVal = 0;
+			int multiplier = 1;
+			for (int p = 1; p <= k; p++) {
+				prevVal += states[r-p][col] * multiplier;			
+				multiplier *= base;
+			}
+			// ** ENTROPY **
+			double p_next = (double) nextCount[nextVal] / (double) observations;
+			// Entropy takes the negative log:
+			localEntropy[r] = - Math.log(p_next) / Math.log(base);
+			averageEntropy += localEntropy[r];
+			if (localEntropy[r] > maxEntropy) {
+				maxEntropy = localEntropy[r];
+			} else if (localEntropy[r] < minEntropy) {
+				minEntropy = localEntropy[r];
+			}
+			// ** ACTIVE INFO **
+			logTerm = ( (double) jointCount[nextVal][prevVal] ) /
+			  		  ( (double) nextCount[nextVal] *
+			  			(double) prevCount[prevVal] );
+			// Now account for the fact that we've
+			//  just used counts rather than probabilities,
+			//  and we've got two counts on the bottom
+			//  but one count on the top:
+			logTerm *= (double) observations;
+			localActive[r] = Math.log(logTerm) / Math.log(base);
+			averageActive += localActive[r];
+			if (localActive[r] > maxActive) {
+				maxActive = localActive[r];
+			} else if (localActive[r] < minActive) {
+				minActive = localActive[r];
+			}
+			// ** ENTROPY RATE **
+			localEntRate[r] = localEntropy[r] - localActive[r];
+			if (localEntRate[r] > maxEntRate) {
+				maxEntRate = localEntRate[r];
+			} else if (localEntRate[r] < minEntRate) {
+				minEntRate = localEntRate[r];
+			}
+		}
+		averageActive = averageActive/(double) (rows - k);
+		averageEntropy = averageEntropy/(double) (rows - k);
+		averageEntRate = averageEntropy - averageActive;
+
+		// Package results ready for return
+		CombinedActiveEntRateLocalResults results = new CombinedActiveEntRateLocalResults();
+		results.localActiveInfo = localActive;
+		results.localEntropyRate = localEntRate;
+		results.localEntropy = localEntropy;
+
+		return results;
+		
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute local values across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param history - parameter k
+	 * @param base - base of the states
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public CombinedActiveEntRateLocalSTResults computeLocal(int history, int base, int states[][]) {
+		
+		initialise(history, base);
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute average local values across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param history - parameter k
+	 * @param base - base of the states
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public void computeAverageLocal(int history, int base, int states[][]) {
+		
+		initialise(history, base);
+		addObservations(states);
+		computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local values for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param base - base of the states
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public CombinedActiveEntRateLocalResults computeLocal(int history, int base, int states[][], int col) {
+		
+		initialise(history, base);
+		addObservations(states, col);
+		return computeLocalFromPreviousObservations(states, col);
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute average local values
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * 
+	 * @param history - parameter k
+	 * @param base - base of the states
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public void computeAverageLocal(int history, int base, int states[][], int col) {
+		
+		initialise(history, base);
+		addObservations(states, col);
+		computeAverageLocalOfObservations();
+	}
+
+	private void resetOverallStats() {
+		averageActive = 0;
+		maxActive = 0;
+		minActive = 0;
+		averageEntRate = 0;
+		maxEntRate = 0;
+		minEntRate = 0;
+		averageEntropy = 0;
+		maxEntropy = 0;
+		minEntropy = 0;
+	}
+	
+	/*
+	 * Accessors for last active information computation
+	 */
+	public double getLastAverageActive() {
+		return averageActive;
+	}
+	public double getLastMaxActive() {
+		return maxActive;
+	}
+	public double getLastMinActive() {
+		return minActive;
+	}
+
+	/*
+	 * Accessors for last entropy rate computation
+	 */
+	public double getLastAverageEntRate() {
+		return averageEntRate;
+	}
+	public double getLastMaxEntRate() {
+		return maxEntRate;
+	}
+	public double getLastMinEntRate() {
+		return minEntRate;
+	}
+
+	/*
+	 * Accessors for last entropy computation
+	 */
+	public double getLastAverageEntropy() {
+		return averageEntropy;
+	}
+	public double getLastMaxEntropy() {
+		return maxEntropy;
+	}
+	public double getLastMinEntropy() {
+		return minEntropy;
+	}
+	
+	// Compute the metrics in several different ways over test data.
+	// Designed to be called from a JUnit test.
+	public double[][] test() {
+		// TODO
+		return null;
+	}
+}

java/source/infodynamics/measures/discrete/CompleteTransferEntropyCalculator.java

@@ -0,0 +1,837 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+
+/**
+ * Implements complete transfer entropy (see Lizier et al, PRE, 2008)
+ * Complete transfer entropy  = transfer entropy conditioned on all causal information
+ *  contributors to the destination.
+ * This class can also be used for incrementally conditioned mutual information terms 
+ *  (see Lizier et al, Chaos 2010) by only supplying a limited
+ *  number of the causal information contributors in the array of other agents to be 
+ *  conditioned on.
+ * The causal information contributors (either their offsets or their absolute column numbers)
+ *  should be supplied in the same order in every method call, otherwise the answer supplied will
+ *  be incorrect.
+ * 
+ * Ideally, this class would extend ContextOfPastMeasure, however
+ *  by conditioning on other info contributors, we need to alter
+ *  the arrays pastCount and nextPastCount to consider all
+ *  conditioned variables (i.e. other sources) also.
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class CompleteTransferEntropyCalculator extends InfoMeasureCalculator {
+
+	protected int k = 0; // history length k.
+	protected int base_power_k = 0;
+	protected int base_power_num_others = 0;
+	protected int numOtherInfoContributors = 0;
+	protected int[][][][] sourceDestPastOthersCount = null;	// count for (i-j[n],i[n+1],i[n]^k,others) tuples
+	protected int[][][] sourcePastOthersCount = null;			// count for (i-j[n],i[n]^k,others) tuples
+	protected int[][][] destPastOthersCount = null; // Count for (i[n+1], i[n]^k,others) tuples
+	protected int[][] pastOthersCount = null; // Count for (i[n]^k,others)
+	protected int[] maxShiftedValue = null; // states * (base^(k-1))
+
+	/**
+	 * First time step at which we can take an observation
+	 *  (needs to account for k previous steps)
+	 */
+	protected int startObservationTime = 1;
+	
+	/**
+	 * User to create new instances through this factory method.
+	 * This allows us to return an efficient calculator for
+	 * base 2, for example, without the user needing to have 
+	 * knowledge of this.
+	 * @param base
+	 * @param history
+	 * @param numOtherInfoContributors
+	 * 
+	 * @return
+	 */
+	public static CompleteTransferEntropyCalculator
+		newInstance(int base, int history, int numOtherInfoContributors) {
+		
+		return new CompleteTransferEntropyCalculator
+					(base, history, numOtherInfoContributors);
+		
+		// Old code for an attempted optimisation:
+		/*
+		if (isPowerOf2(base)) {
+			return new CompleteTransferEntropyCalculatorBase2
+						(base, history, numOtherInfoContributors);
+		} else {
+			return new CompleteTransferEntropyCalculator
+					(base, history, numOtherInfoContributors);
+		}
+		*/
+	}
+	
+	/**
+	 * 
+	 * 
+	 * @param base
+	 * @param history
+	 * @param numOtherInfoContributors number of information contributors
+	 *   (other than the past of the destination, if history < 1,
+	 *   of the source) to condition on.
+	 */
+	public CompleteTransferEntropyCalculator
+		(int base, int history, int numOtherInfoContributors) {
+
+		super(base);
+		
+		k = history;
+		this.numOtherInfoContributors = numOtherInfoContributors;
+		base_power_k = MathsUtils.power(base, k);
+		base_power_num_others = MathsUtils.power(base, numOtherInfoContributors);
+		
+		// Relaxing this assumption so we can use this calculation as
+		//  a time-lagged conditional MI at will:
+		//if (k < 1) {
+		//	throw new RuntimeException("History k " + history + " is not >= 1 a ContextOfPastMeasureCalculator");
+		//}
+		
+		// Which time step do we start taking observations from?
+		// Normally this is k (to allow k previous time steps)
+		//  but if k==0 (becoming a lagged MI), it's 1.
+		startObservationTime = Math.max(k, 1);
+
+		// check that we can convert the base tuple into an integer ok
+		if (k > Math.log(Integer.MAX_VALUE) / log_base) {
+			throw new RuntimeException("Base and history combination too large");
+		}
+		if (numOtherInfoContributors < 1) {
+			throw new RuntimeException("Number of other info contributors < 1 for CompleteTECalculator");
+		}
+		
+		// Create storage for counts of observations
+		sourceDestPastOthersCount = new int[base][base][base_power_k][base_power_num_others];
+		sourcePastOthersCount = new int[base][base_power_k][base_power_num_others];
+		destPastOthersCount = new int [base][base_power_k][base_power_num_others];
+		pastOthersCount = new int[base_power_k][base_power_num_others];
+		
+		// Create constants for tracking prevValues
+		maxShiftedValue = new int[base];
+		for (int v = 0; v < base; v++) {
+			maxShiftedValue[v] = v * MathsUtils.power(base, k-1);
+		}
+	}
+
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 *
+	 */
+	public void initialise(){
+		super.initialise();
+		
+		MatrixUtils.fill(sourceDestPastOthersCount, 0);
+		MatrixUtils.fill(sourcePastOthersCount, 0);
+		MatrixUtils.fill(destPastOthersCount, 0);
+		MatrixUtils.fill(pastOthersCount, 0);
+	}
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs, and all are assumed
+ 	 *  to have other info contributors at same offsets.
+ 	 * 
+	 * @param states
+	 * @param j - number of columns to compute transfer entropy across
+	 * 	(i.e. src i-j, dest i: transfer is j cells to the right) 
+	 * @param othersOffsets offsets of the other information contributors.
+	 *        othersOffsets is permitted to include j, it will be ignored.
+	 *        Offset is signed the same way as j.
+	 */
+	public void addObservations(int states[][], int j, int othersOffsets[]) {
+		addObservations(states, j, othersOffsets, false);
+	}
+	private void addObservations(int states[][], int j, int othersOffsets[], boolean cleanedOthers) {
+		
+		int[] cleanedOthersOffsets;
+		if (cleanedOthers) {
+			cleanedOthersOffsets = othersOffsets;
+		} else {
+			cleanedOthersOffsets = cleanOffsetOthers(othersOffsets, j, k > 0);
+			// This call made redundant by cleanOffsetOthers:
+			// confirmEnoughOffsetOthers(othersOffsets, j);
+		}
+		
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - startObservationTime)*columns; 
+		
+		// Initialise and store the current previous value for each column
+		int[] pastVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			pastVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				pastVal[c] *= base;
+				pastVal[c] += states[p][c];
+			}
+		}
+		
+		// 1. Count the tuples observed
+		int destVal, sourceVal, othersVal;
+		for (int r = startObservationTime; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add to the count for this particular transition:
+				// (cell's assigned as above)
+				destVal = states[r][c];
+				sourceVal = states[r-1][(c-j+columns) % columns];
+				othersVal = 0;
+				for (int o = 0; o < cleanedOthersOffsets.length; o++) {
+					// Include this other contributor
+					othersVal *= base;
+					othersVal += states[r-1][(c-cleanedOthersOffsets[o]+columns) % columns];
+				}
+				sourceDestPastOthersCount[sourceVal][destVal][pastVal[c]][othersVal]++;
+				sourcePastOthersCount[sourceVal][pastVal[c]][othersVal]++;
+				destPastOthersCount[destVal][pastVal[c]][othersVal]++;
+				pastOthersCount[pastVal[c]][othersVal]++;
+				// Update the previous value:
+				if (k > 0) {
+					pastVal[c] -= maxShiftedValue[states[r-k][c]];
+					pastVal[c] *= base;
+					pastVal[c] += states[r][c];
+				}
+			}
+		}		
+	}
+	
+	/**
+ 	 * Add observations for a single source-destination pair of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states
+	 */
+	public void addObservations(int states[][], int destCol, int sourceCol, int[] othersAbsolute) {
+		addObservations(states, destCol, sourceCol, othersAbsolute, false);
+	}
+	private void addObservations(int states[][], int destCol, int sourceCol, int[] othersAbsolute, boolean cleanedOthers) {
+
+		int[] cleanedOthersAbsolute;
+		if (cleanedOthers) {
+			cleanedOthersAbsolute = othersAbsolute;
+		} else {
+			cleanedOthersAbsolute = cleanAbsoluteOthers(othersAbsolute, destCol,
+					sourceCol, k > 0);
+			// This call made redundant by cleanOffsetOthers:
+			// confirmEnoughAbsoluteOthers(othersAbsolute, destCol, sourceCol);
+		}
+		
+		int rows = states.length;
+		// increment the count of observations:
+		observations += (rows - startObservationTime); 
+		
+		// Initialise and store the current previous value for each column
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += states[p][destCol];
+		}
+		
+		// 1. Count the tuples observed
+		int destVal, sourceVal, othersVal;
+		for (int r = startObservationTime; r < rows; r++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			destVal = states[r][destCol];
+			sourceVal = states[r-1][sourceCol];
+			othersVal = 0;
+			for (int o = 0; o < cleanedOthersAbsolute.length; o++) {
+				// Include this other contributor
+				othersVal *= base;
+				othersVal += states[r-1][cleanedOthersAbsolute[o]];
+			}
+			sourceDestPastOthersCount[sourceVal][destVal][pastVal][othersVal]++;
+			sourcePastOthersCount[sourceVal][pastVal][othersVal]++;
+			destPastOthersCount[destVal][pastVal][othersVal]++;
+			pastOthersCount[pastVal][othersVal]++;
+			// Update the previous value:
+			if (k > 0) {
+				pastVal -= maxShiftedValue[states[r-k][destCol]];
+				pastVal *= base;
+				pastVal += states[r][destCol];
+			}
+		}
+	}
+
+	/**
+	 * Returns the average local transfer entropy from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double te = 0.0;
+		double teCont = 0.0;
+
+		max = 0;
+		min = 0;
+		double meanSqLocals = 0;
+		for (int othersVal = 0; othersVal < this.base_power_num_others; othersVal++) {
+			for (int pastVal = 0; pastVal < base_power_k; pastVal++) {
+				for (int destVal = 0; destVal < base; destVal++) {
+					for (int sourceVal = 0; sourceVal < base; sourceVal++) {
+						// Compute TE contribution:
+						if (sourceDestPastOthersCount[sourceVal][destVal][pastVal][othersVal] != 0) {
+							/* Double check: should never happen
+							if ((sourcePastCount[sourceVal][pastVal][othersVal] == 0) ||
+								(destPastCount[destVal][pastVal][othersVal] == 0) ||
+								(pastCount[pastVal][othersVal] == 0)) {
+								throw new RuntimeException("one subcount was zero!!");
+							}
+							*/
+							// compute p(source,dest,past)
+							double p_source_dest_past_others = (double)
+								sourceDestPastOthersCount[sourceVal][destVal][pastVal][othersVal] / (double) observations;
+							
+							double logTerm = ((double) sourceDestPastOthersCount[sourceVal][destVal][pastVal][othersVal] / (double) sourcePastOthersCount[sourceVal][pastVal][othersVal]) /
+							 	((double) destPastOthersCount[destVal][pastVal][othersVal] / (double) pastOthersCount[pastVal][othersVal]);
+							double localValue = Math.log(logTerm) / log_2;
+							teCont = p_source_dest_past_others * localValue;
+							if (localValue > max) {
+								max = localValue;
+							} else if (localValue < min) {
+								min = localValue;
+							}
+							// Add this contribution to the mean 
+							//  of the squared local values
+							meanSqLocals += teCont * localValue;
+						} else {
+							teCont = 0.0;
+						}
+						te += teCont;
+					}
+				}
+			}
+		}
+		
+		average = te;
+		std = Math.sqrt(meanSqLocals - average * average);
+		return te;
+	}
+	
+	/**
+	 * Compute the significance of obtaining the given average TE from the given observations
+	 * 
+	 * 	This is as per Chavez et. al., "Statistical assessment of nonlinear causality:
+	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128.
+	 *  except that we've using conditional/complete TE here.
+	 *
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) {
+		double actualTE = computeAverageLocalOfObservations();
+		
+		// Reconstruct the source values (not necessarily in order)
+		int[] sourceValues = new int[observations];
+		int t_s = 0;
+		for (int sourceVal = 0; sourceVal < base; sourceVal++) {
+			// Count up the number of times this source value was observed:
+			int numberOfSamples = 0;
+			for (int pastVal = 0; pastVal < base_power_k; pastVal++) {
+				for (int othersVal = 0; othersVal < this.base_power_num_others; othersVal++) {
+					numberOfSamples += sourcePastOthersCount[sourceVal][pastVal][othersVal];
+				}
+			}
+			// Now add all of these as unordered observations:
+			MatrixUtils.fill(sourceValues, sourceVal, t_s, numberOfSamples);
+			t_s += numberOfSamples;
+		}
+		
+		// And construct unordered (dest,past,others) tuples.
+		// It doesn't matter that we've appeared to destroy the ordering here because
+		//  the joint distribution destPastOthersCount is actually preserved in
+		//  our construction of pastVal and destValues and othersValues together here.
+		int[] destValues = new int[observations];
+		int[] pastValues = new int[observations];
+		int[] othersValues = new int[observations];
+		int t_d = 0;
+		int t_p = 0;
+		int t_o = 0;
+		for (int pastVal = 0; pastVal < base_power_k; pastVal++) {
+			for (int othersVal = 0; othersVal < this.base_power_num_others; othersVal++) {
+				// Add in pastOthersCount[pastVal][othersVal] dummy past values
+				MatrixUtils.fill(pastValues, pastVal, t_p,
+						pastOthersCount[pastVal][othersVal]);
+				t_p += pastOthersCount[pastVal][othersVal];
+				// Add in pastOthersCount[pastVal][othersVal] dummy others values
+				MatrixUtils.fill(othersValues, othersVal, t_o,
+						pastOthersCount[pastVal][othersVal]);
+				t_o += pastOthersCount[pastVal][othersVal];
+				for (int destVal = 0; destVal < base; destVal++) {
+					MatrixUtils.fill(destValues, destVal, t_d,
+						destPastOthersCount[destVal][pastVal][othersVal]);
+					t_d += destPastOthersCount[destVal][pastVal][othersVal];
+				}
+			}
+		}
+		
+		// Construct new source orderings based on the source probabilities only
+		// Generate the re-ordered indices:
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(observations, numPermutationsToCheck);
+
+		CompleteTransferEntropyCalculator cte = newInstance(base, k, numOtherInfoContributors);
+		cte.initialise();
+		cte.observations = observations;
+		cte.pastOthersCount = pastOthersCount;
+		cte.destPastOthersCount = destPastOthersCount;
+		int countWhereTeIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the source
+			int[] newSourceData = MatrixUtils.extractSelectedTimePoints(sourceValues, newOrderings[p]);
+			// compute the joint probability distributions
+			MatrixUtils.fill(cte.sourceDestPastOthersCount, 0);
+			MatrixUtils.fill(cte.sourcePastOthersCount, 0);
+			for (int t = 0; t < observations; t++) {
+				cte.sourcePastOthersCount[newSourceData[t]][pastValues[t]][othersValues[t]]++;
+				cte.sourceDestPastOthersCount[newSourceData[t]][destValues[t]]
+				                            [pastValues[t]][othersValues[t]]++;
+			}
+			// And get a TE value for this realisation:
+			double newTe = cte.computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newTe;
+			if (newTe >= actualTE) {
+				countWhereTeIsMoreSignificantThanOriginal++;
+			}
+
+		}
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereTeIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualTE;
+		return measDistribution;
+	}
+	
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method.
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations
+		(int states[][], int j, int othersOffsets[]){
+		
+		return computeLocalFromPreviousObservations(states, j, othersOffsets, false);
+	}
+	private double[][] computeLocalFromPreviousObservations
+		(int states[][], int j, int othersOffsets[], boolean cleanedOthers){
+		
+		int[] cleanedOthersOffsets;
+		if (cleanedOthers) {
+			cleanedOthersOffsets = othersOffsets;
+		} else {
+			cleanedOthersOffsets = cleanOffsetOthers(othersOffsets, j, k > 0);
+			// This call made redundant by cleanOffsetOthers:
+			// confirmEnoughOffsetOthers(othersOffsets, j);
+		}
+
+		int rows = states.length;
+		int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[][] localTE = new double[rows][columns];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int[] pastVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			pastVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				pastVal[c] *= base;
+				pastVal[c] += states[p][c];
+			}
+		}
+		int destVal, sourceVal, othersVal;
+		double logTerm;
+		for (int r = startObservationTime; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				destVal = states[r][c];
+				sourceVal = states[r-1][(c-j+columns) % columns];
+				othersVal = 0;
+				for (int o = 0; o < cleanedOthersOffsets.length; o++) {
+					// Include this other contributor
+					othersVal *= base;
+					othersVal += states[r-1][(c-cleanedOthersOffsets[o]+columns) % columns];
+				}
+
+				// Now compute the local value
+				logTerm = ((double) sourceDestPastOthersCount[sourceVal][destVal][pastVal[c]][othersVal] / (double) sourcePastOthersCount[sourceVal][pastVal[c]][othersVal]) /
+			 		((double) destPastOthersCount[destVal][pastVal[c]][othersVal] / (double) pastOthersCount[pastVal[c]][othersVal]);
+				localTE[r][c] = Math.log(logTerm) / log_2;
+				average += localTE[r][c];
+				if (localTE[r][c] > max) {
+					max = localTE[r][c];
+				} else if (localTE[r][c] < min) {
+					min = localTE[r][c];
+				}
+				// Update the previous value:
+				if (k > 0) {
+					pastVal[c] -= maxShiftedValue[states[r-k][c]];
+					pastVal[c] *= base;
+					pastVal[c] += states[r][c];
+				}
+			}
+		}		
+
+		average = average/(double) (columns * (rows - startObservationTime));
+		
+		return localTE;
+	}
+	
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method.
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations
+		(int states[][], int destCol, int sourceCol, int[] othersAbsolute){
+		
+		return computeLocalFromPreviousObservations(states, destCol, sourceCol, othersAbsolute, false);
+	}
+	private double[] computeLocalFromPreviousObservations
+		(int states[][], int destCol, int sourceCol, int[] othersAbsolute, boolean cleanedOthers){
+
+		int[] cleanedOthersAbsolute;
+		if (cleanedOthers) {
+			cleanedOthersAbsolute = othersAbsolute;
+		} else {
+			cleanedOthersAbsolute = cleanAbsoluteOthers(othersAbsolute,
+					destCol, sourceCol, k > 0);
+			// This call made redundant by cleanOffsetOthers:
+			// confirmEnoughAbsoluteOthers(othersAbsolute, destCol, sourceCol);
+		}
+		
+		int rows = states.length;
+		// int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localTE = new double[rows];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int pastVal = 0; 
+		pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += states[p][destCol];
+		}
+		int destVal, sourceVal, othersVal;
+		double logTerm;
+		for (int r = startObservationTime; r < rows; r++) {
+			destVal = states[r][destCol];
+			sourceVal = states[r-1][sourceCol];
+			othersVal = 0;
+			for (int o = 0; o < cleanedOthersAbsolute.length; o++) {
+				// Include this other contributor
+				othersVal *= base;
+				othersVal += states[r-1][cleanedOthersAbsolute[o]];
+			}
+			// Now compute the local value
+			logTerm = ((double) sourceDestPastOthersCount[sourceVal][destVal][pastVal][othersVal] / (double) sourcePastOthersCount[sourceVal][pastVal][othersVal]) /
+		 		((double) destPastOthersCount[destVal][pastVal][othersVal] / (double) pastOthersCount[pastVal][othersVal]);
+			localTE[r] = Math.log(logTerm) / log_2;
+			average += localTE[r];
+			if (localTE[r] > max) {
+				max = localTE[r];
+			} else if (localTE[r] < min) {
+				min = localTE[r];
+			}
+			// Update the previous value:
+			if (k > 0) {
+				pastVal -= maxShiftedValue[states[r-k][destCol]];
+				pastVal *= base;
+				pastVal += states[r][destCol];
+			}
+		}
+
+		average = average/(double) (rows - startObservationTime);
+		
+		return localTE;
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local transfer entropy across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param states - 2D array of states
+	 * @param j - TE across j cells to the right
+	 * @param othersOffsets - column offsets for other causal info contributors
+	 * @return
+	 */
+	public double[][] computeLocal(int states[][], int j, int[] othersOffsets) {
+		
+		initialise();
+		int[] cleanedOthersOffsets = cleanOffsetOthers(othersOffsets, j, k > 0);
+		addObservations(states, j, cleanedOthersOffsets, true);
+		return computeLocalFromPreviousObservations(states, j, cleanedOthersOffsets, true);
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute average local transfer entropy across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param states - 2D array of states
+	 * @param j - TE across j cells to the right
+	 * @param othersOffsets - column offsets for other causal info contributors
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][], int j, int[] othersOffsets) {
+		
+		initialise();
+		addObservations(states, j, othersOffsets);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local transfer entropy across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method suitable for heterogeneous agents
+	 * 
+	 * @param states - 2D array of states
+	 * @param destCol - column index for the destination agent
+	 * @param sourceCol - column index for the source agent
+	 * @param othersAbsolute - column indices for other causal info contributors
+	 * @return
+	 */
+	public double[] computeLocal(int states[][], int destCol, int sourceCol, int[] othersAbsolute) {
+		
+		initialise();
+		int[] cleanedOthers = cleanAbsoluteOthers(othersAbsolute, destCol,
+				sourceCol, k > 0);
+		addObservations(states, destCol, sourceCol, cleanedOthers, true);
+		return computeLocalFromPreviousObservations(states, destCol, sourceCol, cleanedOthers, true);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local transfer entropy across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * 
+	 * @param states - 2D array of states
+	 * @param destCol - column index for the destination agent
+	 * @param sourceCol - column index for the source agent
+	 * @param othersAbsolute - column indices for other causal info contributors
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][], int destCol, int sourceCol, int[] othersAbsolute) {
+		
+		initialise();
+		addObservations(states, destCol, sourceCol, othersAbsolute);
+		return computeAverageLocalOfObservations();
+	}
+	
+	/**
+	 * Counts the information contributors to this node which
+	 * are not equal to the offset j or the node itself (offset 0,
+	 * node itself not included only when removeDest is set to true)
+	 * 
+	 * @param othersOffsets
+	 * @param j
+	 * @param removeDest remove the destination itself from the count
+	 *     of offset others.
+	 * @return
+	 */
+	public static int countOfOffsetOthers(int[] othersOffsets, int j,
+			boolean removeDest) {
+		int countOfOthers = 0;
+		for (int index = 0; index < othersOffsets.length; index++) {
+			if ((othersOffsets[index] != j) && 
+				((othersOffsets[index] != 0) || !removeDest)) {
+				countOfOthers++;
+			}
+		}
+		return countOfOthers;
+	}
+	
+	/**
+	 * Counts the information contributors to the dest which
+	 * are not equal to src or the node itself (offset 0,
+	 * node itself not included only when removeDest is set to true)
+	 * 
+	 * @param others
+	 * @param dest
+	 * @param src
+	 * @param removeDest remove the destination itself from the count
+	 *     of absolute others.
+	 * @return
+	 */
+	public static int countOfAbsoluteOthers(int[] others, int dest, int src,
+			boolean removeDest) {
+		int countOfOthers = 0;
+		for (int index = 0; index < others.length; index++) {
+			if ((others[index] != src) &&
+				((others[index] != dest) || !removeDest)) {
+				countOfOthers++;
+			}
+		}
+		return countOfOthers;
+	}
+
+	/**
+	 * Check that the supplied array of offsets as other info 
+	 * contributors is long enough compared to our expectation
+	 * 
+	 * @param othersOffsets
+	 * @param j
+	 * @param removeDest remove the destination itself from the count
+	 *     of absolute others.
+	 * @return
+	 */
+	public boolean confirmEnoughOffsetOthers(int[] othersOffsets, int j,
+			boolean removeDest) {
+		if (countOfOffsetOthers(othersOffsets, j, removeDest) !=
+				numOtherInfoContributors) {
+			throw new RuntimeException("Incorrect number of others in offsets");
+		}
+		return true;
+	}
+
+	/**
+	 * Check that the supplied array of absolutes as other info 
+	 * contributors is long enough compared to our expectation
+	 * 
+	 * @param othersAbsolute
+	 * @param dest
+	 * @param src
+	 * @param removeDest remove the destination itself from the count
+	 *     of absolute others.
+	 * @return
+	 */
+	public boolean confirmEnoughAbsoluteOthers(int[] othersAbsolute, int dest,
+			int src, boolean removeDest) {
+		if (countOfAbsoluteOthers(othersAbsolute, dest, src, removeDest) !=
+				numOtherInfoContributors) {
+			throw new RuntimeException("Incorrect number of others in absolutes");
+		}
+		return true;
+	}
+	
+	/**
+	 * Returns the information contributors to this node which
+	 * are not equal to the offset j or the node itself (offset 0,
+	 * removed only if removeDest is set to true).
+	 * Checks that there are enough other information contributors.
+	 * 
+	 * @param othersOffsets
+	 * @param j
+	 * @param removeDest Remove the destination itself from the cleaned
+	 *           other sources (if it is there). Should not be done
+	 *           if k == 0 (because then the destination is not included
+	 *           in the past history)
+	 * @return
+	 */
+	public int[] cleanOffsetOthers(int[] othersOffsets, int j, boolean removeDest) {
+		int[] cleaned = new int[numOtherInfoContributors];
+		int countOfOthers = 0;
+		for (int index = 0; index < othersOffsets.length; index++) {
+			if ((othersOffsets[index] != j) &&
+				((othersOffsets[index] != 0) || !removeDest)) {
+				// Add this candidate source to the cleaned sources
+				if (countOfOthers == numOtherInfoContributors) {
+					// We've already taken all the other info 
+					//  contributors we expected
+					countOfOthers++;
+					break;
+				}
+				cleaned[countOfOthers] = othersOffsets[index];
+				countOfOthers++;
+			}
+		}
+		if (countOfOthers < numOtherInfoContributors) {
+			throw new RuntimeException("Too few others in offsets");
+		} else if (countOfOthers > numOtherInfoContributors) {
+			throw new RuntimeException("Too many others in offsets");
+		}
+		return cleaned;
+	}
+	
+	/**
+	 * Returns the information contributors to the dest which
+	 * are not equal to src or the node itself (offset 0,
+	 * removed only if removeDest is true).
+	 * Checks that there are enough other information contributors.
+	 * 
+	 * @param others
+	 * @param dest
+	 * @param src
+	 * @param removeDest Remove the destination itself from the cleaned
+	 *           other sources (if it is there). Should not be done
+	 *           if k == 0 (because then the destination is not included
+	 *           in the past history)
+	 * @return
+	 */
+	public int[] cleanAbsoluteOthers(int[] others, int dest, int src,
+			boolean removeDest) {
+		int[] cleaned = new int[numOtherInfoContributors];
+		int countOfOthers = 0;
+		for (int index = 0; index < others.length; index++) {
+			if ((others[index] != src) &&
+				((others[index] != dest) || !removeDest)) {
+				// Add this candidate source to the cleaned sources
+				if (countOfOthers == numOtherInfoContributors) {
+					// We've already taken all the other info 
+					//  contributors we expected
+					countOfOthers++;
+					break;
+				}
+				cleaned[countOfOthers] = others[index];
+				countOfOthers++;
+			}
+		}
+		if (countOfOthers < numOtherInfoContributors) {
+			throw new RuntimeException("Too few others in absolutes");
+		} else if (countOfOthers > numOtherInfoContributors) {
+			throw new RuntimeException("Too many others in absolutes");
+		}
+		return cleaned;
+	}
+}

java/source/infodynamics/measures/discrete/ContextOfPastMeasureCalculator.java

@@ -0,0 +1,153 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * @author Joseph Lizier
+ * 
+ * Info theoretic measure calculator base class for 
+ *  measures which require the context of the past
+ *  history of the destination variable.
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations before computing :
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations() or computeAverageLocalOfObservations()
+ * 2. Standalone computation from a single set of observations:
+ *   Call: computeLocal() or computeAverageLocal()
+ * 
+ * @author Joseph Lizier
+ * joseph.lizier at gmail.com
+ * http://lizier.me/joseph/
+ *
+ */
+public abstract class ContextOfPastMeasureCalculator extends
+		InfoMeasureCalculator {
+
+	protected int k = 0; // history length k.
+	protected boolean noObservationStorage = false;
+	protected int[][] nextPastCount = null; // Count for (i[t+1], i[t]) tuples
+	protected int[] pastCount = null; // Count for i[t]
+	protected int[] nextCount = null; // count for i[t+1]
+	protected int[] maxShiftedValue = null; // states * (base^(k-1))
+	protected int base_power_k = 0;
+
+	/**
+	 * @param base
+	 */
+	public ContextOfPastMeasureCalculator(int base, int history) {
+		this(base, history, false);
+	}
+
+	/**
+	 * Constructor to be used by child classes only.
+	 * In general, only needs to be explicitly called if child classes
+	 *  do not wish to create the observation arrays.
+	 * 
+	 * @param base
+	 * @param history
+	 * @param dontCreateObsStorage
+	 */
+	protected ContextOfPastMeasureCalculator(int base, int history, boolean dontCreateObsStorage) {
+		super(base);
+
+		k = history;
+		base_power_k = MathsUtils.power(base, k);
+		
+		// Relax the requirement that k >= 1, so that we can 
+		//  eliminate considering the history at will ...
+		//if (k < 1) {
+		//	throw new RuntimeException("History k " + history + " is not >= 1 a ContextOfPastMeasureCalculator");
+		//}
+		
+		// Check that we can convert the history value into an integer ok: 
+		if (k > Math.log(Integer.MAX_VALUE) / log_base) {
+			throw new RuntimeException("Base and history combination too large");
+		}
+
+		// Create constants for tracking prevValues
+		maxShiftedValue = new int[base];
+		for (int v = 0; v < base; v++) {
+			maxShiftedValue[v] = v * MathsUtils.power(base, k-1);
+		}
+		
+		noObservationStorage = dontCreateObsStorage;
+		if (!dontCreateObsStorage) {
+			// Create storage for counts of observations
+			nextPastCount = new int[base][base_power_k];
+			pastCount = new int[base_power_k];
+			nextCount = new int[base];
+		}
+	}
+	
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 *
+	 */
+	public void initialise() {
+		super.initialise();
+		
+		if (!noObservationStorage) {
+			MatrixUtils.fill(nextPastCount, 0);
+			MatrixUtils.fill(pastCount, 0);
+			MatrixUtils.fill(nextCount, 0);
+		}
+	}
+	
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[] x, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p];
+		}
+		return pastVal;
+	}
+
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param agentNumber
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[][] x, int agentNumber, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p][agentNumber];
+		}
+		return pastVal;
+	}
+
+	/**
+	 * Utility function to compute the combined past values of x up to and including time step t
+	 *  (i.e. (x_{t-k+1}, ... ,x_{t-1},x_{t}))
+	 * 
+	 * @param x
+	 * @param agentNumber
+	 * @param t
+	 * @return
+	 */
+	public int computePastValue(int[][][] x, int agentRow, int agentColumn, int t) {
+		int pastVal = 0;
+		for (int p = 0; p < k; p++) {
+			pastVal *= base;
+			pastVal += x[t - k + 1 + p][agentRow][agentColumn];
+		}
+		return pastVal;
+	}
+}

java/source/infodynamics/measures/discrete/EntropyCalculator.java

@@ -0,0 +1,518 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * Single state entropy calculator
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class EntropyCalculator extends InfoMeasureCalculator
+				implements SingleAgentMeasure
+{
+
+	protected int[] stateCount = null; // Count for i[t]
+
+	/**
+	 * User to create new instances through this factory method.
+	 * This allows us to return an efficient calculator for
+	 * base 2, for example, without the user needing to have 
+	 * knowledge of this. (This is obselete though)
+	 * @param base
+	 * @param blocksize
+	 * 
+	 * @return
+	 */
+	public static EntropyCalculator newInstance(int base, int blocksize) {
+		if (blocksize > 1) {
+			return BlockEntropyCalculator.newInstance(blocksize, base);
+		} else {
+			return EntropyCalculator.newInstance(base);
+		}
+	}
+	public static EntropyCalculator newInstance(int base) {
+		return new EntropyCalculator(base);
+	}
+	
+	/**
+	 * 
+	 * @param base
+	 */
+	public EntropyCalculator(int base) {
+
+		super(base);
+		
+		// Create storage for counts of observations
+		stateCount = new int[base];
+	}
+	
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 */
+	public void initialise(){
+		super.initialise();
+		MatrixUtils.fill(stateCount, 0);
+	}
+		
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][]) {
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += rows * columns; 
+		
+		// 1. Count the tuples observed
+		for (int r = 0; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add to the count for this particular state:
+				stateCount[states[r][c]]++;					
+			}
+		}		
+	}
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 */
+	public void addObservations(int states[][][]) {
+		int timeSteps = states.length;
+		if (timeSteps == 0) {
+			return;
+		}
+		int agentRows = states[0].length;
+		if (agentRows == 0) {
+			return;
+		}
+		int agentColumns = states[0][0].length;
+		// increment the count of observations:
+		observations += timeSteps * agentRows * agentColumns; 
+		
+		// 1. Count the tuples observed
+		for (int t = 0; t < timeSteps; t++) {
+			for (int i = 0; i < agentRows; i++) {
+				for (int j = 0; j < agentColumns; j++) {
+					// Add to the count for this particular state:
+					stateCount[states[t][i][j]]++;
+				}
+			}
+		}		
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @param agentNumber
+	 */
+	public void addObservations(int states[][], int agentNumber) {
+		int rows = states.length;
+		// increment the count of observations:
+		observations += rows; 
+		
+		// 1. Count the tuples observed
+		for (int r = 0; r < rows; r++) {
+			// Add to the count for this particular state:
+			stateCount[states[r][agentNumber]]++;					
+		}
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 * @param agentIndex1
+	 * @param agentIndex2
+	 */
+	public void addObservations(int states[][][], int agentIndex1, int agentIndex2) {
+		int timeSteps = states.length;
+		// increment the count of observations:
+		observations += timeSteps; 
+		
+		// 1. Count the tuples observed
+		for (int r = 0; r < timeSteps; r++) {
+			// Add to the count for this particular state:
+			stateCount[states[r][agentIndex1][agentIndex2]]++;					
+		}
+	}
+
+	/**
+	 * 
+	 * @param stateVal
+	 * @return count of observations of the given state
+	 */
+	public int getStateCount(int stateVal) {
+		return stateCount[stateVal];
+	}
+	
+	/**
+	 * 
+	 * @param stateVal
+	 * @return probability of the given state
+	 */
+	public double getStateProbability(int stateVal) {
+		return (double) stateCount[stateVal] / (double) observations;
+	}
+
+	/**
+	 * Returns the average local entropy from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double ent = 0.0;
+		double entCont = 0.0;
+
+		max = 0;
+		min = 0;
+		for (int stateVal = 0; stateVal < base; stateVal++) {
+			// compute p_state
+			double p_state = (double) stateCount[stateVal] / (double) observations;
+			if (p_state > 0.0) {
+				// Entropy takes the negative log:
+				double localValue = - Math.log(p_state) / log_2;
+				entCont = p_state * localValue;
+				if (localValue > max) {
+					max = localValue;
+				} else if (localValue < min) {
+					min = localValue;
+				}
+			} else {
+				entCont = 0.0;
+			}
+			ent += entCont;
+		}
+		
+		average = ent;
+		return ent;
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations(int states[][]){
+		int rows = states.length;
+		int columns = states[0].length;
+		
+		// Allocate for all rows even though we may not be assigning all of them
+		double[][] localEntropy = new double[rows][columns];
+		average = 0;
+		max = 0;
+		min = 0;
+		for (int r = 0; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				double p_state = (double) stateCount[states[r][c]] / (double) observations;
+				// Entropy takes the negative log:
+				localEntropy[r][c] = - Math.log(p_state) / log_2;
+				average += localEntropy[r][c];
+				if (localEntropy[r][c] > max) {
+					max = localEntropy[r][c];
+				} else if (localEntropy[r][c] < min) {
+					min = localEntropy[r][c];
+				}
+			}
+		}
+		average = average/(double) (columns * rows);
+		
+		return localEntropy;
+		
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[][][] computeLocalFromPreviousObservations(int states[][][]){
+		int timeSteps = states.length;
+		int agentRows, agentColumns;
+		if (timeSteps == 0) {
+			agentRows = 0;
+			agentColumns = 0;
+		} else {
+			agentRows = states[0].length;
+			if (agentRows == 0) {
+				agentColumns = 0;
+			} else {
+				agentColumns = states[0][0].length;
+			}
+		}
+
+		// Allocate for all rows even though we may not be assigning all of them
+		double[][][] localEntropy = new double[timeSteps][agentRows][agentColumns];
+		average = 0;
+		max = 0;
+		min = 0;
+		for (int r = 0; r < timeSteps; r++) {
+			for (int i = 0; i < agentRows; i++) {
+				for (int j = 0; j < agentColumns; j++) {
+					double p_state = (double) stateCount[states[r][i][j]] / (double) observations;
+					// Entropy takes the negative log:
+					localEntropy[r][i][j] = - Math.log(p_state) / log_2;
+					average += localEntropy[r][i][j];
+					if (localEntropy[r][i][j] > max) {
+						max = localEntropy[r][i][j];
+					} else if (localEntropy[r][i][j] < min) {
+						min = localEntropy[r][i][j];
+					}
+				}
+			}
+		}
+		average = average/(double) (agentRows * agentColumns * timeSteps);
+		
+		return localEntropy;
+		
+	}
+
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states
+	 * @param agentNumber
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][], int agentNumber){
+		int rows = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntropy = new double[rows];
+		average = 0;
+		max = 0;
+		min = 0;
+		for (int r = 0; r < rows; r++) {
+			double p_state = (double) stateCount[states[r][agentNumber]] / (double) observations;
+			// Entropy takes the negative log:
+			localEntropy[r] = - Math.log(p_state) / log_2;
+			average += localEntropy[r];
+			if (localEntropy[r] > max) {
+				max = localEntropy[r];
+			} else if (localEntropy[r] < min) {
+				min = localEntropy[r];
+			}
+		}
+		average = average/(double) (rows);
+		
+		return localEntropy;
+		
+	}
+	
+	/**
+	 * Computes local entropy for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @param agentIndex1
+	 * @param agentIndex2
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][][], int agentIndex1, int agentIndex2){
+		int timeSteps = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntropy = new double[timeSteps];
+		average = 0;
+		max = 0;
+		min = 0;
+		for (int r = 0; r < timeSteps; r++) {
+			double p_state = (double) stateCount[states[r][agentIndex1][agentIndex2]] / (double) observations;
+			// Entropy takes the negative log:
+			localEntropy[r] = - Math.log(p_state) / log_2;
+			average += localEntropy[r];
+			if (localEntropy[r] > max) {
+				max = localEntropy[r];
+			} else if (localEntropy[r] < min) {
+				min = localEntropy[r];
+			}
+		}
+		average = average/(double) (timeSteps);
+		
+		return localEntropy;
+		
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public final double[][] computeLocal(int states[][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure across a 3D spatiotemporal
+	 *  array of the states of 2D homogeneous agents
+	 * Return a 3D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public final double[][][] computeLocal(int states[][][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public final double computeAverageLocal(int states[][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure across a 3D spatiotemporal
+	 *  array of the states of 2D homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public final double computeAverageLocal(int states[][][]) {
+		
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public final double[] computeLocalAtAgent(int states[][], int col) {
+		
+		initialise();
+		addObservations(states, col);
+		return computeLocalFromPreviousObservations(states, col);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @param agentIndex1
+	 * @param agentIndex2
+	 * @return
+	 */
+	public final double[] computeLocalAtAgent(int states[][][], int agentIndex1, int agentIndex2) {
+		
+		initialise();
+		addObservations(states, agentIndex1, agentIndex2);
+		return computeLocalFromPreviousObservations(states, agentIndex1, agentIndex2);
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure 
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * @param states - 2D array of states
+	 * @param col - column number of the agent in the states array
+	 * 
+	 * @return
+	 */
+	public final double computeAverageLocalAtAgent(int states[][], int col) {
+		initialise();
+		addObservations(states, col);
+		return computeAverageLocalOfObservations();
+	}
+
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure 
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * @param states - 2D array of states
+	 * @param agentIndex1
+	 * @param agentIndex2
+	 * 
+	 * @return
+	 */
+	public final double computeAverageLocalAtAgent(int states[][][], int agentIndex1, int agentIndex2) {
+		initialise();
+		addObservations(states, agentIndex1, agentIndex2);
+		return computeAverageLocalOfObservations();
+	}
+}

java/source/infodynamics/measures/discrete/EntropyRateCalculator.java

@@ -0,0 +1,460 @@
+package infodynamics.measures.discrete;
+
+/**
+ * Compute average and local entropy rates
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localActiveInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class EntropyRateCalculator extends SingleAgentMeasureInContextOfPastCalculator {
+
+	/**
+	 * User to create new instances through this factory method.
+	 * This allows us to return an efficient calculator for
+	 * base 2, for example, without the user needing to have 
+	 * knowledge of this. (This is obselete anyway)
+	 * @param base
+	 * @param history
+	 * 
+	 * @return
+	 */
+	public static EntropyRateCalculator newInstance(int base, int history) {
+		return new EntropyRateCalculator(base, history);
+	}
+	
+	public EntropyRateCalculator(int base, int history) {
+		
+		super(base, history);
+
+	}		
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][]) {
+		int rows = states.length;
+		int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - k)*columns; 
+		
+		// Initialise and store the current previous value for each column
+		int[] prevVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			prevVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				prevVal[c] *= base;
+				prevVal[c] += states[p][c];
+			}
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				// Add to the count for this particular transition:
+				// (cell's assigned as above)
+				nextVal = states[r][c];
+				nextPastCount[nextVal][prevVal[c]]++;
+				pastCount[prevVal[c]]++;
+				// Update the previous value:
+				prevVal[c] -= maxShiftedValue[states[r-k][c]];
+				prevVal[c] *= base;
+				prevVal[c] += states[r][c];
+			}
+		}		
+	}
+	
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 */
+	public void addObservations(int states[][][]) {
+		int timeSteps = states.length;
+		if (timeSteps == 0) {
+			return;
+		}
+		int agentRows = states[0].length;
+		if (agentRows == 0) {
+			return;
+		}
+		int agentColumns = states[0][0].length;
+		// increment the count of observations:
+		observations += (timeSteps - k) * agentRows * agentColumns; 
+		
+		// Initialise and store the current previous value for each column
+		int[][] prevVal = new int[agentRows][agentColumns];
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				prevVal[r][c] = 0;
+				for (int p = 0; p < k; p++) {
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[p][r][c];
+				}
+			}
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					// Add to the count for this particular transition:
+					// (cell's assigned as above)
+					nextVal = states[t][r][c];
+					nextPastCount[nextVal][prevVal[r][c]]++;
+					pastCount[prevVal[r][c]]++;
+					// Update the previous value:
+					prevVal[r][c] -= maxShiftedValue[states[t-k][r][c]];
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[t][r][c];
+				}
+			}
+		}		
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][], int col) {
+		int rows = states.length;
+		// increment the count of observations:
+		observations += (rows - k); 
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][col];
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int r = k; r < rows; r++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			nextVal = states[r][col];
+			nextPastCount[nextVal][prevVal]++;
+			pastCount[prevVal]++;
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[r-k][col]];
+			prevVal *= base;
+			prevVal += states[r][col];
+		}
+	}
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 */
+	public void addObservations(int states[][][], int agentIndex1, int agentIndex2) {
+		int timeSteps = states.length;
+		// increment the count of observations:
+		observations += (timeSteps - k); 
+		
+		// Initialise and store the current previous value for this column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][agentIndex1][agentIndex2];
+		}
+		
+		// 1. Count the tuples observed
+		int nextVal;
+		for (int r = k; r < timeSteps; r++) {
+			// Add to the count for this particular transition:
+			// (cell's assigned as above)
+			nextVal = states[r][agentIndex1][agentIndex2];
+			nextPastCount[nextVal][prevVal]++;
+			pastCount[prevVal]++;
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[r-k][agentIndex1][agentIndex2]];
+			prevVal *= base;
+			prevVal += states[r][agentIndex1][agentIndex2];
+		}
+	}
+
+	/**
+	 * Returns the average local active information storage from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double entRate = 0.0;
+		double entRateCont = 0.0;
+
+		max = 0;
+		min = 0;
+		double logTerm = 0;
+		for (int nextVal = 0; nextVal < base; nextVal++) {
+			for (int prevVal = 0; prevVal < base_power_k; prevVal++) {
+				// compute p_prev
+				double p_prev = (double) pastCount[prevVal] / (double) observations;
+				// compute p(prev, next)
+				double p_joint = (double) nextPastCount[nextVal][prevVal] / (double) observations;
+				// Compute entropy rate contribution:
+				if (p_joint > 0.0) {
+					logTerm = p_joint / p_prev;
+					// Entropy rate takes the negative log:
+					double localValue = - Math.log(logTerm) / log_2;
+					entRateCont = p_joint * localValue;
+					if (localValue > max) {
+						max = localValue;
+					} else if (localValue < min) {
+						min = localValue;
+					}
+				} else {
+					entRateCont = 0.0;
+				}
+				entRate += entRateCont;
+			}
+		}
+		
+		average = entRate;
+		return entRate;
+	}
+	
+	/**
+	 * Computes local entropy rate for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations(int states[][]){
+		int rows = states.length;
+		int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[][] localEntRate = new double[rows][columns];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int[] prevVal = new int[columns]; 
+		for (int c = 0; c < columns; c++) {
+			prevVal[c] = 0;
+			for (int p = 0; p < k; p++) {
+				prevVal[c] *= base;
+				prevVal[c] += states[p][c];
+			}
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int r = k; r < rows; r++) {
+			for (int c = 0; c < columns; c++) {
+				nextVal = states[r][c];
+				logTerm = ( (double) nextPastCount[nextVal][prevVal[c]] ) /
+				  		  ( (double) pastCount[prevVal[c]] );
+				// Entropy rate takes the negative log:
+				localEntRate[r][c] = - Math.log(logTerm) / log_2;
+				average += localEntRate[r][c];
+				if (localEntRate[r][c] > max) {
+					max = localEntRate[r][c];
+				} else if (localEntRate[r][c] < min) {
+					min = localEntRate[r][c];
+				}
+				// Update the previous value:
+				prevVal[c] -= maxShiftedValue[states[r-k][c]];
+				prevVal[c] *= base;
+				prevVal[c] += states[r][c];
+			}
+		}
+		average = average/(double) (columns * (rows - k));
+		
+		return localEntRate;
+		
+	}
+	
+	/**
+	 * Computes local entropy rate for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 * @return
+	 */
+	public double[][][] computeLocalFromPreviousObservations(int states[][][]){
+		int timeSteps = states.length;
+		int agentRows = states[0].length;
+		int agentColumns = states[0][0].length;
+
+		// Allocate for all time steps even though we'll leave the first ones as zeros
+		double[][][] localEntRate = new double[timeSteps][agentRows][agentColumns];
+		average = 0;
+		max = 0;
+		min = 0;
+
+		// Initialise and store the current previous value for each column
+		int[][] prevVal = new int[agentRows][agentColumns];
+		for (int r = 0; r < agentRows; r++) {
+			for (int c = 0; c < agentColumns; c++) {
+				prevVal[r][c] = 0;
+				for (int p = 0; p < k; p++) {
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[p][r][c];
+				}
+			}
+		}
+		
+		int nextVal;
+		double logTerm = 0.0;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = 0; r < agentRows; r++) {
+				for (int c = 0; c < agentColumns; c++) {
+					nextVal = states[t][r][c];
+					logTerm = ( (double) nextPastCount[nextVal][prevVal[r][c]] ) /
+					  		  ( (double) pastCount[prevVal[r][c]] );
+					// Entropy rate takes the negative log:
+					localEntRate[t][r][c] = - Math.log(logTerm) / log_2;
+					average += localEntRate[t][r][c];
+					if (localEntRate[t][r][c] > max) {
+						max = localEntRate[t][r][c];
+					} else if (localEntRate[t][r][c] < min) {
+						min = localEntRate[t][r][c];
+					}
+					// Update the previous value:
+					prevVal[r][c] -= maxShiftedValue[states[t-k][r][c]];
+					prevVal[r][c] *= base;
+					prevVal[r][c] += states[t][r][c];
+				}
+			}
+		}
+		average = average/(double) (agentRows * agentColumns * (timeSteps - k));
+		
+		return localEntRate;
+		
+	}
+
+	/**
+	 * Computes local entropy rate for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][], int col){
+		int rows = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntRate = new double[rows];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][col];
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int r = k; r < rows; r++) {
+			nextVal = states[r][col];
+			logTerm = ( (double) nextPastCount[nextVal][prevVal] ) /
+			  		  ( (double) pastCount[prevVal] );
+			// Entropy rate takes the negative log:
+			localEntRate[r] = - Math.log(logTerm) / log_2;
+			average += localEntRate[r];
+			if (localEntRate[r] > max) {
+				max = localEntRate[r];
+			} else if (localEntRate[r] < min) {
+				min = localEntRate[r];
+			}
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[r-k][col]];
+			prevVal *= base;
+			prevVal += states[r][col];
+		}
+		average = average/(double) (rows - k);
+		
+		return localEntRate;
+		
+	}
+
+	/**
+	 * Computes local entropy rate for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * @param states 1st index is time, 2nd and 3rd index give the 2D agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][][], int agentIndex1, int agentIndex2){
+		int timeSteps = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localEntRate = new double[timeSteps];
+		average = 0;
+		max = 0;
+		min = 0;
+		
+		// Initialise and store the current previous value for each column
+		int prevVal = 0; 
+		prevVal = 0;
+		for (int p = 0; p < k; p++) {
+			prevVal *= base;
+			prevVal += states[p][agentIndex1][agentIndex2];
+		}
+		int nextVal;
+		double logTerm = 0.0;
+		for (int t = k; t < timeSteps; t++) {
+			nextVal = states[t][agentIndex1][agentIndex2];
+			logTerm = ( (double) nextPastCount[nextVal][prevVal] ) /
+			  		  ( (double) pastCount[prevVal] );
+			// Entropy rate takes the negative log:
+			localEntRate[t] = - Math.log(logTerm) / log_2;
+			average += localEntRate[t];
+			if (localEntRate[t] > max) {
+				max = localEntRate[t];
+			} else if (localEntRate[t] < min) {
+				min = localEntRate[t];
+			}
+			// Update the previous value:
+			prevVal -= maxShiftedValue[states[t-k][agentIndex1][agentIndex2]];
+			prevVal *= base;
+			prevVal += states[t][agentIndex1][agentIndex2];
+		}
+		average = average/(double) (timeSteps - k);
+		
+		return localEntRate;
+		
+	}	
+}

java/source/infodynamics/measures/discrete/InfoMeasureCalculator.java

@@ -0,0 +1,123 @@
+package infodynamics.measures.discrete;
+
+/**
+ * Info theoretic measure calculator base class
+ * 
+ * Usage:
+ * 1. Continuous accumulation of observations before computing :
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations() or computeAverageLocalOfObservations()
+ * 2. Standalone computation from a single set of observations:
+ *   Call: computeLocal() or computeAverageLocal()
+ * 
+ * @author Joseph Lizier
+ * joseph.lizier at gmail.com
+ * http://lizier.me/joseph/
+ *
+ */
+public abstract class InfoMeasureCalculator {
+
+	protected double average = 0.0;
+	protected double max = 0.0;
+	protected double min = 0.0;
+	protected double std = 0.0;
+	protected int observations = 0;
+	protected int base = 0; // number of individual states. Need initialised to 0 for changedSizes
+
+	protected double log_base = 0;
+	protected double log_2 = Math.log(2.0);
+	protected boolean power_of_2_base = false;
+	protected int log_2_base = 0;
+	
+	protected boolean debug = false;
+	
+	/**
+	 * 
+	 * @param blocksize
+	 * @param base
+	 */
+	protected InfoMeasureCalculator(int base) {
+		
+		this.base = base;
+		log_base = Math.log(base);
+
+		if (base < 2) {
+			throw new RuntimeException("Can't calculate info theoretic measures for base " + base);
+		}
+		
+		// Check if we've got a power of 2
+		power_of_2_base = isPowerOf2(base);
+		if (power_of_2_base) {
+			log_2_base = (int) Math.round(Math.log(base) / Math.log(2));
+		}
+	}
+	
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 * Should be called prior to any of the addObservations() methods.
+	 * You can reinitialise without needing to create a new object.
+	 */
+	public void initialise(){
+		average = 0.0;
+		max = 0.0;
+		min = 0.0;
+		std = 0.0;
+		observations = 0;
+	}
+	
+	public final double getLastAverage() {
+		return average;
+	}
+
+	/**
+	 * Not declaring this final so that separable calculator
+	 *  can throw an exception on it since it does not support it
+	 * 
+	 * @return
+	 */
+	public double getLastMax() {
+		return max;
+	}
+
+	/**
+	 * Not declaring this final so that separable calculator
+	 *  can throw an exception on it since it does not support it
+	 * 
+	 * @return
+	 */
+	public double getLastMin() {
+		return min;
+	}
+	
+	public final double getLastStd() {
+		return std;
+	}
+	
+	public final int getNumObservations() {
+		return observations;
+	}
+	
+	public final static boolean isPowerOf2(int num) {
+		int bits = 0;
+		int shiftedValue = num;
+		for (int b = 0; b < Integer.SIZE; b ++) {
+			if ((shiftedValue & 0x01) > 0) {
+				// LSB is a 1
+				bits++;
+				if (bits > 1) {
+					// Encountered more than 1 bit set to 1.
+					// Is not a power of 2
+					return false;
+				}
+			}
+			// Shift a new bit down into the LSB
+			shiftedValue = shiftedValue >> 1;
+		}
+		// Post: num has either 1 bit set to 1 or 0 bits set to 1
+		if (bits == 0) {
+			return false;
+		}
+		return true;
+	}
+}

java/source/infodynamics/measures/discrete/MultiInformationCalculator.java

@@ -0,0 +1,179 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localMultiInformation()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MultiInformationCalculator extends InfoMeasureCalculator {
+	
+	private int[]	jointCount = null; // jointCount[jointState]
+	private int[][] marginalCounts = null; // marginalCounts[marginalIndex][state]
+	
+	private int numVars;
+	private int jointStates;
+	private boolean checkedFirst = false;
+
+	/**
+	 * Null constructor
+	 *
+	 */
+	public MultiInformationCalculator(int base, int numVars) {
+		super(base);
+		this.numVars = numVars;
+		jointStates = MathsUtils.power(base, numVars);
+		jointCount = new int[jointStates];
+		marginalCounts = new int[numVars][base];
+	}
+
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 *
+	 */
+	public void initialise(){
+		super.initialise();
+		MatrixUtils.fill(jointCount, 0);
+		MatrixUtils.fill(marginalCounts, 0);
+	}
+	
+	/**
+	 * Add observations of the variables based at every point, with the set defined by 
+	 *  the group offsets array
+	 *  
+	 * @param states
+	 * @param groupOffsets
+	 */
+	public void addObservations(int[] states, int[] groupOffsets) {
+		for (int c = 0; c < states.length; c++) {
+			// Add the marginal observations in, and compute the joint state value
+			int jointValue = 0;
+			for (int i = 0; i < numVars; i++) {
+				int thisValue = states[(c + groupOffsets[i] + states.length) % states.length];
+				marginalCounts[i][thisValue]++;
+				jointValue *= base;
+				jointValue += thisValue;
+			}
+			jointCount[jointValue]++;
+			observations++;
+		}
+	}
+	
+	/**
+	 * Add observations of the variables based at every point, with the set defined by 
+	 *  the group offsets array
+	 *  
+	 * @param states
+	 * @param groupOffsets
+	 */
+	public void addObservations(int[] states, int destinationIndex, int[] groupOffsets) {
+		// Add the marginal observations in, and compute the joint state value
+		int jointValue = 0;
+		for (int i = 0; i < numVars; i++) {
+			int thisValue = states[(destinationIndex + groupOffsets[i] + states.length) % states.length];
+			marginalCounts[i][thisValue]++;
+			jointValue *= base;
+			jointValue += thisValue;
+		}
+		jointCount[jointValue]++;
+		observations++;
+	}
+
+	/**
+	 * Add observations of the variables based at every point, with the set defined by 
+	 *  the group offsets array
+	 *  
+	 * @param states
+	 * @param groupOffsets
+	 */
+	public void addObservations(int[][] states, int[] groupOffsets) {
+		for (int t = 0; t < states.length; t++) {
+			for (int c = 0; c < states[t].length; c++) {
+				// Add the marginal observations in, and compute the joint state value
+				int jointValue = 0;
+				for (int i = 0; i < numVars; i++) {
+					int thisValue = states[t][(c + groupOffsets[i] + states.length) % states.length];
+					marginalCounts[i][thisValue]++;
+					jointValue *= base;
+					jointValue += thisValue;
+				}
+				jointCount[jointValue]++;
+				observations++;
+			}
+		}
+	}
+	
+	/**
+	 * Returns the average local multi information storage from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		
+		int[] jointTuple = new int[numVars];
+		checkedFirst = false;
+		average = computeMiOfGivenTupleFromVarIndex(jointTuple, 0);
+				
+		return average;
+	}
+	
+	/**
+	 * Compute the contribution to the MI for all tuples starting with tuple[0..(fromIndex-1)].
+	 * 
+	 * @param tuple
+	 * @param fromIndex
+	 * @return
+	 */
+	public double computeMiOfGivenTupleFromVarIndex(int[] tuple, int fromIndex) {
+		double miCont = 0;
+		if (fromIndex == numVars) {
+			// The whole tuple is filled in, so compute the contribution to the MI from this tuple
+			double prodMarginalProbs = 1.0;
+			int jointValue = 0;
+			for (int i = 0; i < numVars; i++) {
+				prodMarginalProbs *= (double) marginalCounts[i][tuple[i]] / (double) observations;
+				jointValue *= base;
+				jointValue += tuple[i];
+			}
+			if (jointCount[jointValue] == 0) {
+				// This joint state does not occur, so it makes no contribution here
+				return 0;
+			}
+			double jointProb = (double) jointCount[jointValue] / (double) observations;
+			double logValue = jointProb / prodMarginalProbs;
+			double localValue = Math.log(logValue) / log_base;
+			if (jointProb > 0.0) {
+				if (!checkedFirst) {
+					max = localValue;
+					min = localValue;
+					checkedFirst = true;
+				} else {
+					if (localValue > max) {
+						max = localValue;
+					}
+					if (localValue < min) {
+						min = localValue;
+					}
+				}
+			}
+			miCont = jointProb * localValue;
+		} else {
+			// Fill out the next part of the tuple and make the recursive calls
+			for (int v = 0; v < base; v++) {
+				tuple[fromIndex] = v;
+				miCont += computeMiOfGivenTupleFromVarIndex(tuple, fromIndex + 1);
+			}
+		}
+		return miCont;
+	}
+}

java/source/infodynamics/measures/discrete/MutualInformationCalculator.java

@@ -0,0 +1,307 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+
+
+/**
+ * Usage:
+ * 1. Continuous accumulation of observations:
+ *   Call: a. initialise()
+ *         b. addObservations() several times over
+ *         c. computeLocalFromPreviousObservations()
+ * 2. Standalone:
+ *   Call: localMutualInformation()
+ * 
+ * @author Joseph Lizier, jlizier at it.usyd.edu.au
+ *
+ */
+public class MutualInformationCalculator extends InfoMeasureCalculator 
+	implements ChannelCalculator {
+
+	private int timeDiff = 0;
+	private int[][]	jointCount = null; // Count for (i[t-timeDiff], j[t]) tuples
+	private int[] iCount = null; // Count for i[t-timeDiff]		
+	private int[] jCount = null; // Count for j[t]
+
+	public MutualInformationCalculator(int base, int timeDiff) {
+		super(base);
+		this.timeDiff = timeDiff;
+		jointCount = new int[base][base];
+		iCount = new int[base];
+		jCount = new int[base];
+	}
+
+	/**
+	 * Initialise calculator, preparing to take observation sets in
+	 *
+	 */
+	public void initialise(){
+		super.initialise();
+		MatrixUtils.fill(iCount, 0);
+		MatrixUtils.fill(jCount, 0);
+		MatrixUtils.fill(jointCount, 0);
+	}
+	
+	/**
+	 * Add more observations in to our estimates of the pdfs
+	 * Pairs are between the arrays var1 and var2, separated in time by timeDiff (i is first)
+	 *
+	 */
+	public void addObservations(int[] var1, int[] var2) {
+		int timeSteps = var1.length;
+		// int columns = states[0].length;
+		// increment the count of observations:
+		observations += (timeSteps - timeDiff); 
+		
+		// 1. Count the tuples observed
+		int iVal, jVal;
+		for (int r = timeDiff; r < timeSteps; r++) {
+			// Add to the count for this particular pair:
+			iVal = var1[r-timeDiff];
+			jVal = var2[r];
+			jointCount[iVal][jVal]++;
+			iCount[iVal]++;
+			jCount[jVal]++;
+		}
+	}
+
+	/**
+	 * Add more observations in to our estimates of the pdfs
+	 * Pairs are between columns iCol and jCol, separated in time by timeDiff (i is first)
+	 *
+	 */
+	public void addObservations(int states[][], int iCol, int jCol) {
+		int rows = states.length;
+		// int columns = states[0].length;
+		// increment the count of observations:
+		observations += (rows - timeDiff); 
+		
+		// 1. Count the tuples observed
+		int iVal, jVal;
+		for (int r = timeDiff; r < rows; r++) {
+			// Add to the count for this particular pair:
+			iVal = states[r-timeDiff][iCol];
+			jVal = states[r][jCol];
+			jointCount[iVal][jVal]++;
+			iCount[iVal]++;
+			jCount[jVal]++;
+		}
+	}
+	
+	/**
+	 * Returns the average local mutual information storage from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations() {
+		double mi = 0.0;
+		double miCont = 0.0;
+
+		max = 0;
+		min = 0;
+		double meanSqLocals = 0;
+		for (int i = 0; i < base; i++) {
+			// compute p_i
+			double probi = (double) iCount[i] / (double) observations;
+			for (int j = 0; j < base; j++) {
+				// compute p_j
+				double probj = (double) jCount[j] / (double) observations;
+				// compute p(veci=i, vecj=j)
+				double jointProb = (double) jointCount[i][j] / (double) observations;
+				// Compute MI contribution:
+				if (jointProb * probi * probj > 0.0) {
+					double localValue = Math.log(jointProb / (probi * probj)) / log_base;
+					miCont = jointProb * localValue;
+					if (localValue > max) {
+						max = localValue;
+					} else if (localValue < min) {
+						min = localValue;
+					}
+					// Add this contribution to the mean 
+					//  of the squared local values
+					meanSqLocals += miCont * localValue;
+				} else {
+					miCont = 0.0;
+				}
+				mi += miCont;
+			}
+		}
+		
+		average = mi;
+		std = Math.sqrt(meanSqLocals - average * average);
+
+		return mi;
+	}
+
+	public double debugLocalOfObservations() {
+		double mi = 0.0;
+		double miCont = 0.0;
+
+		max = 0;
+		min = 0;
+		double meanSqLocals = 0;
+		System.out.println("i\tj\tp_i\tp_j\tp_joint\tlocal");
+		for (int i = 0; i < base; i++) {
+			// compute p_i
+			double probi = (double) iCount[i] / (double) observations;
+			for (int j = 0; j < base; j++) {
+				// compute p_j
+				double probj = (double) jCount[j] / (double) observations;
+				// compute p(veci=i, vecj=j)
+				double jointProb = (double) jointCount[i][j] / (double) observations;
+				// Compute MI contribution:
+				if (jointProb * probi * probj > 0.0) {
+					double localValue = Math.log(jointProb / (probi * probj)) / log_base;
+					miCont = jointProb * localValue;
+					System.out.printf("%d\t%d\t%.4f\t%.4f\t%.4f\t%.4f\n",
+							i, j, probi, probj, jointProb, localValue);
+					if (localValue > max) {
+						max = localValue;
+					} else if (localValue < min) {
+						min = localValue;
+					}
+					// Add this contribution to the mean 
+					//  of the squared local values
+					meanSqLocals += miCont * localValue;
+				} else {
+					miCont = 0.0;
+				}
+				mi += miCont;
+			}
+		}
+		
+		average = mi;
+		std = Math.sqrt(meanSqLocals - average * average);
+
+		return mi;
+	}
+
+	/**
+	 * Compute the significance of obtaining the given average from the given observations
+	 * 
+	 * @param numPermutationsToCheck number of new orderings of the source values to compare against
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(int numPermutationsToCheck) {
+		RandomGenerator rg = new RandomGenerator();
+		int[][] newOrderings = rg.generateDistinctRandomPerturbations(observations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	/**
+	 * Compute the significance of obtaining the given average from the given observations
+	 * 
+	 * @param newOrderings the reorderings to use
+	 * @return
+	 */
+	public MeasurementDistribution computeSignificance(int[][] newOrderings) {
+		double actualMI = computeAverageLocalOfObservations();
+		
+		int numPermutationsToCheck = newOrderings.length;
+		
+		// Reconstruct the values of the first and second variables (not necessarily in order)
+		int[] iValues = new int[observations];
+		int[] jValues = new int[observations];
+		int t_i = 0;
+		int t_j = 0;
+		for (int iVal = 0; iVal < base; iVal++) {
+			int numberOfSamplesI = iCount[iVal];
+			MatrixUtils.fill(iValues, iVal, t_i, numberOfSamplesI);
+			t_i += numberOfSamplesI;
+			int numberOfSamplesJ = jCount[iVal];
+			MatrixUtils.fill(jValues, iVal, t_j, numberOfSamplesJ);
+			t_j += numberOfSamplesJ;
+		}
+				
+		MutualInformationCalculator mi2 = new MutualInformationCalculator(base, timeDiff);
+		mi2.initialise();
+		mi2.observations = observations;
+		mi2.iCount = iCount;
+		mi2.jCount = jCount;
+		int countWhereMIIsMoreSignificantThanOriginal = 0;
+		MeasurementDistribution measDistribution = new MeasurementDistribution(numPermutationsToCheck);
+		for (int p = 0; p < numPermutationsToCheck; p++) {
+			// Generate a new re-ordered data set for the i variable
+			int[] newDataI = MatrixUtils.extractSelectedTimePoints(iValues, newOrderings[p]);
+			// compute the joint probability distribution
+			MatrixUtils.fill(mi2.jointCount, 0);
+			for (int t = 0; t < observations; t++) {
+				mi2.jointCount[newDataI[t]][jValues[t]]++;
+			}
+			// And get an MI value for this realisation:
+			double newMI = mi2.computeAverageLocalOfObservations();
+			measDistribution.distribution[p] = newMI;
+			if (newMI >= actualMI) {
+				countWhereMIIsMoreSignificantThanOriginal++;
+			}
+
+		}
+		
+		// And return the significance
+		measDistribution.pValue = (double) countWhereMIIsMoreSignificantThanOriginal / (double) numPermutationsToCheck;
+		measDistribution.actualValue = actualMI;
+		return measDistribution;
+	}
+
+	/**
+	 * Computes local active information storage for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 *  
+	 * @param states
+	 * @return
+	 */
+	double[] computeLocalFromPreviousObservations(int states[][], int iCol, int jCol){
+		int rows = states.length;
+		//int columns = states[0].length;
+
+		// Allocate for all rows even though we'll leave the first ones as zeros
+		double[] localMI = new double[rows];
+		int iVal, jVal;
+		double logTerm = 0.0;
+		for (int r = timeDiff; r < rows; r++) {
+			iVal = states[r-timeDiff][iCol];
+			jVal = states[r][jCol];
+			logTerm = ( (double) jointCount[iVal][jVal] ) /
+			  		  ( (double) jCount[jVal] *
+			  			(double) iCount[iVal] );
+			// Now account for the fact that we've
+			//  just used counts rather than probabilities,
+			//  and we've got two counts on the bottom
+			//  but one count on the top:
+			logTerm *= (double) observations;
+			localMI[r] = Math.log(logTerm) / log_base;
+			average += localMI[r];
+			if (localMI[r] > max) {
+				max = localMI[r];
+			} else if (localMI[r] < min) {
+				min = localMI[r];
+			}
+		}
+		average = average/(double) observations;
+		
+		return localMI;
+		
+	}
+	
+	/**
+	 * Standalone routine to 
+	 * compute local active information storage across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param states - 2D array of states
+	 * @return
+	 */
+	public double[] localMutualInformation(int states[][], int iCol, int jCol) {
+		initialise();
+		addObservations(states, iCol, jCol);
+		return computeLocalFromPreviousObservations(states, iCol, jCol);
+		
+	}
+	
+}

java/source/infodynamics/measures/discrete/SeparableInfoCalculatorByAddition.java

@@ -0,0 +1,739 @@
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MatrixUtils;
+
+public class SeparableInfoCalculatorByAddition extends SeparableInfoCalculator {
+
+	ActiveInformationCalculator aiCalc;
+	ApparentTransferEntropyCalculator[] ateCalcs;
+	
+	private boolean localStatsValid = true;
+	
+	public SeparableInfoCalculatorByAddition(int base, int history,
+			int numInfoContributors) {
+		// Create super class without creating any storage
+		//  for the observations
+		super(base, history, numInfoContributors, true);
+		
+		// Create the calculators
+		aiCalc = ActiveInformationCalculator.newInstance(base, history);
+	}
+
+	/**
+	 * Explicitly create the transfer entropy calculators
+	 *  when required (this can save much memory in certain circumstances,
+	 *  e.g. calling computeAverageLocal methods).
+	 *
+	 */
+	private void createAppTeCalculators() {
+		ateCalcs = new ApparentTransferEntropyCalculator[numSources];
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i] = ApparentTransferEntropyCalculator.newInstance(base, k);
+			ateCalcs[i].setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		}
+	}
+	
+	@Override
+	public void initialise() {
+		super.initialise();
+		aiCalc.initialise();
+		if (ateCalcs == null) {
+			createAppTeCalculators();
+		}
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i].initialise();
+		}		
+	}
+
+	@Override
+	public void addObservations(int[][] states, int destCol, int[] sourcesAbsolute) {
+		aiCalc.addObservations(states);
+		int[] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destCol);
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i].addObservations(states, destCol, cleanedSourcesAbsolute[i]);
+		}
+	}
+
+	@Override
+	public void addObservations(int[][] states, int[] offsetOfDestFromSources) {
+		aiCalc.addObservations(states);
+		int[] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i].addObservations(states, cleanedOffsets[i]);
+		}
+	}
+
+	@Override
+	public void addObservations(int[][][] states, int destAgentRow, int destAgentColumn, int[][] sourcesAbsolute) {
+		aiCalc.addObservations(states, destAgentRow, destAgentColumn);
+		int[][] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destAgentRow, destAgentColumn);
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i].addObservations(states, destAgentRow, destAgentColumn,
+					cleanedSourcesAbsolute[i][ROW_INDEX], cleanedSourcesAbsolute[i][COLUMN_INDEX]);
+		}
+	}
+
+	@Override
+	public void addObservations(int[][][] states, int[][] offsetOfDestFromSources) {
+		aiCalc.addObservations(states);
+		int[][] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		for (int i = 0; i < numSources; i++) {
+			ateCalcs[i].addObservations(states, cleanedOffsets[i][ROW_INDEX],
+					cleanedOffsets[i][COLUMN_INDEX]);
+		}
+	}
+
+	@Override
+	public synchronized double computeAverageLocalOfObservations() {
+		average = aiCalc.computeAverageLocalOfObservations();
+		for (int i = 0; i < numSources; i++) {
+			average += ateCalcs[i].computeAverageLocalOfObservations();
+		}
+		localStatsValid = false;
+		return average;
+	}
+
+	@Override
+	public double[] computeLocalFromPreviousObservations(int[][] states, int destCol, int[] sourcesAbsolute) {
+		double[] local = aiCalc.computeLocalFromPreviousObservations(states, destCol);
+		double[][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destCol);
+		double[] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalcs[i].computeLocalFromPreviousObservations(states, destCol, cleanedSourcesAbsolute[i]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics and if not periodic boundary conditions, 
+		//  clean up points which don't get all information contributors
+		setStatistics(local, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+
+	@Override
+	public double[][] computeLocalFromPreviousObservations(int[][] states, int[] offsetOfDestFromSources) {
+		double[][] local = aiCalc.computeLocalFromPreviousObservations(states);
+		double[][][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		double[][] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalcs[i].computeLocalFromPreviousObservations(
+					states, cleanedOffsets[i]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics and if not periodic boundary conditions, 
+		//  clean up points which don't get all information contributors
+		setStatistics(local, cleanedOffsets, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+
+	@Override
+	public double[] computeLocalFromPreviousObservations(int[][][] states,
+			int destAgentRow, int destAgentColumn, int[][] sourcesAbsolute) {
+		double[] local = aiCalc.computeLocalFromPreviousObservations(states, destAgentRow, destAgentColumn);
+		double[][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[][] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destAgentRow, destAgentColumn);
+		double[] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalcs[i].computeLocalFromPreviousObservations(
+					states, destAgentRow, destAgentColumn,
+					cleanedSourcesAbsolute[i][ROW_INDEX],
+					cleanedSourcesAbsolute[i][COLUMN_INDEX]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics
+		setStatistics(local, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+
+	@Override
+	public double[][][] computeLocalFromPreviousObservations(int[][][] states, int[][] offsetOfDestFromSources) {
+		double[][][] local = aiCalc.computeLocalFromPreviousObservations(states);
+		double[][][][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][][][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[][] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		double[][][] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalcs[i].computeLocalFromPreviousObservations(states,
+					cleanedOffsets[i][ROW_INDEX],
+					cleanedOffsets[i][COLUMN_INDEX]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics and if not periodic boundary conditions, 
+		//  clean up points which don't get all information contributors
+		setStatistics(local, cleanedOffsets, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+	
+	@Override
+	public double[][] computeLocal(int states[][], int[] offsetOfDestFromSources) {
+		double[][] local = aiCalc.computeLocal(states);
+		double[][][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		double[][] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalc.computeLocal(
+					states, cleanedOffsets[i]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics and if not periodic boundary conditions, 
+		//  clean up points which don't get all information contributors
+		setStatistics(local, cleanedOffsets, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+	
+	@Override
+	public double[][][] computeLocal(int states[][][], int[][] offsetOfDestFromSources) {
+		double[][][] local = aiCalc.computeLocal(states);
+		double[][][][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][][][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[][] cleanedOffsets = cleanOffsetOfDestFromSources(offsetOfDestFromSources);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		double[][][] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalc.computeLocal(states,
+					cleanedOffsets[i][ROW_INDEX],
+					cleanedOffsets[i][COLUMN_INDEX]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics and if not periodic boundary conditions, 
+		//  clean up points which don't get all information contributors
+		setStatistics(local, cleanedOffsets, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+	
+	@Override
+	public double computeAverageLocal(int[][] states, int[] sourceOffsets) {
+		// One could allow the call to defer here, however it will
+		//  be much less memory intensive if we can use a single 
+		//  ApparentTransferEntropy object
+		average = aiCalc.computeAverageLocal(states);
+		int[] cleanedOffsets = cleanOffsetOfDestFromSources(sourceOffsets);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		for (int i = 0; i < numSources; i++) {
+			average += ateCalc.computeAverageLocal(states, cleanedOffsets[i]);
+		}
+		localStatsValid = false;
+		return average;
+	}
+	
+	@Override
+	public double computeAverageLocal(int[][][] states, int[][] sourceOffsets) {
+		// One could allow the call to defer here, however it will
+		//  be much less memory intensive if we can use a single 
+		//  ApparentTransferEntropy object
+		average = aiCalc.computeAverageLocal(states);
+		int[][] cleanedOffsets = cleanOffsetOfDestFromSources(sourceOffsets);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		for (int i = 0; i < numSources; i++) {
+			average += ateCalc.computeAverageLocal(states,
+					cleanedOffsets[i][ROW_INDEX],
+					cleanedOffsets[i][COLUMN_INDEX]);
+		}
+		localStatsValid = false;
+		return average;
+	}
+	
+	@Override
+	public double[] computeLocal(int states[][], int destCol, int[] sourcesAbsolute) {
+		double[] local = aiCalc.computeLocal(states, destCol);
+		double[][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[] cleanedOffsets = cleanAbsoluteSources(sourcesAbsolute, destCol);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		double[] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalc.computeLocal(
+					states, destCol, cleanedOffsets[i]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics
+		setStatistics(local, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+	
+	@Override
+	public double[] computeLocal(int states[][][], int destAgentRow,
+			int destAgentColumn, int[][] sourcesAbsolute) {
+		double[] local = aiCalc.computeLocal(states, destAgentRow, destAgentColumn);
+		double[][] localComponents = null;
+		if (computeMultiInfoCoherence) {
+			localComponents = new double[numSources + 1][];
+			// Keep a link to the local active info
+			localComponents[0] = local;
+		}
+		int[][] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destAgentRow, destAgentColumn);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		double[] temp;
+		for (int i = 0; i < numSources; i++) {
+			temp = ateCalc.computeLocal(states, destAgentRow, destAgentColumn,
+					cleanedSourcesAbsolute[i][ROW_INDEX],
+					cleanedSourcesAbsolute[i][COLUMN_INDEX]);
+			if (computeMultiInfoCoherence) {
+				// Keep a link to this apparent transfer
+				localComponents[1 + numSources] = temp;
+			}
+			try {
+				local = MatrixUtils.add(local, temp);
+			} catch (Exception e) {
+				// Exception only thrown where arrays were not
+				//  of the same length - should not happen here.
+				return null;
+			}
+		}
+		// Set statistics
+		setStatistics(local, localComponents);
+		localStatsValid = true;
+		return local;
+	}
+
+	@Override
+	public double computeAverageLocal(int[][] states, int destCol, int[] sourcesAbsolute) {
+		// One could allow the call to defer here, however it will
+		//  be much less memory intensive if we can use a single 
+		//  ApparentTransferEntropy object
+		average = aiCalc.computeAverageLocal(states, destCol);
+		int[] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destCol);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		for (int i = 0; i < numSources; i++) {
+			average += ateCalc.computeAverageLocal(states, destCol, cleanedSourcesAbsolute[i]);
+		}
+		localStatsValid = false;
+		return average;
+	}
+
+	@Override
+	public double computeAverageLocal(int states[][][], int destAgentRow,
+			int destAgentColumn, int[][] sourcesAbsolute) {
+		// One could allow the call to defer here, however it will
+		//  be much less memory intensive if we can use a single 
+		//  ApparentTransferEntropy object
+		average = aiCalc.computeAverageLocal(states);
+		int[][] cleanedSourcesAbsolute = cleanAbsoluteSources(sourcesAbsolute, destAgentRow, destAgentColumn);
+		ApparentTransferEntropyCalculator ateCalc = 
+			ApparentTransferEntropyCalculator.newInstance(base, k);
+		ateCalc.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		for (int i = 0; i < numSources; i++) {
+			average += ateCalc.computeAverageLocal(states,
+					cleanedSourcesAbsolute[i][ROW_INDEX],
+					cleanedSourcesAbsolute[i][COLUMN_INDEX]);
+		}
+		localStatsValid = false;
+		return average;
+	}
+	
+	@Override
+	public void setPeriodicBoundaryConditions(boolean periodicBoundaryConditions) {
+		super.setPeriodicBoundaryConditions(periodicBoundaryConditions);
+		if (ateCalcs != null) {
+			for (int i = 0; i < numSources; i++) {
+				ateCalcs[i].setPeriodicBoundaryConditions(periodicBoundaryConditions);
+			}
+		}
+	}
+
+	@Override
+	public double getLastMax() {
+		if (localStatsValid) {
+			return super.getLastMax();
+		}
+		throw new RuntimeException("Last maximum is not valid after previous computation by SeparableInfoCalculatorByAddition");
+	}
+
+	@Override
+	public double getLastMin() {
+		if (localStatsValid) {
+			return super.getLastMin();
+		}
+		throw new RuntimeException("Last minimum is not valid after previous computation by SeparableInfoCalculatorByAddition");
+	}
+
+	@Override
+	public double getLastAverageNegative() {
+		if (localStatsValid) {
+			return super.getLastAverageNegative();
+		}
+		throw new RuntimeException("Last average negative sep is not valid after previous computation by SeparableInfoCalculatorByAddition");
+	}
+
+	@Override
+	public double getLastAveragePositive() {
+		if (localStatsValid) {
+			return super.getLastAveragePositive();
+		}
+		throw new RuntimeException("Last average positive sep is not valid after previous computation by SeparableInfoCalculatorByAddition");
+	}
+
+	private void setStatistics(double[] localValues, double[][] localComponents) {
+		int timeSteps = localValues.length;
+
+		// Compute average, max, min, avPositiveLocal, avNegativeLocal
+		average = 0;
+		max = localValues[k];
+		min = localValues[k];
+		avPositiveLocal = 0;
+		avNegativeLocal = 0;
+		for (int t = k; t < timeSteps; t++) {
+			average += localValues[t];
+			if (localValues[t] > 0) {
+				avPositiveLocal += localValues[t];
+			} else {
+				avNegativeLocal += localValues[t];
+			}
+			if (localValues[t] > max) {
+				max = localValues[t];
+			} else if (localValues[t] < min) {
+				min = localValues[t];
+			}
+		}
+		average /= (double) (timeSteps - k);
+		avPositiveLocal /= (double) (timeSteps - k);
+		avNegativeLocal /= (double) (timeSteps - k);
+		
+		// Now compute the coherence of computation
+		if (computeMultiInfoCoherence) {
+			miCalc.startIndividualObservations();
+			double[] miTuple = new double[numSources + 1];
+			for (int t = k; t < timeSteps; t++) {
+				// Construct the multi-info tuple
+				for (int i = 0; i < numSources + 1; i++) {
+					miTuple[i] = localComponents[i][t];
+				}
+				// Add this tuple to the observations
+				miCalc.addObservation(miTuple);
+			}
+			try {
+				miCalc.endIndividualObservations();
+			} catch (Exception e) {
+				// an exception would only be thrown if we changed the number of causal contributors here
+				//  which simply will not happen. Just in case it does, we'll throw a runtime exception
+				throw new RuntimeException("Number of causal contributors changed from intialisation to calculation!");
+			}
+		}
+	}
+
+	private void setStatistics(double[][] localValues, int[] cleanedSourcesOffsets,
+			double[][][] localComponents) {
+		int timeSteps = localValues.length;
+		int numAgents = localValues[0].length;
+		
+		// Clean up the point which didn't have all of the local information contributors
+		// if we're doing non-peridoic boundary conditions
+		int minAgentOffset = MatrixUtils.min(cleanedSourcesOffsets);
+		int maxAgentOffset = MatrixUtils.max(cleanedSourcesOffsets);
+		int nonPeriodicStartAgent = Math.max(0, maxAgentOffset);
+		int nonPeriodicEndAgent = numAgents - 1 + Math.min(0, minAgentOffset);
+
+		// Clean up if required
+		if (!periodicBoundaryConditions) {
+			for (int t = k; t < timeSteps; t++) {
+				for (int r = 0; r < nonPeriodicStartAgent; r++) {
+					localValues[t][r] = 0;
+				}
+				for (int r = nonPeriodicEndAgent + 1; r < numAgents; r++) {
+					localValues[t][r] = 0;
+				}
+			}
+		}
+
+		// Compute average, max, min, avPositiveLocal, avNegativeLocal
+		average = 0;
+		max = localValues[k][0];
+		min = localValues[k][0];
+		avPositiveLocal = 0;
+		avNegativeLocal = 0;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = periodicBoundaryConditions ? 0 : nonPeriodicStartAgent;
+			 r < (periodicBoundaryConditions ? numAgents : nonPeriodicEndAgent + 1);
+			 r++) {
+				average += localValues[t][r];
+				if (localValues[t][r] > 0) {
+					avPositiveLocal += localValues[t][r];
+				} else {
+					avNegativeLocal += localValues[t][r];
+				}
+				if (localValues[t][r] > max) {
+					max = localValues[t][r];
+				} else if (localValues[t][r] < min) {
+					min = localValues[t][r];
+				}
+			}
+		}
+		if (periodicBoundaryConditions) {
+			average /= (double) ((timeSteps - k) * numAgents);
+			avPositiveLocal /= (double) ((timeSteps - k) * numAgents);
+			avNegativeLocal /= (double) ((timeSteps - k) * numAgents);
+		} else {
+			average /= (double) ((timeSteps - k) * (nonPeriodicEndAgent - nonPeriodicStartAgent + 1));
+			avPositiveLocal /= (double) ((timeSteps - k) * (nonPeriodicEndAgent - nonPeriodicStartAgent + 1));
+			avNegativeLocal /= (double) ((timeSteps - k) * (nonPeriodicEndAgent - nonPeriodicStartAgent + 1));
+		}
+
+		// Now compute the coherence of computation
+		if (computeMultiInfoCoherence) {
+			miCalc.startIndividualObservations();
+			double[] miTuple = new double[numSources + 1];
+			for (int t = k; t < timeSteps; t++) {
+				for (int r = periodicBoundaryConditions ? 0 : nonPeriodicStartAgent;
+				 r < (periodicBoundaryConditions ? numAgents : nonPeriodicEndAgent + 1);
+				 r++) {
+					// Construct the multi-info tuple
+					for (int i = 0; i < numSources + 1; i++) {
+						miTuple[i] = localComponents[i][t][r];
+					}
+					// Add this tuple to the observations
+					miCalc.addObservation(miTuple);
+				}
+			}
+			try {
+				miCalc.endIndividualObservations();
+			} catch (Exception e) {
+				// an exception would only be thrown if we changed the number of causal contributors here
+				//  which simply will not happen. Just in case it does, we'll throw a runtime exception
+				throw new RuntimeException("Number of causal contributors changed from intialisation to calculation!");
+			}
+		}
+	}
+
+	private void setStatistics(double[][][] localValues, int[][] cleanedSourcesOffsets,
+			double[][][][] localComponents) {
+		int timeSteps = localValues.length;
+		int numAgentRows = localValues[0].length;
+		int numAgentColumns = localValues[0][0].length;
+		
+		// Clean up the point which didn't have all of the local information contributors
+		// if we're doing non-peridoic boundary conditions
+		int minRowOffset = MatrixUtils.min(cleanedSourcesOffsets, ROW_INDEX);
+		int maxRowOffset = MatrixUtils.max(cleanedSourcesOffsets, ROW_INDEX);
+		int nonPeriodicStartRow = Math.max(0, maxRowOffset);
+		int nonPeriodicEndRow = numAgentRows - 1 + Math.min(0, minRowOffset);
+		int minColumnOffset = MatrixUtils.min(cleanedSourcesOffsets, COLUMN_INDEX);
+		int maxColumnOffset = MatrixUtils.max(cleanedSourcesOffsets, COLUMN_INDEX);
+		int nonPeriodicStartColumn = Math.max(0, maxColumnOffset);
+		int nonPeriodicEndColumn = numAgentColumns - 1 + Math.min(0, minColumnOffset);
+
+		System.out.println(periodicBoundaryConditions + " " + nonPeriodicStartRow + " " +
+				nonPeriodicEndRow + " " + nonPeriodicStartColumn + " " + nonPeriodicEndColumn);
+		
+		// Clean up if required
+		if (!periodicBoundaryConditions) {
+			for (int t = k; t < timeSteps; t++) {
+				for (int r = 0; r < nonPeriodicStartRow; r++) {
+					for (int c = 0; c < nonPeriodicStartColumn; c++) {
+						localValues[t][r][c] = 0;
+					}
+					for (int c = nonPeriodicEndColumn + 1; c < numAgentColumns; c++) {
+						localValues[t][r][c] = 0;
+					}
+				}
+				for (int r = nonPeriodicEndRow + 1; r < numAgentRows; r++) {
+					for (int c = 0; c < nonPeriodicStartColumn; c++) {
+						localValues[t][r][c] = 0;
+					}
+					for (int c = nonPeriodicEndColumn + 1; c < numAgentColumns; c++) {
+						localValues[t][r][c] = 0;
+					}
+				}
+			}
+		}
+		
+		// Compute average, max, min, avPositiveLocal, avNegativeLocal
+		average = 0;
+		max = localValues[k][0][0];
+		min = localValues[k][0][0];
+		avPositiveLocal = 0;
+		avNegativeLocal = 0;
+		for (int t = k; t < timeSteps; t++) {
+			for (int r = periodicBoundaryConditions ? 0 : nonPeriodicStartRow;
+			 r < (periodicBoundaryConditions ? numAgentRows : nonPeriodicEndRow + 1);
+			 r++) {
+				for (int c = periodicBoundaryConditions ? 0 : nonPeriodicStartColumn;
+				c < (periodicBoundaryConditions ? numAgentColumns : nonPeriodicEndColumn + 1);
+				c++) {
+					average += localValues[t][r][c];
+					if (localValues[t][r][c] > 0) {
+						avPositiveLocal += localValues[t][r][c];
+					} else {
+						avNegativeLocal += localValues[t][r][c];
+					}
+					if (localValues[t][r][c] > max) {
+						max = localValues[t][r][c];
+					} else if (localValues[t][r][c] < min) {
+						min = localValues[t][r][c];
+					}
+				}
+			}
+		}
+		if (periodicBoundaryConditions) {
+			average /= (double) ((timeSteps - k) * numAgentRows * numAgentColumns);
+			avPositiveLocal /= (double) ((timeSteps - k) * numAgentRows * numAgentColumns);
+			avNegativeLocal /= (double) ((timeSteps - k) * numAgentRows * numAgentColumns);
+		} else {
+			average /= (double) ((timeSteps - k) * (nonPeriodicEndRow - nonPeriodicStartRow + 1) *
+					(nonPeriodicEndColumn - nonPeriodicStartColumn + 1));
+			avPositiveLocal /= (double) ((timeSteps - k) * (nonPeriodicEndRow - nonPeriodicStartRow + 1) *
+					(nonPeriodicEndColumn - nonPeriodicStartColumn + 1));
+			avNegativeLocal /= (double) ((timeSteps - k) * (nonPeriodicEndRow - nonPeriodicStartRow + 1) *
+					(nonPeriodicEndColumn - nonPeriodicStartColumn + 1));
+		}
+		
+		// Now compute the coherence of computation
+		if (computeMultiInfoCoherence) {
+			miCalc.startIndividualObservations();
+			double[] miTuple = new double[numSources + 1];
+			for (int t = k; t < timeSteps; t++) {
+				for (int r = periodicBoundaryConditions ? 0 : nonPeriodicStartRow;
+				 r < (periodicBoundaryConditions ? numAgentRows : nonPeriodicEndRow + 1);
+				 r++) {
+					for (int c = periodicBoundaryConditions ? 0 : nonPeriodicStartColumn;
+					c < (periodicBoundaryConditions ? numAgentColumns : nonPeriodicEndColumn + 1);
+					c++) {
+						// Construct the multi-info tuple
+						for (int i = 0; i < numSources + 1; i++) {
+							miTuple[i] = localComponents[i][t][r][c];
+						}
+						// Add this tuple to the observations
+						miCalc.addObservation(miTuple);
+					}
+				}
+			}
+			try {
+				miCalc.endIndividualObservations();
+			} catch (Exception e) {
+				// an exception would only be thrown if we changed the number of causal contributors here
+				//  which simply will not happen. Just in case it does, we'll throw a runtime exception
+				throw new RuntimeException("Number of causal contributors changed from intialisation to calculation!");
+			}
+		}
+	}
+
+	@Override
+	public boolean canComputeMultiInfoCoherenceFromAverageOfObservations() {
+		return false;
+	}
+	
+	
+}

java/source/infodynamics/measures/discrete/SingleAgentMeasure.java

@@ -0,0 +1,185 @@
+package infodynamics.measures.discrete;
+
+/**
+ * Interface to define adding observations and calculating
+ * local and average values of info theoretic measures
+ * for single agent metrics (e.g. entropy, active information).
+ * Would ideally be an abstract class to be inherited from, but
+ * it's more important for us to have inheritance from
+ * ContextOfPastCalculator, and since java doesn't allow multiple
+ * inheritance, one of them has to miss out.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public interface SingleAgentMeasure {
+
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 */
+	public void addObservations(int states[][]);
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @param col index of agent
+	 */
+	public void addObservations(int states[][], int col);
+
+	/**
+ 	 * Add observations in to our estimates of the pdfs.
+ 	 * This call suitable only for homogeneous agents, as all
+ 	 *  agents will contribute to single pdfs.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 */
+	public void addObservations(int states[][][]);
+
+	/**
+ 	 * Add observations for a single agent of the multi-agent system
+ 	 *  to our estimates of the pdfs.
+ 	 * This call should be made as opposed to addObservations(int states[][])
+ 	 *  for computing active info for heterogeneous agents.
+	 *
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @param index1 first index of agent
+	 * @param index2 index of agent in 2nd dimension
+	 */
+	public void addObservations(int states[][][], int index1, int index2);
+
+	/**
+	 * Returns the average information theoretic measure from
+	 *  the observed values which have been passed in previously. 
+	 *  
+	 * Must set average, min and max
+	 * 
+	 * @return
+	 */
+	public double computeAverageLocalOfObservations();
+
+	/**
+	 * Computes local information theoretic measure for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * Must set average, min and max
+	 * 
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocalFromPreviousObservations(int states[][]);
+
+	/**
+	 * Computes local information theoretic measure for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * Must set average, min and max
+	 * 
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][], int col);
+
+	/**
+	 * Computes local information theoretic measure for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method to be used for homogeneous agents only
+	 *  
+	 * Must set average, min and max
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[][][] computeLocalFromPreviousObservations(int states[][][]);
+
+	/**
+	 * Computes local information theoretic measure for the given
+	 *  states, using pdfs built up from observations previously
+	 *  sent in via the addObservations method 
+	 * This method is suitable for heterogeneous agents
+	 *  
+	 * Must set average, min and max
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[] computeLocalFromPreviousObservations(int states[][][], int index1, int index2);
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double[][] computeLocal(int states[][]);
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * 
+	 * @param states 1st index is time, 2nd and 3rd index are agent number
+	 * @return
+	 */
+	public double[][][] computeLocal(int states[][][]);
+
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure across a 2D spatiotemporal
+	 *  array of the states of homogeneous agents
+	 * Return the average
+	 * This method to be called for homogeneous agents only
+	 * 
+	 * @param states -1st index is time, 2nd index is agent number
+	 * @return
+	 */
+	public double computeAverageLocal(int states[][]);
+
+	/**
+	 * Standalone routine to 
+	 * compute local information theoretic measure for one agent in a 2D spatiotemporal
+	 *  array of the states of agents
+	 * Return a 2D spatiotemporal array of local values.
+	 * First history rows are zeros
+	 * This method should be used for heterogeneous agents
+	 * 
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @param col - column number of the agent in the states array
+	 * @return
+	 */
+	public double[] computeLocalAtAgent(int states[][], int col);
+	
+	/**
+	 * Standalone routine to 
+	 * compute average local information theoretic measure 
+	 * for a single agent
+	 * Returns the average
+	 * This method suitable for heterogeneous agents
+	 * @param blocksize - Size of blocks to compute entropy over
+	 * @param base - base of the states
+	 * @param states 1st index is time, 2nd index is agent number
+	 * @param col - column number of the agent in the states array
+	 * 
+	 * @return
+	 */
+	public double computeAverageLocalAtAgent(int states[][], int col);
+
+}

java/source/infodynamics/measures/discrete/SingleAgentMeasureInContextOfPastCalculator.java

@@ -0,0 +1,59 @@
+package infodynamics.measures.discrete;
+
+/**
+ * Combines functionality for single agents with functionality
+ * required in the context of the past.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public abstract class SingleAgentMeasureInContextOfPastCalculator extends
+		ContextOfPastMeasureCalculator implements SingleAgentMeasure {
+
+	public SingleAgentMeasureInContextOfPastCalculator(int base, int history) {
+		super(base, history);
+	}
+
+	public final double[][] computeLocal(int[][] states) {
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	public final double[][][] computeLocal(int[][][] states) {
+		initialise();
+		addObservations(states);
+		return computeLocalFromPreviousObservations(states);
+	}
+
+	public final double computeAverageLocal(int[][] states) {
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	public final double computeAverageLocal(int[][][] states) {
+		initialise();
+		addObservations(states);
+		return computeAverageLocalOfObservations();
+	}
+
+	public final double[] computeLocalAtAgent(int[][] states, int col) {
+		initialise();
+		addObservations(states, col);
+		return computeLocalFromPreviousObservations(states, col);
+	}
+
+	public final double[] computeLocalAtAgent(int[][][] states, int index1, int index2) {
+		initialise();
+		addObservations(states, index1, index2);
+		return computeLocalFromPreviousObservations(states, index1, index2);
+	}
+
+	public final double computeAverageLocalAtAgent(int[][] states, int col) {
+		initialise();
+		addObservations(states, col);
+		return computeAverageLocalOfObservations();
+	}
+
+}

java/source/infodynamics/networkinference/interregional/InterregionalChannelMeasure.java

@@ -0,0 +1,825 @@
+package infodynamics.networkinference.interregional;
+
+import infodynamics.measures.continuous.ChannelCalculatorMultiVariate;
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.measures.continuous.TransferEntropyCalculatorMultiVariate;
+import infodynamics.utils.ArrayFileReader;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+import infodynamics.utils.ParsedProperties;
+import infodynamics.utils.RandomGenerator;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.PrintStream;
+import java.util.Properties;
+
+
+/**
+ * <p>Compute a significance score for an inter-regional channel measure
+ * between two sets, based on looking at the channel measure between elements taken 
+ * jointVars1 and jointVars2 at a time.
+ * </p>
+ * <p>Usage:
+ * 	<ol>
+ * 		<li>newInstance()</li>
+ * 		<li>initialise</li>
+ * 		<li>setObservations</li>
+ * 		<li>computeMean, or computeSignificance, or computeLocals</li>
+ *  </ol>
+ * </p>
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com 
+ *
+ */
+public abstract class InterregionalChannelMeasure {
+
+	protected ChannelCalculatorMultiVariate channelCalc;
+	/**
+	 * The number of joint variables to consider from each region in each calculation
+	 */
+	protected int jointVars1; // destination
+	protected int jointVars2; // source
+	/**
+	 * The total number of variables in each region
+	 */
+	protected boolean useAverageRegion1 = false;
+	protected int totalVars1;
+	protected boolean useAverageRegion2 = false;
+	protected int totalVars2;
+	protected int maxNumSubsets;
+	protected boolean debug;
+	protected boolean writeResults;
+
+	protected double[][] region1;
+	protected double[][] region2;
+	protected boolean allValid;
+	protected boolean validityForIndividualElements;
+	protected boolean[] jointValidity1;
+	protected boolean[] jointValidity2;
+	protected boolean[][] individualValidity1;
+	protected boolean[][] individualValidity2;
+	
+	protected double lastAverage;
+	
+	protected RandomGenerator rg;
+	protected long seed;
+	
+	// Subsets to use
+	protected int[][] subsetsForEachRegion1; 
+	protected int[][] subsetsForEachRegion2; 
+	protected int numOfSets;
+	
+	// Track whether each subset pair is known to be significant or not
+	protected double[] significanceLevelsForEachSubset;
+	// Track the t-score for each subset.
+	protected double[] tScoreForEachSubject;
+
+	private String calculatorClass;
+	protected Properties calculatorProperties;
+
+	public static final String PROP_CALCULATOR_PROPERTIES_PREFIX = "props.interregionalChannel.calculatorProperties.";
+	public static final String PROP_CALCULATOR_CLASS = "props.interregionalChannel.calculator";
+	public static final String PROP_JOINT_VARS_1 = "props.interregionalChannel.jointVars1";
+	public static final String PROP_JOINT_VARS_2 = "props.interregionalChannel.jointVars2";
+	public static final String PROP_NUM_SUBSETS = "props.interregionalChannel.maxNumSets";
+	public static final String PROP_SEED = "props.interregionalChannel.seed";
+	public static final String PROP_GET_SIGNIFICANCE = "props.interregionalChannel.directRun.getSignificance";
+	public static final String PROP_SIG_REORDERINGS = "props.interregionalChannel.directRun.numReorderingsForSignificance";
+	public static final String PROP_SIGNIFICANCE_COMPARE_POPULATIONS = "props.interregionalChannel.directRun.compareSubsetPopulationsForSignificance";
+	public static final String PROP_WRITE_RESULTS = "props.interregionalChannel.writeResultsToStdOut";
+	public static final String PROP_DIRECT_FILE1 = "props.interregionalChannel.directRun.file1";
+	public static final String PROP_DIRECT_FILE2 = "props.interregionalChannel.directRun.file2";
+	public static final String PROP_DIRECT_OUTPUT_PREFIX = "props.interregionalChannel.directRun.outputPrefix";
+	public static final String PROP_DEBUG = "props.debug";
+	
+	public static final String JOINT_VARS_SELECT_AVERAGE = "Av";
+
+	/**
+	 * Data structure to store a distribution of channel measurements. 
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	public class ChannelMeasurementDistribution {
+		public double[] channelMeasurements;
+		public double mean;
+		public double std;
+	}
+	
+	/**
+	 * Data structure to store the means and standard deviations of local values of the 
+	 *  channel measurement in time. The averages are taken across a number of subsets
+	 *  selected from each region.
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	public class LocalChannelMeasurementDistribution {
+		public double[] meanLocalChannelMeasurements;
+		public double[] stdLocalChannelMeasurements;
+		public boolean hasLocalsForSignificantChannelsOnly;
+		public int countOfSignificantChannels;
+		public double[] meanLocalForSignificantChannelsOnly;
+		public double[] stdLocalForSignificantChannelsOnly;
+	}
+
+	public static InterregionalChannelMeasure newInstance(ParsedProperties props) throws Exception {
+		// Generate a new instance based on the calculator name
+		String calcName = props.getStringProperty(PROP_CALCULATOR_CLASS);
+		return newInstance(calcName);
+	}
+	
+	public static InterregionalChannelMeasure newInstance(String calcName) throws Exception {
+		// Generate a new instance based on the calculator name
+		ChannelCalculatorMultiVariate channelCalcInstance = (ChannelCalculatorMultiVariate)
+			Class.forName(calcName).newInstance();
+		if (TransferEntropyCalculatorMultiVariate.class.isInstance(channelCalcInstance)) {
+			return new InterregionalTransferEntropy();
+		} else if (MutualInfoCalculatorMultiVariate.class.isInstance(channelCalcInstance)) {
+			return new InterregionalMutualInfo();
+		} else {
+			throw new Exception("Calculator name not recognised");
+		}
+	}
+
+	protected InterregionalChannelMeasure() {
+		super();
+	}
+	
+	public void initialise(ParsedProperties props) throws Exception {
+		setCalculatorName(props.getStringProperty(PROP_CALCULATOR_CLASS));
+		useAverageRegion1 = props.getProperty(PROP_JOINT_VARS_1).
+								equalsIgnoreCase(JOINT_VARS_SELECT_AVERAGE);
+		if (useAverageRegion1) {
+			jointVars1 = 1;
+		} else {
+			setJointVars1(props.getIntProperty(PROP_JOINT_VARS_1));
+		}
+		useAverageRegion2 = props.getProperty(PROP_JOINT_VARS_2).
+								equalsIgnoreCase(JOINT_VARS_SELECT_AVERAGE);
+		if (useAverageRegion2) {
+			jointVars2 = 1;
+		} else {
+			setJointVars2(props.getIntProperty(PROP_JOINT_VARS_2));
+		}
+		setDebug(props.getBooleanProperty(PROP_DEBUG));
+		if (useAverageRegion1 && useAverageRegion2) {
+			// Since we're taking the average of each region, 
+			//  there are no subsets to take
+			setMaxNumSubsets(1);
+		} else {
+			setMaxNumSubsets(props.getIntProperty(PROP_NUM_SUBSETS));
+		}
+		setSeed(props.getLongProperty(PROP_SEED));
+		setWriteResults(props.getBooleanProperty(PROP_WRITE_RESULTS));
+		readInCalculatorProperties(props);
+		initialise();
+	}
+
+	public void initialise(int jointVars1, int jointVars2, int maxNumSubsets) throws Exception {
+		setJointVars1(jointVars1);
+		setJointVars2(jointVars2);
+		useAverageRegion1 = false;
+		useAverageRegion2 = false;
+		setMaxNumSubsets(maxNumSubsets);
+		initialise();
+	}
+
+	public void initialise() throws Exception {
+		// Create our channel calculator
+		channelCalc = (ChannelCalculatorMultiVariate)
+					Class.forName(calculatorClass).newInstance();
+		channelCalc.setDebug(debug);
+		setPropertiesOnCalculator();
+		rg = new RandomGenerator();
+		rg.setSeed(seed);
+		subsetsForEachRegion1 = null;
+		subsetsForEachRegion2= null;
+		significanceLevelsForEachSubset = null;
+		tScoreForEachSubject = null;
+		numOfSets = 0;
+	}
+	
+	private void readInCalculatorProperties(ParsedProperties props) {
+		calculatorProperties = new Properties();
+		for (Object keyObject : props.getProperties().keySet()) {
+			String key = (String) keyObject;
+			if (key.startsWith(PROP_CALCULATOR_PROPERTIES_PREFIX)) {
+				// Add this key to the calculator properties
+				String propertyName = key.replaceFirst(PROP_CALCULATOR_PROPERTIES_PREFIX, "");
+				calculatorProperties.setProperty(propertyName, props.getProperty(key));
+			}
+		}
+	}
+
+	public void setObservations(double[][] region1, double[][] region2) {
+		setRegion1And2Data(region1, region2);
+		allValid = true;
+		finaliseSetObservations();
+	}
+	
+	public void setObservations(double[][] region1, double[][] region2,
+			boolean[] validity1, boolean[] validity2) {
+		setRegion1And2Data(region1, region2);
+		jointValidity1 = validity1;
+		jointValidity2 = validity2;
+		allValid = false;
+		validityForIndividualElements = false;
+		finaliseSetObservations();
+	}
+
+	public void setObservations(double[][] region1, double[][] region2,
+			boolean[][] validity1, boolean[][] validity2) {
+		setRegion1And2Data(region1, region2);
+		individualValidity1 = validity1;
+		individualValidity2 = validity2;
+		allValid = false;
+		validityForIndividualElements = true;
+		finaliseSetObservations();
+	}
+	
+	protected void setRegion1And2Data(double[][] region1, double[][] region2) {
+		if (useAverageRegion1) {
+			// We're taking the average across all of the variables in the region
+			this.region1 = new double[region1.length][1];
+			try {
+				MatrixUtils.copyIntoColumn(this.region1, 0,
+						MatrixUtils.meansOfRows(region1));
+			} catch (Exception e) {
+				// This should never happen
+				throw new RuntimeException(e);
+			}
+		} else {
+			// We're using all of the individual variables
+			this.region1 = region1;
+		}
+		if (useAverageRegion2) {
+			// We're taking the average across all of the variables in the region
+			this.region2 = new double[region2.length][1];
+			try {
+				MatrixUtils.copyIntoColumn(this.region2, 0,
+						MatrixUtils.meansOfRows(region2));
+			} catch (Exception e) {
+				// This should never happen
+				throw new RuntimeException(e);
+			}
+		} else {
+			// We're using all of the individual variables
+			this.region2 = region2;
+		}
+	}
+	
+	protected void finaliseSetObservations() {
+		totalVars1 = region1[0].length;
+		totalVars2 = region2[0].length;
+	}
+
+	/**
+	 * Initialise the calculator used for the calculation between each subset.
+	 * Should be overridden by each subclass if if needs a different type of initialisation.
+	 *
+	 */
+	protected void initialiseCalculator() throws Exception {
+		channelCalc.initialise(jointVars1, jointVars2);
+	}
+	
+	/**
+	 * Set properties for the calculator object to match those we're holding.
+	 * 
+	 * @throws Exception
+	 */
+	protected void setPropertiesOnCalculator()
+		throws Exception {
+		
+		boolean oldDebug = debug;
+		channelCalc.setDebug(true);
+		for (Object keyObject : calculatorProperties.keySet()) {
+			channelCalc.setProperty((String) keyObject,
+					calculatorProperties.getProperty((String) keyObject));
+		}
+		channelCalc.setDebug(oldDebug);
+	}
+	
+	/**
+	 * Check that the subsets for each region have been generated
+	 *
+	 */
+	protected void checkSubsetsAreGenerated() {
+		// Grab a set of subsets for each region if required:
+		if (subsetsForEachRegion1 == null) {
+			subsetsForEachRegion1 = rg.generateNRandomSets(totalVars1, jointVars1, maxNumSubsets); 
+			subsetsForEachRegion2 = rg.generateNRandomSets(totalVars2, jointVars2, maxNumSubsets); 
+			numOfSets = Math.min(subsetsForEachRegion1.length, subsetsForEachRegion2.length);
+		}
+	}
+	
+	/**
+	 * Compute the mean transfer entropy across maxNumSubsets subsets of 
+	 * joint variables from each region
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public ChannelMeasurementDistribution computeMean() throws Exception {
+		// Set up the calculator
+		setPropertiesOnCalculator();
+		
+		checkSubsetsAreGenerated();
+		
+		double[] tes = new double[numOfSets];
+		// Store all the results
+		for (int s = 0; s < numOfSets; s++) {
+			// Select the subset of region 1:
+			double[][] r1Subset = MatrixUtils.selectColumns(region1, subsetsForEachRegion1[s]);
+			// Select the subset of region 2:
+			double[][] r2Subset = MatrixUtils.selectColumns(region2, subsetsForEachRegion2[s]);
+			initialiseCalculator();
+			// Then add them in as observations
+			if (allValid) {
+				// all the observations are valid
+				channelCalc.setObservations(r1Subset, r2Subset);
+			} else if (validityForIndividualElements) {
+				// we've been given validity for each individual sub-variable
+				// compute the joint validities
+				boolean[] computedJointValidity1 = MatrixUtils.andRowsOverSelectedColumns(
+						individualValidity1, subsetsForEachRegion1[s]);
+				boolean[] computedJointValidity2 = MatrixUtils.andRowsOverSelectedColumns(
+						individualValidity2, subsetsForEachRegion2[s]);
+				channelCalc.setObservations(r1Subset, r2Subset, computedJointValidity1, computedJointValidity2);
+			} else {
+				// we've got joint validity for each time series
+				channelCalc.setObservations(r1Subset, r2Subset, jointValidity1, jointValidity2);
+			}
+			
+			// Compute the TEs
+			tes[s] = channelCalc.computeAverageLocalOfObservations();
+			if (writeResults) {
+				System.out.print("Done average for subset " + s + " ");
+				printSubset(System.out, subsetsForEachRegion1[s]);
+				System.out.print(" -> ");
+				printSubset(System.out, subsetsForEachRegion2[s]);
+				System.out.println(": average = " +
+						tes[s]);
+			}
+		}
+
+		ChannelMeasurementDistribution ted = new ChannelMeasurementDistribution();
+		ted.channelMeasurements = tes;
+		ted.mean = MatrixUtils.mean(tes);
+		ted.std = MatrixUtils.stdDev(tes, ted.mean);
+		lastAverage = ted.mean;
+		if (writeResults) {
+			System.out.println("Average across " + numOfSets + " subsets was " + lastAverage);
+		}
+		return ted;
+	}
+	
+	/**
+	 * <p>Generate reorderingsForSignificance reorderings for the observations supplied
+	 * here. Supplied where a user wants to generate the reordering once, and thereafter
+	 * supply to a number of InterregionalChannelMeasure objects.
+	 * </p> 
+	 * 
+	 * @param reorderingsForSignificance
+	 * @return
+	 * @throws Exception
+	 */
+	public int[][] generateReorderings(int reorderingsForSignificance) throws Exception {
+		// Compute the reorderings
+		int numObservationsToReorder = computeNumObservationsToReorder();
+		return rg.generateDistinctRandomPerturbations(numObservationsToReorder,
+				reorderingsForSignificance);
+	}
+	
+	/**
+	 * Compute the significance of the average of the channel measure across all subsets, 
+	 *  by comparing it to the population of averages that could be obtained where the
+	 *  first region is reordered against the second.
+	 * Each average in the population of reordering is made with each underlying subset
+	 *  reordered the same way. 
+	 * 
+	 * @param reorderingsForSignificance the number of reorderings to consider
+	 * @return the distribution of SxP surrogate measurements for each subset S and permutation P 
+	 * @throws Exception
+	 */
+	public MeasurementDistributionPermutationsOverSubsets computeSignificance(int reorderingsForSignificance) throws Exception {
+		return computeSignificance(generateReorderings(reorderingsForSignificance));
+	}
+	
+	/**
+	 * Compute the significance of the average of the channel measure across all subsets, 
+	 *  by comparing it to the population of averages that could be obtained where the
+	 *  first region is reordered against the second.
+	 * Each average in the population of reordering is made with each underlying subset
+	 *  reordered the same way. 
+	 * 
+	 * @param reorderings the reorderings to consider
+	 * @return the distribution of SxP surrogate measurements for each subset S and permutation P 
+	 * @throws Exception
+	 */
+	public MeasurementDistributionPermutationsOverSubsets computeSignificance(int[][] reorderings) throws Exception {
+		int reorderingsForSignificance = reorderings.length;
+		
+		// Set up the calculator
+		setPropertiesOnCalculator();
+		
+		checkSubsetsAreGenerated();
+
+		double[] measureForEachSet = new double[numOfSets];
+		double[][] reorderedTesForAllSubsets = new double[numOfSets][];
+		
+		// Track the significance for each subset
+		significanceLevelsForEachSubset = new double[numOfSets];
+		tScoreForEachSubject = new double[numOfSets];
+		
+		// Store all the results
+		for (int s = 0; s < numOfSets; s++) {
+			// Select the subset of region 1:
+			double[][] r1Subset = MatrixUtils.selectColumns(region1, subsetsForEachRegion1[s]);
+			// Select the subset of region 2:
+			double[][] r2Subset = MatrixUtils.selectColumns(region2, subsetsForEachRegion2[s]);
+			initialiseCalculator();
+			// Then add them in as observations
+			if (allValid) {
+				// all the observations are valid
+				channelCalc.setObservations(r1Subset, r2Subset);
+			} else if (!validityForIndividualElements) {
+				// we've got joint validity for each time series
+				channelCalc.setObservations(r1Subset, r2Subset, jointValidity1, jointValidity2);
+			} else {
+				// we've been given validity for each individual sub-variable
+				channelCalc.setObservations(r1Subset, r2Subset, individualValidity1, individualValidity2);
+			}
+			
+			// Compute the measure for set s
+			measureForEachSet[s] = channelCalc.computeAverageLocalOfObservations();
+
+			MeasurementDistribution measDist;
+			if (allValid || !validityForIndividualElements) {
+				// Ask the TE calculator to work out the significance for us
+				measDist = channelCalc.computeSignificance(reorderings);
+			} else {
+				// We need to explicitly reorder including the individual validities.
+				// Can't ask the calculator to do it, as it will only use the source-dest
+				//  pairings it originally used, which won't match across all subsets
+				measDist = new MeasurementDistribution(reorderingsForSignificance);
+				measDist.actualValue = measureForEachSet[s];
+				int countWhereReorderedIsMoreSignificantThanOriginal = 0;
+				for (int p = 0; p < reorderingsForSignificance; p++) {
+					double[][] reorderedR1Subset =
+						MatrixUtils.extractSelectedTimePointsReusingArrays(r1Subset, reorderings[p]);
+					boolean[][] reorderedIndividualValidity1 = 
+						MatrixUtils.extractSelectedTimePointsReusingArrays(individualValidity1, reorderings[p]);
+					initialiseCalculator();
+					channelCalc.setObservations(reorderedR1Subset, r2Subset,
+							reorderedIndividualValidity1, individualValidity2);
+					measDist.distribution[p] = channelCalc.computeAverageLocalOfObservations();
+					if (measDist.distribution[p] >= measureForEachSet[s]) {
+						countWhereReorderedIsMoreSignificantThanOriginal++;
+					}
+				}
+				measDist.pValue = (double) countWhereReorderedIsMoreSignificantThanOriginal /
+										(double) reorderingsForSignificance;
+			}
+			reorderedTesForAllSubsets[s] = measDist.distribution;
+			significanceLevelsForEachSubset[s] = measDist.pValue;
+			tScoreForEachSubject[s] = measDist.getTSscore();
+			if (writeResults) {
+				double meanOfDist = MatrixUtils.mean(measDist.distribution);
+				double stdOfDist = MatrixUtils.stdDev(measDist.distribution, meanOfDist);
+				System.out.print("Significance for subset " + s + " ");
+				printSubset(System.out, subsetsForEachRegion1[s]);
+				System.out.print(" -> ");
+				printSubset(System.out, subsetsForEachRegion2[s]);
+				System.out.printf(" was %.3f (%.4f compared to %.4f +/- %.4f, t=%.4f, factor of %.2f)\n",
+						measDist.pValue,
+						measDist.actualValue,
+						meanOfDist, stdOfDist, tScoreForEachSubject[s], 
+						measDist.actualValue / meanOfDist);
+			}
+		}
+
+		// Compute the averages over subsets for each reordering, and
+		//  compute the significance = probability that we could have gotten a 
+		//  value >= by chance alone
+		MeasurementDistributionPermutationsOverSubsets measDist = 
+			new MeasurementDistributionPermutationsOverSubsets(
+					reorderedTesForAllSubsets, measureForEachSet);
+		
+		lastAverage = measDist.actualValue;
+		if (writeResults) {
+			System.out.println("Significance across " + reorderingsForSignificance + " permutations of " + 
+					numOfSets + " subsets averaged was " + measDist.pValue);
+		}
+		return measDist;
+	}
+
+	/**
+	 * Compute the average local transfer entropy values
+	 *  (local in time, averaged across all subsets)
+	 * from the source region to the destination.
+	 * 
+	 * @return object containing means and std deviations
+	 *  of local values in time.
+	 */
+	public LocalChannelMeasurementDistribution computeLocals() throws Exception {
+		return computeLocals(-1.0, true);
+	}
+	
+	/**
+	 * Compute the average local transfer entropy values
+	 *  (local in time, averaged across all subsets)
+	 * from the source region to the destination.
+	 * 
+	 * @param cutoff level at which to count an individual channel as significant
+	 *   for the return distribution. If &lt; 0 don't look at the significance of individual
+	 *   channels. If &gt;= 0 one of the computeSignificance methods should have already
+	 *   been called.
+	 * @param cutoffIsSignificance whether the cutoff is the cutoff significance of the TE value
+	 *  or on the cutoff z score
+	 * @return object containing means and std deviations
+	 *  of local values in time.
+	 */
+	public LocalChannelMeasurementDistribution computeLocals(double cutoff, boolean cutoffIsSignificance) throws Exception {
+		// Set up the calculator
+		setPropertiesOnCalculator();
+		
+		checkSubsetsAreGenerated();
+
+		lastAverage = 0.0;
+		double[] averageLocalTes = null;
+		double[] stdLocalTes = null;
+		int countOfSignificantChannels = 0;
+		double[] averageLocalTesForSigChannelsOnly = null;
+		double[] stdLocalTesForSigChannelsOnly = null;
+		// Store all the results
+		for (int s = 0; s < numOfSets; s++) {
+			// Select the subset of region 1:
+			double[][] r1Subset = MatrixUtils.selectColumns(region1, subsetsForEachRegion1[s]);
+			// Select the subset of region 2:
+			double[][] r2Subset = MatrixUtils.selectColumns(region2, subsetsForEachRegion2[s]);
+			initialiseCalculator();
+			// Then add them in as observations
+			if (allValid) {
+				// all the observations are valid
+				channelCalc.setObservations(r1Subset, r2Subset);
+			} else if (validityForIndividualElements) {
+				// we've been given validity for each individual sub-variable
+				// compute the joint validities
+				boolean[] computedJointValidity1 = MatrixUtils.andRowsOverSelectedColumns(
+						individualValidity1, subsetsForEachRegion1[s]);
+				boolean[] computedJointValidity2 = MatrixUtils.andRowsOverSelectedColumns(
+						individualValidity2, subsetsForEachRegion2[s]);
+				channelCalc.setObservations(r1Subset, r2Subset, computedJointValidity1, computedJointValidity2);
+				// TODO I don't think adding in place will work for this case
+				//  because each subset pair will have a different number of valid observations,
+				//  which means averaging over all the local time series doesn't make much sense ??
+				// Could average into 0..t-1 but not make a contribution if we don't 
+				//  have a valid local value there.
+			} else {
+				// we've got joint validity for each time series
+				channelCalc.setObservations(r1Subset, r2Subset, jointValidity1, jointValidity2);
+			}
+			
+			// Compute the local TEs
+			channelCalc.setDebug(true);
+			double[] localTes = channelCalc.computeLocalOfPreviousObservations();
+			channelCalc.setDebug(debug);
+			lastAverage += channelCalc.getLastAverage();
+			
+			// And add these into the sums for each time step
+			if (averageLocalTes == null) {
+				// Create the space to store the average 
+				//  and std dev of local TEs.
+				// Creating here because outside the loop
+				//  we don't know the length of arrays to create
+				averageLocalTes = new double[localTes.length];
+				stdLocalTes = new double[localTes.length];
+			}
+			MatrixUtils.addInPlace(averageLocalTes, localTes);
+			// stdLocalTes holds the sum of squares for now
+			MatrixUtils.addSquaresInPlace(stdLocalTes, localTes);
+			if ((cutoff >= 0.0) && (significanceLevelsForEachSubset != null)) {
+				// We've been asked to give a result for significant channels only,
+				//  and the significance of each channel has been calculated.
+				if (averageLocalTesForSigChannelsOnly == null) {
+					averageLocalTesForSigChannelsOnly = new double[localTes.length];
+					stdLocalTesForSigChannelsOnly = new double[localTes.length];
+				}
+				// Store these locals if the channel was previously found to be significant
+				if ((cutoffIsSignificance && significanceLevelsForEachSubset[s] <= cutoff) ||
+					(!cutoffIsSignificance && tScoreForEachSubject[s] >= cutoff)) {
+					countOfSignificantChannels++;
+					MatrixUtils.addInPlace(averageLocalTesForSigChannelsOnly, localTes);
+					// stdLocalTes holds the sum of squares for now
+					MatrixUtils.addSquaresInPlace(stdLocalTesForSigChannelsOnly, localTes);
+				}
+			}
+			if (writeResults) {
+				System.out.print("Done locals for subset " + s + " ");
+				printSubset(System.out, subsetsForEachRegion1[s]);
+				System.out.print(" -> ");
+				printSubset(System.out, subsetsForEachRegion2[s]);
+				System.out.printf(": (average = %.5f)\n",
+						channelCalc.getLastAverage());
+			}
+		}
+
+		// Now take the average across all subsets
+		//  and compute the standard devs
+		for (int t = 0; t < averageLocalTes.length; t++) {
+			averageLocalTes[t] /= (double) numOfSets;
+			stdLocalTes[t] = Math.sqrt(
+					stdLocalTes[t] / (double) numOfSets - 
+					averageLocalTes[t] * averageLocalTes[t]);
+		}
+		lastAverage /= (double) numOfSets;
+		if (writeResults) {
+			System.out.println("Average (from locals) across " + numOfSets + " subsets was "
+					+ lastAverage);
+		}
+		LocalChannelMeasurementDistribution lcmd = new
+			LocalChannelMeasurementDistribution();
+		lcmd.meanLocalChannelMeasurements = averageLocalTes;
+		lcmd.stdLocalChannelMeasurements = stdLocalTes;
+		lcmd.hasLocalsForSignificantChannelsOnly = (cutoff >= 0.0) && (significanceLevelsForEachSubset != null);
+		// Include the results for significant channels only if requested
+		if (lcmd.hasLocalsForSignificantChannelsOnly) {
+			lcmd.countOfSignificantChannels = countOfSignificantChannels;
+			if (countOfSignificantChannels > 0) {
+				// Find the mean and std for each point in time
+				for (int t = 0; t < averageLocalTes.length; t++) {
+					averageLocalTesForSigChannelsOnly[t] /= (double) countOfSignificantChannels;
+					stdLocalTesForSigChannelsOnly[t] = Math.sqrt(
+							stdLocalTesForSigChannelsOnly[t] / (double) countOfSignificantChannels - 
+							averageLocalTesForSigChannelsOnly[t] * averageLocalTesForSigChannelsOnly[t]);
+				}
+			}
+			lcmd.meanLocalForSignificantChannelsOnly = averageLocalTesForSigChannelsOnly;
+			lcmd.stdLocalForSignificantChannelsOnly = stdLocalTesForSigChannelsOnly;
+		}
+		return lcmd;
+	}
+	
+	/**
+	 * Compute the number of observations that need to be reordered in a significance
+	 *  computation, based on how the validities have been supplied here, and
+	 *  what type of calculator we are used (this is why it is left abstract).
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	protected abstract int computeNumObservationsToReorder() throws Exception;
+	
+	/**
+	 * Compute the time indices for the local observations generated here
+	 * 
+	 * @return
+	 * @throws Exception
+	 */
+	public abstract int[] computeTimeIndicesForLocalValues() throws Exception;
+	
+	/**
+	 * Print the subset out to the given stream
+	 * 
+	 * @param out
+	 * @param subset
+	 */
+	private void printSubset(PrintStream out, int[] subset) {
+		out.print("{");
+		boolean printedOne = false;
+		for (int i = 0; i < subset.length; i++) {
+			if (printedOne) {
+				out.print(", ");
+			}
+			out.print(subset[i]);
+			printedOne = true;
+		}
+		out.print("}");
+	}
+
+	public void setCalculatorName(String calculatorName) {
+		calculatorClass = calculatorName;
+	}
+	
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+	public void setJointVars1(int jointVars1) {
+		this.jointVars1 = jointVars1;
+	}
+
+	public void setJointVars2(int jointVars2) {
+		this.jointVars2 = jointVars2;
+	}
+	
+	public void setMaxNumSubsets(int maxNumSubsets) {
+		this.maxNumSubsets = maxNumSubsets;
+	}
+	
+	public void setSeed(long seed) {
+		this.seed = seed;	
+	}
+
+	private void setWriteResults(boolean writeResults) {
+		this.writeResults = writeResults;
+	}
+
+	public double getLastAverage() {
+		return lastAverage;
+	}
+	
+	/**
+	 * A simple method to evaluate the interregional channel measure 
+	 *  for a given pair of files.
+	 * 
+	 * @param args first arg is a property filename
+	 */
+	public static void main(String[] args) throws Exception {
+		
+		if (args.length < 1) {
+			System.err.println("Usage: InterregionalChannelMeasure <propsFile>");
+			return;
+		}
+		String propertiesFilename = args[0];
+
+		Properties properties = new Properties();
+		properties.load(new FileInputStream(new File(propertiesFilename)));
+		ParsedProperties props = new ParsedProperties(properties);
+		InterregionalChannelMeasure interregionalCalculator =
+			InterregionalChannelMeasure.newInstance(props);
+
+		// Load in the data
+		// for the one region pair
+		ArrayFileReader afr1 = new ArrayFileReader(props.getStringProperty(PROP_DIRECT_FILE1));
+		double[][] region1 = afr1.getDouble2DMatrix();
+		ArrayFileReader afr2 = new ArrayFileReader(props.getStringProperty(PROP_DIRECT_FILE2));
+		double[][] region2 = afr2.getDouble2DMatrix();
+
+		interregionalCalculator.initialise(props);
+		// System.out.println("Using all " + region1.length + " observations");
+		interregionalCalculator.setObservations(region1, region2);
+		
+		ChannelMeasurementDistribution meanDistribution =
+			interregionalCalculator.computeMean();
+		
+		/*
+		System.out.print("MeanActuals,StdActuals");
+		if (props.getBooleanProperty(PROP_GET_SIGNIFICANCE)) {
+			System.out.println(",MeanPsOvS,StdPsOvS,t,degFree");
+		}
+		*/
+		try {
+			String outputPrefix = props.getStringProperty(PROP_DIRECT_OUTPUT_PREFIX);
+			System.out.printf("%s", outputPrefix);
+		} catch (Exception e) {
+			// there was no output prefix to print out
+		}
+		
+		System.out.printf("%.4f,%.4f", meanDistribution.mean, meanDistribution.std);
+		if (props.getBooleanProperty(PROP_GET_SIGNIFICANCE)) {
+			// Generate the reorderings once and use a common set across all subsets
+			//  and subject;
+			// Unless we are using only pre and post steps, in which case the number
+			//  of observations will be different across subjects, so we will need
+			//  to regenerate the reorderings subject by subject.
+			int permutations = props.getIntProperty(PROP_SIG_REORDERINGS);
+			MeasurementDistributionPermutationsOverSubsets measDist =
+				interregionalCalculator.computeSignificance(permutations);
+			if (props.getBooleanProperty(PROP_SIGNIFICANCE_COMPARE_POPULATIONS)) {
+				// Non-standard method that we investigated with Bernstein on Tracking data set.
+				//  Compares the *population* of subset measures against the population of surrogate measures.
+				//  It is not recommended that you do this.
+				int numSubsets = measDist.avDistributionForSubsets.length;
+				double meanOfMeanPermsForSubsets = MatrixUtils.mean(measDist.avDistributionForSubsets);
+				// TODO I think using the std of mean permutations for subsets makes the standard dev here
+				//  artificially small. This is because we're taken a std of averages, rather than of raw values.
+				// We could approximately correct for this by multiplying by sqrt(numSubsets).
+				// Of course this alters the number of samples used in its calculation to numSubsets * numPermutations,
+				//  which alters s2SqrOnN and degFreedon and tVal.
+				double stdOfMeanPermsForSubsets = MatrixUtils.stdDev(
+						measDist.avDistributionForSubsets, meanOfMeanPermsForSubsets);
+				// t-test of populations with unequal variance: see
+				//  http://en.wikipedia.org/wiki/Student%27s_t-test
+				double s1SqrOnN = (meanDistribution.std * meanDistribution.std) / numSubsets;
+				double s2SqrOnN = (stdOfMeanPermsForSubsets * stdOfMeanPermsForSubsets) / numSubsets;
+				double s = Math.sqrt(s1SqrOnN + s2SqrOnN);
+				double degFreedom = (s1SqrOnN + s2SqrOnN) * (s1SqrOnN + s2SqrOnN) * (numSubsets - 1) /
+									( s1SqrOnN*s1SqrOnN + s2SqrOnN*s2SqrOnN);
+				double tValForPopsOverSubsets = (meanDistribution.mean - meanOfMeanPermsForSubsets) / s;
+				System.out.printf(",%.4f,%.4f,%.4f,%.1f", meanOfMeanPermsForSubsets,
+						stdOfMeanPermsForSubsets, tValForPopsOverSubsets, degFreedom);
+			} else {
+				// Default statistical significance measurement:
+				// Comparing the single interregional value to the distribution of interregional values
+				//  for each permutation.
+				double tValForInterregionalMeasure = (meanDistribution.mean - measDist.getMeanOfDistribution()) / measDist.getStdOfDistribution();
+				System.out.printf(",%.4f,%.4f,%.4f,%d", measDist.getMeanOfDistribution(),
+						measDist.getStdOfDistribution(), tValForInterregionalMeasure, permutations);
+			}
+		}
+		System.out.println();
+	}
+}

java/source/infodynamics/networkinference/interregional/InterregionalMutualInfo.java

@@ -0,0 +1,60 @@
+package infodynamics.networkinference.interregional;
+
+import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
+import infodynamics.utils.ParsedProperties;
+
+/**
+ * <p>Compute a significance score for the inter-regional mutual information
+ * between two sets, based on looking at transfer between elements taken 
+ * e at a time.
+ * </p> 
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com 
+ *
+ */
+public class InterregionalMutualInfo extends InterregionalChannelMeasure {
+
+	private int timeDiff = 0;
+	
+	public InterregionalMutualInfo() {
+		super();
+	}
+	
+	public void initialise(ParsedProperties props) throws Exception {
+		String timeDiffPropName = InterregionalChannelMeasure.PROP_CALCULATOR_PROPERTIES_PREFIX + 
+						MutualInfoCalculatorMultiVariate.PROP_TIME_DIFF;
+		if (props.containsProperty(timeDiffPropName)) {
+			timeDiff = props.getIntProperty(timeDiffPropName);
+		}
+		super.initialise(props);
+	}
+
+	@Override
+	protected int computeNumObservationsToReorder() throws Exception {
+		if (allValid) {
+			// all the observations are valid
+			return region1.length - timeDiff;
+		} else if (!validityForIndividualElements) {
+			// we've got joint validity for each time series
+			// Check how many of the time steps are both valid
+			int numValidTimePoints = 0;
+			for (int t = 0; t < jointValidity1.length - timeDiff; t++) {
+				if (jointValidity1[t] && jointValidity2[t + timeDiff]) {
+					numValidTimePoints++;
+				}
+			}
+			return numValidTimePoints;
+		} else {
+			// We've been given validity for each individual sub-variable.
+			// There is a different number of observations for each sub-variable.
+			// It is best if we just reorder all of the observations here
+			//  for every subset.
+			return region1.length;
+		}
+	}
+
+	@Override
+	public int[] computeTimeIndicesForLocalValues() throws Exception {
+		throw new Exception("Not implemented yet");
+	}
+}

java/source/infodynamics/networkinference/interregional/InterregionalTransferEntropy.java

@@ -0,0 +1,128 @@
+package infodynamics.networkinference.interregional;
+
+import infodynamics.measures.continuous.TransferEntropyCalculatorMultiVariate;
+import infodynamics.measures.continuous.kraskov.TransferEntropyCalculatorMultiVariateKraskovByMulti;
+import infodynamics.utils.ParsedProperties;
+
+import java.util.Vector;
+
+
+/**
+ * <p>Compute a significance score for the inter-regional transfer entropy
+ * between two sets, based on looking at transfer between elements taken 
+ * e at a time.
+ * </p> 
+ * 
+ * @author Joseph Lizier, joseph.lizier at gmail.com 
+ *
+ */
+public class InterregionalTransferEntropy extends InterregionalChannelMeasure {
+
+	protected int k;
+	protected static final String PROP_K_TE = "props.interregionalChannel.te.k";
+
+	public InterregionalTransferEntropy() {
+		super();
+	}
+	
+	public void initialise(ParsedProperties props) throws Exception {
+		setK(props.getIntProperty(PROP_K_TE));
+		super.initialise(props);
+	}
+
+	public void initialise(int k, int jointVars1, int jointVars2, int maxNumSubsets) throws Exception {
+		setK(k);
+		super.initialise(jointVars1, jointVars2, maxNumSubsets);
+	}
+
+	protected void initialiseCalculator() throws Exception {
+		TransferEntropyCalculatorMultiVariate teChannelCalc =
+			(TransferEntropyCalculatorMultiVariate) channelCalc;
+		teChannelCalc.initialise(k, jointVars2, jointVars1);
+	}
+
+	public void setK(int k) {
+		this.k = k;
+	}
+
+	@Override
+	public int[] computeTimeIndicesForLocalValues() throws Exception {
+		int[] localtimeIndices;
+		if (allValid) {
+			localtimeIndices = new int[region1.length - k];
+			for (int t = k; t < region1.length; t++) {
+				localtimeIndices[t - k] = t;
+			}
+		} else if (!validityForIndividualElements) {
+			int numObs = computeNumberOfObservations(jointValidity1, jointValidity2);
+			localtimeIndices = new int[numObs];
+			TransferEntropyCalculatorMultiVariateKraskovByMulti tecmvKras =
+				new TransferEntropyCalculatorMultiVariateKraskovByMulti();
+			tecmvKras.initialise(k);
+			Vector<int[]> startAndEndTimePairs = tecmvKras.computeStartAndEndTimePairs(
+					jointValidity1, jointValidity2);
+			// We've found the set of start and end times for this pair
+			int obsNum = 0;
+			for (int[] timePair : startAndEndTimePairs) {
+				int startTime = timePair[0];
+				int endTime = timePair[1];
+				for (int t = startTime + k; t <= endTime; t++) {
+					localtimeIndices[obsNum++] = t;
+				}
+			}
+
+		} else {
+			// TODO Need to implement this properly in the super class.
+			// for the moment I am assuming it gets implemented by putting locals
+			//  at all time points
+			localtimeIndices = new int[region1.length - k];
+			for (int t = k; t < region1.length; t++) {
+				localtimeIndices[t - k] = t;
+			}
+		}
+		return localtimeIndices;
+	}	
+	
+	@Override
+	protected int computeNumObservationsToReorder() throws Exception {
+		if (allValid) {
+			// all the observations are valid
+			return region1.length - k;
+		} else if (!validityForIndividualElements) {
+			// we've got joint validity for each time series
+			return computeNumberOfObservations(jointValidity1, jointValidity2);
+		} else {
+			// We've been given validity for each individual sub-variable.
+			// There is a different number of observations for each sub-variable.
+			// It is best if we just reorder all of the observations here
+			//  for every subset.
+			return region1.length;
+		}
+	}
+
+	/**
+	 * Dummy method to compute the number of observations for the given validities
+	 * 
+	 * @param sourceValid
+	 * @param destValid
+	 * @return
+	 * @throws Exception
+	 */
+	private int computeNumberOfObservations(boolean[] sourceValid, boolean[] destValid) throws Exception {
+		TransferEntropyCalculatorMultiVariateKraskovByMulti tecmvKras =
+			new TransferEntropyCalculatorMultiVariateKraskovByMulti();
+		
+		tecmvKras.initialise(k);
+		Vector<int[]> startAndEndTimePairs = tecmvKras.computeStartAndEndTimePairs(
+				sourceValid, destValid);
+		
+		// We've found the set of start and end times for this pair
+		int numObservations = 0;
+		for (int[] timePair : startAndEndTimePairs) {
+			int startTime = timePair[0];
+			int endTime = timePair[1];
+			numObservations += endTime - startTime + 1 - k;
+		}
+		return numObservations;
+	}
+}

java/source/infodynamics/networkinference/interregional/MeasurementDistributionPermutationsOverSubsets.java

@@ -0,0 +1,68 @@
+/**
+ * 
+ */
+package infodynamics.networkinference.interregional;
+
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MeasurementDistribution;
+
+/**
+ * Extends MeasurementDistribution for computations over 
+ *  large sets where we generate measures over permutations of subsets.
+ * 
+ * The member distribution refers to averages over subsets for each permutation.
+ * The new member distributionForSubsets refers to averages over permutations
+ *   for each subset.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MeasurementDistributionPermutationsOverSubsets extends MeasurementDistribution {
+
+	// The true measurements for each subset s
+	double[] actualValues;
+	
+	// Distribution over [subsets] then [permutations]
+	double[][] distributionOverSubsetsAndPermutations;
+	
+	// The average over permutations for each subset s
+	double[] avDistributionForSubsets;
+	
+	/**
+	 * 
+	 */
+	public MeasurementDistributionPermutationsOverSubsets(int size) {
+		super(size);
+	}
+
+	public MeasurementDistributionPermutationsOverSubsets
+		(double[][] theDistribution, double[] actualValues) {
+
+		super(theDistribution[0].length);
+
+		this.actualValues = actualValues;
+		this.actualValue = MatrixUtils.mean(actualValues);
+		distributionOverSubsetsAndPermutations = theDistribution;
+		
+		int subsets = theDistribution.length;
+		int permutations = theDistribution[0].length;
+		// distribution, which is av over permutations, was created in the
+		//  super constructor.
+		avDistributionForSubsets = new double[subsets];
+		
+		// distribution will hold the averages for each permutation i
+		int avValuesFromDistributionGreaterThanActualAvs = 0;
+		for (int i = 0; i < permutations; i++) {
+			distribution[i] = MatrixUtils.mean(theDistribution, i);
+			if (distribution[i] >= actualValue) {
+				avValuesFromDistributionGreaterThanActualAvs++;
+			}
+		}
+		pValue = 
+			(double) avValuesFromDistributionGreaterThanActualAvs / (double) permutations;
+		
+		for (int s = 0; s < subsets; s++) {
+			avDistributionForSubsets[s] = MatrixUtils.mean(theDistribution[s]);
+		}
+	}
+}

java/source/infodynamics/utils/ArrayFileReader.java

@@ -0,0 +1,384 @@
+package infodynamics.utils;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+
+public class ArrayFileReader {
+	private String filename;
+	private BufferedReader br;
+	private int rows;
+	private int columns;
+	private boolean isOpen = false;
+	
+	static final String DELIMITER = "[ \t,]";
+	
+	/**
+	 * Creates the ArrayFileReader and opens the given file for reading.
+	 * 
+	 * @param arrayFilename
+	 */
+	public ArrayFileReader(String arrayFilename) {
+		filename = arrayFilename;
+	}
+	
+	public ArrayFileReader(File file) {
+		filename = file.getPath();
+	}
+	
+	private void openArrayFile()
+		throws FileNotFoundException, IOException {
+		if (isOpen) {
+			return;
+		}
+		
+		// Work out how many columns and how many rows we have
+		br = new BufferedReader(new FileReader(filename));
+		rows = 0;
+		columns = 0;
+		for (String line = br.readLine(); line != null; line = br.readLine()) {
+			// Strip leading whitespace
+			line = line.replaceFirst("^[ \t]*", "");
+			if (line.startsWith("#")) {
+				// comment line
+				continue;
+			}
+			String[] stringValues = line.split(DELIMITER);
+			if (stringValues.length == 0) {
+				// Nothing on this line
+				continue;
+			}
+			// Check that there is something other than whitespace here
+			int columnsHere = 0;
+			for (int i = 0; i < stringValues.length; i++) {
+				if (!stringValues[i].equals("")) {
+					// Got a non-empty entity
+					columnsHere++;
+				}
+			}
+			if (columnsHere == 0) {
+				// Nothing on this line
+				continue;
+			}
+			// Got something here
+			if (columns == 0) {
+				// Count the number of columns on the first line
+				columns = columnsHere;
+			}
+			// Now make sure that the number of columns here
+			//  matches the number on previous lines
+			if (columnsHere != columns) {
+				throw new IOException("Number of columns " + columnsHere +
+						" on row " + rows +
+						" does not match number on previous lines " + columns);
+			}
+			// Else this row is OK
+			rows++;
+		}
+		br.close();
+		
+		// Now have the file open and ready to read in
+		br = new BufferedReader(new FileReader(filename));
+		isOpen = true;
+	}
+	
+	private void closeArrayFile() throws IOException {
+		br.close();
+		isOpen = false;
+	}
+
+	public int[][] getInt2DMatrix() throws Exception {
+		openArrayFile();
+		// Create the return array
+		int[][] values = new int[rows][columns];
+		
+		int r = 0;
+		for (String line = br.readLine(); line != null; line = br.readLine()) {
+			// Strip leading whitespace
+			line = line.replaceFirst("^[ \t]*", "");
+			if (line.startsWith("#")) {
+				// comment line
+				continue;
+			}
+			String[] stringValues = line.split(DELIMITER);
+			if (stringValues.length == 0) {
+				// Nothing on this line
+				continue;
+			}
+			// Check that there is something other than whitespace here
+			int c = 0;
+			for (int i = 0; i < stringValues.length; i++) {
+				if (!stringValues[i].equals("")) {
+					// Got a non-empty entity
+					// Not checking array bounds since this
+					//  was already checked by openArrayFile().
+					values[r][c] = Integer.parseInt(stringValues[i]);
+					c++;
+				}
+			}
+			if (c == 0) {
+				// Nothing was on this line
+				continue;
+			}
+			// Got some values here.
+			// Now make sure that the number of columns here
+			//  matches the number on previous lines
+			if (c != columns) {
+				// This should not occur as it was already checked
+				//  in the openArrayFile() method
+				throw new IOException("Number of columns " + c +
+						" on row " + rows +
+						" does not match number on previous lines " + columns);
+			}
+			// Else this row is OK
+			r++;
+		}
+
+		closeArrayFile();
+		return values;
+	}
+
+	public int[] getIntArray() throws Exception {
+		openArrayFile();
+		
+		if ((rows != 1) && (columns != 1)) {
+			throw new Exception("Cannot get an int array when neither the number of rows nor columns equals 1");
+		}
+		boolean alongRows = (rows > columns);
+		int length = alongRows ? rows : columns;
+		
+		// Create the return array
+		int[] values = new int[length];
+		
+		if (alongRows) {
+			int r = 0;
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				// Strip leading whitespace
+				line = line.replaceFirst("^[ \t]*", "");
+				if (line.startsWith("#")) {
+					// comment line
+					continue;
+				}
+				String[] stringValues = line.split(DELIMITER);
+				if (stringValues.length == 0) {
+					// Nothing on this line
+					continue;
+				}
+				// Check that there is something other than whitespace here
+				boolean readOneOnThisLine = false;
+				for (int i = 0; i < stringValues.length; i++) {
+					if (!stringValues[i].equals("")) {
+						if (readOneOnThisLine) {
+							// We've already read an integer on this line - this is an error condition
+							throw new IOException("Cannot have multiple values on each line for a column vector");
+						}
+						// Got a non-empty entity
+						// Not checking array bounds since this
+						//  was already checked by openArrayFile().
+						values[r] = Integer.parseInt(stringValues[i]);
+						readOneOnThisLine = true;
+					}
+				}
+				if (!readOneOnThisLine) {
+					// Nothing was on this line
+					continue;
+				}
+				// Got some value here.
+				r++;
+			}
+		} else {
+			boolean readFromOneLine = false;
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				// Strip leading whitespace
+				line = line.replaceFirst("^[ \t]*", "");
+				if (line.startsWith("#")) {
+					// comment line
+					continue;
+				}
+				String[] stringValues = line.split(DELIMITER);
+				if (stringValues.length == 0) {
+					// Nothing on this line
+					continue;
+				}
+				// Check that there is something other than whitespace here
+				int c = 0;
+				for (int i = 0; i < stringValues.length; i++) {
+					if (!stringValues[i].equals("")) {
+						if ((c == 0) && readFromOneLine) {
+							// We're about to start reading from this line, but it's not the first
+							//  line we've read from
+							throw new IOException("Cannot have data on multiple lines for a row vector");
+						}
+						// Got a non-empty entity - read all from this line
+						// Not checking array bounds since this
+						//  was already checked by openArrayFile().
+						values[c] = Integer.parseInt(stringValues[i]);
+						readFromOneLine = true;
+						c++;
+					}
+				}
+				if (c == 0) {
+					// Nothing was on this line
+					continue;
+				}
+				// Got at least one value here - check if we read enough
+				if (c != columns) {
+					// We did not read the correct number of columns here:
+					throw new IOException("Not enough elements in the row vector");
+				}
+			}
+		}
+		
+		closeArrayFile();
+		return values;
+	}
+
+	public double[][] getDouble2DMatrix() throws Exception {
+		openArrayFile();
+		// Create the return array
+		double[][] values = new double[rows][columns];
+		
+		int r = 0;
+		for (String line = br.readLine(); line != null; line = br.readLine()) {
+			// Strip leading whitespace
+			line = line.replaceFirst("^[ \t]*", "");
+			if (line.startsWith("#")) {
+				// comment line
+				continue;
+			}
+			String[] stringValues = line.split(DELIMITER);
+			if (stringValues.length == 0) {
+				// Nothing on this line
+				continue;
+			}
+			// Check that there is something other than whitespace here
+			int c = 0;
+			for (int i = 0; i < stringValues.length; i++) {
+				if (!stringValues[i].equals("")) {
+					// Got a non-empty entity
+					// Not checking array bounds since this
+					//  was already checked by openArrayFile().
+					values[r][c] = Double.parseDouble(stringValues[i]);
+					c++;
+				}
+			}
+			if (c == 0) {
+				// Nothing was on this line
+				continue;
+			}
+			// Got some values here.
+			// Now make sure that the number of columns here
+			//  matches the number on previous lines
+			if (c != columns) {
+				// This should not occur as it was already checked
+				//  in the openArrayFile() method
+				throw new IOException("Number of columns " + c +
+						" on row " + rows +
+						" does not match number on previous lines " + columns);
+			}
+			// Else this row is OK
+			r++;
+		}
+
+		closeArrayFile();
+		return values;
+	}
+
+	public double[] getDoubleArray() throws Exception {
+		openArrayFile();
+		
+		if ((rows != 1) && (columns != 1)) {
+			throw new Exception("Cannot get an double array when neither the number of rows nor columns equals 1");
+		}
+		boolean alongRows = (rows > columns);
+		int length = alongRows ? rows : columns;
+		
+		// Create the return array
+		double[] values = new double[length];
+		
+		if (alongRows) {
+			int r = 0;
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				// Strip leading whitespace
+				line = line.replaceFirst("^[ \t]*", "");
+				if (line.startsWith("#")) {
+					// comment line
+					continue;
+				}
+				String[] stringValues = line.split(DELIMITER);
+				if (stringValues.length == 0) {
+					// Nothing on this line
+					continue;
+				}
+				// Check that there is something other than whitespace here
+				boolean readOneOnThisLine = false;
+				for (int i = 0; i < stringValues.length; i++) {
+					if (!stringValues[i].equals("")) {
+						if (readOneOnThisLine) {
+							// We've already read an double on this line - this is an error condition
+							throw new IOException("Cannot have multiple values on each line for a column vector");
+						}
+						// Got a non-empty entity
+						// Not checking array bounds since this
+						//  was already checked by openArrayFile().
+						values[r] = Double.parseDouble(stringValues[i]);
+						readOneOnThisLine = true;
+					}
+				}
+				if (!readOneOnThisLine) {
+					// Nothing was on this line
+					continue;
+				}
+				// Got some value here.
+				r++;
+			}
+		} else {
+			boolean readFromOneLine = false;
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				// Strip leading whitespace
+				line = line.replaceFirst("^[ \t]*", "");
+				if (line.startsWith("#")) {
+					// comment line
+					continue;
+				}
+				String[] stringValues = line.split(DELIMITER);
+				if (stringValues.length == 0) {
+					// Nothing on this line
+					continue;
+				}
+				// Check that there is something other than whitespace here
+				int c = 0;
+				for (int i = 0; i < stringValues.length; i++) {
+					if (!stringValues[i].equals("")) {
+						if ((c == 0) && readFromOneLine) {
+							// We're about to start reading from this line, but it's not the first
+							//  line we've read from
+							throw new IOException("Cannot have data on multiple lines for a row vector");
+						}
+						// Got a non-empty entity - read all from this line
+						// Not checking array bounds since this
+						//  was already checked by openArrayFile().
+						values[c] = Double.parseDouble(stringValues[i]);
+						readFromOneLine = true;
+						c++;
+					}
+				}
+				if (c == 0) {
+					// Nothing was on this line
+					continue;
+				}
+				// Got at least one value here - check if we read enough
+				if (c != columns) {
+					// We did not read the correct number of columns here:
+					throw new IOException("Not enough elements in the row vector");
+				}
+			}
+		}
+		
+		closeArrayFile();
+		return values;
+	}
+
+}

java/source/infodynamics/utils/ArrayFileWriter.java

@@ -0,0 +1,135 @@
+package infodynamics.utils;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class ArrayFileWriter {
+	
+	/**
+	 * Outputs different types of 2D Matrix to a file
+	 * 
+	 * @param outputFileName and matrix
+	 */
+	
+	public static void makeBooleanMatrixFile(boolean matrix[][],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+		int colSize = matrix[0].length;
+		
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			for(int j=0;j<colSize;j++){
+				if (matrix[i][j]==false){
+					out.write("0\t");
+				}else{
+					out.write("1\t");
+				}
+				if (j==colSize-1){
+					out.write("\n");
+				}
+			}
+		}
+		out.close();
+	}
+	
+	public static void makeDoubleMatrixFile(double matrix[][],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+		int colSize = matrix[0].length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			for(int j=0;j<colSize;j++){
+				out.write(String.valueOf(matrix[i][j])+"\t");
+				if (j==colSize-1){
+					out.write("\n");
+				}
+			}
+		}
+		out.close();
+	}
+
+	public static void makeDoubleMatrixFile(double matrix[][],String outputFileName, int decimalPlaces) throws IOException {
+		String template = String.format("%%.%df\t", decimalPlaces);
+    	int rowSize = matrix.length;
+		int colSize = matrix[0].length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			for(int j=0;j<colSize;j++){
+				out.write(String.format(template, matrix[i][j]));
+				if (j==colSize-1){
+					out.write("\n");
+				}
+			}
+		}
+		out.close();
+	}
+
+	public static void makeIntMatrixFile(int matrix[][],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+		int colSize = matrix[0].length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			for(int j=0;j<colSize;j++){
+				out.write(String.valueOf(matrix[i][j])+"\t");
+				if (j==colSize-1){
+					out.write("\n");
+				}
+			}
+		}
+		out.close();
+	}
+
+	public static void makeMatrixFile(double matrix[],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			out.write(String.valueOf(matrix[i]) + "\n");
+		}
+		out.close();
+	}
+
+	public static void makeMatrixFile(Number matrix[][],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+		int colSize = matrix[0].length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			for(int j=0;j<colSize;j++){
+				out.write(String.valueOf(matrix[i][j])+"\t");
+				if (j==colSize-1){
+					out.write("\n");
+				}
+			}
+		}
+		out.close();
+	}
+
+	public static void makeMatrixFile(Number matrix[],String outputFileName) throws IOException {
+    	int rowSize = matrix.length;
+
+		createDirectories(outputFileName);
+		BufferedWriter out = new BufferedWriter(new FileWriter(outputFileName));
+		for(int i=0;i<rowSize;i++){
+			out.write(String.valueOf(matrix[i]) + "\n");
+		}
+		out.close();
+	}
+	
+	private static void createDirectories(String filename) {
+		File file = new File(filename);
+		File parentDir = file.getParentFile();
+		if ((parentDir != null) && !parentDir.isDirectory()) {
+			parentDir.mkdirs();
+		}
+	}
+}

java/source/infodynamics/utils/EuclideanUtils.java

@@ -0,0 +1,335 @@
+package infodynamics.utils;
+
+/**
+ * 
+ * A set of utilities for manipulating vectors in Euclidean space.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class EuclideanUtils {
+
+	public static int MAX_TIMESTEPS_FOR_FAST_DISTANCE = 2000;
+	
+	public static final int NORM_EUCLIDEAN = 0;
+	public static final String NORM_EUCLIDEAN_STRING = "EUCLIDEAN";
+	public static final int NORM_EUCLIDEAN_NORMALISED = 1;
+	public static final String NORM_EUCLIDEAN_NORMALISED_STRING = "EUCLIDEAN_NORMALISED";
+	public static final int NORM_MAX_NORM = 2;
+	public static final String NORM_MAX_NORM_STRING = "MAX_NORM";
+	private static int normToUse = 0;
+
+	public EuclideanUtils() {
+		super();
+	}
+
+	public static double[] computeMinEuclideanDistances(double[][] observations) {
+		if (observations.length <= MAX_TIMESTEPS_FOR_FAST_DISTANCE) {
+			return EuclideanUtils.computeMinEuclideanDistancesFast(observations);
+		} else {
+			return computeMinEuclideanDistancesNaive(observations);
+		}
+	}
+	
+	
+	/**
+	 * Naive method for computing minimum distance - slower but needs less memory.
+	 * Made public for debugging only. O(d.n^2) speed
+	 * 
+	 * @param observations
+	 * @return
+	 */
+	public static double[] computeMinEuclideanDistancesNaive(double[][] observations) {
+		int numObservations = observations.length;
+		int dimensions = observations[0].length;
+		double[] distances = new double[numObservations];
+		for (int t = 0; t < numObservations; t++) {
+			double minDistance = Double.POSITIVE_INFINITY;
+			for (int t2 = 0; t2 < numObservations; t2++) {
+				if (t == t2) {
+					continue;
+				}
+				double thisDistance = 0.0;
+				for (int d = 0; (d < dimensions) && (thisDistance < minDistance); d++) {
+					double distanceOnThisVar = (observations[t][d] - observations[t2][d]);
+					thisDistance += distanceOnThisVar * distanceOnThisVar;
+				}
+				// Now we need to sqrt the distance sum
+				thisDistance = Math.sqrt(thisDistance);
+				// Now check if this is a lower distance
+				if (thisDistance < minDistance) {
+					minDistance = thisDistance;
+				}
+			}
+			distances[t] = minDistance;
+		}
+		return distances;
+	}
+
+	/**
+	 * Return the minimum Euclidean distance from each point to any other observation.
+	 * Computes this faster than using naive computation.
+	 * 
+	 * Exposed as a public method for debugging purposes only.
+	 * 
+	 * @param observations
+	 * @return
+	 */
+	public static double[] computeMinEuclideanDistancesFast(double[][] observations) {
+		
+		int dimensions  = observations[0].length;
+		
+		int timeSteps = observations.length;
+		// Hold the sqr distance from index1 to index2 ...
+		double[][] sqrDistance = new double[timeSteps][timeSteps];
+		// ... computed over this many of the variables so far
+		int[][] addedInUpToVariable = new int[timeSteps][timeSteps];
+		double[] minDistance = new double[timeSteps];
+		
+		for (int t1 = 0; t1 < timeSteps; t1++) {
+			// Current minimum distance from this index to another point:
+			double minSqrDistance = Double.POSITIVE_INFINITY;
+
+			// First grab the minimum distance from nodes for which the distance might
+			//  have already been measured
+			for (int t2 = 0; t2 < t1; t2++) {
+				if (addedInUpToVariable[t2][t1] == dimensions) {
+					// We have previously computed this distance from t2 to t1
+					sqrDistance[t1][t2] = sqrDistance[t2][t1];
+					// unnecessary, since we won't be looking at [t1][t2] later:
+					addedInUpToVariable[t1][t2] = dimensions;
+					if (sqrDistance[t1][t2] < minSqrDistance) {
+						minSqrDistance = sqrDistance[t1][t2];
+					}
+				}
+			}
+			// Now check the previously considered source nodes which didn't have their full distance
+			//  computed in case we need to compute them
+			for (int t2 = 0; t2 < t1; t2++) {
+				if (addedInUpToVariable[t2][t1] != dimensions) {
+					// We have not finished computing this distance from t1
+					addedInUpToVariable[t1][t2] = addedInUpToVariable[t2][t1];
+					sqrDistance[t1][t2] = sqrDistance[t2][t1];
+					for (; (sqrDistance[t1][t2] < minSqrDistance) &&
+							(addedInUpToVariable[t1][t2] < dimensions);
+							addedInUpToVariable[t1][t2]++) {
+						double distOnThisVar = observations[t1][addedInUpToVariable[t1][t2]] -
+												observations[t2][addedInUpToVariable[t1][t2]]; 
+						sqrDistance[t1][t2] += distOnThisVar * distOnThisVar;
+					}
+					if (sqrDistance[t1][t2] < minSqrDistance) {
+						// we finished the calculation and t2 is now the closest observation to t1
+						minSqrDistance = sqrDistance[t1][t2];
+					}
+				}
+			}
+			// Now check any source nodes t2 for which there is no chance we've looked at the
+			//  the distance back to t1 yet
+			for (int t2 = t1 + 1; t2 < timeSteps; t2++) {
+				for (; (sqrDistance[t1][t2] < minSqrDistance) &&
+						(addedInUpToVariable[t1][t2] < dimensions);
+						addedInUpToVariable[t1][t2]++) {
+					double distOnThisVar = observations[t1][addedInUpToVariable[t1][t2]] -
+											observations[t2][addedInUpToVariable[t1][t2]]; 
+					sqrDistance[t1][t2] += distOnThisVar * distOnThisVar;
+				}
+				if (sqrDistance[t1][t2] < minSqrDistance) {
+					// we finished the calculation and  t2 is now the closest observation to t1
+					minSqrDistance = sqrDistance[t1][t2];
+				}
+			}
+			
+			minDistance[t1] = Math.sqrt(minSqrDistance);
+		}
+		return minDistance;
+	}
+
+	public static double maxJointSpaceNorm(double[] x1, double[] y1,
+			   double[] x2, double[] y2) {
+		return Math.max(norm(x1, x2), norm(y1,y2));
+	}
+
+	/**
+	 * Computing the configured norm between vectors x1 and x2.
+	 * 
+	 * @param x1
+	 * @param x2
+	 * @return
+	 */
+	public static double norm(double[] x1, double[] x2) {
+		switch (normToUse) {
+		case NORM_EUCLIDEAN_NORMALISED:
+			return euclideanNorm(x1, x2) / Math.sqrt(x1.length);
+		case NORM_MAX_NORM:
+			return maxNorm(x1, x2);
+		case NORM_EUCLIDEAN:
+		default:
+			return euclideanNorm(x1, x2);
+		}
+	}
+	
+	/**
+	 * Computing the norm as the Euclidean norm.
+	 * 
+	 * @param x1
+	 * @param x2
+	 * @return
+	 */
+	public static double euclideanNorm(double[] x1, double[] x2) {
+		double distance = 0.0;
+		for (int d = 0; d < x1.length; d++) {
+			double difference = x1[d] - x2[d];
+			distance += difference * difference;
+		}
+		return Math.sqrt(distance);
+	}
+
+	/**
+	 * Computing the norm as the Max norm.
+	 * 
+	 * @param x1
+	 * @param x2
+	 * @return
+	 */
+	public static double maxNorm(double[] x1, double[] x2) {
+		double distance = 0.0;
+		for (int d = 0; d < x1.length; d++) {
+			double difference = x1[d] - x2[d];
+			// Take the abs
+			if (difference < 0) {
+				difference = -difference;
+			}
+			if (difference > distance) {
+				distance = difference;
+			}
+		}
+		return distance;
+	}
+
+	/**
+	 * Compute the x and y norms of all other points from
+	 *  the data points at time step t.
+	 * Puts norms of t from itself as infinity, which is useful
+	 *  when counting the number of points closer than epsilon say.
+	 * 
+	 * @param mvTimeSeries1
+	 * @param mvTimeSeries2
+	 * @return
+	 */
+	public static double[][] computeNorms(double[][] mvTimeSeries1,
+			double[][] mvTimeSeries2, int t) {
+		
+		int timeSteps = mvTimeSeries1.length;
+		double[][] norms = new double[timeSteps][2];
+		for (int t2 = 0; t2 < timeSteps; t2++) {
+			if (t2 == t) {
+				norms[t2][0] = Double.POSITIVE_INFINITY;
+				norms[t2][1] = Double.POSITIVE_INFINITY;
+				continue;
+			}
+			// Compute norm in first direction
+			norms[t2][0] = norm(mvTimeSeries1[t], mvTimeSeries1[t2]);
+			// Compute norm in second direction
+			norms[t2][1] = norm(mvTimeSeries2[t], mvTimeSeries2[t2]);
+		}
+		return norms;
+	}
+	
+	/**
+	 * Compute the x, y and z norms of all other points from
+	 *  the data points at time step t.
+	 * Puts norms of t from itself as infinity, which is useful
+	 *  when counting the number of points closer than epsilon say.
+	 * 
+	 * @param mvTimeSeries1
+	 * @param mvTimeSeries2
+	 * @param mvTimeSeries3
+	 * @return
+	 */
+	public static double[][] computeNorms(double[][] mvTimeSeries1,
+			double[][] mvTimeSeries2, double[][] mvTimeSeries3, int t) {
+		
+		int timeSteps = mvTimeSeries1.length;
+		double[][] norms = new double[timeSteps][3];
+		for (int t2 = 0; t2 < timeSteps; t2++) {
+			if (t2 == t) {
+				norms[t2][0] = Double.POSITIVE_INFINITY;
+				norms[t2][1] = Double.POSITIVE_INFINITY;
+				norms[t2][2] = Double.POSITIVE_INFINITY;
+				continue;
+			}
+			// Compute norm in first direction
+			norms[t2][0] = norm(mvTimeSeries1[t], mvTimeSeries1[t2]);
+			// Compute norm in second direction
+			norms[t2][1] = norm(mvTimeSeries2[t], mvTimeSeries2[t2]);
+			// Compute norm in third direction
+			norms[t2][2] = norm(mvTimeSeries3[t], mvTimeSeries3[t2]);
+		}
+		return norms;
+	}
+
+	/**
+	 * Compute the norms for each marginal variable for all other points from
+	 *  the data points at time step t.
+	 * Puts norms of t from itself as infinity, which is useful
+	 *  when counting the number of points closer than epsilon say.
+	 * 
+	 * @param mvTimeSeries
+	 * @return
+	 */
+	public static double[][] computeNorms(double[][] mvTimeSeries, int t) {
+		
+		int timeSteps = mvTimeSeries.length;
+		int variables = mvTimeSeries[0].length;
+		
+		double[][] norms = new double[timeSteps][variables];
+		for (int t2 = 0; t2 < timeSteps; t2++) {
+			if (t2 == t) {
+				for (int v = 0; v < variables; v++) {
+					norms[t2][v] = Double.POSITIVE_INFINITY;
+				}
+				continue;
+			}
+			for (int v = 0; v < variables; v++) {
+				norms[t2][v] = Math.abs(mvTimeSeries[t][v] - mvTimeSeries[t2][v]);
+			}
+		}
+		return norms;
+	}
+
+	/**
+	 * Sets which type of norm will be used by calls to norm()
+	 * 
+	 * @param normType
+	 */
+	public static void setNormToUse(int normType) {
+		normToUse = normType;
+	}
+
+	/**
+	 * Sets which type of norm will be used by calls to norm()
+	 * 
+	 * @param normType
+	 */
+	public static void setNormToUse(String normType) {
+		if (normType.equalsIgnoreCase(NORM_EUCLIDEAN_NORMALISED_STRING)) {
+			normToUse = NORM_EUCLIDEAN_NORMALISED;
+		} else if (normType.equalsIgnoreCase(NORM_MAX_NORM_STRING)) {
+			normToUse = NORM_MAX_NORM;
+		} else {
+			normToUse = NORM_EUCLIDEAN;
+		}
+	}
+	
+	public static String getNormInUse() {
+		switch (normToUse) {
+		case NORM_EUCLIDEAN_NORMALISED:
+			return NORM_EUCLIDEAN_NORMALISED_STRING;
+		case NORM_MAX_NORM:
+			return NORM_MAX_NORM_STRING;
+		default:
+		case NORM_EUCLIDEAN:
+			return NORM_EUCLIDEAN_STRING;
+		}
+	}
+}

java/source/infodynamics/utils/FirstIndexComparatorDouble.java

@@ -0,0 +1,35 @@
+package infodynamics.utils;
+
+/**
+ * <p>Class used to sort an array of array of doubles based on the first element in each array.
+ * </p> 
+ * 
+ * <p>Usage: <code>java.util.Arrays.sort(double[][], FirstIndexComparatorDouble.getInstance())</code>;
+ * </p>
+ * 
+ * @author Joseph Lizier
+ */
+public class FirstIndexComparatorDouble implements java.util.Comparator<double[]> {
+	
+	private static FirstIndexComparatorDouble instance = null;
+	
+	private FirstIndexComparatorDouble() {
+	}
+
+	public static FirstIndexComparatorDouble getInstance() {
+		if (instance == null) {
+			instance = new FirstIndexComparatorDouble();
+		}
+		return instance;
+	}
+	
+	public int compare(double[] a1, double[] a2) {
+		if (a1[0] < a2[0]) {
+			return -1;
+		} else if (a1[0] == a2[0]) {
+			return 0;
+		} else {
+			return 1;
+		}
+	}
+}

java/source/infodynamics/utils/FirstIndexComparatorInteger.java

@@ -0,0 +1,35 @@
+package infodynamics.utils;
+
+/**
+ * <p>Class used to sort an array of array of doubles based on the first element in each array.
+ * </p> 
+ * 
+ * <p>Usage: <code>java.util.Arrays.sort(int[][], FirstIndexComparatorInteger.getInstance())</code>;
+ * </p>
+ * 
+ * @author Joseph Lizier
+ */
+public class FirstIndexComparatorInteger implements java.util.Comparator<int[]> {
+	
+	private static FirstIndexComparatorInteger instance = null;
+	
+	private FirstIndexComparatorInteger() {
+	}
+
+	public static FirstIndexComparatorInteger getInstance() {
+		if (instance == null) {
+			instance = new FirstIndexComparatorInteger();
+		}
+		return instance;
+	}
+	
+	public int compare(int[] a1, int[] a2) {
+		if (a1[0] < a2[0]) {
+			return -1;
+		} else if (a1[0] == a2[0]) {
+			return 0;
+		} else {
+			return 1;
+		}
+	}
+}

java/source/infodynamics/utils/IntArrayWrapper.java

@@ -0,0 +1,67 @@
+package infodynamics.utils;
+
+import java.util.Arrays;
+
+/**
+ * 
+ * @author Joseph Lizier
+ *
+ * LongArrayWrapper is used to wrap a long array so as to be able to give it a hash value
+ *  for placement into a hash table.
+ */
+public class IntArrayWrapper {
+
+	private int[] array;
+	private int firstCols = 0;
+	
+	public IntArrayWrapper() {
+		super();
+	}
+
+	public IntArrayWrapper(int[] newArray) {
+		super();
+		setArray(newArray, newArray.length);
+	}
+
+	public IntArrayWrapper(int[] newArray, int firstcolumns) {
+		super();
+		setArray(newArray, firstcolumns);
+	}
+
+	public int hashCode() {
+		return Arrays.hashCode(array);
+	}
+	
+	public int[] getArray() {
+		return array;
+	}
+
+	public void setArray(int[] newArray, int firstcolumns) {
+		array = newArray;
+		firstCols = firstcolumns;
+	}
+	
+	public int getArrayUsedLength() {
+		return firstCols;
+	}
+	
+	public boolean equals(Object o) {
+		if (o == this) {
+			return true;
+		}
+		
+		IntArrayWrapper iaw2 = (IntArrayWrapper) o;
+		if (iaw2.getArrayUsedLength() != firstCols) {
+			return false;
+		}
+		
+		// check deeply inside arrays
+		int[] inArray = iaw2.getArray();
+		for (int i = 0; i < firstCols; i++) {
+			if (array[i] != inArray[i]) {
+				return false;
+			}
+		}
+		return true;
+	}
+}

java/source/infodynamics/utils/MathsUtils.java

@@ -0,0 +1,381 @@
+package infodynamics.utils;
+
+public class MathsUtils {
+
+	private static final double EULER_MASCHERONI_CONSTANT = 0.5772156;
+	
+	private static int highestDigammaArgCalced = 0;
+	private static final int NUM_STORED_DIGAMMAS = 10000;
+	private static double[] storedDigammas;
+	
+	public MathsUtils() {
+		super();
+		// TODO Auto-generated constructor stub
+	}
+
+	/**
+	 * Returns the integer result of base^power
+	 * 
+	 * @param base base integer of the operation
+	 * @param power power that base is raised to
+	 * @return base raised to exponent power (rounded by integer operations)
+	 */
+	public static int power(int base, int power) {
+		int result = 1;
+		int absPower = Math.abs(power);
+		for (int p = 0; p < absPower; p++) {
+			result *= base;
+		}
+		if (power < 0) {
+			// This will be zero for any base except 1 or -1
+			result = 1 / result;
+		}
+		return result;
+	}
+	
+	/**
+	 * Returns the integer result of base^power
+	 * 
+	 * Tested - works.
+	 * 
+	 * @param base
+	 * @param power
+	 * @return
+	 */
+	public static long power(long base, long power) {
+		long result = 1;
+		long absPower = Math.abs(power);
+		for (long p = 0; p < absPower; p++) {
+			result *= base;
+		}
+		if (power < 0) {
+			// This will be zero for any base except 1 or -1
+			result = 1 / result;
+		}
+		return result;
+	}
+	
+	public static long factorial(int n) {
+		long result = 1;
+		for (int i = 1; i <= n; i++) {
+			result *= (long) i;
+		}
+		return result;
+	}
+	
+	public static int factorialCheckBounds(int n) throws Exception {
+		long result = 1;
+		for (int i = 1; i <= n; i++) {
+			result *= (long) i;
+			if (result > Integer.MAX_VALUE) {
+				throw new Exception("n! causes integer overflow");
+			}
+		}
+		return (int) result;
+	}
+
+	public static double factorialAsDouble(int n) {
+		double result = 1;
+		for (int i = 1; i <= n; i++) {
+			result *= (double) i;
+		}
+		return result;		
+	}
+	
+	/**
+	 * Computes n! as a double (to provide extended range over a long).
+	 * 
+	 * We include the divisor here since n! hits with n at about 340.
+	 * So if there is anything the result would have been divided by, we include it here,
+	 *  thus extending the range of n the function is suitable for. 
+	 * 
+	 * @param n
+	 * @param divisor
+	 * @return
+	 */
+	public static double factorialAsDoubleIncludeDivisor(int n, double divisor) {
+		double result = 1.0 / divisor;
+		for (int i = 1; i <= n; i++) {
+			result *= (double) i;
+		}
+		return result;		
+	}
+
+	/**
+	 * n!!
+	 * see http://en.wikipedia.org/wiki/Double_factorial#Double_factorial
+	 * 
+	 * @param n
+	 * @return
+	 */
+	public static long doubleFactorial(int n) {
+		long result = 1;
+		int startValue;
+		if (n % 2 == 0) {
+			// n even
+			startValue = 2;
+		} else {
+			// n odd
+			startValue = 3;
+		}
+		for (int i = startValue; i <= n; i += 2) {
+			result *= (long) i;
+		}
+		return result;
+	}
+
+	/**
+	 * n!!
+	 * see http://en.wikipedia.org/wiki/Double_factorial#Double_factorial
+	 * 
+	 * @param n
+	 * @return
+	 */
+	public static double doubleFactorialAsDouble(int n) {
+		double result = 1.0;
+		int startValue;
+		if (n % 2 == 0) {
+			// n even
+			startValue = 2;
+		} else {
+			// n odd
+			startValue = 3;
+		}
+		for (int i = startValue; i <= n; i += 2) {
+			result *= (double) i;
+		}
+		return result;
+	}
+	
+	/**
+	 * n!!
+	 * see http://en.wikipedia.org/wiki/Double_factorial#Double_factorial
+	 * 
+	 * We include the divisor here since the gamma(d/2+1) hits Inf with d just over 300.
+	 * So if there is anything the result would have been divided by, we include it here,
+	 *  thus extending the range of d the function is suitable for. 
+	 * 
+	 * @param n
+	 * @param divisor
+	 * @return
+	 */
+	public static double doubleFactorialAsDoublewithDivisor(int n, double divisor) {
+		double result = 1.0 / divisor;
+		int startValue;
+		if (n % 2 == 0) {
+			// n even
+			startValue = 2;
+		} else {
+			// n odd
+			startValue = 3;
+		}
+		for (int i = startValue; i <= n; i += 2) {
+			result *= (double) i;
+		}
+		return result;
+	}
+
+	/**
+	 * Computes gamma(d/2 + 1)
+	 * See http://en.wikipedia.org/wiki/Gamma_function
+	 *  for description of the analytical result for d odd.
+	 * For d even, we have gamma of an integer, which is equal to
+	 *  (d/2)!
+	 * 
+	 * @param d
+	 * @return
+	 */
+	public static double gammaOfArgOn2Plus1(int d) {
+		if (d % 2 == 0) {
+			// d even
+			return factorialAsDouble(d/2);
+		} else {
+			// d odd
+			return Math.sqrt(Math.PI) * (double) doubleFactorialAsDouble(d) / 
+				(double) Math.pow(2, ((double) (d + 1)) / 2.0);
+		}
+	}
+
+	/**
+	 * Computes gamma(d/2 + 1)/divisor
+	 * See http://en.wikipedia.org/wiki/Gamma_function
+	 *  for description of the analytical result for d odd.
+	 * For d even, we have gamma of an integer, which is equal to
+	 *  (d/2)!
+	 * 
+	 * We include the divisor here since the gamma(d/2+1) hits Inf with d just over 300.
+	 * So if there is anything the result would have been divided by, we include it here,
+	 *  thus extending the range of d the function is suitable for. 
+	 * 
+	 * @param d
+	 * @param divisor
+	 * @return
+	 */
+	public static double gammaOfArgOn2Plus1IncludeDivisor(int d, double divisor) {
+		if (d % 2 == 0) {
+			// d even
+			return factorialAsDoubleIncludeDivisor(d/2, divisor);
+		} else {
+			// d odd
+			return doubleFactorialAsDoublewithDivisor(d,
+					divisor * Math.pow(2, ((double) (d + 1)) / 2.0) / Math.sqrt(Math.PI));
+		}
+	}
+
+	/**
+	 * Compute digamma(d).
+	 * 
+	 * Stores previous calculations to speed up computation here, though some precision may
+	 *  be lost because we're adding in larger numbers first.
+	 * 
+	 * @param d
+	 * @return
+	 * @throws Exception
+	 */
+	public static double digamma(int d) throws Exception {
+		if (d < 1) {
+			return Double.NaN;
+		}
+		if (storedDigammas == null) {
+			// allocate space to store our results
+			storedDigammas = new double[NUM_STORED_DIGAMMAS];
+			storedDigammas[0] = Double.NaN;
+			storedDigammas[1] = -EULER_MASCHERONI_CONSTANT;
+			highestDigammaArgCalced = 1;
+		}
+		if (d <= highestDigammaArgCalced) {
+			// We've already calculated this one
+			return storedDigammas[d];
+		}
+		// else need to calculate it
+		double result = storedDigammas[highestDigammaArgCalced];
+		for (int n = highestDigammaArgCalced + 1; n <= d; n++) {
+			result += 1.0 / (double) (n-1);
+			if (d < NUM_STORED_DIGAMMAS) {
+				storedDigammas[n] = result;
+			}
+		}
+		if (d < NUM_STORED_DIGAMMAS) {
+			highestDigammaArgCalced = d;
+		} else {
+			highestDigammaArgCalced = NUM_STORED_DIGAMMAS - 1;
+		}
+		return result;		
+	}
+	
+	/**
+	 * Compute the digamma function from first principles
+	 * 
+	 * @param d
+	 * @return
+	 * @throws Exception
+	 */
+	public static double digammaByDefinition(int d) throws Exception {
+		if (d < 1) {
+			return Double.NaN;
+		}
+		double result = 0;
+		for (int n = d; n > 1; n--) {
+			result += 1.0 / (double) (n-1);
+		}
+		// Now add in result for n == 1
+		result += -EULER_MASCHERONI_CONSTANT;
+		return result;
+	}
+
+	/**
+	 * Return the number of possible combinations of p from n (i.e. n choose p)
+	 * 
+	 * @param n
+	 * @param p
+	 * @return
+	 * @throws Exception if the number would be greater than Integer.MAX_INT
+	 */
+	public static int numOfSets(int n, int p) throws Exception {
+		// Compute how many sets there will be
+		long counter = n;
+		long numSets = 1;
+		for (int x = 1; x <= p; x++) {
+			numSets *= counter;
+			numSets /= x;
+			if (numSets > Integer.MAX_VALUE) {
+				throw new Exception("nCp causes integer overflow");
+			}
+			counter--;
+		}
+		// numSets counts the number of permutations of n.
+		// Need to get rid of repeats to make is combinations:
+		return (int) numSets;
+	}
+
+	/**
+	 * Return an array of all possible combinations of p from n 
+	 * 
+	 * @param n
+	 * @param p
+	 * @return
+	 * @throws Exception when the number of sets is greaterr than Integer.MAX_INT
+	 */
+	public static int[][] generateAllSets(int n, int p) throws Exception {
+		int numOfSets = numOfSets(n,p);
+		int[][] sets = new int[numOfSets][p];
+		int[] currentSet = new int[p];
+		writeSetsIn(n, p, 0, 0, currentSet, sets, 0);
+		return sets;
+	}
+
+	/**
+	 * Recursive call used by generateAllSets.
+	 * 
+	 * @param n
+	 * @param p
+	 * @param currentIndexInSet current index in currentSet that we are writing into
+	 * @param currentSet current set containing indices already written into the upper parts
+	 * @param sets array to write generated sets into
+	 * @param upToSetNum
+	 * @return new value of upToSetNum
+	 */
+	private static int writeSetsIn(int n, int p, int currentIndexInSet,
+			int firstCandidate, int[] currentSet, int[][] sets, int upToSetNum) {
+		/*
+		String indent = "";
+		for (int i = 0; i < currentIndexInSet; i++) {
+			indent += " ";
+		}
+		System.out.println(indent + String.format("currentIndex=%d", currentIndexInSet));
+		*/
+		// Put every candidate into this position:
+		for (int candidate = firstCandidate; candidate < n - p + currentIndexInSet + 1; candidate++) {
+			// System.out.println(indent + candidate);
+			currentSet[currentIndexInSet] = candidate;
+			if (currentIndexInSet == p - 1) {
+				// We just wrote the last index, so copy this one in and return
+				// System.out.println(indent + "writing into line " + upToSetNum);
+				System.arraycopy(currentSet, 0, sets[upToSetNum++], 0, p);
+			} else {
+				// There are more indices to be written in, so make a recursive call to write the 
+				//  next ones in
+				upToSetNum = writeSetsIn(n, p, currentIndexInSet + 1, candidate + 1, currentSet, sets, upToSetNum);
+			}
+		}
+		return upToSetNum;
+	}
+
+	public static void main(String args[]) throws Exception {
+		/*
+		System.out.println(numOfSets(158,4));
+		System.out.println(numOfSets(158,3));
+		System.out.println(numOfSets(158,2));
+		*/
+		// int[][] sets = generateAllSets(6,4);
+		// MatrixUtils.printMatrix(System.out, sets);
+		
+		System.out.printf("digamma()  digammaOld()\n");
+		for (int n = 0; n < 100; n++) {
+			System.out.printf("%d  %.3f  %.3f\n", n, MathsUtils.digamma(n), MathsUtils.digammaByDefinition(n));
+		}
+		for (int n = 0; n < 101; n++) {
+			System.out.printf("%d  %.3f  %.3f\n", n, MathsUtils.digamma(n), MathsUtils.digammaByDefinition(n));
+		}
+	}
+}

java/source/infodynamics/utils/MeasurementDistribution.java

@@ -0,0 +1,85 @@
+package infodynamics.utils;
+
+/**
+ * 
+ * Structure to hold a distribution of info-theoretic measurements,
+ *  and a significance value for how an original measurement compared 
+ *  with these.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class MeasurementDistribution {
+
+	/**
+	 * Distribution of surrogate measurement values
+	 */
+	public double[] distribution;
+	/**
+	 * Actual observed value of the measurement
+	 */
+	public double actualValue;
+	/**
+	 * Probability that surrogate measurement is greater than
+	 *  the observed value
+	 */
+	public double pValue;
+	
+	protected boolean computedMean = false;
+	protected double meanOfDist;
+	protected double stdOfDist;
+	
+	public MeasurementDistribution(int size) {
+		distribution = new double[size];
+	}
+
+	public MeasurementDistribution(double[] distribution, double actualValue) {
+		this.actualValue = actualValue;
+		this.distribution = distribution;
+		int countWhereActualIsNotGreater = 0;
+		for (int i = 0; i < distribution.length; i++) {
+			if (distribution[i] >= actualValue) {
+				countWhereActualIsNotGreater++;
+			}
+		}
+		pValue = (double) countWhereActualIsNotGreater / (double) distribution.length;
+	}
+	
+	/*
+	public double computeGaussianSignificance() {
+		// Need to conpute the significance based on the assumption of 
+		//  an underlying Gaussian distribution.
+		// Use the t distribution for analysis, since we have a finite
+		//  number of samples to comptue the mean and std from.
+		return 0;
+	}
+	*/
+	
+	public double getTSscore() {
+		if (! computedMean) {
+			meanOfDist = MatrixUtils.mean(distribution);
+			stdOfDist = MatrixUtils.stdDev(distribution, meanOfDist);
+			computedMean = true;
+		}
+		double t = (actualValue - meanOfDist) / stdOfDist;
+		return t;
+	}
+	
+	public double getMeanOfDistribution() {
+		if (! computedMean) {
+			meanOfDist = MatrixUtils.mean(distribution);
+			stdOfDist = MatrixUtils.stdDev(distribution, meanOfDist);
+			computedMean = true;
+		}
+		return meanOfDist;
+	}
+
+	public double getStdOfDistribution() {
+		if (! computedMean) {
+			meanOfDist = MatrixUtils.mean(distribution);
+			stdOfDist = MatrixUtils.stdDev(distribution, meanOfDist);
+			computedMean = true;
+		}
+		return stdOfDist;
+	}
+}

java/source/infodynamics/utils/OctaveFileReader.java

@@ -0,0 +1,426 @@
+package infodynamics.utils;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.reflect.Array;
+
+/**
+ * Octave file reader.
+ * Usage:
+ * 1. call constructor
+ * 2. to retrieve a variable, call the get method appropriate to the
+ *    data type of that variable.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class OctaveFileReader {
+	private String filename;
+	private BufferedReader br;
+	private boolean isOpen = false;
+	
+	static final String SCALAR_HEADER = "# type: scalar";
+	static final String GLOBAL_SCALAR_HEADER = "# type: scalar";
+	static final String MATRIX_HEADER = "# type: matrix";
+	static final String GLOBAL_MATRIX_HEADER = "# type: global matrix";
+	static final String OCTAVE_DELIMITER = "[ \t]";
+	
+	public OctaveFileReader(String octaveFilename) throws FileNotFoundException {
+		filename = octaveFilename;
+		openOctaveFile();
+	}
+	
+	private void openOctaveFile() throws FileNotFoundException {
+		if (isOpen) {
+			return;
+		}
+		br = new BufferedReader(new FileReader(filename));
+		isOpen = true;
+	}
+	
+	private void closeOctaveFile() throws IOException {
+		br.close();
+		isOpen = false;
+	}
+	
+	/*
+	public OctaveFileReader(FileReader fileReader) {
+		br = new BufferedReader(fileReader);
+	}
+	
+	public OctaveFileReader(File file) throws FileNotFoundException {
+		br = new BufferedReader(new FileReader(file));
+	}
+	*/
+	
+	public int getInt(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a scalar
+					line = br.readLine();
+					if (!line.equals(SCALAR_HEADER) && !line.equals(GLOBAL_SCALAR_HEADER)) {
+						// The variable is not a scalar - it is not compatible with the
+						//  int type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with int");
+					}
+					// The next line contains the variable's value
+					line = br.readLine();
+					closeOctaveFile();
+					// Now, since we want to allow a call to getInt even if this is a double,
+					// then we'll call parse double then cast it to an int.
+					double value = Double.parseDouble(line);
+					return (int) value;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			// let the exception go
+			throw e;
+		}
+	}
+	
+	public double getDouble(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a scalar
+					line = br.readLine();
+					if (!line.equals(SCALAR_HEADER) && !line.equals(GLOBAL_SCALAR_HEADER)) {
+						// The variable is not a scalar - it is not compatible with the
+						//  int type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with double");
+					}
+					// The next line contains the variable's value
+					line = br.readLine();
+					closeOctaveFile();
+					return Double.parseDouble(line);
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public int[] getIntArray(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a matrix
+					line = br.readLine();
+					if (!line.equals(MATRIX_HEADER) && !line.equals(GLOBAL_MATRIX_HEADER)) {
+						// The variable is not a matrix - it is not compatible with the
+						//  int [] type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with int[] - not a matrix");
+					}
+					// The next line contains the number of rows
+					line = br.readLine();
+					String[] parts = line.split(" ");
+					int rows = Integer.parseInt(parts[2]);
+					// The next line contains the number of columns
+					line = br.readLine();
+					parts = line.split(" ");
+					int columns = Integer.parseInt(parts[2]);
+					// Now check that it either has one row or one column
+					if (!(rows == 1) && !(columns == 1)) {
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with int[] - not a single dimension (" +
+								rows + " x " + columns + ")");
+					}
+					int theLength = (rows > columns) ? rows : columns;
+					int[] values = new int[theLength];
+					// The next lines contain each row of values
+					int l = 0;
+					for (int r = 0; r < rows; r++) {
+						// Grab the text for the next row - values separated by spaces
+						line = br.readLine();
+						String[] stringValues = line.split(OCTAVE_DELIMITER);
+						// ignore stringValues[0] since the line starts with a delimiter
+						if ((stringValues.length + 1) < columns) {
+							throw new Exception("Not enough values for row " + r + " of variable " + name);
+						}
+						for (int c = 0; c < columns; c++) {
+							// Parse the next value
+							values[l] = Integer.parseInt(stringValues[c + 1]);
+							l++;
+						}
+					}
+					closeOctaveFile();
+					return values;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public long[] getLongArray(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a matrix
+					line = br.readLine();
+					if (!line.equals(MATRIX_HEADER) && !line.equals(GLOBAL_MATRIX_HEADER)) {
+						// The variable is not a matrix - it is not compatible with the
+						//  int [] type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with long[] - not a matrix");
+					}
+					// The next line contains the number of rows
+					line = br.readLine();
+					String[] parts = line.split(" ");
+					int rows = Integer.parseInt(parts[2]);
+					// The next line contains the number of columns
+					line = br.readLine();
+					parts = line.split(" ");
+					int columns = Integer.parseInt(parts[2]);
+					// Now check that it either has one row or one column
+					if ((rows != 1) && (columns != 1)) {
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with long[] - not a single dimension (" +
+								rows + " x " + columns + ")");
+					}
+					int theLength = (rows > columns) ? rows : columns;
+					long[] values = new long[theLength];
+					// The next lines contain each row of values
+					int l = 0;
+					for (int r = 0; r < rows; r++) {
+						// Grab the text for the next row - values separated by spaces
+						line = br.readLine();
+						String[] stringValues = line.split(OCTAVE_DELIMITER);
+						// ignore stringValues[0] since the line starts with a delimiter
+						if ((stringValues.length + 1) < columns) {
+							throw new Exception("Not enough values for row " + r + " of variable " + name);
+						}
+						for (int c = 0; c < columns; c++) {
+							// Parse the next value
+							values[l] = Long.parseLong(stringValues[c + 1]);
+							l++;
+						}
+					}
+					closeOctaveFile();
+					return values;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public double[] getDoubleArray(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a matrix
+					line = br.readLine();
+					if (!line.equals(MATRIX_HEADER) && !line.equals(GLOBAL_MATRIX_HEADER)) {
+						// The variable is not a matrix - it is not compatible with the
+						//  int [] type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with double[] - not a matrix");
+					}
+					// The next line contains the number of rows
+					line = br.readLine();
+					String[] parts = line.split(" ");
+					int rows = Integer.parseInt(parts[2]);
+					// The next line contains the number of columns
+					line = br.readLine();
+					parts = line.split(" ");
+					int columns = Integer.parseInt(parts[2]);
+					// Now check that it either has one row or one column
+					if (!(rows == 1) && !(columns == 1)) {
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with double[] - not a single dimension (" +
+								rows + " x " + columns + ")");
+					}
+					int theLength = (rows > columns) ? rows : columns;
+					double[] values = new double[theLength];
+					// The next lines contain each row of values
+					int l = 0;
+					for (int r = 0; r < rows; r++) {
+						// Grab the text for the next row - values separated by spaces
+						line = br.readLine();
+						String[] stringValues = line.split(OCTAVE_DELIMITER);
+						// ignore stringValues[0] since the line starts with a delimiter
+						if ((stringValues.length + 1) < columns) {
+							throw new Exception("Not enough values for row " + r + " of variable " + name);
+						}
+						for (int c = 0; c < columns; c++) {
+							// Parse the next value
+							values[l] = Double.parseDouble(stringValues[c + 1]);
+							l++;
+						}
+					}
+					closeOctaveFile();
+					return values;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public int[][] getInt2DMatrix(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a matirx
+					line = br.readLine();
+					if (!line.equals(MATRIX_HEADER) && !line.equals(GLOBAL_MATRIX_HEADER)) {
+						// The variable is not a matrix - it is not compatible with the
+						//  int [][] type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with int[][]");
+					}
+					// The next line contains the number of rows
+					line = br.readLine();
+					String[] parts = line.split(" ");
+					int rows = Integer.parseInt(parts[2]);
+					// The next line contains the number of columns
+					line = br.readLine();
+					parts = line.split(" ");
+					int columns = Integer.parseInt(parts[2]);
+					int[][] values = new int[rows][columns];
+					// The next lines contain each row of values
+					for (int r = 0; r < rows; r++) {
+						// Grab the text for the next row - values separated by spaces
+						line = br.readLine();
+						String[] stringValues = line.split(OCTAVE_DELIMITER);
+						int subFromLength = 0;
+						if ((stringValues.length > 0) && (stringValues[0].equalsIgnoreCase(""))) {
+							// Ignore stringValues[0] since the line starts with a delimiter
+							subFromLength = 1;
+						}
+						if ((stringValues.length - subFromLength) < columns) {
+							throw new Exception("Not enough values for row " + r + " of " + rows + " of variable " + name + "( actual: " +
+									(stringValues.length - subFromLength) + ", required: " + columns + ")");
+						}
+						for (int c = 0; c < columns; c++) {
+							// Parse the next value
+							values[r][c] = Integer.parseInt(stringValues[c + subFromLength]);
+						}
+					}
+					closeOctaveFile();
+					return values;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public double[][] getDouble2DMatrix(String name) throws Exception {
+		openOctaveFile();
+		try {
+			for (String line = br.readLine(); line != null; line = br.readLine()) {
+				if (line.equalsIgnoreCase("# name: " + name)) {
+					// We have the header line for this stored variable
+					// Make sure the next line indicates a matirx
+					line = br.readLine();
+					if (!line.equals(MATRIX_HEADER) && !line.equals(GLOBAL_MATRIX_HEADER)) {
+						// The variable is not a matrix - it is not compatible with the
+						//  int [][] type
+						throw new Exception("Stored Octave variable " + name + " is not of a type compatible with int[][]");
+					}
+					// The next line contains the number of rows
+					line = br.readLine();
+					String[] parts = line.split(" ");
+					int rows = Integer.parseInt(parts[2]);
+					// The next line contains the number of columns
+					line = br.readLine();
+					parts = line.split(" ");
+					int columns = Integer.parseInt(parts[2]);
+					double[][] values = new double[rows][columns];
+					// The next lines contain each row of values
+					for (int r = 0; r < rows; r++) {
+						// Grab the text for the next row - values separated by spaces
+						line = br.readLine();
+						String[] stringValues = line.split(OCTAVE_DELIMITER);
+						// ignore stringValues[0] since the line starts with a delimiter
+						// TODO Need to fix this in case it doesn't start with a delimiter.
+						//  See the 2D integer method. Should fix other methods also. 
+						if ((stringValues.length + 1) < columns) {
+							throw new Exception("Not enough values for row " + r + " of variable " + name);
+						}
+						for (int c = 0; c < columns; c++) {
+							// Parse the next value
+							values[r][c] = Double.parseDouble(stringValues[c + 1]);
+						}
+					}
+					closeOctaveFile();
+					return values;
+				}
+			}
+			// End of file reached and variable not found !
+			throw new Exception("Stored Octave variable " + name + " not found");
+		} catch (Exception e) {
+			// clean up
+			closeOctaveFile();
+			throw e;
+		}
+	}
+
+	public static void main(String argv[]) throws Exception {
+
+		// Run some quick tests - ALL WORKS
+		
+		OctaveFileReader ofr = new OctaveFileReader("C:\\Work\\Investigations\\CATransferEntropy\\base-2\\neighbourhood-5\\rule-133976044.txt");
+		// Now grab a matrix variable
+		int[][] rules = ofr.getInt2DMatrix("settledExecutedRules");
+		for (int r = 0; r < rules.length; r++) {
+			for (int c = 0; c < rules[r].length; c++) {
+				System.out.print(" " + rules[r][c]);
+			}
+			System.out.println();
+		}
+		System.out.println("rules has " + rules.length + " rows and " + rules[0].length + " columns");
+		/* double[][] rulesDouble = ofr.getDouble2DMatrix("settledExecutedRules");
+		for (int r = 0; r < rulesDouble.length; r++) {
+			for (int c = 0; c < rulesDouble[r].length; c++) {
+				System.out.print(" " + rulesDouble[r][c]);
+			}
+			System.out.println();
+		}
+		System.out.println("rulesDouble has " + rulesDouble.length + " rows and " + rulesDouble[0].length + " columns");
+		*/
+		System.out.println("base = " + ofr.getInt("base"));
+		System.out.println("neighbourhood = " + ofr.getInt("neighbourhood"));
+		
+		Object obj = new int[3];
+		if (obj.getClass().isArray()) {
+			System.out.println("got an array of length " + Array.getLength(obj));
+		}
+		System.out.println(obj.getClass() + " " + obj);
+	}
+}

java/source/infodynamics/utils/OctaveFileWriter.java

@@ -0,0 +1,267 @@
+package infodynamics.utils;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Array;
+import java.net.InetAddress;
+import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Octave file format writer.
+ * Usage:
+ * 1. call constructor
+ * 2. call put for each variable to be stored.
+ * 3. call writeFile(outputFilename) or setFilename(outputFilename)
+ *    then writeFile()
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class OctaveFileWriter extends HashMap<String, Object> {
+
+	public static final long serialVersionUID = 1;
+	
+	private boolean writeLFOnly = true;
+	private String octaveFilename = null;
+	private static final String LINE_SEPARATOR_PROPERTY = "line.separator";
+	private static final String LINE_SEPARATOR = "\n";
+	private Hashtable<String, Integer> roundingHT = new Hashtable<String, Integer>();
+		
+	public OctaveFileWriter() {
+	}
+	
+	public void setFilename(String outputFilename) {
+		octaveFilename = outputFilename;
+	}
+	
+	public void writeFile(String outputFilename) throws IOException {
+		setFilename(outputFilename);
+		writeFile();
+	}
+
+	/**
+	 * Writes the contained variables to a file formatted for octave to read
+	 * 
+	 * @param octaveFilename
+	 */
+	public void writeFile() throws IOException {
+		int maxLength = 0; // required for arrays of strings
+		
+		if (octaveFilename == null) {
+			throw new IOException("No filename has been set");
+		}
+		
+		String originalLSValue = null;
+		if (writeLFOnly) {
+			// We only want to write \n. Save the current line separator for later restoration
+			originalLSValue = System.getProperty(LINE_SEPARATOR_PROPERTY);
+			System.setProperty(LINE_SEPARATOR_PROPERTY, LINE_SEPARATOR);
+		}
+		
+		// Create the directory if required
+    	createDirectories(octaveFilename);
+		
+		// Open the file for writing
+		PrintWriter pw = new PrintWriter(new BufferedWriter(new FileWriter(octaveFilename)));
+		
+		// Write header
+		String hostname = "null";
+		try {
+			InetAddress localMachine = InetAddress.getLocalHost();
+			hostname = localMachine.getHostName();
+		} catch (Exception e) {
+			// do nothing
+		}
+		pw.println("# Created on " + hostname + " by infodynamics.utils.OctaveFileWriter, " + (new Date()));
+		
+		// Have a decimal format object ready in case we have to do any rounding
+		DecimalFormat decFormat = new DecimalFormat();
+		int rounding = -1;
+		
+		Set keySet = this.keySet();
+		Iterator iterator = keySet.iterator();
+		for (; iterator.hasNext();) {
+			String key = (String) iterator.next();
+			Object value = this.get(key);
+			try {
+				// Check if we need to round off this value
+				Integer roundObject = roundingHT.get(key);
+				if (roundObject != null) {
+					rounding = roundObject.intValue();
+					decFormat.setMaximumFractionDigits(rounding);
+				} else {
+					rounding = -1;
+				}
+				// Start header for this variable
+				pw.println("# name: " + key);
+				// Check what the type of the object is
+				if (value.getClass().isArray()) {
+					// we have an array item
+					// is it a 1D or 2D matrix? Check if the first item is an array
+					if (Array.getLength(value) > 0) {
+						Object item1 = Array.get(value, 0);
+						// Check whether this is an array of strings
+						boolean isStrings = String.class.isInstance(item1); 
+						if (isStrings) {
+							pw.println("# type: string");
+							// Now need to find the maximum length of all strings in the array
+							maxLength = 0;
+							for (int i = 0; i < Array.getLength(value); i++) {
+								if (((String)Array.get(value, i)).length() > maxLength) {
+									maxLength = ((String)Array.get(value, i)).length();
+								}
+							}
+							pw.println("# elements: " + Array.getLength(value));
+						} else {
+							// Ordinary array
+							pw.println("# type: matrix");
+						}
+						if (item1.getClass().isArray()) {
+							// We have a 2D array - (cannot have 2D arrays of strings in Octave,
+							//  so assume they haven't been passed in)
+							pw.println("# rows: " + Array.getLength(value));
+							pw.println("# columns: " + Array.getLength(item1));
+							for (int i = 0; i < Array.getLength(value); i++) {
+								// Grab the next row
+								Object row = Array.get(value, i);
+								// If the first element is a float or double we'll
+								//  round off every element.
+								for (int j = 0; j < Array.getLength(row); j++) {
+									// Print each index item
+									Object thisValue = Array.get(row, j);
+									if (Boolean.class.isInstance(thisValue)) {
+										pw.print(" " + (((Boolean)thisValue) ? "1" : "0"));
+									} else {
+										if (rounding >= 0) {
+											// print with rounding
+											pw.print(" " + decFormat.format(thisValue));
+										} else {
+											pw.print(" " + thisValue);
+										}
+									}
+								}
+								// Close off the row
+								pw.println();
+							}
+						} else {
+							// We have a 1D array
+							if (isStrings) {
+								for (int i = 0; i < Array.getLength(value); i++) {
+									pw.println("# length: " + maxLength);
+									String thisString = (String) Array.get(value, i); 
+									pw.print(thisString);
+									for (int l = thisString.length(); l < maxLength; l++) {
+										pw.print(' ');
+									}
+									pw.println();
+								}
+							} else {
+								// ordinary array
+								pw.println("# rows: " + Array.getLength(value));
+								pw.println("# columns: 1");
+								for (int i = 0; i < Array.getLength(value); i++) {
+									// Print each index item on it's own row
+									Object thisValue = Array.get(value, i);
+									if (Boolean.class.isInstance(thisValue)) {
+										pw.print(" " + (((Boolean)thisValue) ? "1" : "0"));
+									} else {
+										if (rounding >= 0) {
+											// print with rounding
+											pw.print(" " + decFormat.format(thisValue));
+										} else {
+											pw.println(" " + thisValue);
+										}
+									}
+								}
+							}
+						}
+					} else {
+						// Empty array ...
+						// just write null values. Not 100% sure this is valid ...
+						pw.println("# type: matrix");
+						pw.println("# rows: 0");
+						pw.println("# columns: 0");
+					}
+				} else {
+					if (String.class.isInstance(value)) {
+						pw.println("# type: string");
+						pw.println("# elements: 1");
+						pw.println("# length: " + ((String)value).length());
+						pw.println(value);
+					} else if (Boolean.class.isInstance(value)) {
+						// we have a boolean value
+						pw.println("# type: bool");
+						pw.println(((Boolean)value) ? "1" : "0");
+					} else {
+						// we have a general scalar value
+						pw.println("# type: scalar");
+						if (rounding >= 0) {
+							pw.println(decFormat.format(value));
+						} else {
+							pw.println(value);
+						}
+					}
+				}
+			} catch (Exception e) {
+				System.out.println("Problem writing variable " +
+						key + " to the output file (value = " +
+						value + "):");
+				e.printStackTrace();
+				System.out.println("Continuing with file.");
+			}
+		}
+		// Add one extra newline at the end
+		pw.println();
+		pw.close();
+		
+		if (writeLFOnly) {
+			// Restore the saved line separator
+			System.setProperty(LINE_SEPARATOR_PROPERTY, originalLSValue);
+		}
+	}
+	
+	/**
+	 * JL: I have no idea why i put this method here. Nothing appears
+	 *  to be using it
+	 * 
+	 * @param key
+	 * @param value
+	 * @param roundDPs
+	 */
+	public void put(String key, Object value, int roundDPs) {
+		roundingHT.put(key, new Integer(roundDPs));
+		put(key, value);
+	}
+	
+	/** 
+	 * Create the relevant directories if required
+	 * 
+	 * @param filename
+	 */
+	private static void createDirectories(String filename) {
+		File file = new File(filename);
+		File parentDir = file.getParentFile();
+		if ((parentDir != null) && !parentDir.isDirectory()) {
+			parentDir.mkdirs();
+		}
+	}
+	
+	public void putAll(Properties props) {
+		for (Object keyObject : props.keySet()) {
+			String key = (String) keyObject;
+			// Alter the key name to remvoe "." characters
+			key = key.replaceAll("\\.", "__");
+			// Add this pair in
+			put(key, props.get(keyObject));
+		}
+	}
+}

java/source/infodynamics/utils/ParsedProperties.java

@@ -0,0 +1,129 @@
+package infodynamics.utils;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.Set;
+import java.io.FileOutputStream;
+
+
+public class ParsedProperties {
+	private Properties properties;
+	
+	public ParsedProperties(Properties props) {
+		properties = props;
+	}
+	
+	public ParsedProperties(String propsFile) throws Exception {
+		Properties props = new Properties();
+		props.load(new FileInputStream(propsFile));
+		properties = props;
+	}
+
+	public int getIntProperty(String name) throws Exception {
+		String value = properties.getProperty(name);
+		if (value == null) {
+			throw new Exception("Property " + name + " not found");
+		}
+		return Integer.parseInt(value);
+	}
+
+	public long getLongProperty(String name) throws Exception {
+		String value = properties.getProperty(name);
+		if (value == null) {
+			throw new Exception("Property " + name + " not found");
+		}
+		return Long.parseLong(value);
+	}
+
+	public double getDoubleProperty(String name) throws Exception {
+		String value = properties.getProperty(name);
+		if (value == null) {
+			throw new Exception("Property " + name + " not found");
+		}
+		return Double.parseDouble(value);
+	}
+
+	public boolean getBooleanProperty(String name) throws Exception {
+		String value = properties.getProperty(name);
+		if (value == null) {
+			throw new Exception("Property " + name + " not found");
+		}
+		return Boolean.parseBoolean(value);
+	}
+	
+	/**
+	 * Return an integer array for a comma separated integer list,
+	 * or a specification of "startIndex:endIndex"
+	 * @param name
+	 * @return Integer array for the property
+	 */
+	public int[] getIntArrayProperty(String name) {
+		int[] returnValues;
+		String repeatString = properties.getProperty(name);
+		if (repeatString.indexOf(':') >= 0) {
+			// The repeats are of the format "startIndex:endIndex"
+			String[] indices = repeatString.split(":");
+			int startIndex = Integer.parseInt(indices[0]);
+			int endIndex = Integer.parseInt(indices[1]);
+			returnValues = new int[endIndex - startIndex + 1];
+			for (int i = 0; i < returnValues.length; i++) {
+				returnValues[i] = startIndex + i;
+			}
+		} else {
+			// The repeats are in a comma separated format
+			String[] repeatStrings = repeatString.split(",");
+			returnValues = new int[repeatStrings.length];
+			for (int i = 0; i < returnValues.length; i++) {
+				returnValues[i] = Integer.parseInt(repeatStrings[i]);
+			}
+		}
+		return returnValues;
+	}
+	
+	public String getStringProperty(String name) throws Exception {
+		String value = getProperty(name);
+		if (value == null) {
+			throw new Exception("Property " + name + " not found");
+		}
+		return value;
+	}
+	
+	/**
+	 * Return a string array for a comma separated string list
+	 * 
+	 * @param name
+	 * @return String array for the property
+	 */
+	public String[] getStringArrayProperty(String name) {
+		String repeatString = properties.getProperty(name);
+		// The repeats are in a comma separated format
+		String[] repeatStrings = repeatString.split(",");
+		return repeatStrings;
+	}
+
+	public String getProperty(String name) {
+		return properties.getProperty(name);
+	}
+	
+	public Properties getProperties() {
+		return properties;
+	}
+	
+	public void setProperty(String name, String value) {
+		properties.setProperty(name, value);
+	}
+	
+	public Set<Object> keySet() {
+		return properties.keySet();
+	}
+	                                                 
+	public void writeToFile(String fileName,String comment) throws IOException{
+		FileOutputStream writeFile = new FileOutputStream(fileName);
+		properties.store(writeFile, comment);
+	}
+	
+	public boolean containsProperty(String name) {
+		return properties.containsKey(name);
+	}
+}

java/source/infodynamics/utils/RandomGenerator.java

@@ -0,0 +1,746 @@
+package infodynamics.utils;
+
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Arrays;
+
+/**
+ * Utility to generate arrays of random variables
+ * 
+ * TODO I think I may need to revisit whether I reuse the objects added to the hashtable;
+ * I don't think we should be doing this.
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class RandomGenerator {
+
+	Random random;
+	
+	public RandomGenerator() {
+		random = new Random();
+	}
+
+	public void setSeed(long seed) {
+		random.setSeed(seed);
+	}
+	
+	public double[] generateRandomData(int length){
+		double[] data = new double[length];
+		for (int i = 0; i < length; i++) {
+			data[i] = random.nextDouble();
+		}
+		return data;
+	}
+
+	/**
+	 * Generate an array of random data on the interval [min .. max)
+	 * 
+	 * @param length
+	 * @param min
+	 * @param max
+	 * @return
+	 */
+	public double[] generateRandomData(int length, double min, double max){
+		double[] data = new double[length];
+		for (int i = 0; i < length; i++) {
+			data[i] = min + random.nextDouble() * (max - min);
+		}
+		return data;
+	}
+
+	public double[][] generateRandomData(int length, int dimenions){
+		double[][] data = new double[length][dimenions];
+		for (int i = 0; i < length; i++) {
+			for (int j = 0; j < dimenions; j++) {
+				data[i][j] = random.nextDouble();
+			}
+		}
+		return data;
+	}
+	
+	/**
+	 * <p>
+	 * Generate <i>length</i> random ints, between the values 0..(<i>cap</i>-1)
+	 * </p>
+	 * 
+	 * @param length
+	 * @param cap
+	 * @return
+	 */
+	public double[] generateDistinctRandomInts(int length, int cap){
+		double[] data = new double[length];
+		boolean[] used = new boolean[cap];
+		for (int i = 0; i < length; i++) {
+			int nextAttempt;
+			// Select an int we haven't used yet:
+			for (nextAttempt = random.nextInt(cap); used[nextAttempt]; nextAttempt = random.nextInt(cap)) {
+				// Select the next attempt
+			}
+			data[i] = nextAttempt;
+			used[nextAttempt] = true;
+		}
+		return data;
+	}
+
+	public double[] generateNormalData(int length, double mean, double std){
+		double[] data = new double[length];
+		for (int i = 0; i < length; i++) {
+			data[i] = random.nextGaussian()*std + mean;
+		}
+		return data;
+	}
+
+	public double[][] generateNormalData(int length, int dimenions,
+			double mean, double std){
+		double[][] data = new double[length][dimenions];
+		for (int i = 0; i < length; i++) {
+			for (int j = 0; j < dimenions; j++) {
+				data[i][j] = random.nextGaussian()*std + mean;
+			}
+		}
+		return data;
+	}
+
+	/**
+	 * <p>Generate bivariate Gaussian series with the given covariance.
+	 * See http://mathworld.wolfram.com/BivariateNormalDistribution.html</p>
+	 * <p>
+	 * If we have two normal distributions x1 and x2, we can define</br>
+	 *   <ul>
+	 *   	<li>y1 = mean1 + sigma11*x1 + sigma12*x2</li>
+	 *   	<li>y2 = mean2 + sigma21*x1 + sigma22*x2</li>
+	 *   </ul>
+	 * which are Gaussian distributed with:
+	 * 	<ul>
+	 *  	<li>means (mean1,mean2),</li>
+	 *  	<li>variances (sigma11^2+sigma12^2, sigma21^2+sigma22^2), and</li>
+	 *  	<li>covariance sigma11*sigma21 + sigma12*sigma22</li>
+	 *  </ul>
+	 * So to generate a bivariate series with a desired covariance, means and stds,
+	 * we set sigma12=0, giving sigma11 = std1, solve for sigma21 from the covariance,
+	 * and solve for sigma22 from the std2.
+	 * </p> 
+	 * 
+	 * @param length
+	 * @param mean1
+	 * @param std1
+	 * @param mean2
+	 * @param std2
+	 * @param covariance
+	 * @return a time series with two variables: first index is time step, second index is variable number
+	 */
+	public double[][] generateBivariateNormalData(int length,
+			double mean1, double std1,
+			double mean2, double std2,
+			double covariance){
+
+		double[][] data = new double[length][2];
+		double sigma21 = covariance / std1;
+		double sigma22 = Math.sqrt(std2*std2 - sigma21*sigma21);
+		for (int i = 0; i < length; i++) {
+			double x1 = random.nextGaussian();
+			double x2 = random.nextGaussian();
+			data[i][0] = mean1 + std1*x1;
+			data[i][1] = mean2 + sigma21*x1 + sigma22*x2;
+		}
+		return data;
+	}
+	
+	/**
+	 * Generate a set of covariant gaussians, with the given means and covariances.
+	 * 
+	 * @param length Number of time steps (samples) generated
+	 * @param dimensions Number of gaussians
+	 * @param means Means of the generated gaussians
+	 * @param componentDependencies Underlying depencence matrix A between the generated gaussians.
+	 * 		Covariance C = A * A^T
+	 * @return
+	 */
+	public double[][] generateCovariantGaussians(int length, int dimensions,
+			double[] means, double[][] componentDependencies) {
+		double[][] data = new double[length][dimensions];
+		for (int t = 0; t < length; t++) {
+			// Generate the underlying random values for this time step
+			double[] x = generateNormalData(dimensions, 0, 1);
+			for (int d = 0; d < dimensions; d++) {
+				data[t][d] = means[d];
+				// Combine the underlying random values for variable d
+				for (int d2 = 0; d2 < dimensions; d2++) {
+					data[t][d] += componentDependencies[d][d2] * x[d2];
+				}
+			}
+		}
+		return data;
+	}
+	
+	/**
+	 * Generate (up to) numSets distinct sets of p distinct values in [0..n-1] 
+	 * Done using random guesses as this is designed for high dimension n
+	 *  where its highly unlikely we repeat a set (though this is checked)
+	 * 
+	 * @param n
+	 * @param p
+	 * @param maxNumSets
+	 * @return
+	 */
+	public int[][] generateDistinctRandomSets(int n, int p, int maxNumSets) {
+		
+		// Check what is the max possible number of sets we
+		//  could generate:
+		int maxPossibleNumSets = 0;
+		try {
+			maxPossibleNumSets = MathsUtils.numOfSets(n, p);
+			if (maxNumSets > maxPossibleNumSets) {
+				// Best limit maxNumSets
+				maxNumSets = maxPossibleNumSets;
+				// We want to generate all possible sets
+				return generateAllDistinctSets(n, p);
+			}
+		} catch (Exception e) {
+			// n choose p blew Integer.MAX_INT
+			// therefore there is no way maxNumSets is larger than it
+		}
+		
+		int[][] sets = new int[maxNumSets][p];
+		// Pool of available choices:
+		Vector<Integer> availableChoices = new Vector<Integer>();
+		for (int i = 0; i < n; i++) {
+			availableChoices.add(new Integer(i));
+		}
+		// Pool of choices already used this turn
+		Vector<Integer> thisSet = new Vector<Integer>();
+		// Hashtable tracking sets we've already chosen
+		Hashtable<Vector<Integer>,Integer> chosenSets =
+			new Hashtable<Vector<Integer>,Integer>();
+		for (int s = 0; s < maxNumSets; s++) {
+			// Select set s:
+			for (;;) {
+				// Try to get a new unique set
+				// Reset the pool of choices ready to choose this set
+				availableChoices.addAll(thisSet);
+				thisSet.clear();
+				// System.out.println("Available: " + availableChoices);
+				for (int q = 0; q < p; q++) {
+					// Select the qth index from the available pool to use here:
+					int randIndex = random.nextInt(n - q);
+					// Find out what number this corresponds to, and write it in:
+					Integer nextSelection = availableChoices.remove(randIndex);
+					sets[s][q] = nextSelection.intValue();
+				}
+				// Track the chosen integers in order to avoid duplicates
+				Arrays.sort(sets[s]);
+				for (int q = 0; q < p; q++) {
+					// And track it in thisSet for hashing and 
+					//  adding back to the pool
+					thisSet.add(new Integer(sets[s][q]));
+				}
+				if (chosenSets.get(thisSet) == null) {
+					// We haven't included this set yet:
+					chosenSets.put(thisSet, new Integer(0));
+					// System.out.println(" Chosen: " + thisSet);
+					break;
+				}
+				// else we have already added this set,
+				//  so we need to try again.
+				// System.out.println(" Attempted: " + thisSet);
+				// System.out.println(" Need to try again");
+			}
+		}
+		return sets;
+	}
+	
+	/**
+	 * Generate exactly N random sets of p numbers from [0 .. n-1],
+	 *  allowing repeats if nCp < N.
+	 * 
+	 * @param n
+	 * @param p
+	 * @param N
+	 * @return
+	 */
+	public int[][] generateNRandomSets(int n, int p,
+			int N) {
+		int[][] distinctSets = generateDistinctRandomSets(n, p, N);
+		if (distinctSets.length == N) {
+			// Fine - return it
+			return distinctSets;
+		} else if (distinctSets.length > N) {
+			// Error condition - generateDistinctRandomSets
+			//  should not do this
+			throw new RuntimeException(
+				"generateDistinctRandomSets generated more than " +
+				N + " distinct sets when asked for " + N + 
+				"; note n=" + n + " p=" + p);
+		}
+		// Else we now scale up the available distinct rows
+		//  to fill the whole N required sets
+		int[][] randomSets = new int[N][];
+		for (int i = 0; i < N; i++) {
+			// Select one of the distinct sets at random:
+			randomSets[i] = distinctSets[random.nextInt(distinctSets.length)];
+		}
+		return randomSets;
+	}
+
+	/**
+	 * Generate (up to) numSets distinct sets of p distinct values in [0..n-1].
+	 * Done using random guesses as this is designed for high dimension n
+	 *  where its highly unlikely we repeat a set (though this is checked).
+	 * Here we avoid overlapping the sets with any elements of the corresponding
+	 *  row of setsToAvoidOverlapWith
+	 * 
+	 * @param n
+	 * @param p
+	 * @param maxNumSets
+	 * @param setsToAvoidOverlapWith
+	 * @return
+	 */
+	public int[][] generateDistinctRandomSets(int n, int p, int maxNumSets, int[][] setsToAvoidOverlapWith) {
+		
+		// TODO Write this
+		
+		if (true) {
+			throw new RuntimeException("Not implemented yet");
+		}
+		
+		// Not sure whether we need to make this call beforehand or not - have a think about
+		
+		
+		// Check what is the max possible number of sets we
+		//  could generate:
+		int maxPossibleNumSets = 0;
+		try {
+			maxPossibleNumSets = MathsUtils.numOfSets(n, p);
+			if (maxNumSets > maxPossibleNumSets) {
+				// Best limit maxNumSets
+				maxNumSets = maxPossibleNumSets;
+				// We want to generate all possible sets
+				return generateAllDistinctSets(n, p);
+			}
+		} catch (Exception e) {
+			// n choose p blew Integer.MAX_INT
+			// therefore there is no way maxNumSets is larger than it
+		}
+		
+		int[][] sets = new int[maxNumSets][p];
+		// Pool of available choices:
+		Vector<Integer> availableChoices = new Vector<Integer>();
+		for (int i = 0; i < n; i++) {
+			availableChoices.add(new Integer(i));
+		}
+		// Pool of choices already used this turn
+		Vector<Integer> thisSet = new Vector<Integer>();
+		// Hashtable tracking sets we've already chosen
+		Hashtable<Vector<Integer>,Integer> chosenSets =
+			new Hashtable<Vector<Integer>,Integer>();
+		for (int s = 0; s < maxNumSets; s++) {
+			// Select set s:
+			for (;;) {
+				// Try to get a new unique set
+				// Reset the pool of choices ready to choose this set
+				availableChoices.addAll(thisSet);
+				// And remove the integers already chosen in the corresponding row of setsToAvoidOverlapWith:
+				if (setsToAvoidOverlapWith != null) {
+					for (int i = 0; i < p; i++) {
+						availableChoices.remove(new Integer(setsToAvoidOverlapWith[s][p]));
+					}
+				}
+				thisSet.clear();
+				// System.out.println("Available: " + availableChoices);
+				for (int q = 0; q < p; q++) {
+					// Select the qth index from the available pool to use here:
+					int randIndex = random.nextInt(n - q);
+					// Find out what number this corresponds to, and write it in:
+					Integer nextSelection = availableChoices.remove(randIndex);
+					sets[s][q] = nextSelection.intValue();
+				}
+				// Track the chosen integers in order to avoid duplicates
+				Arrays.sort(sets[s]);
+				for (int q = 0; q < p; q++) {
+					// And track it in thisSet for hashing and 
+					//  adding back to the pool
+					thisSet.add(new Integer(sets[s][q]));
+				}
+				if (chosenSets.get(thisSet) == null) {
+					// We haven't included this set yet.
+					chosenSets.put(thisSet, new Integer(0));
+					// System.out.println(" Chosen: " + thisSet);
+					break;
+				}
+				// else we have already added this set,
+				//  so we need to try again.
+				// System.out.println(" Attempted: " + thisSet);
+				// System.out.println(" Need to try again");
+			}
+		}
+		return sets;
+	}
+
+	/**
+	 * Generate exactly N sets of p integers chosen from [0..n-1].
+	 * Make sure that set i does not have any integers overlapping with set i
+	 *  from setsToAvoidOverlapWith.
+	 * 
+	 * @param n
+	 * @param p
+	 * @param N
+	 * @param setsToAvoidOverlapWith
+	 * @return
+	 */
+	public int[][] generateNRandomSetsNoOverlap(int n, int p,
+			int N, int[][] setsToAvoidOverlapWith) {
+		// We generate N random sets of p numbers from n,
+		//  allowing repeats if nCp < N.
+		int[][] distinctSets = generateDistinctRandomSets(n, p, N, setsToAvoidOverlapWith);
+		if (distinctSets.length == N) {
+			// Fine - return it
+			return distinctSets;
+		} else if (distinctSets.length > N) {
+			// Error condition - generateDistinctRandomSets
+			//  should not do this
+			throw new RuntimeException(
+				"generateDistinctRandomSets generated more than " +
+				N + " distinct sets when asked for " + N + 
+				"; note n=" + n + " p=" + p);
+		}
+		// Else we now scale up the available distinct rows
+		//  to fill the whole N required sets
+		int[][] randomSets = new int[N][];
+		for (int i = 0; i < N; i++) {
+			// Select one of the distinct sets at random:
+			randomSets[i] = distinctSets[random.nextInt(distinctSets.length)];
+		}
+		return randomSets;
+	}
+
+	/**
+	 * Generate all nCp sets (assuming this doesn't blow our memory
+	 * 
+	 * @param n
+	 * @param p
+	 * @return
+	 */
+	public int[][] generateAllDistinctSets(int n, int p) throws Exception {
+		int numSets;
+		try {
+			numSets = MathsUtils.numOfSets(n, p);
+		} catch (Exception e) {
+			// n choose p blew Integer.MAX_INT
+			throw new Exception("nCp too large");
+		}
+		// allocate space for the distinct sets
+		int[][] sets = new int[numSets][p];
+		int[] workingSet = new int[p];
+		addToDistinctSets(sets, n, p, 0, workingSet, 0, 0);
+		return sets;
+	}
+	
+	/**
+	 * Using the workingSet which is filled up to (but not including)
+	 *  fromIndex, add new distinct sets of nCp to the sets matrix,
+	 *  from the setNumber index
+	 * 
+	 * @param sets
+	 * @param n
+	 * @param p
+	 * @param setNumber
+	 * @param workingSet
+	 * @param fromIndex
+	 */
+	protected int addToDistinctSets(int[][] sets, int n, int p, int setNumber, 
+			int[] workingSet, int fromIndex, int selectFrom) {
+		if (fromIndex == p) {
+			// The workingSet is ready to go, so copy it in
+			// MatrixUtils.printArray(System.out, workingSet);
+			System.arraycopy(workingSet, 0, sets[setNumber], 0, p);
+			setNumber++;
+		} else {
+			// Add to the working set and pass it on:
+			for (int c = selectFrom; c < n; c++) {
+				workingSet[fromIndex] = c;
+				setNumber = addToDistinctSets(sets, n, p, setNumber,
+						workingSet, fromIndex + 1, c + 1);
+			}
+		}
+		return setNumber;
+	}
+	
+	/**
+	 * Generate numberOfPerturbations perturbations of [0..n-1]
+	 * 
+	 * @param n
+	 * @param numberOfPerturbations
+	 * @return an array of dimensions [numberOfPerturbations][n], with each row
+	 *  being one perturbation of the elements
+	 */
+	public int[][] generateDistinctRandomPerturbations(int n, int numberOfPerturbations) {
+		// Check what is the max possible number of perturbations we
+		//  could generate:
+		int maxPossibleNumPerturbations = 0;
+		try {
+			maxPossibleNumPerturbations = MathsUtils.factorialCheckBounds(n);
+			if (numberOfPerturbations > maxPossibleNumPerturbations) {
+				// Best limit maxNumSets
+				numberOfPerturbations = maxPossibleNumPerturbations;
+				// We want to generate all possible sets
+				return generateAllDistinctPerturbations(n);
+			}
+		} catch (Exception e) {
+			// n! blew Integer.MAX_INT
+			// therefore there is no way numberOfPerturbations is larger than it
+		}
+		
+		int[][] sets = new int[numberOfPerturbations][n];
+		// Pool of available choices:
+		Vector<Integer> availableChoices = new Vector<Integer>();
+		for (int i = 0; i < n; i++) {
+			availableChoices.add(new Integer(i));
+		}
+		// Pool of choices already used this turn
+		Vector<Integer> thisSet = new Vector<Integer>();
+		// Hashtable tracking sets we've already chosen
+		Hashtable<Vector<Integer>,Integer> chosenSets =
+			new Hashtable<Vector<Integer>,Integer>();
+		for (int s = 0; s < numberOfPerturbations; s++) {
+			// Select set s:
+			for (;;) {
+				// Try to get a new unique set
+				// Reset the pool of choices ready to choose this set
+				availableChoices.addAll(thisSet);
+				thisSet.clear();
+				// System.out.println("Available: " + availableChoices);
+				for (int q = 0; q < n; q++) {
+					// Select the qth index from the available pool to use here:
+					int randIndex = random.nextInt(n - q);
+					// Find out what number this corresponds to, and write it in:
+					Integer nextSelection = availableChoices.remove(randIndex);
+					sets[s][q] = nextSelection.intValue();
+				}
+				// Track the chosen integers (in their selected order) to avoid duplicates
+				for (int q = 0; q < n; q++) {
+					// And track it in thisSet for hashing and 
+					//  adding back to the pool
+					thisSet.add(new Integer(sets[s][q]));
+				}
+				if (chosenSets.get(thisSet) == null) {
+					// We haven't included this set yet:
+					chosenSets.put(thisSet, new Integer(0));
+					// System.out.println(" Chosen: " + thisSet);
+					break;
+				}
+				// else we have already added this set,
+				//  so we need to try again.
+				// System.out.println(" Attempted: " + thisSet);
+				// System.out.println(" Need to try again");
+			}
+		}
+		return sets;
+	}
+	
+	/**
+	 * Generate all n! perturbations (assuming this doesn't blow our memory
+	 * 
+	 * @param n
+	 * @return
+	 */
+	public int[][] generateAllDistinctPerturbations(int n) throws Exception {
+		int numSets;
+		try {
+			numSets = MathsUtils.factorialCheckBounds(n);
+		} catch (Exception e) {
+			// n! blew Integer.MAX_INT
+			throw new Exception("n! too large");
+		}
+		// allocate space for the distinct sets
+		int[][] sets = new int[numSets][n];
+		
+		int[] workingSet = new int[n];
+		Vector<Integer> availableChoices = new Vector<Integer>();
+		for (int i = 0; i < n; i++) {
+			availableChoices.add(new Integer(i));
+		}
+		addToDistinctPerturbations(sets, n, 0, workingSet, 0, availableChoices);
+		return sets;
+	}
+
+	/**
+	 * Using the workingSet which is filled up to (but not including)
+	 *  fromIndex, add new distinct sets of nCp to the sets matrix,
+	 *  from the setNumber index
+	 * 
+	 * @param sets
+	 * @param n
+	 * @param setNumber
+	 * @param workingSet
+	 * @param fromIndex
+	 */
+	protected int addToDistinctPerturbations(int[][] sets, int n, int setNumber, 
+			int[] workingSet, int fromIndex, Vector<Integer> availableChoices) {
+		if (fromIndex == n) {
+			// The workingSet is ready to go, so copy it in
+			// MatrixUtils.printArray(System.out, workingSet);
+			System.arraycopy(workingSet, 0, sets[setNumber], 0, n);
+			setNumber++;
+		} else {
+			// Iterate over a copy of the available choices, in case altering it during the loop
+			//  causes problems.
+			Vector<Integer> copyOfAvailableChoices = (Vector<Integer>) availableChoices.clone();
+			// Add to the working set and pass it on:
+			for (Integer nextInteger : copyOfAvailableChoices) {
+				int nextInt = nextInteger.intValue();
+				workingSet[fromIndex] = nextInt;
+				// Remove this element as an available choice
+				availableChoices.remove(nextInteger);
+				// And keep filling out the array
+				setNumber = addToDistinctPerturbations(sets, n, setNumber,
+						workingSet, fromIndex + 1, availableChoices);
+				// Put this integer back in as an available choice
+				availableChoices.add(nextInteger);
+			}
+		}
+		return setNumber;
+	}
+
+	public static void main(String[] args) throws Exception {
+		// This code demonstrates that the Hashtable is hashing the 
+		//  pointer rather than the array values -
+		//  actually, it's hard to say, but if it is hashing the values then it's not doing .equals
+		//  properly on the array because it isn't implemented.
+		//  Should use a vector or an array object wrapper.
+		java.util.Hashtable<int[],Integer> hashtable = new java.util.Hashtable<int[],Integer>();
+		int[] array1 = {1,2,3,4,5};
+		int[] array2 = {1,2,3,4,5};
+		int[] array3 = {1,2,3,5,5};
+		hashtable.put(array1, 0);
+		hashtable.put(array3, 1);
+		System.out.println(hashtable.get(array1));
+		System.out.println(hashtable.get(array2));
+		System.out.println(hashtable.get(array3));
+		// This demonstrates is works on the values if we use vectors:
+		Vector<Integer> vec1 = new Vector();
+		Vector<Integer> vec2 = new Vector();
+		Vector<Integer> vec3 = new Vector();
+		for (int i = 0; i < array1.length; i++) {
+			vec1.add(new Integer(array1[i]));
+			vec2.add(new Integer(array2[i]));
+			vec3.add(new Integer(array3[i]));
+		}
+		java.util.Hashtable<Vector<Integer>,Integer> hashtable2 = new java.util.Hashtable<Vector<Integer>,Integer>();
+		hashtable2.put(vec1, 1);
+		hashtable2.put(vec3, 3);
+		System.out.println(hashtable2.get(vec1));
+		System.out.println(hashtable2.get(vec2));
+		System.out.println(hashtable2.get(vec3));
+		
+		RandomGenerator rg = new RandomGenerator();
+		// And test out generating distinct random set:
+		//MatrixUtils.printMatrix(System.out, rg.generateDistinctRandomSets(5, 2, 9));
+		// Test out generating whole sets:
+		MatrixUtils.printMatrix(System.out, rg.generateAllDistinctSets(5,3));
+
+		System.out.println("Generating all distinct perturbations of 5:");
+		MatrixUtils.printMatrix(System.out, rg.generateAllDistinctPerturbations(5));
+		System.out.println("Generating 10 distinct perturbations of 4:");
+		MatrixUtils.printMatrix(System.out, rg.generateDistinctRandomPerturbations(4, 10));
+	}
+	
+	public class RandomPairs {
+		public int n1, n2, p1, p2, N;
+		public int[][] sets1;
+		public int[][] sets2;
+		
+		public RandomPairs(int n1, int n2, int p1, int p2, int N) {
+			this.n1 = n1;
+			this.n2 = n2;
+			this.p1 = p1;
+			this.p2 = p2;
+			this.N = N;
+			sets1 = null;
+			sets2 = null;
+		}
+	}
+	
+	/**
+	 * Generate up to N <b>distinct<b/> pairs of p1 numbers from [0 .. n1-1]
+	 *  and p2 numbers from [0 .. n2 - 1].
+	 * 
+	 * @param n1
+	 * @param n2
+	 * @param p1
+	 * @param p2
+	 * @param N
+	 * @return 
+	 */
+	public RandomPairs generateDistinctPairsOfRandomSets(int n1, int n2, int p1, int p2,
+			int N) {
+		RandomPairs randPairs = new RandomPairs(n1, n2, p1, p2, N);
+		int numOfPossibleSets1, numOfPossibleSets2;
+		try {
+			numOfPossibleSets1 = MathsUtils.numOfSets(n1, p1);
+		} catch (Exception e) {
+			// n1 choose p1 blew Integer.MAX_INT
+			numOfPossibleSets1 = Integer.MAX_VALUE;
+		}
+		try {
+			numOfPossibleSets2 = MathsUtils.numOfSets(n2, p2);
+		} catch (Exception e) {
+			// n2 choose p2 blew Integer.MAX_INT
+			numOfPossibleSets2 = Integer.MAX_VALUE;
+		}
+
+		int[][] sets1 = generateDistinctRandomSets(n1, p1, N);
+		int[][] sets2 = generateDistinctRandomSets(n2, p2, N);
+		if ((numOfPossibleSets1 < N) || (numOfPossibleSets2 < N)) {
+			// One pair does not have enough.
+			// Use a long to avoid overflow
+			long totalPossiblePairs = numOfPossibleSets1 * numOfPossibleSets2;
+			if (totalPossiblePairs < N) {
+				// We can return the product of the pairs
+				randPairs.sets1 = new int[(int) totalPossiblePairs][];
+				randPairs.sets2 = new int[(int) totalPossiblePairs][];
+				int pairIndex = 0;
+				for (int i1 = 0; i1 < sets1.length; i1++) {
+					for (int i2 = 0; i2 < sets2.length; i2++) {
+						randPairs.sets1[pairIndex] = sets1[i1];
+						randPairs.sets2[pairIndex] = sets2[i2];
+						pairIndex++;
+					}
+				}
+			} else {
+				// Need to randomly select pairs out of the ones we've already got here
+				randPairs.sets1 = new int[N][];
+				randPairs.sets2 = new int[N][];
+				// Hashtable tracking the pairs we've already chosen
+				HashSet<Vector<Integer>> alreadyChosen = new HashSet<Vector<Integer>>();
+				for (int i = 0; i < N; i++) {
+					// Select one of each of the distinct sets at random
+					//  for the i-th pair
+					for (;;) {
+						// Select a candidate pair
+						Vector<Integer> candidate = new Vector<Integer>();
+						int candidate1 = random.nextInt(sets1.length);
+						int candidate2 = random.nextInt(sets2.length);
+						candidate.clear();
+						candidate.add(candidate1);
+						candidate.add(candidate2);
+						if (!alreadyChosen.contains(candidate)) {
+							// We're clear to add this candidate pair
+							randPairs.sets1[i] = sets1[candidate1];
+							randPairs.sets2[i] = sets2[candidate2];
+							alreadyChosen.add(candidate);
+							break;
+						}
+					}
+				}
+			}
+		} else {
+			// Both sets have enough pairs, so we can just use these without any repeats
+			randPairs.sets1 = sets1;
+			randPairs.sets2 = sets2;
+		}
+		return randPairs;
+	}
+}