787 lines
26 KiB
Java
787 lines
26 KiB
Java
|
package edu.nju.ics.frontier.learning;
|
||
|
|
|
||
|
|
import com.google.gson.Gson;
|
||
|
|
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||
|
|
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||
|
|
import edu.nju.ics.frontier.util.Assertion;
|
||
|
|
import weka.core.*;
|
||
|
|
|
||
|
|
import java.text.SimpleDateFormat;
|
||
|
|
import java.util.*;
|
||
|
|
|
||
|
|
public class TimeDatabase {
|
||
|
|
private String[] users;
|
||
|
|
private int[] classes;
|
||
|
|
private String[] featureNames;
|
||
|
|
private String labelName;
|
||
|
|
private List<TimeSeries> timeSeries;
|
||
|
|
|
||
|
|
public TimeDatabase() {
|
||
|
|
this.timeSeries = new ArrayList<TimeSeries>();
|
||
|
|
}
|
||
|
|
|
||
|
|
public void load(String csvPath) {
|
||
|
|
OkTextReader reader = new OkTextReader();
|
||
|
|
reader.open(csvPath);
|
||
|
|
|
||
|
|
// head
|
||
|
|
String[] head = reader.readLine().split(",");
|
||
|
|
this.featureNames = new String[head.length - 10];
|
||
|
|
for (int i = 0, l = this.featureNames.length; i < l; i++) {
|
||
|
|
this.featureNames[i] = head[i + 8];
|
||
|
|
}
|
||
|
|
this.labelName = head[head.length - 2];
|
||
|
|
|
||
|
|
// body
|
||
|
|
Set<Integer> classSet = new HashSet<Integer>();
|
||
|
|
String line;
|
||
|
|
while ((line = reader.readLine()) != null) {
|
||
|
|
// parse data
|
||
|
|
String[] info = line.split(",");
|
||
|
|
String team = info[0];
|
||
|
|
String user = info[1];
|
||
|
|
// 徐敏敏组, 江超组, 缪忍忍组
|
||
|
|
if (team.equals("徐敏敏组")) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
long timestamp = Long.parseLong(info[3]);
|
||
|
|
int sessionId = Integer.parseInt(info[7]);
|
||
|
|
double[] features = new double[info.length - 10];
|
||
|
|
for (int i = 0, l = features.length; i < l; i++) {
|
||
|
|
features[i] = Double.parseDouble(info[i + 8]);
|
||
|
|
}
|
||
|
|
// zero-based classes
|
||
|
|
int label = Integer.parseInt(info[info.length - 2]);
|
||
|
|
if (label != -1) {
|
||
|
|
label = label - 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
// add data into database
|
||
|
|
TimeSeries ts = findTimeSeriesByUser(user);
|
||
|
|
if (ts == null) {
|
||
|
|
ts = new TimeSeries(team, user);
|
||
|
|
timeSeries.add(ts);
|
||
|
|
}
|
||
|
|
ts.addTimePoint(new TimePoint(timestamp, sessionId, features, label));
|
||
|
|
|
||
|
|
// update classes
|
||
|
|
if (label != -1) {
|
||
|
|
classSet.add(label);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
reader.close();
|
||
|
|
|
||
|
|
// assign value to users
|
||
|
|
this.users = new String[this.timeSeries.size()];
|
||
|
|
for (int i = 0, l = this.users.length; i < l; i++) {
|
||
|
|
this.users[i] = this.timeSeries.get(i).getUser();
|
||
|
|
}
|
||
|
|
|
||
|
|
// assign value to classes
|
||
|
|
List<Integer> classList = new ArrayList<Integer>(classSet);
|
||
|
|
Collections.sort(classList, new Comparator<Integer>() {
|
||
|
|
public int compare(Integer o1, Integer o2) {
|
||
|
|
return o1 - o2;
|
||
|
|
}
|
||
|
|
});
|
||
|
|
this.classes = new int[classList.size()];
|
||
|
|
for (int i = 0, l = this.classes.length; i < l; i++) {
|
||
|
|
this.classes[i] = classList.get(i);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public Instances defineDataFormat() {
|
||
|
|
ArrayList<Attribute> attrs = new ArrayList<Attribute>();
|
||
|
|
// add features
|
||
|
|
for (String attrName : this.featureNames) {
|
||
|
|
attrs.add(new Attribute(attrName));
|
||
|
|
}
|
||
|
|
// add label
|
||
|
|
ArrayList<String> classes = new ArrayList<String>();
|
||
|
|
for (int classValue : this.classes) {
|
||
|
|
classes.add(String.valueOf(classValue));
|
||
|
|
}
|
||
|
|
attrs.add(new Attribute(this.labelName, classes));
|
||
|
|
return new Instances("work_engagement", attrs, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
public Instances getLabeledInstances() {
|
||
|
|
Instances instances = defineDataFormat();
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (TimePoint tp : tps) {
|
||
|
|
if (!tp.isLabeled()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
Instance instance = tp.transferToInstance();
|
||
|
|
instances.add(instance);
|
||
|
|
instance.setDataset(instances);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
instances.setClassIndex(instances.numAttributes() - 1);
|
||
|
|
return instances.isEmpty() ? null : instances;
|
||
|
|
}
|
||
|
|
|
||
|
|
public Instances getLabeledAndUnlabeledInstances() {
|
||
|
|
Instances instances = defineDataFormat();
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (TimePoint tp : tps) {
|
||
|
|
if (tp.isEmpty()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
Instance instance = tp.transferToInstance();
|
||
|
|
instances.add(instance);
|
||
|
|
instance.setDataset(instances);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
instances.setClassIndex(instances.numAttributes() - 1);
|
||
|
|
return instances.isEmpty() ? null : instances;
|
||
|
|
}
|
||
|
|
|
||
|
|
public void positiveNeutralNegative() {
|
||
|
|
this.classes = new int[]{0, 1, 2};
|
||
|
|
|
||
|
|
if (this.timeSeries.isEmpty()) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
if (tp.isLabeled()) {
|
||
|
|
int label = tp.getLabel();
|
||
|
|
if (label <= 1) {
|
||
|
|
tp.setLabel(0);
|
||
|
|
} else if (label == 2) {
|
||
|
|
tp.setLabel(1);
|
||
|
|
} else {
|
||
|
|
tp.setLabel(2);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public void positiveNegative() {
|
||
|
|
this.classes = new int[]{0, 1};
|
||
|
|
|
||
|
|
if (this.timeSeries.isEmpty()) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
if (tp.isLabeled()) {
|
||
|
|
int label = tp.getLabel();
|
||
|
|
if (label <= 1) {
|
||
|
|
tp.setLabel(0);
|
||
|
|
} else if (label == 2) {
|
||
|
|
tp.setLabel(-1);
|
||
|
|
} else {
|
||
|
|
tp.setLabel(1);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public TimeDatabase copy() {
|
||
|
|
TimeDatabase newDb = new TimeDatabase();
|
||
|
|
newDb.users = this.users;
|
||
|
|
newDb.classes = this.classes;
|
||
|
|
newDb.featureNames = this.featureNames;
|
||
|
|
newDb.labelName = this.labelName;
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
newTs.addTimePoint(new TimePoint(tp));
|
||
|
|
}
|
||
|
|
newDb.timeSeries.add(newTs);
|
||
|
|
}
|
||
|
|
return newDb;
|
||
|
|
}
|
||
|
|
|
||
|
|
public TimeDatabase align() {
|
||
|
|
TimeDatabase newDb = new TimeDatabase();
|
||
|
|
newDb.users = this.users;
|
||
|
|
newDb.classes = this.classes;
|
||
|
|
newDb.featureNames = this.featureNames;
|
||
|
|
newDb.labelName = this.labelName;
|
||
|
|
|
||
|
|
// retrieve all users' timestamps, and sort in ascending order
|
||
|
|
Set<Long> timestampSet = new HashSet<Long>();
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (TimePoint tp : tps) {
|
||
|
|
timestampSet.add(tp.getTimestamp());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
List<Long> timestampList = new ArrayList<Long>(timestampSet);
|
||
|
|
Collections.sort(timestampList, new Comparator<Long>() {
|
||
|
|
public int compare(Long o1, Long o2) {
|
||
|
|
if (o1 < o2) {
|
||
|
|
return -1;
|
||
|
|
} else if (o1 > o2) {
|
||
|
|
return 1;
|
||
|
|
} else {
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
// align all users' time points by adding empty time points as placeholders
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||
|
|
for (long timestamp : timestampList) {
|
||
|
|
TimePoint newTp = ts.findClosestTimePoint(timestamp, 0, true, true, false);
|
||
|
|
if (newTp == null) {
|
||
|
|
newTp = new TimePoint(timestamp, -1, null, -1);
|
||
|
|
}
|
||
|
|
newTs.addTimePoint(newTp);
|
||
|
|
}
|
||
|
|
newDb.timeSeries.add(newTs);
|
||
|
|
}
|
||
|
|
|
||
|
|
return newDb;
|
||
|
|
}
|
||
|
|
|
||
|
|
public boolean fillingEmpty(boolean[] classFlags, long threshold) {
|
||
|
|
if (this.timeSeries.isEmpty()) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
boolean isDbChanged = false;
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
if (!tp.isEmpty()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
TimePoint closestTp = ts.findClosestTimePoint(tp.getTimestamp(), threshold, true, true, false);
|
||
|
|
if (closestTp == null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (closestTp.isLabeled()) {
|
||
|
|
int label = closestTp.getLabel();
|
||
|
|
if (classFlags[argWhere(this.classes, label)]) {
|
||
|
|
tp.setFeatures(closestTp.getFeatures());
|
||
|
|
tp.setLabel(label);
|
||
|
|
isDbChanged = true;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
tp.setFeatures(closestTp.getFeatures());
|
||
|
|
isDbChanged = true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return isDbChanged;
|
||
|
|
}
|
||
|
|
|
||
|
|
public boolean spreadLabels(boolean[] classFlags, long threshold) {
|
||
|
|
if (this.timeSeries.isEmpty()) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
boolean isDbChanged = false;
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
if (!tp.isUnlabeled()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
int label = ts.findClosestLabel(tp.getTimestamp(), threshold);
|
||
|
|
if (label != -1 && classFlags[argWhere(this.classes, label)]) {
|
||
|
|
tp.setLabel(label);
|
||
|
|
isDbChanged = true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return isDbChanged;
|
||
|
|
}
|
||
|
|
|
||
|
|
public TimeDatabase slice(int[] indices) {
|
||
|
|
TimeDatabase newDb = new TimeDatabase();
|
||
|
|
newDb.users = this.users;
|
||
|
|
newDb.classes = this.classes;
|
||
|
|
newDb.featureNames = this.featureNames;
|
||
|
|
newDb.labelName = this.labelName;
|
||
|
|
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (int index : indices) {
|
||
|
|
newTs.addTimePoint(new TimePoint(tps.get(index)));
|
||
|
|
}
|
||
|
|
newDb.timeSeries.add(newTs);
|
||
|
|
}
|
||
|
|
|
||
|
|
return newDb;
|
||
|
|
}
|
||
|
|
|
||
|
|
public String simpleDescribe(String tag) {
|
||
|
|
StringBuilder builder = new StringBuilder();
|
||
|
|
int[] classCount = zeros(this.classes.length);
|
||
|
|
int labeledCount = 0;
|
||
|
|
int unlabeledCount = 0;
|
||
|
|
int emptyCount = 0;
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
if (tp.isLabeled()) {
|
||
|
|
labeledCount++;
|
||
|
|
int index = argWhere(this.classes, tp.getLabel());
|
||
|
|
classCount[index]++;
|
||
|
|
} else if (tp.isUnlabeled()) {
|
||
|
|
unlabeledCount++;
|
||
|
|
} else {
|
||
|
|
emptyCount++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
builder.append(tag).
|
||
|
|
append("{all:").append(labeledCount + unlabeledCount + emptyCount).
|
||
|
|
append(",lbl:").append(labeledCount).
|
||
|
|
append(",unl:").append(unlabeledCount).
|
||
|
|
append(",emp:").append(emptyCount).
|
||
|
|
append(",cls[").append(classCount[0]);
|
||
|
|
for (int i = 1; i < classCount.length; i++) {
|
||
|
|
builder.append(",").append(classCount[i]);
|
||
|
|
}
|
||
|
|
builder.append("]}");
|
||
|
|
return builder.toString();
|
||
|
|
}
|
||
|
|
|
||
|
|
public String describe(String tag) {
|
||
|
|
StringBuilder builder = new StringBuilder();
|
||
|
|
builder.append("id | #all #lbl #unl #emp |");
|
||
|
|
for (int cls : this.classes) {
|
||
|
|
builder.append(" #(").append(cls).append(")");
|
||
|
|
}
|
||
|
|
builder.append(" | #workday | team#user <- ").append(tag).append("\n");
|
||
|
|
|
||
|
|
int allLblNum = 0;
|
||
|
|
int allUnlNum = 0;
|
||
|
|
int[] allClsNums = zeros(this.classes.length);
|
||
|
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
|
||
|
|
|
||
|
|
for (int tsIndex = 0, tsLength = this.timeSeries.size(); tsIndex < tsLength; tsIndex++) {
|
||
|
|
int lblNum = 0;
|
||
|
|
int unlNum = 0;
|
||
|
|
int empNum = 0;
|
||
|
|
int[] clsNums = zeros(this.classes.length);
|
||
|
|
Set<String> workdays = new HashSet<String>();
|
||
|
|
|
||
|
|
TimeSeries ts = this.timeSeries.get(tsIndex);
|
||
|
|
String team = ts.getTeam();
|
||
|
|
String user = ts.getUser();
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (TimePoint tp : tps) {
|
||
|
|
switch (tp.getType()) {
|
||
|
|
case TimePoint.LABELED:
|
||
|
|
lblNum++;
|
||
|
|
clsNums[argWhere(this.classes, tp.getLabel())]++;
|
||
|
|
workdays.add(sdf.format(new Date(tp.getTimestamp())));
|
||
|
|
break;
|
||
|
|
case TimePoint.UNLABELED:
|
||
|
|
unlNum++;
|
||
|
|
workdays.add(sdf.format(new Date(tp.getTimestamp())));
|
||
|
|
break;
|
||
|
|
case TimePoint.EMPTY:
|
||
|
|
empNum++;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
int allNum = tps.size();
|
||
|
|
allLblNum += lblNum;
|
||
|
|
allUnlNum += unlNum;
|
||
|
|
addVector(allClsNums, clsNums);
|
||
|
|
int workdayNum = workdays.size();
|
||
|
|
|
||
|
|
builder.append(String.format("%2d | %4d %4d %4d %4d |",
|
||
|
|
(tsIndex + 1), allNum, lblNum, unlNum, empNum));
|
||
|
|
for (int clsNum : clsNums) {
|
||
|
|
if (clsNum > 0) {
|
||
|
|
builder.append(String.format(" %4d", clsNum));
|
||
|
|
} else {
|
||
|
|
builder.append(" ");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
builder.append(String.format(" | %8d | %s#%s\n", workdayNum, team, user));
|
||
|
|
}
|
||
|
|
|
||
|
|
builder.append(String.format(" | %4d %4d |", allLblNum, allUnlNum));
|
||
|
|
for (int allClsNum : allClsNums) {
|
||
|
|
if (allClsNum > 0) {
|
||
|
|
builder.append(String.format(" %4d", allClsNum));
|
||
|
|
} else {
|
||
|
|
builder.append(" ");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
builder.append(" |");
|
||
|
|
return builder.toString();
|
||
|
|
}
|
||
|
|
|
||
|
|
public String describe() {
|
||
|
|
return describe("");
|
||
|
|
}
|
||
|
|
|
||
|
|
public Map<String, KStepTransitionMatrix> getTemporalCorrelationMatrix(int n, long offset) {
|
||
|
|
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||
|
|
for (String user : this.users) {
|
||
|
|
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(user, user, n, offset);
|
||
|
|
Assertion.assertNotNull(transitionMatrix);
|
||
|
|
matricesMap.put(String.format("%s->%s", user, user), transitionMatrix);
|
||
|
|
}
|
||
|
|
return matricesMap;
|
||
|
|
}
|
||
|
|
|
||
|
|
public Map<String, KStepTransitionMatrix> getSpatialCorrelationMatrix(int n, long offset) {
|
||
|
|
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||
|
|
for (String fromUser : this.users) {
|
||
|
|
for (String toUser : this.users) {
|
||
|
|
if (toUser.equals(fromUser)) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(fromUser, toUser, n, offset);
|
||
|
|
Assertion.assertNotNull(transitionMatrix);
|
||
|
|
matricesMap.put(String.format("%s->%s", fromUser, toUser), transitionMatrix);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return matricesMap;
|
||
|
|
}
|
||
|
|
|
||
|
|
public Map<String, KStepTransitionMatrix> getKStepTransitionMatrix(int n, long offset) {
|
||
|
|
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||
|
|
for (String fromUser : this.users) {
|
||
|
|
for (String toUser : this.users) {
|
||
|
|
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(fromUser, toUser, n, offset);
|
||
|
|
Assertion.assertNotNull(transitionMatrix);
|
||
|
|
matricesMap.put(String.format("%s->%s", fromUser, toUser), transitionMatrix);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return matricesMap;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* get the 1- to n-order state transition matrices from {@code fromUser} to {@code toUser}.
|
||
|
|
* @param fromUser requester
|
||
|
|
* @param toUser recipient
|
||
|
|
* @param n n-order Markov Chain Model.
|
||
|
|
* @param offset upper limitation of the time difference between 1-step pair of time point.
|
||
|
|
* @return 1- to n-order state transition matrices from {@code fromUser} to {@code toUser},
|
||
|
|
* or {@code null} if {@code n} less than 1.
|
||
|
|
*/
|
||
|
|
public KStepTransitionMatrix getKStepTransitionMatrix(String fromUser, String toUser, int n, long offset) {
|
||
|
|
if (fromUser == null || toUser == null || n < 1) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
TimeSeries fromTs = findTimeSeriesByUser(fromUser);
|
||
|
|
TimeSeries toTs = findTimeSeriesByUser(toUser);
|
||
|
|
if (fromTs == null || toTs == null) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
List<TimePoint> fromTps = fromTs.getTimePoints();
|
||
|
|
List<TimePoint> toTps = toTs.getTimePoints();
|
||
|
|
if (fromTps.size() != toTps.size()) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
// initialize F
|
||
|
|
int[][][] F = new int[n][this.classes.length][this.classes.length];
|
||
|
|
for (int i = 0; i < F.length; i++) {
|
||
|
|
for (int j = 0; j < F[i].length; j++) {
|
||
|
|
for (int k = 0; k < F[i][j].length; k++) {
|
||
|
|
F[i][j][k] = 0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// statistic F
|
||
|
|
for (int step = 1; step <= n; step++) {
|
||
|
|
long threshold = offset * step;
|
||
|
|
for (int i = 0, j = step, l = toTps.size(); j < l; i++, j++) {
|
||
|
|
TimePoint fromTp = fromTps.get(i);
|
||
|
|
TimePoint toTp = toTps.get(j);
|
||
|
|
if (!(fromTp.isLabeled() && toTp.isLabeled())) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
long deltaTime = toTp.getTimestamp() - fromTp.getTimestamp();
|
||
|
|
Assertion.assertPositive(deltaTime);
|
||
|
|
if (deltaTime > threshold) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
int fromIndex = argWhere(this.classes, fromTp.getLabel());
|
||
|
|
int toIndex = argWhere(this.classes, toTp.getLabel());
|
||
|
|
F[step - 1][fromIndex][toIndex]++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// transform F to Q_hat
|
||
|
|
double[][][] Q_hat = new double[n][this.classes.length][this.classes.length];
|
||
|
|
for (int i = 0; i < F.length; i++) {
|
||
|
|
for (int j = 0; j < F[i].length; j++) {
|
||
|
|
int sum = sumVector(F[i][j]);
|
||
|
|
if (sum == 0) {
|
||
|
|
for (int k = 0; k < F[i][j].length; k++) {
|
||
|
|
Q_hat[i][j][k] = 0;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
for (int k = 0; k < F[i][j].length; k++) {
|
||
|
|
Q_hat[i][j][k] = ((double) F[i][j][k]) / ((double) sum);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return new KStepTransitionMatrix(fromUser, toUser, Q_hat);
|
||
|
|
}
|
||
|
|
|
||
|
|
public String[] getUsers() {
|
||
|
|
return users;
|
||
|
|
}
|
||
|
|
|
||
|
|
public int[] getClasses() {
|
||
|
|
return classes;
|
||
|
|
}
|
||
|
|
|
||
|
|
public String[] getFeatureNames() {
|
||
|
|
return featureNames;
|
||
|
|
}
|
||
|
|
|
||
|
|
public String getLabelName() {
|
||
|
|
return labelName;
|
||
|
|
}
|
||
|
|
|
||
|
|
public List<TimeSeries> getTimeSeries() {
|
||
|
|
return timeSeries;
|
||
|
|
}
|
||
|
|
|
||
|
|
public int getNumberOfTimeSeries() {
|
||
|
|
return this.timeSeries.size();
|
||
|
|
}
|
||
|
|
|
||
|
|
public int[] getNumberOfTimePointsOfEachUser() {
|
||
|
|
if (this.timeSeries.isEmpty()) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
int[] result = new int[this.timeSeries.size()];
|
||
|
|
for (int i = 0; i < result.length; i++) {
|
||
|
|
result[i] = this.timeSeries.get(i).getTimePoints().size();
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
public int getNumberOfTimePoints() {
|
||
|
|
int[] tmp = getNumberOfTimePointsOfEachUser();
|
||
|
|
if (tmp == null) {
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
for (int i = 1; i < tmp.length; i++) {
|
||
|
|
if (tmp[i] != tmp[0]) {
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return tmp[0];
|
||
|
|
}
|
||
|
|
|
||
|
|
public TimeSeries findTimeSeriesByUser(String user) {
|
||
|
|
if (timeSeries.isEmpty() || user == null) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
for (TimeSeries ts : timeSeries) {
|
||
|
|
if (ts.getUser().equals(user)) {
|
||
|
|
return ts;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static int[] zeros(int length) {
|
||
|
|
int[] result = new int[length];
|
||
|
|
for (int i = 0; i < length; i++) {
|
||
|
|
result[i] = 0;
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static int argWhere(int[] array, int value) {
|
||
|
|
for (int i = 0, l = array.length; i < l; i++) {
|
||
|
|
if (array[i] == value) {
|
||
|
|
return i;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return -1;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void addVector(int[] dest, int[] src) {
|
||
|
|
Assertion.assertTrue(dest.length == src.length);
|
||
|
|
for (int i = 0, l = dest.length; i < l; i++) {
|
||
|
|
dest[i] += src[i];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public static int sumVector(int[] vector) {
|
||
|
|
int sum = 0;
|
||
|
|
for (int i = 0, l = vector.length; i < l; i++) {
|
||
|
|
sum += vector[i];
|
||
|
|
}
|
||
|
|
return sum;
|
||
|
|
}
|
||
|
|
|
||
|
|
public static boolean isAllZero(int[] vector) {
|
||
|
|
for (int value : vector) {
|
||
|
|
if (value != 0) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
public void cleanCache() {
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
for (TimePoint tp : ts.getTimePoints()) {
|
||
|
|
tp.cleanCache();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public void erasePartialLabels(double rate) {
|
||
|
|
Assertion.assertTrue(rate > 0.0 && rate < 1.0);
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (int cls : this.classes) {
|
||
|
|
List<Integer> indices = ts.getIndicesWithClass(cls);
|
||
|
|
if (indices == null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
int eraseNum = (int) (indices.size() * rate);
|
||
|
|
if (eraseNum <= 0) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
int[] eraseIndices = randInt(indices, eraseNum);
|
||
|
|
for (int eraseIndex : eraseIndices) {
|
||
|
|
TimePoint tp = tps.get(eraseIndex);
|
||
|
|
Assertion.assertTrue(tp.isLabeled() && tp.getLabel() == cls);
|
||
|
|
tp.setLabel(-1);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets n random integers fall in the interval of [min, max).
|
||
|
|
* @param min inclusive
|
||
|
|
* @param max exclusive
|
||
|
|
* @param length number of random to generate
|
||
|
|
* @return
|
||
|
|
*/
|
||
|
|
public static int[] randInt(int min, int max, int length) {
|
||
|
|
Assertion.assertTrue(min < max && max - min >= length);
|
||
|
|
List<Integer> tmp = new ArrayList<Integer>();
|
||
|
|
for (int i = min; i < max; i++) {
|
||
|
|
tmp.add(i);
|
||
|
|
}
|
||
|
|
int[] result = new int[length];
|
||
|
|
for (int i = 0; i < length; i++) {
|
||
|
|
int index = (int) (Math.random() * tmp.size());
|
||
|
|
result[i] = tmp.remove(index);
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets n random integers from a specific integer list.
|
||
|
|
* @param list
|
||
|
|
* @param length number of random to generate
|
||
|
|
* @return
|
||
|
|
*/
|
||
|
|
public static int[] randInt(List<Integer> list, int length) {
|
||
|
|
Assertion.assertNotNull(list);
|
||
|
|
Assertion.assertTrue(list.size() >= length);
|
||
|
|
int[] indices = randInt(0, list.size(), length);
|
||
|
|
int[] result = new int[length];
|
||
|
|
for (int i = 0; i < length; i++) {
|
||
|
|
result[i] = list.get(indices[i]);
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
public String drawSparseDatabase() {
|
||
|
|
StringBuilder builder = new StringBuilder();
|
||
|
|
int tpNum = this.getNumberOfTimePoints();
|
||
|
|
int notLabeledTpNum = 0;
|
||
|
|
List<Integer> tmp = new ArrayList<Integer>();
|
||
|
|
for (int i = 0; i < tpNum; i++) {
|
||
|
|
boolean isAllNotLabeledInAColumn = true;
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
if (ts.getTimePoints().get(i).isLabeled()) {
|
||
|
|
isAllNotLabeledInAColumn = false;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (isAllNotLabeledInAColumn) {
|
||
|
|
notLabeledTpNum++;
|
||
|
|
} else {
|
||
|
|
if (notLabeledTpNum > 0) {
|
||
|
|
tmp.add(-notLabeledTpNum);
|
||
|
|
}
|
||
|
|
tmp.add(i);
|
||
|
|
notLabeledTpNum = 0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (notLabeledTpNum > 0) {
|
||
|
|
tmp.add(-notLabeledTpNum);
|
||
|
|
}
|
||
|
|
|
||
|
|
// print head
|
||
|
|
for (int value : tmp) {
|
||
|
|
if (value < 0) {
|
||
|
|
builder.append(String.format("[%4d]", -value));
|
||
|
|
} else {
|
||
|
|
builder.append("*");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
builder.append("\n");
|
||
|
|
|
||
|
|
// print body
|
||
|
|
for (TimeSeries ts : this.timeSeries) {
|
||
|
|
List<TimePoint> tps = ts.getTimePoints();
|
||
|
|
for (int value : tmp) {
|
||
|
|
if (value < 0) {
|
||
|
|
builder.append(" ");
|
||
|
|
} else {
|
||
|
|
TimePoint tp = tps.get(value);
|
||
|
|
if (tp.isLabeled()) {
|
||
|
|
builder.append(tp.getLabel());
|
||
|
|
} else {
|
||
|
|
builder.append("|");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
builder.append("<-").append(ts.getTeam()).append("#").append(ts.getUser()).append("\n");
|
||
|
|
}
|
||
|
|
return builder.toString();
|
||
|
|
}
|
||
|
|
|
||
|
|
public static void serialize(TimeDatabase db, String path) {
|
||
|
|
Gson gson = new Gson();
|
||
|
|
String json = gson.toJson(db);
|
||
|
|
OkTextWriter writer = new OkTextWriter();
|
||
|
|
writer.open(path);
|
||
|
|
writer.println(json);
|
||
|
|
writer.close();
|
||
|
|
}
|
||
|
|
|
||
|
|
public static TimeDatabase deserialize(String path) {
|
||
|
|
OkTextReader reader = new OkTextReader();
|
||
|
|
reader.open(path);
|
||
|
|
String json = reader.readLine();
|
||
|
|
reader.close();
|
||
|
|
Gson gson = new Gson();
|
||
|
|
return gson.fromJson(json, TimeDatabase.class);
|
||
|
|
}
|
||
|
|
}
|