jixuan1989
/
IoTDB
mirror of https://github.com/apache/iotdb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Transfer codes from pre-project

This commit is contained in:
xingtanzjr 2017-05-07 14:50:07 +08:00
commit a3a5a392e3
45 changed files with 5391 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.gitignore vendored Normal file
View File

@ -0,0 +1,30 @@
# Eclipse IDE files
**/.classpath
**/.project
**/.settings/
# intellij IDE files
**/*.iml
**/.idea/
# Apple OS X related
**/.DS_Store
# build generated
**/target/
**/build/
# intermediately generated locally
**/logs/
tsfile-timeseries/src/main/resources/logback.out.out.xml
tsfile-timeseries/src/main/resources/logback.out.xml
tsfile-service/derby-tsfile-db/
tsfile-timeseries/src/test/resources/data
src/main/resources/metadata/mlog.txt
tsfile-jdbc/src/main/resources/output/queryRes.csv
*.jar
*.gz
*.tar.gz
*.tar

72
pom.xml Normal file
View File

@ -0,0 +1,72 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>cn.edu.thu</groupId>
<artifactId>tsfiledb</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>tsfiledb</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<compiler.version>1.8</compiler.version>
<hadoop.version>2.6.0</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>cn.edu.thu</groupId>
<artifactId>tsfile</artifactId>
<version>0.1.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>${compiler.version}</source>
<target>${compiler.version}</target>
<encoding>UTF-8</encoding>
</configuration>
<!-- compile java files in src/it/java -->
<executions>
<execution>
<id>compile-integration-test</id>
<phase>pre-integration-test</phase>
<goals>
<goal>testCompile</goal>
</goals>
<configuration>
<testIncludes>
<testInclude>**/*.java</testInclude>
</testIncludes>
<outputDirectory>${project.build.directory}/it-classes</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

13
src/main/java/cn/edu/thu/tsfiledb/App.java Normal file
View File

@ -0,0 +1,13 @@
package cn.edu.thu.tsfiledb;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args )
{
System.out.println( "Hello World!" );
}
}

10
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/Action.java Normal file
View File

@ -0,0 +1,10 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
/**
* @author kangrong
*
*/
@FunctionalInterface
public interface Action {
void act();
}

46
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIOWriter.java Normal file
View File

@ -0,0 +1,46 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData;
import cn.edu.thu.tsfile.timeseries.write.io.TSFileIOWriter;
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
/**
* @author kangrong
*
*/
public class BufferWriteIOWriter extends TSFileIOWriter {
/*
* The backup list is used to store the rowgroup's metadata whose data has
* been flushed into file.
*/
private final List<RowGroupMetaData> backUpList = new ArrayList<RowGroupMetaData>();
public BufferWriteIOWriter(FileSchema schema, TSRandomAccessFileWriter output) throws IOException {
super(schema, output);
}
/**
* <b>Note that</b>,the method is not thread safe.
*/
public void addNewRowGroupMetaDataToBackUp() {
backUpList.add(rowGroups.get(rowGroups.size() - 1));
}
/**
* <b>Note that</b>, the method is not thread safe. You mustn't do any
* change on the return.<br>
*
* @return
*/
public List<RowGroupMetaData> getCurrentRowGroupMetaList() {
List<RowGroupMetaData> ret = new ArrayList<>();
backUpList.forEach(ret::add);
return ret;
}
}

35
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIndex.java Normal file
View File

@ -0,0 +1,35 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
/**
* The function of this interface is to store and index TSRecord in memory
* temporarily
*
* @author kangrong
*
*/
public interface BufferWriteIndex {
/**
* insert a tsRecord
*
* @param tsRecord
*/
void insert(TSRecord tsRecord);
/**
* Get the DynamicOneColumnData from the buffer index
*
* @param deltaObjectId
* @param measurementId
* @return
*/
public DynamicOneColumnData query(String deltaObjectId, String measurementId);
/**
* clear all data written in the bufferindex structure which will be used
* for next stage
*/
void clear();
}

190
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteManager.java Normal file
View File

@ -0,0 +1,190 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.function.BiFunction;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.edu.thu.tsfile.common.conf.TSFileConfig;
import cn.edu.thu.tsfile.common.conf.TSFileDescriptor;
import cn.edu.thu.tsfile.common.constant.JsonFormatConstant;
import cn.edu.thu.tsfile.common.utils.Pair;
import cn.edu.thu.tsfile.common.utils.RandomAccessOutputStream;
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
import cn.edu.thu.tsfile.timeseries.write.TSRecordWriteSupport;
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
import cn.edu.thu.tsfiledb.exception.ProcessorException;
import cn.edu.thu.tsfiledb.metadata.ColumnSchema;
import cn.edu.thu.tsfiledb.metadata.MManager;
/**
* BufferWrite manager manage a list of bufferwrite processor
*
* @author kangrong
* @author liukun
*/
public class BufferWriteManager extends LRUManager<BufferWriteProcessor> {
private static Logger LOG = LoggerFactory.getLogger(BufferWriteManager.class);
private static TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
protected static BufferWriteManager instance = new BufferWriteManager(conf.maxBufferWriteNodeNum,
MManager.getInstance(), conf.BufferWriteDir);
public static BufferWriteManager getInstance() {
return instance;
}
/**
* Initialize the instance of BufferWriteManager using special parameters
*
* @param maxLRUNumber
* @param mManager
* @param normalDataDir
*/
public synchronized static void init(int maxLRUNumber, MManager mManager, String normalDataDir) {
instance = new BufferWriteManager(maxLRUNumber, mManager, normalDataDir);
}
private BufferWriteManager(int maxLRUNumber, MManager mManager, String normalDataDir) {
super(maxLRUNumber, mManager, normalDataDir);
}
/*
* If the buffer write processor is not exist before, it will be constructed
* using name space path and key-value information(args)
*/
@Override
protected BufferWriteProcessor constructNewProcessor(String namespacePath, Map<String, Object> args)
throws ProcessorException, IOException, WriteProcessException {
long startTime = (Long) args.get(FileNodeConstants.TIMESTAMP_KEY);
Pair<String, String> fileNamePair = prepareMeasurePathFileName.apply(namespacePath,
Long.valueOf(startTime).toString());
TSRecordWriterParameter param = prepareTSRecordWriterParameter.apply(namespacePath, mManager, fileNamePair);
BufferWriteIOWriter bufferTSFileWriter = new BufferWriteIOWriter(param.fileSchema, param.outputStream);
LOG.info("Construct a new buffer write processor, fileName:{}", fileNamePair);
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
return new BufferWriteProcessor(conf, bufferTSFileWriter, param.writeSupport, param.fileSchema, namespacePath,
fileNamePair);
}
@Override
protected void initProcessor(BufferWriteProcessor processor, String namespacePath, Map<String, Object> args)
throws ProcessorException {
if (processor.getCloseAction() == null && args == null) {
LOG.error(
"The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args");
throw new RuntimeException(
"The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args");
}
if (args != null && args.containsKey(FileNodeConstants.CLOSE_ACTION))
processor.setCloseAction((Action) args.get(FileNodeConstants.CLOSE_ACTION));
}
/**
* @param namespacePath
* -this path is a concept in name space tree. like
* root.laptop.xxx. The absolute file path is deltaDataDir +
* namespacepath + number.
* @param MManager
* @param Pair<String,String>
*
* @return TSRecordWriterParameter
*/
public TriFunction<String, MManager, Pair<String, String>, TSRecordWriterParameter> prepareTSRecordWriterParameter = (
namespacePath, inputMManager, filePathPair) -> {
Optional<List<ColumnSchema>> meaSchema;
String deltaObjectType;
try {
deltaObjectType = mManager.getDeltaObjectTypeByPath(namespacePath);
meaSchema = Optional.ofNullable(inputMManager.getSchemaForOneType(deltaObjectType));
} catch (PathErrorException e) {
throw new RuntimeException(e);
}
FileSchema fSchema = null;
try {
if (meaSchema == null) {
throw new ProcessorException("Measures schema list is null");
}
fSchema = getFileSchemaFromColumnSchema(meaSchema.orElse(new ArrayList<ColumnSchema>()), deltaObjectType);
} catch (ProcessorException e) {
LOG.error("The measures schema list is {}", meaSchema);
throw new RuntimeException("Get the File schema failed");
}
File file = new File(filePathPair.left);
WriteSupport<TSRecord> writeSupport = new TSRecordWriteSupport();
TSRandomAccessFileWriter outputStream;
try {
outputStream = new RandomAccessOutputStream(file);
} catch (IOException e) {
LOG.error("Initialized the data output stream failed, the file path is {}", filePathPair.left);
throw new RuntimeException(new ProcessorException(
"IOException: " + e.getMessage() + " create data output stream failed:" + file));
}
return new TSRecordWriterParameter(fSchema, outputStream, writeSupport);
};
/**
* @param namespacePath
* -this path is a concept in name space tree. like
* root.laptop.xxx. The absolute file path is deltaDataDir +
* namespacepath + number.
* @param filename
* - start time for this file
* @return Pair<String,String> left - data file right -error data file
*/
public BiFunction<String, String, Pair<String, String>> prepareMeasurePathFileName = (namespacePath, fileName) -> {
String absolutePathDir = normalDataDir + namespacePath;
File dir = new File(absolutePathDir);
if (!dir.exists()) {
dir.mkdirs();
}
String absolutePath = dir.getAbsolutePath() + File.separator + fileName;
return new Pair<>(absolutePath, "null");
};
/**
* Construct one FileSchema from a list of columnschema and the deltaobject
* type
*
* @param schemaList
* @param deltaObjectType
* @return
*/
private FileSchema getFileSchemaFromColumnSchema(List<ColumnSchema> schemaList, String deltaObjectType) throws WriteProcessException {
JSONArray rowGroup = new JSONArray();
for (ColumnSchema col : schemaList) {
JSONObject measurement = new JSONObject();
measurement.put(JsonFormatConstant.MEASUREMENT_UID, col.name);
measurement.put(JsonFormatConstant.DATA_TYPE, col.dataType.toString());
measurement.put(JsonFormatConstant.MEASUREMENT_ENCODING, col.encoding.toString());
for (Entry<String, String> entry : col.getArgsMap().entrySet()) {
if (JsonFormatConstant.ENUM_VALUES.equals(entry.getKey())) {
String[] valueArray = entry.getValue().split(",");
measurement.put(JsonFormatConstant.ENUM_VALUES, new JSONArray(valueArray));
} else
measurement.put(entry.getKey(), entry.getValue().toString());
}
rowGroup.put(measurement);
}
JSONObject jsonSchema = new JSONObject();
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, rowGroup);
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, deltaObjectType);
return new FileSchema(jsonSchema);
}
}

330
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteProcessor.java Normal file
View File

@ -0,0 +1,330 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.edu.thu.tsfile.common.conf.TSFileConfig;
import cn.edu.thu.tsfile.common.utils.Pair;
import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import cn.edu.thu.tsfile.timeseries.write.TSRecordWriter;
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
import cn.edu.thu.tsfile.timeseries.write.record.DataPoint;
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
import cn.edu.thu.tsfile.timeseries.write.series.IRowGroupWriter;
import cn.edu.thu.tsfiledb.engine.utils.FlushState;
/**
* BufferWriteProcessor is used to write one data point or one tsRecord whose
* timestamp is greater than than the last update time in
* {@code FileNodeProcessor}<br>
*
* @author kangrong
* @author liukun
*/
public class BufferWriteProcessor extends LRUProcessor {
private static final Logger LOG = LoggerFactory.getLogger(BufferWriteProcessor.class);
private BufferWriteIndex workingBufferIndex;
private BufferWriteIndex flushingBufferIndex;
private boolean isFlushingSync = false;
private FlushState flushState = new FlushState();
private BufferWriteIOWriter bufferIOWriter;
private ReadWriteLock convertBufferLock = new ReentrantReadWriteLock(false);
private BufferWriteRecordWriter recordWriter = null;
private Pair<String, String> fileNamePair;
private boolean isNewProcessor = false;
private Action closeAction;
BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport<TSRecord> writeSupport,
FileSchema schema, String nsPath) {
super(nsPath);
this.workingBufferIndex = new MemoryBufferWriteIndexImpl();
this.flushingBufferIndex = new MemoryBufferWriteIndexImpl();
this.bufferIOWriter = ioFileWriter;
this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema);
}
BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport<TSRecord> writeSupport,
FileSchema schema, String nsPath, Pair<String, String> fileNamePair) {
super(nsPath);
this.workingBufferIndex = new MemoryBufferWriteIndexImpl();
this.flushingBufferIndex = new MemoryBufferWriteIndexImpl();
this.bufferIOWriter = ioFileWriter;
this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema);
this.fileNamePair = fileNamePair;
this.isNewProcessor = true;
}
/**
* Write a data point
*
* @param deltaObjectId
* @param measurementId
* @param timestamp
* @param deltaObjectType
* @param dataType
* @param value
* @throws WriteProcessException
* @throws IOException
*/
public void write(String deltaObjectId, String measurementId, long timestamp, String deltaObjectType,
TSDataType dataType, String value) throws IOException, WriteProcessException {
TSRecord record = new TSRecord(timestamp, deltaObjectId);
DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementId, value);
record.addTuple(dataPoint);
write(record);
}
/**
* Write a tsRecord
*
* @param tsRecord
* @throws WriteProcessException
* @throws IOException
*/
public void write(TSRecord tsRecord) throws IOException, WriteProcessException {
workingBufferIndex.insert(tsRecord);
recordWriter.write(tsRecord);
}
private void switchIndexFromWorkingToFlushing() {
BufferWriteIndex temp = workingBufferIndex;
workingBufferIndex = flushingBufferIndex;
flushingBufferIndex = temp;
}
private void switchIndexFromFlushingToWorking() {
flushingBufferIndex.clear();
bufferIOWriter.addNewRowGroupMetaDataToBackUp();
flushState.setUnFlushing();
}
/**
* Get the result of DynamicOneColumnData from work index and flush index
*
* @param deltaObjectId
* @param measurementId
* @return
*/
private DynamicOneColumnData mergeTwoDynamicColumnData(String deltaObjectId, String measurementId) {
DynamicOneColumnData working = workingBufferIndex.query(deltaObjectId, measurementId);
DynamicOneColumnData ret = (working == null || working.length == 0) ? new DynamicOneColumnData()
: new DynamicOneColumnData(working.dataType, true);
if (flushState.isFlushing()) {
DynamicOneColumnData flushing = flushingBufferIndex.query(deltaObjectId, measurementId);
ret.mergeRecord(flushing);
}
ret.mergeRecord(working);
return ret;
}
@Override
public void close() {
isFlushingSync = true;
try {
recordWriter.close();
isFlushingSync = false;
} catch (IOException e) {
LOG.error("Close BufferWriteProcessor error in namespace {}. Message:{}", nameSpacePath, e.getMessage());
e.printStackTrace();
}
closeAction.act();
/*
* try { WriteLogManager.getInstance().bufferFlush(); } catch (Exception
* e) { LOG.error("Write log for bufferredWrite error", e);
* System.exit(1); }
*/
}
public void setCloseAction(Action action) {
this.closeAction = action;
}
public Action getCloseAction() {
return this.closeAction;
}
@Override
public boolean canBeClosed() {
LOG.debug("{} can be closed-{}", nameSpacePath, !flushState.isFlushing());
return !flushState.isFlushing();
}
public Pair<DynamicOneColumnData, List<RowGroupMetaData>> getIndexAndRowGroupList(String deltaObjectId,
String measurementId) {
convertBufferLock.readLock().lock();
DynamicOneColumnData index = null;
List<RowGroupMetaData> list = null;
try {
index = mergeTwoDynamicColumnData(deltaObjectId, measurementId);
list = bufferIOWriter.getCurrentRowGroupMetaList();
} finally {
convertBufferLock.readLock().unlock();
}
return new Pair<>(index, list);
}
public Pair<String, String> getPair() {
return fileNamePair;
}
public void setIsNewProcessor(boolean isNewProcessor) {
this.isNewProcessor = isNewProcessor;
}
public boolean isNewProcessor() {
return isNewProcessor;
}
private class BufferWriteRecordWriter extends TSRecordWriter {
private Map<String, IRowGroupWriter> flushingRowGroupWriters;
private Set<String> flushingRowGroupSet;
private long flushingRecordCount;
BufferWriteRecordWriter(TSFileConfig conf, BufferWriteIOWriter ioFileWriter,
WriteSupport<TSRecord> writeSupport, FileSchema schema) {
super(conf, ioFileWriter, writeSupport, schema);
}
/**
* insert a list of data value in form of TimePair.
*
* @param record
* - TSRecord to be written
* @throws WriteProcessException
* @throws IOException
*/
@Override
public void write(TSRecord record) throws IOException, WriteProcessException {
super.write(record);
}
@Override
protected void flushRowGroup(boolean isFillRowGroup) throws IOException {
if (recordCount > 0) {
synchronized (flushState) {
// This thread wait until the subThread flush finished
while (flushState.isFlushing()) {
try {
flushState.wait();
} catch (InterruptedException e) {
LOG.error("Interrupt error when waiting flush,processor:{}. Message: {}", nameSpacePath,
e.getMessage());
e.printStackTrace();
}
}
}
if (isFlushingSync) {
try {
super.flushRowGroup(false);
} catch (IOException e) {
LOG.error("Flush row group to store failed, processor:{}. Message: {}", nameSpacePath,
e.getMessage());
/*
* There should be added system log by CGF and throw
* exception
*/
e.printStackTrace();
System.exit(1);
}
} else {
flushState.setFlushing();
switchIndexFromWorkingToFlushing();
switchRecordWriterFromWorkingToFlushing();
Runnable flushThread;
flushThread = () -> {
LOG.info("Asynchronous flushing start,-Thread id {}", Thread.currentThread().getId());
try {
asyncFlushRowGroupToStore();
} catch (IOException e) {
/*
* There should be added system log by CGF and throw
* exception
*/
LOG.error("{} Asynchronous flush error, sleep this thread-{}. Message:{}", nameSpacePath,
Thread.currentThread().getId(), e.getMessage());
e.printStackTrace();
while (true) {
try {
Thread.sleep(3000);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
}
}
switchRecordWriterFromFlushingToWorking();
convertBufferLock.writeLock().lock();
// notify the thread which is waiting for asynchronous
// flush
synchronized (flushState) {
switchIndexFromFlushingToWorking();
flushState.notify();
}
convertBufferLock.writeLock().unlock();
LOG.info("Asynchronous flushing end,-Thread is {}", Thread.currentThread().getId());
};
Thread flush = new Thread(flushThread);
flush.start();
}
}
}
private void asyncFlushRowGroupToStore() throws IOException {
if (flushingRecordCount > 0) {
String deltaObjectType = schema.getDeltaType();
long totalMemStart = deltaFileWriter.getPos();
for (String deltaObjectId : flushingRowGroupSet) {
long rowGroupStart = deltaFileWriter.getPos();
deltaFileWriter.startRowGroup(flushingRecordCount, deltaObjectId, deltaObjectType);
IRowGroupWriter groupWriter = flushingRowGroupWriters.get(deltaObjectId);
groupWriter.flushToFileWriter(deltaFileWriter);
deltaFileWriter.endRowGroup(deltaFileWriter.getPos() - rowGroupStart);
}
long actualTotalRowGroupSize = deltaFileWriter.getPos() - totalMemStart;
fillInRowGroupSize(actualTotalRowGroupSize);
LOG.info("Asynchronous total row group size:{}, actual:{}, less:{}", primaryRowGroupSize,
actualTotalRowGroupSize, primaryRowGroupSize - actualTotalRowGroupSize);
LOG.info("Asynchronous write row group end");
}
}
private void switchRecordWriterFromWorkingToFlushing() {
flushingRowGroupWriters = groupWriters;
flushingRowGroupSet = schema.getDeltaObjectAppearedSet();
for (String DeltaObjectId : schema.getDeltaObjectAppearedSet()) {
flushingRowGroupSet.add(DeltaObjectId);
}
flushingRecordCount = recordCount;
// reset
groupWriters = new HashMap<String, IRowGroupWriter>();
schema.getDeltaObjectAppearedSet().clear();
recordCount = 0;
}
private void switchRecordWriterFromFlushingToWorking() {
flushingRowGroupSet = null;
flushingRowGroupWriters = null;
flushingRecordCount = -1;
}
}
}

19
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/FileNodeConstants.java Normal file
View File

@ -0,0 +1,19 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
/**
* Constants for using in bufferwrite，overflow and filenode
*
* @author liukun
*
*/
public class FileNodeConstants {
public static final String FILE_NODE_OPERATOR_TYPE = "OPERATOR_TYPE";
public static final String TIMESTAMP_KEY = "TIMESTAMP";
public static final String FILE_NODE = "FILE_NODE";
public static final String CLOSE_ACTION = "CLOSE_ACTION";
public static final String ERR_EXTENSION = "err";
public static final String PATH_SEPARATOR = ".";
}

322
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUManager.java Normal file
View File

@ -0,0 +1,322 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
import cn.edu.thu.tsfiledb.exception.ErrorDebugException;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
import cn.edu.thu.tsfiledb.exception.ProcessorException;
import cn.edu.thu.tsfiledb.metadata.MManager;
/**
* <p>
* LRUManager manage a list of processor: overflow/filenode/bufferwrite
* processor<br>
*
* @author kangrong
* @author liukun
*/
public abstract class LRUManager<T extends LRUProcessor> {
private static Logger LOG = LoggerFactory.getLogger(LRUManager.class);
private LinkedList<T> processorLRUList;
private Map<String, T> processorMap;
private final int maxLRUNodeNumber;
protected final MManager mManager;
protected final String normalDataDir;
private static final long removeCheckInterval = 1000;
protected LRUManager(int maxLRUNumber, MManager mManager, String normalDataDir) {
this.mManager = mManager;
this.maxLRUNodeNumber = maxLRUNumber;
LOG.info("The max of LRUProcessor number of {} is {}", this.getClass().getSimpleName(), maxLRUNumber);
processorLRUList = new LinkedList<>();
processorMap = new HashMap<>();
if (normalDataDir.charAt(normalDataDir.length() - 1) != File.separatorChar)
normalDataDir += File.separatorChar;
this.normalDataDir = normalDataDir;
File dir = new File(normalDataDir);
if (dir.mkdirs()) {
LOG.info("{} dir home is not exists, create it", this.getClass().getSimpleName());
}
}
/**
* <p>
* Get the data directory
*
* @return data directory
*/
public String getNormalDataDir() {
return normalDataDir;
}
/**
* <p>
* Get processor just using the metadata path
*
* @param path
* @param isWriteLock
* @return LRUProcessor
* @throws ProcessorException
* @throws IOException
*/
public T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock)
throws ProcessorException, IOException, WriteProcessException {
return getProcessorWithDeltaObjectIdByLRU(path, isWriteLock, null);
}
/**
* check whether the given path's exists. If it doesn't exist, add it to
* list. If the list has been full(list.size == maxLRUNodeNumber), remove
* the last one. If it exists, raise this processor to the first position
*
* @param path
* - measurement path in name space.
* @param isWriteLock
* @param args
* - save some key-value information used for constructing a
* special processor
* @return return processor which has the specified nsPath
* @throws ProcessorException
* @throws IOException
*/
public synchronized T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock, Map<String, Object> args)
throws ProcessorException, IOException, WriteProcessException {
String nsPath;
try {
nsPath = mManager.getFileNameByPath(path);
} catch (PathErrorException e) {
throw new ProcessorException(e);
}
return getProcessorByLRU(nsPath, isWriteLock, args);
}
/**
* check the processor exists or not by namespacepath
*
* @param namespacePath
* @return
*/
public synchronized boolean containNamespacePath(String namespacePath) {
return processorMap.containsKey(namespacePath);
}
/**
* Get processor using namespacepath
*
* @param namespacePath
* @param isWriteLock
* true if add a write lock on return {@code T}
* @return
* @throws ProcessorException
* @throws IOException
*/
public T getProcessorByLRU(String namespacePath, boolean isWriteLock) throws ProcessorException, IOException, WriteProcessException {
return getProcessorByLRU(namespacePath, isWriteLock, null);
}
/**
*
* check whether the given namespace path exists. If it doesn't exist, add
* it to list. If the list has been full(list.size == maxLRUNodeNumber),
* remove the last one. If it exists, raise this processor to the first
* position
*
* @param namespacePath
* @param isWriteLock
* @param args
* @return
* @throws ProcessorException
* throw it for meeting InterruptedException while removing
* unused processor
* @throws IOException
*/
public synchronized T getProcessorByLRU(String namespacePath, boolean isWriteLock, Map<String, Object> args)
throws ProcessorException, IOException, WriteProcessException {
LOG.debug("Try to getProcessorByLRU, {}, nsPath:{}-Thread id {}", this.getClass().getSimpleName(),
namespacePath, Thread.currentThread().getId());
T processor;
if (processorMap.containsKey(namespacePath)) {
LOG.debug("{} contains nsPath:{}", this.getClass().getSimpleName(), namespacePath);
processor = processorMap.get(namespacePath);
processor.lock(isWriteLock);
initProcessor(processor, namespacePath, args);
processorLRUList.remove(processor);
processorLRUList.addFirst(processor);
LOG.debug("Use {}:{}, raise it to top", processor.getClass().getSimpleName(), processor.getNameSpacePath());
} else {
if (processorLRUList.size() == maxLRUNodeNumber) {
LOG.debug(
"LRU list doesn't contain the processor and is full, remove the oldest unused processor firstly");
try {
removeLastUnusedProcessor();
} catch (InterruptedException e) {
throw new ProcessorException(
"thread is interrupted. nsPath:" + namespacePath + " reason: " + e.getMessage());
}
}
LOG.debug("Construct and init a new processor in {}:{}", this.getClass().getSimpleName(), namespacePath);
processor = constructNewProcessor(namespacePath, args);
initProcessor(processor, namespacePath, args);
processorLRUList.addFirst(processor);
processorMap.put(namespacePath, processor);
processor.lock(isWriteLock);
LOG.debug("{} LUR list: size is {}, content:{}", this.getClass().getSimpleName(), processorLRUList.size(),
processorLRUList);
}
return processor;
}
/**
* remove the last unused processor if the LRU list is full
*
* @throws InterruptedException
*/
private void removeLastUnusedProcessor() throws InterruptedException {
while (true) {
int index = -1;
for (int i = processorLRUList.size() - 1; i >= 0; i--) {
T pro = processorLRUList.get(i);
if (pro.tryWriteLock()) {
try {
LOG.debug("Try write lock successfully, class:{},namespace:{}", pro.getClass().getSimpleName(),
pro.getNameSpacePath());
if (pro.canBeClosed()) {
index = i;
LOG.debug("find a removable processor, index:{}, namespace:{} break loop", index,
pro.getNameSpacePath());
break;
} else {
LOG.debug("{}:{} cannot be removed", pro.getClass().getSimpleName(),
pro.getNameSpacePath());
}
} finally {
pro.writeUnlock();
}
} else {
LOG.info("Can't remove the processor {} namespace path is {}", pro.getClass().getSimpleName(),
pro.getNameSpacePath());
}
}
if (index != -1) {
T pro = processorLRUList.get(index);
pro.close();
processorLRUList.remove(index);
String proNsPath = pro.getNameSpacePath();
processorMap.remove(proNsPath);
LOG.debug("Remove the last processor {}:{}", pro.getClass().getSimpleName(), proNsPath);
break;
} else {
LOG.debug("All processors are occupied, sleep {} millisecond", removeCheckInterval);
Thread.sleep(removeCheckInterval);
}
}
}
/**
* <p>
* Try to close the processor of the nsPath<br>
*
* @param nsPath
* @param isSync
* true must close this processor, false close this processor if
* possible
* @return
*/
public synchronized boolean close(String nsPath, boolean isSync) {
if (!processorMap.containsKey(nsPath))
return true;
LRUProcessor pro = processorMap.get(nsPath);
if (!isSync && !pro.tryWriteLock()) {
return false;
}
try {
do {
if (pro.canBeClosed()) {
pro.close();
processorMap.remove(nsPath);
processorLRUList.remove(pro);
LOG.info("Close {} file:{}", pro.getClass().getSimpleName(), nsPath);
return true;
} else if (isSync) {
try {
LOG.info("Can't close {} file:{}, sleep 1000ms", pro.getClass(), nsPath);
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new ErrorDebugException(e);
}
} else
return false;
} while (true);
} finally {
pro.writeUnlock();
}
}
/**
* <p>
* close all processor still exists in memory<br>
*/
public synchronized void close() {
List<String> cannotRemoveList = new ArrayList<>();
do {
cannotRemoveList.clear();
for (String nsPath : processorMap.keySet()) {
cannotRemoveList.add(nsPath);
}
for (String nsPath : cannotRemoveList) {
close(nsPath, false);
}
if (!processorMap.isEmpty()) {
LOG.info("{}:{} Can't be remove, waiting 1000ms", processorMap.get(cannotRemoveList.get(0)),
cannotRemoveList);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
throw new ErrorDebugException(e);
}
} else {
processorMap.clear();
processorLRUList.clear();
break;
}
} while (true);
}
/**
* <p>
* construct processor using namespacepath and key-value object<br>
*
* @param namespacePath
* @param args
* @return
* @throws ProcessorException
* @throws IOException
*/
protected abstract T constructNewProcessor(String namespacePath, Map<String, Object> args)
throws ProcessorException, IOException, WriteProcessException;
/**
* <p>
* initialize the processor with the key-value object<br>
*
* @param processor
* @param namespacePath
* @param args
* @throws ProcessorException
*/
protected void initProcessor(T processor, String namespacePath, Map<String, Object> args)
throws ProcessorException {
}
}

155
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUProcessor.java Normal file
View File

@ -0,0 +1,155 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* <p>
* LRUProcessor is used for implementing different processor with different
* operation.<br>
*
* @see BufferWriteProcessor
* @see OverflowProcessor
* @see FileNodeProcessor
*
* @author kangrong
*
*/
public abstract class LRUProcessor {
private final static Logger LOG = LoggerFactory.getLogger(LRUProcessor.class);
protected String nameSpacePath;
private final ReadWriteLock lock;
/**
* Construct processor using name space path
*
* @param nameSpacePath
*/
public LRUProcessor(String nameSpacePath) {
this.nameSpacePath = nameSpacePath;
this.lock = new ReentrantReadWriteLock();
}
/**
* Release the read lock
*/
public void readUnlock() {
LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
lock.readLock().unlock();
}
/**
* Acquire the read lock
*/
public void readLock() {
LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
lock.readLock().lock();
}
/**
* Acquire the write lock
*/
public void writeLock() {
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
lock.writeLock().lock();
}
/**
* Release the write lock
*/
public void writeUnlock() {
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
lock.writeLock().unlock();
}
/**
* @param isWriteLock
* true acquire write lock, false acquire read lock
*/
public void lock(boolean isWriteLock) {
if (isWriteLock) {
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(),
Thread.currentThread().getId());
lock.writeLock().lock();
} else {
LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(),
Thread.currentThread().getId());
lock.readLock().lock();
}
}
/**
* @param isWriteUnlock
* true release write lock, false release read unlock
*/
public void unlock(boolean isWriteUnlock) {
if (isWriteUnlock) {
LOG.debug("{}: lru write unlock-Thread id {}", this.getClass().getSimpleName(),
Thread.currentThread().getId());
lock.writeLock().unlock();
} else {
LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(),
Thread.currentThread().getId());
lock.readLock().unlock();
}
}
/**
* Get the name space path
*
* @return
*/
public String getNameSpacePath() {
return nameSpacePath;
}
/**
* Try to get the write lock
*
* @return
*/
public boolean tryWriteLock() {
return lock.writeLock().tryLock();
}
/**
* Try to get the read lock
*
* @return
*/
public boolean tryReadLock() {
return lock.readLock().tryLock();
}
/**
* Judge whether this processor can be closed.
*
* @return true if subclass doesn't have other implementation.
*/
public abstract boolean canBeClosed();
@Override
public int hashCode() {
return nameSpacePath.hashCode();
}
@Override
public boolean equals(Object pro) {
if (pro == null)
return false;
if (!pro.getClass().equals(this.getClass()))
return false;
LRUProcessor lruPro = (LRUProcessor) pro;
return nameSpacePath.equals(lruPro.getNameSpacePath());
}
/**
* Close the processor.<br>
* Notice: Thread is not safe
*/
public abstract void close();
}

77
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/MemoryBufferWriteIndexImpl.java Normal file
View File

@ -0,0 +1,77 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import java.util.HashMap;
import java.util.Map;
import cn.edu.thu.tsfile.common.utils.Binary;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import cn.edu.thu.tsfile.timeseries.write.record.DataPoint;
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
/**
* Implement the {@code BufferWriteIndex}<br>
* This class is used to store bufferwrite data in memory, also index data
* easily<br>
*
* @author kangrong
* @author liukun
*
*/
public class MemoryBufferWriteIndexImpl implements BufferWriteIndex {
private Map<String, DynamicOneColumnData> indexMap;
public MemoryBufferWriteIndexImpl() {
indexMap = new HashMap<String, DynamicOneColumnData>();
}
@Override
public DynamicOneColumnData query(String deltaObjectId, String measurementId) {
return indexMap.get(concatPath(deltaObjectId, measurementId));
}
@Override
public void insert(TSRecord tsRecord) {
String deltaObjectId = tsRecord.deltaObjectId;
for (DataPoint dp : tsRecord.dataPointList) {
String measurementId = dp.getMeasurementId();
String path = concatPath(deltaObjectId, measurementId);
if (!indexMap.containsKey(path))
indexMap.put(path, new DynamicOneColumnData(dp.getType(), true));
DynamicOneColumnData deltaMea = indexMap.get(path);
Object value = dp.getValue();
deltaMea.putTime(tsRecord.time);
switch (dp.getType()) {
case INT32:
deltaMea.putInt((Integer) value);
break;
case INT64:
deltaMea.putLong((Long) value);
break;
case FLOAT:
deltaMea.putFloat((Float) value);
break;
case DOUBLE:
deltaMea.putDouble((Double) value);
break;
case BYTE_ARRAY:
deltaMea.putBinary((Binary) value);
break;
case BOOLEAN:
deltaMea.putBoolean((boolean) value);
break;
default:
throw new UnsupportedOperationException("no support type:" + dp.getType() + "record:" + tsRecord);
}
}
}
@Override
public void clear() {
indexMap.clear();
}
private String concatPath(String deltaObjectId, String measurementId) {
return deltaObjectId + FileNodeConstants.PATH_SEPARATOR + measurementId;
}
}

25
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TSRecordWriterParameter.java Normal file
View File

@ -0,0 +1,25 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
/**
* @author kangrong
* @author liukun
*
*/
public class TSRecordWriterParameter {
public final FileSchema fileSchema;
public final TSRandomAccessFileWriter outputStream;
public final WriteSupport<TSRecord> writeSupport;
public TSRecordWriterParameter(FileSchema fileSchema, TSRandomAccessFileWriter outputStream,
WriteSupport<TSRecord> writeSupport) {
this.fileSchema = fileSchema;
this.outputStream = outputStream;
this.writeSupport = writeSupport;
}
}

16
src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TriFunction.java Normal file
View File

@ -0,0 +1,16 @@
package cn.edu.thu.tsfiledb.engine.bufferwrite;
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
/**
* @author kangrong
*
* @param <U>
* @param <T>
* @param <C>
* @param <R>
*/
@FunctionalInterface
public interface TriFunction<U, T, C, R> {
R apply(U u, T t, C c) throws WriteProcessException;
}

109
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IIntervalTreeOperator.java Normal file
View File

@ -0,0 +1,109 @@
package cn.edu.thu.tsfiledb.engine.overflow;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
/**
* we advise the implementation class of this interface can be constructed by two ways:<br>
* 1.construct a empty index without parameters. 2.construct a empty index with a input stream for
* restoring.
*
* @author kangrong
*
*/
public interface IIntervalTreeOperator {
/**
* insert a value to a time point. Insert doesn't support time range insert
*
* @param t - time
* @param value - value
*/
void insert(long t, byte[] value);
/**
* update a value to a time point or a time range.
*
* @param s - start time.
* @param e - end time.
* @param value - value to be updated.
*/
void update(long s, long e, byte[] value);
/**
* The implementation maintains an overflow index in memory. The data in the index is prior to
* <em>newerMemoryData</em>, which means the overflow operators corresponding to the index are covered with
* <em>newerMemoryData</em>. This function merges current index into <em>newerMemoryData</em> and return the
* merged result.
*
* @param timeFilter - timeFilter is specified by user.
* @param valueFilter - valueFilter is specified by user.
* @param freqFilter - freqFilter is specified by user.
* @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.)
* @param dataType - data type of this series
* @param newerMemoryData - newer overflow data.
*
* @return merged result.
*/
DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize,
TSDataType dataType, DynamicOneColumnData newerMemoryData);
/**
* This function merges the older data which deserialized from given parameter <em>in</em> into <em>newerData</em>
* and return the merged result. The data in <em>in</em> is prior to <em>newerData</em>, which means the overflow
* operators corresponding to <em>in</em> are covered with <em>newerData</em>.
*
* @param timeFilter - timeFilter is specified by user.
* @param valueFilter - valueFilter is specified by user.
* @param freqFilter - freqFilter is specified by user.
* @param in - the inputstream to be merged into newerData which contains older overflow data .
* @param newerData - newer overflow data.
* @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.)
* @param dataType - data type of this series
*
* @return merged result.
*/
DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream in,
DynamicOneColumnData newerData, int valueSize, TSDataType dataType) throws IOException;
/**
* Get List<Object>(insert operations, update operations and delete operations which meet the expression of time filter,
* value filter and frequency filter in DynamicOneColumnData data.)
*
* @param timeFilter - timeFilter is specified by user.
* @param valueFilter - valueFilter is specified by user.
* @param freqFilter - freqFilter is specified by user.
* @param data - a DynamicOneColumnData information.
* @param dataType - TSDataType
* @return - List<Object>
*/
List<Object> getDynamicList(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter,
DynamicOneColumnData data, TSDataType dataType);
/**
* delete all values earlier than timestamp.
*
* @param timestamp - delete timestamp
*/
void delete(long timestamp);
/**
* given an outputstream, serialize the index into it.
*
* @param out - serialization output stream.
*/
void toBytes(OutputStream out) throws IOException;
/**
* @return the memory size for this index
*/
long calcMemSize();
}

991
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IntervalTreeOperation.java Normal file
View File

@ -0,0 +1,991 @@
package cn.edu.thu.tsfiledb.engine.overflow;
import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException;
import cn.edu.thu.tsfile.common.utils.Binary;
import cn.edu.thu.tsfile.common.utils.BytesUtils;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory;
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType;
import cn.edu.thu.tsfile.timeseries.filter.definition.operators.And;
import cn.edu.thu.tsfile.timeseries.filter.definition.operators.GtEq;
import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval;
import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier;
import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitor;
import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitorFactory;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation;
import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalRelation;
import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalTree;
import cn.edu.thu.tsfiledb.engine.overflow.utils.MergeStatus;
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
import cn.edu.thu.tsfiledb.exception.UnSupportedOverflowOpTypeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils.readUnsignedVarInt;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
/**
* @author CGF
*/
public class IntervalTreeOperation implements IIntervalTreeOperator {
private static final Logger LOG = LoggerFactory.getLogger(IntervalTreeOperation.class);
private IntervalTree index = null;
private TSDataType dataType; // All operations data type in IntervalTreeOperation.
public IntervalTreeOperation(TSDataType dataType) {
index = new IntervalTree(dataType);
this.dataType = dataType;
}
@Override
public void insert(long t, byte[] value) {
index.update(new TimePair(t, t, value, OverflowOpType.INSERT, dataType));
}
@Override
public void update(long s, long e, byte[] value) {
// s must >= e !
index.update(new TimePair(s, e, value, OverflowOpType.UPDATE, dataType));
}
@Override
public void delete(long timestamp) {
// timestamp-1 for TSFile sql expression
// in TSFile sql expression, there only exists "DELETE X where Y < Z"
if (timestamp > 0)
index.update(new TimePair(0, timestamp - 1, null, OverflowOpType.DELETE, dataType));
else
// for "DELETE X where Y = 0"
index.update(new TimePair(0, timestamp, null, OverflowOpType.DELETE, dataType));
}
/**
* Notice that serialization file doesn't store the value when DELETE operation.
*
* @param out - serialization output stream.
* @throws IOException - serialization problem
*/
@Override
public void toBytes(OutputStream out) throws IOException {
index.midOrderSerialize(out);
index = new IntervalTree();
}
/**
* used for queryFileBlock()
*
* @param doc - DynamicOneColumn to be put
* @param s - start time
* @param e - end time
* @param value - overflow value
* @param overflowOpType - OverflowOpType
* @param dataType - TSDataType
*/
private void putValueUsingFileData(DynamicOneColumnData doc, long s, long e,
byte[] value, OverflowOpType overflowOpType, TSDataType dataType) {
switch (overflowOpType) {
case INSERT:
doc.putTimePair(s, -e);
break;
case UPDATE:
doc.putTimePair(s, e);
break;
case DELETE:
doc.putTimePair(-s, -e);
break;
default:
LOG.error("Unsupported Overflow operation type.");
throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type.");
}
switch (dataType) {
case INT32:
if (overflowOpType != OverflowOpType.DELETE)
doc.putInt(BytesUtils.bytesToInt(value));
else
doc.putInt(0);
break;
case INT64:
if (overflowOpType != OverflowOpType.DELETE)
doc.putLong(BytesUtils.bytesToLong(value));
else
doc.putLong(0L);
break;
case FLOAT:
if (overflowOpType != OverflowOpType.DELETE)
doc.putFloat(BytesUtils.bytesToFloat(value));
else
doc.putFloat(0.0f);
break;
case DOUBLE:
if (overflowOpType != OverflowOpType.DELETE)
doc.putDouble(BytesUtils.bytesToDouble(value));
else
doc.putDouble(0);
break;
case BYTE_ARRAY:
if (overflowOpType != OverflowOpType.DELETE)
doc.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
else
doc.putDouble(0);
break;
default:
LOG.error("Unsupported TSFile data type.");
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
}
}
/**
* Put value into newer DynamicOneColumnData.
*
* @param ansData new DynamicOneColumn
* @param startTime - start time
* @param endTime - end time
* @param doc - previous DynamicOneColumn
* @param i - index
* @param dataType - TSDataType
*/
private void putValueUseDynamic(DynamicOneColumnData ansData, long startTime, long endTime,
DynamicOneColumnData doc, int i, OverflowOpType overflowOpType, TSDataType dataType) {
// don't care the plus or minus of the value of start time or end time.
switch (overflowOpType) {
case INSERT:
ansData.putTimePair(Math.abs(startTime), -Math.abs(endTime));
break;
case UPDATE:
ansData.putTimePair(Math.abs(startTime), Math.abs(endTime));
break;
case DELETE:
ansData.putTimePair(-Math.abs(startTime), -Math.abs(endTime));
break;
default:
LOG.error("Unsupported Overflow operation type.");
throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type.");
}
switch (dataType) {
case INT32:
ansData.putInt(doc.getInt(i));
break;
case INT64:
ansData.putLong(doc.getLong(i));
break;
case FLOAT:
ansData.putFloat(doc.getFloat(i));
break;
case DOUBLE:
ansData.putDouble(doc.getDouble(i));
break;
case BYTE_ARRAY:
ansData.putBinary(doc.getBinary(i));
break;
default:
LOG.error("Unsupported TSFile data type.");
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
}
}
/**
* Notice that both s and e are >= 0. </br>
*
* @param s - start time
* @param e - end time
* @param value - time pair value
* @param status - time pair merge status
* @return - TimePair tp has both positive start time and end time values.
*/
private TimePair constructTimePair(long s, long e, byte[] value, MergeStatus status) {
if (s <= 0 && e <= 0) // s<=0 && e<=0 for delete operation
return new TimePair(-s, -e, value, OverflowOpType.DELETE, status);
else if (s > 0 && e < 0) {
return new TimePair(s, -e, value, OverflowOpType.INSERT, status);
} else {
return new TimePair(s, e, value, OverflowOpType.UPDATE, status);
}
}
/**
* Notice that both s and e are >= 0. </br>
* <p>
* Return the time pair using given start time value and end time value.
*
* @param startTime - start time
* @param endTime - end time
* @param status - time pair merge status
* @return - TimePair tp has both positive start time and end time values.
*/
private TimePair constructTimePair(long startTime, long endTime, MergeStatus status) {
if (startTime <= 0 && endTime <= 0) // s<=0 && e<=0 for delete operation
return new TimePair(-startTime, -endTime, null, OverflowOpType.DELETE, status);
else if (startTime > 0 && endTime < 0) {
return new TimePair(startTime, -endTime, null, OverflowOpType.INSERT, status);
} else {
return new TimePair(startTime, endTime, null, OverflowOpType.UPDATE, status);
}
}
/**
* Read data from overflow file which has been serialized,
* return the correspond time pair structure. </br>
*
* @param inputStream - InputStream
* @param valueSize - value byte size
* @return - TimePair
* @throws IOException - IOException
*/
private TimePair readTimePairFromOldOverflow(InputStream inputStream, int valueSize) throws IOException {
long s = BytesUtils.readLong(inputStream);
long e = BytesUtils.readLong(inputStream);
if (s <= 0 && e < 0) { // DELETE OPERATION. s may < 0.
return constructTimePair(s, e, null, MergeStatus.MERGING);
} else { // INSERT or UPDATE OPERATION
if (valueSize == -1) { // var length read method
int len = readUnsignedVarInt(inputStream);
byte[] stringBytes = new byte[len];
inputStream.read(stringBytes);
return constructTimePair(s, e, stringBytes, MergeStatus.MERGING);
}
return constructTimePair(s, e, BytesUtils.safeReadInputStreamToBytes(valueSize, inputStream), MergeStatus.MERGING);
}
}
@Override
public DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream inputStream,
DynamicOneColumnData newData, int valueSize, TSDataType dataType) throws IOException {
DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true); // merge answer
int i = 0;
TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE);
TimePair newTimePair = constructTimePair(-1, -1, MergeStatus.DONE);
long L = Long.MIN_VALUE;
long R = Long.MIN_VALUE;
while (i < newData.length) {
if (newTimePair.mergestatus == MergeStatus.DONE) {
// (L,R) represent new time pair range.
L = newData.getTime(i * 2);
R = newData.getTime(i * 2 + 1);
newTimePair = constructTimePair(newData.getTime(i * 2), newData.getTime(i * 2 + 1), MergeStatus.MERGING);
}
if (inputStream.available() == 0 && oldTimePair.mergestatus == MergeStatus.DONE) { // old overflow file is empty, but newData is not empty
putValueUseDynamic(ans, L, R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
continue;
}
if (L <= 0 && R < 0) { // DELETE OPERATION
long rightTime = -R;
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
}
if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10]
oldTimePair.reset();
} else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16]
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType);
newTimePair.reset();
break;
} else { // e.g. [0, 12] [10, 20]
oldTimePair.s = rightTime + 1;
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType);
newTimePair.reset();
break;
}
}
} else if (L > 0 && R < 0) { // INSERT OPERATION
R = -R;
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
}
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
if (relation == CrossRelation.RCOVERSL) {
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
} else if (oldTimePair.s == L) {
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (oldTimePair.e == R) {
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
} else {
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
}
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
newTimePair.reset();
break;
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.reset();
} else {
// relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible
// relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible
// relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible
LOG.error("unreachable method");
}
}
}
} else { // UPDATE OPERATION
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
}
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair
if (oldTimePair.opType == OverflowOpType.INSERT) {
if (oldTimePair.s == L) {
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
L = oldTimePair.s + 1;
oldTimePair.reset();
} else if (oldTimePair.e == R) {
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
i++;
newTimePair.reset();
oldTimePair.reset();
} else {
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
}
} else if (oldTimePair.opType == OverflowOpType.DELETE) {
if (oldTimePair.s == L) {
putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.DELETE, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
} else if (oldTimePair.e == R) {
// the start time of DELETE time pair > 0
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
i++;
newTimePair.reset();
oldTimePair.reset();
} else {
// the start time of DELETE time pair > 0
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
}
} else {
oldTimePair.reset();
}
} else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
i++;
oldTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
putValueUseDynamic(ans, L, -R, newData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
}
} else if (oldTimePair.s == L) {
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUsingFileData(ans, oldTimePair.s, R, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.s = R + 1;
i++;
newTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else {
// oldTimePair.opType == OverflowOpType.INSERT
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
} else if (oldTimePair.e == R) {
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.reset();
i++;
newTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
} else {
if (oldTimePair.opType == OverflowOpType.DELETE) {
i++;
newTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
}
} else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross
// old TimePair could not be INSERT, DELETE
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross
if (oldTimePair.opType == OverflowOpType.DELETE) {
// delete operation need to be added to ans
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
} else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ??
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.reset();
}
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
break;
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.reset();
}
}
}
}
}
// newData is empty, but overflow file still has data.
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
}
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
oldTimePair.reset();
}
return ans;
}
@Override
public DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize, TSDataType dataType
, DynamicOneColumnData newerMemoryData) {
if (newerMemoryData == null) {
return index.dynamicQuery(timeFilter, dataType);
}
DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true);
DynamicOneColumnData oldData = index.dynamicQuery(timeFilter, dataType);
int i = 0, j = 0; // i represents newMemoryData, j represents oldMemoryData
TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE);
TimePair newTimePair = new TimePair(-1, -1, MergeStatus.DONE);
long L = Long.MIN_VALUE;
long R = Long.MIN_VALUE;
while (i < newerMemoryData.length) {
if (newTimePair.mergestatus == MergeStatus.DONE) {
// (L,R) represent new time pair range.
L = newerMemoryData.getTime(i * 2);
R = newerMemoryData.getTime(i * 2 + 1);
newTimePair = constructTimePair(newerMemoryData.getTime(i * 2), newerMemoryData.getTime(i * 2 + 1), MergeStatus.MERGING);
}
if (oldTimePair.mergestatus == MergeStatus.DONE && j >= oldData.length) { // old overflow file is empty, but newData is not empty
putValueUseDynamic(ans, L, R, newerMemoryData, i++, newTimePair.opType, dataType);
newTimePair.reset();
continue;
}
if (L <= 0 && R < 0) { // DELETE OPERATION
long rightTime = -R;
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
}
if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10]
oldTimePair.reset();
j++;
} else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16]
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType);
newTimePair.reset();
break;
} else { // e.g. [0, 12] [10, 20]
oldTimePair.s = rightTime + 1;
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType);
newTimePair.reset();
break;
}
}
} else if (L > 0 && R < 0) { // INSERT OPERATION
R = -R;
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
}
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
if (relation == CrossRelation.RCOVERSL) {
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.reset();
j++;
break;
} else if (oldTimePair.s == L) {
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (oldTimePair.e == R) {
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j++, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.reset();
break;
} else {
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
}
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
break;
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
oldTimePair.reset();
j++;
} else {
// relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible
// relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible
// relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible
LOG.error("unreachable method");
}
}
}
} else { // UPDATE OPERATION
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
}
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair
if (oldTimePair.opType == OverflowOpType.INSERT) {
if (oldTimePair.s == L) {
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
L = oldTimePair.s + 1;
oldTimePair.reset();
j++;
} else if (oldTimePair.e == R) {
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
i++;
newTimePair.reset();
oldTimePair.reset();
j++;
} else {
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
j++;
}
} else if (oldTimePair.opType == OverflowOpType.DELETE) {
if (oldTimePair.s == L) {
putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.DELETE, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
j++;
} else if (oldTimePair.e == R) {
// the start time of DELETE time pair > 0
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
i++;
newTimePair.reset();
oldTimePair.reset();
j++;
} else {
// the start time of DELETE time pair > 0
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
j++;
}
} else {
oldTimePair.reset();
j++;
}
} else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
i++;
oldTimePair.reset();
j++;
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.reset();
j++;
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
newTimePair.reset();
oldTimePair.reset();
j++;
break;
}
} else if (oldTimePair.s == L) {
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUseDynamic(ans, oldTimePair.s, R, oldData, j, oldTimePair.opType, dataType);
oldTimePair.s = R + 1;
i++;
newTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else {
// oldTimePair.opType == OverflowOpType.INSERT
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
} else if (oldTimePair.e == R) {
if (oldTimePair.opType == OverflowOpType.DELETE) {
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
oldTimePair.reset();
j++;
newTimePair.reset();
i++;
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
i++;
oldTimePair.reset();
j++;
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
} else {
if (oldTimePair.opType == OverflowOpType.DELETE) {
i++;
newTimePair.reset();
break;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
LOG.error("unreachable method");
}
}
} else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross
// old TimePair could not be INSERT, DELETE
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
oldTimePair.s = R + 1;
break;
} else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross
if (oldTimePair.opType == OverflowOpType.DELETE) {
// delete operation need to be added to ans
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
L = oldTimePair.e + 1;
oldTimePair.reset();
j++;
} else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ??
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
oldTimePair.reset();
j++;
}
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
newTimePair.reset();
break;
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
oldTimePair.reset();
j++;
}
}
}
}
}
// newData is empty, but old memory still has data.
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
if (oldTimePair.mergestatus == MergeStatus.DONE) {
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
}
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
oldTimePair.reset();
j++;
}
return ans;
}
/**
* To determine whether a time pair is satisfy the demand of the SingleSeriesFilterExpression.
*
* @param valueFilter - value filter
* @param data - DynamicOneColumnData
* @param i - index
* @return - boolean
*/
private boolean isIntervalSatisfy(SingleSeriesFilterExpression valueFilter, DynamicOneColumnData data, int i) {
if (valueFilter == null) {
return true;
}
SingleValueVisitor<?> visitor = new SingleValueVisitor(valueFilter);
switch (valueFilter.getFilterSeries().getSeriesDataType()) {
case INT32:
return visitor.verify(data.getInt(i));
case INT64:
return visitor.verify(data.getLong(i));
case FLOAT:
return visitor.verify(data.getFloat(i));
case DOUBLE:
return visitor.verify(data.getDouble(i));
case BYTE_ARRAY:
return SingleValueVisitorFactory.getSingleValueVistor(TSDataType.BYTE_ARRAY).satisfyObject(data.getBinary(i).getStringValue(), valueFilter);
default:
LOG.error("Unsupported TSFile data type.");
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
}
}
/**
* put data from DynamicOneColumnData[data] to DynamicOneColumnData[ope]
* <p>
* value in ope must > 0
*/
private void putDynamicValue(long s, long e, TSDataType dataType, DynamicOneColumnData ope
, DynamicOneColumnData data, int i) {
if (s > 0 && e < 0) { // INSERT OPERATION, storage single point
ope.putTime(s < 0 ? -s : s);
} else if (s > 0 && e > 0) { // UPDATE OPERATION
ope.putTime(s < 0 ? -s : s);
ope.putTime(e < 0 ? -e : e);
}
switch (dataType) {
case INT32:
ope.putInt(data.getInt(i));
break;
case INT64:
ope.putLong(data.getLong(i));
break;
case FLOAT:
ope.putFloat(data.getFloat(i));
break;
case DOUBLE:
ope.putDouble(data.getDouble(i));
break;
case BYTE_ARRAY:
ope.putBinary(data.getBinary(i));
break;
default:
LOG.error("Unsupported tsfile data type.");
throw new UnSupportedDataTypeException("Unsupported tsfile data type.");
}
}
/**
* Given time filter, value filter and frequency filter,
* return the correspond data which meet all the filters expression. </br>
* List<Object> stores three DynamicOneColumnData structures, insertAdopt, updateAdopt
* and updateNotAdopt.
*
* @param timeFilter - time filter specified by user
* @param valueFilter - value filter specified by user
* @param freqFilter - frequency filter specified by user
* @param overflowData - overflow data
* @param dataType - TSDataType
* @return - List<Object>
*/
@Override
public List<Object> getDynamicList(SingleSeriesFilterExpression timeFilter,
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, DynamicOneColumnData overflowData, TSDataType dataType) {
long deleteMaxLength = -1;
if (timeFilter == null) {
timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries(
"NoName", "NoName", FilterSeriesType.TIME_FILTER), 0L, true);
}
List<Object> ans = new ArrayList<>();
DynamicOneColumnData insertAdopt = new DynamicOneColumnData(dataType, true);
DynamicOneColumnData updateAdopt = new DynamicOneColumnData(dataType, true);
DynamicOneColumnData updateNotAdopt = new DynamicOneColumnData(dataType, true);
LongInterval filterInterval = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter);
for (int i = 0; i < overflowData.length; i++) {
long L = overflowData.getTime(i * 2);
long R = overflowData.getTime(i * 2 + 1);
for (int j = 0; j < filterInterval.count; j += 2) {
TimePair exist = constructTimePair(L, R, MergeStatus.MERGING);
TimePair filterTimePair = constructTimePair(filterInterval.v[j], filterInterval.v[j + 1], MergeStatus.MERGING);
CrossRelation crossRelation = IntervalRelation.getRelation(filterTimePair, exist);
if (L > 0 && R < 0) { // INSERT
switch (crossRelation) {
case LCOVERSR:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(L, R, dataType, insertAdopt, overflowData, i);
} else {
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
}
break;
case RCOVERSL:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, insertAdopt, overflowData, i);
} else {
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
}
break;
case LFIRSTCROSS:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(exist.s, filterTimePair.e, dataType, insertAdopt, overflowData, i);
} else {
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
}
break;
case RFIRSTCROSS:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(filterTimePair.s, exist.e, dataType, insertAdopt, overflowData, i);
} else {
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
}
break;
}
} else if (L > 0 && R > 0) { // UPDATE
switch (crossRelation) {
case LCOVERSR:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(L, R, dataType, updateAdopt, overflowData, i);
} else {
putDynamicValue(L, R, dataType, updateNotAdopt, overflowData, i);
}
break;
case RCOVERSL:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateAdopt, overflowData, i);
} else {
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i);
}
break;
case LFIRSTCROSS:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(exist.s, filterTimePair.e, dataType, updateAdopt, overflowData, i);
} else {
putDynamicValue(exist.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i);
}
break;
case RFIRSTCROSS:
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
putDynamicValue(filterTimePair.s, exist.e, dataType, updateAdopt, overflowData, i);
} else {
putDynamicValue(filterTimePair.s, exist.e, dataType, updateNotAdopt, overflowData, i);
}
break;
}
} else { // DELETE
// LOG.info("getDynamicList max length:{} delete length:{}", deleteMaxLength, endTime);
deleteMaxLength = Math.max(deleteMaxLength, -R);
}
}
}
ans.add(insertAdopt);
ans.add(updateAdopt);
ans.add(updateNotAdopt);
GtEq<Long> deleteFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries(
"Any", "Any", FilterSeriesType.TIME_FILTER), deleteMaxLength, false);
And and = (And) FilterFactory.and(timeFilter, deleteFilter);
ans.add(and);
return ans;
}
/**
* both start time and end time are "long" datatype and occupy 16 bytes.
*
* @return - memory occupation
*/
@Override
public long calcMemSize() {
if (index.getValueSize() == 0) {
return 16;
} else {
return index.getNodeSize() * (16 + index.getValueSize());
}
}
/**
* reset the status of IntervalTreeOperation.
*/
public void reset() {
index = new IntervalTree();
}
}

15
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/CrossRelation.java Normal file
View File

@ -0,0 +1,15 @@
package cn.edu.thu.tsfiledb.engine.overflow.index;
/**
* An enum represent the cross relation between two time pairs.
*
* @author CGF
*/
public enum CrossRelation {
LCOVERSR, // left time pair covers right time pair. e.g. [1, 10], [5, 8]
RCOVERSL, // right time pair covers (or equals) left time pair. e.g. [5, 8], [1, 10]
LFIRSTCROSS, // left time pair first cross right time pair. e.g. [1, 10], [8, 13]
RFIRSTCROSS, // right time pair first covers left time pair. e.g. [8, 13], [1, 10]
LFIRST, // left time pair is on the left of right time pair. e.g. [1, 10], [15, 18]
RFIRST // right time pair is on the left of left time pair. e.g. [15, 18], [1, 10]
}

69
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalRelation.java Normal file
View File

@ -0,0 +1,69 @@
package cn.edu.thu.tsfiledb.engine.overflow.index;
import static cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation.*;
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
/**
* This class is used to determine the relation between two time pairs.
*
* @author CGF
*/
public class IntervalRelation {
/**
* To determine the relation between the two time pairs.
*
* @param left - left time pair
* @param right - right time pair
*/
private static CrossRelation getCrossRelation(TimePair left, TimePair right) {
if (right.s <= left.s && right.e >= left.e) { // right covers left | right equals left
return RCOVERSL;
} else if (right.s >= left.s && right.e <= left.e) { // left covers right
return LCOVERSR;
} else if (right.s > left.s) { // left first cross
return LFIRSTCROSS;
} else { // right first cross
return RFIRSTCROSS;
}
}
private static CrossRelation getCrossRelation(long s1, long e1, long s2, long e2) {
if (s2 <= s1 && e2 >= e1) { // right covers left | right equals left
return RCOVERSL;
} else if (s2 >= s1 && e2 <= e1) { // left covers right
return LCOVERSR;
} else if (s2 > s1) { // left first cross
return LFIRSTCROSS;
} else { // right first cross
return RFIRSTCROSS;
}
}
/**
* @param left - left time pair
* @param right - right time pair
* @return CrossRelation
*/
public static CrossRelation getRelation(TimePair left, TimePair right) {
if (left.e < right.s) { // left first
return LFIRST;
} else if (right.e < left.s) { // right first
return RFIRST;
} else
return getCrossRelation(left, right);
}
public static CrossRelation getRelation(long s1, long e1, long s2, long e2) {
if (e1 < s2) { // left first
return LFIRST;
} else if (e2 < s1) { // right first
return RFIRST;
} else
return getCrossRelation(s1, e1, s2, e2);
}
}

588
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalTree.java Normal file
View File

@ -0,0 +1,588 @@
package cn.edu.thu.tsfiledb.engine.overflow.index;
import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException;
import cn.edu.thu.tsfile.common.utils.Binary;
import cn.edu.thu.tsfile.common.utils.BytesUtils;
import cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory;
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType;
import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval;
import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier;
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
import cn.edu.thu.tsfiledb.exception.OverflowWrongParameterException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
/**
* IntervalTree is a data structure implemented used Treap. </br>
* An IntervalTree stores many TreeNodes, a TreeNode saves a time range operation. </br>
* A time range operation is explained in OverflowOpType. </br>
* see https://en.wikipedia.org/wiki/Treap.
*
* @author CGF
*/
public class IntervalTree {
private static final Logger LOG = LoggerFactory.getLogger(IntervalTree.class);
private int nodeSize = 0; // num of tree nodes
private int valueSize = 0; // num of bytes of value
private TreeNode root = null;
private TSDataType dataType = null; // The data type of IntervalTree structure.
public IntervalTree() {
}
public IntervalTree(TSDataType dataType) {
this.dataType = dataType;
if (dataType == TSDataType.BYTE_ARRAY) {
this.valueSize = -1; // -1 represents var length encoding.
}
}
/**
* TimePair must be no conflict with current IntervalTree.
*
* @param tp TimePair to be inserted.
*/
private void insert(TimePair tp) {
nodeSize++;
if (root == null) {
root = new TreeNode(tp.s, tp.e, tp.v, tp.opType);
} else {
insertNoConflict(tp, root, null, false);
}
}
/**
* Used to insert the no conflicting TreeNode.
*
* @param timePair time pair to be inserted
* @param currentNode current TreeNode
* @param parentNode parent TreeNode
* @param left whether the pNode is left child of paNode.
*/
private void insertNoConflict(TimePair timePair, TreeNode currentNode, TreeNode parentNode, boolean left) {
if (currentNode.start > timePair.e) {
if (currentNode.left == null) {
currentNode.left = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType);
} else {
insertNoConflict(timePair, currentNode.left, currentNode, true);
}
if (currentNode.fix > currentNode.left.fix)
rightTurn(currentNode, parentNode, left);
} else if (currentNode.end < timePair.s) {
if (currentNode.right == null) {
currentNode.right = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType);
} else {
insertNoConflict(timePair, currentNode.right, currentNode, false);
}
if (currentNode.fix > currentNode.right.fix)
leftTurn(currentNode, parentNode, left);
}
}
private void leftTurn(TreeNode currentNode, TreeNode parentNode, boolean left) {
if (parentNode == null) {
root = currentNode.right;
} else if (left) {
parentNode.left = currentNode.right;
} else {
parentNode.right = currentNode.right;
}
TreeNode tmpNode = currentNode.right.left;
currentNode.right.left = currentNode;
currentNode.right = tmpNode;
}
private void rightTurn(TreeNode currentNode, TreeNode parentNode, boolean left) {
if (parentNode == null) {
root = currentNode.left;
} else if (left) {
parentNode.left = currentNode.left;
} else {
parentNode.right = currentNode.left;
}
TreeNode tmpNode = currentNode.left.right;
currentNode.left.right = currentNode;
currentNode.left = tmpNode;
}
private void delete(TreeNode currentNode, TreeNode parentNode, boolean left) {
if (parentNode == null) {
if (currentNode.left == null && currentNode.right == null) {
root = null;
nodeSize--;
return;
} else if (currentNode.left == null) {
root = currentNode.right;
nodeSize--;
return;
} else if (currentNode.right == null) {
root = currentNode.left;
nodeSize--;
return;
}
}
if (left) {
if (currentNode.left == null) {
parentNode.left = currentNode.right;
nodeSize--;
} else if (currentNode.right == null) {
parentNode.left = currentNode.left;
nodeSize--;
} else {
if (currentNode.left.fix < currentNode.right.fix) {
TreeNode tmpNode = currentNode.left;
rightTurn(currentNode, parentNode, left);
delete(currentNode, tmpNode, false);
} else {
TreeNode tmpNode = currentNode.right;
leftTurn(currentNode, parentNode, left);
delete(currentNode, tmpNode, true);
}
}
} else {
if (currentNode.left == null) {
parentNode.right = currentNode.right;
nodeSize--;
} else if (currentNode.right == null) {
parentNode.right = currentNode.left;
nodeSize--;
} else {
if (currentNode.left.fix < currentNode.right.fix) {
TreeNode tmpNode = currentNode.left;
rightTurn(currentNode, parentNode, left);
delete(currentNode, tmpNode, false);
} else {
TreeNode tmpNode = currentNode.right;
leftTurn(currentNode, parentNode, left);
delete(currentNode, tmpNode, true);
}
}
}
}
/**
* Update operation api.
*
* @param tp TimePair to be updated
*/
public void update(TimePair tp) {
if (this.dataType == null) {
if (tp.opType != OverflowOpType.DELETE) {
this.valueSize = tp.v.length;
}
this.dataType = tp.dataType;
} else if (tp.dataType != this.dataType){
LOG.error("IntervalTree wrong time pair parameters");
throw new OverflowWrongParameterException("IntervalTree wrong time pair parameters");
}
// When an UPDATE operation covers root(INSERT) node,
// we delete root(INSERT) node and stores it in insertList temporary.
List<TimePair> insertList = new ArrayList<>();
while (root != null) {
if (update(tp, root, null, false, insertList)) {
break;
}
}
if (root == null) {
insert(tp);
}
for (TimePair t : insertList) {
while (root != null) {
if (update(t, root, null, false, null)) {
break;
}
}
}
}
/**
* IntervalTree update operation.
*
* @param tp TimePair to be updated
* @param currentNode current TreeNode
* @param parentNode parent node of pNode
* @param left whether pNode is left child of paNode
* @return whether update process is over
*/
private boolean update(TimePair tp, TreeNode currentNode, TreeNode parentNode, boolean left, List insertList) {
if (currentNode == null) {
insert(tp);
return true;
}
CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(currentNode.start, currentNode.end));
switch (relation) {
case LCOVERSR: // tp covers currentNode
if (currentNode.opType == OverflowOpType.INSERT && tp.opType == OverflowOpType.UPDATE) {
insertList.add(new TimePair(currentNode.start, currentNode.end, tp.v,
OverflowOpType.INSERT));
}
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
tp.s = currentNode.end + 1;
} else {
delete(currentNode, parentNode, left);
}
return false;
case RCOVERSL: // currentNode covers tp
if (tp.s == currentNode.start && currentNode.end == tp.e) {
if (currentNode.opType == OverflowOpType.INSERT
&& tp.opType == OverflowOpType.UPDATE) {
currentNode.value = tp.v;
} else if (currentNode.opType == OverflowOpType.DELETE
&& tp.opType == OverflowOpType.UPDATE) {
return true;
} else {
currentNode.opType = tp.opType;
currentNode.value = tp.v;
}
} else if (tp.s == currentNode.start) {
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
return true;
} else {
currentNode.start = tp.e + 1;
insert(tp);
}
} else if (tp.e == currentNode.end) {
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
return true;
} else {
currentNode.end = tp.s - 1;
insert(tp);
}
} else { // (currentNode tp currentNode)
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
return true;
} else {
long tmp = currentNode.end;
currentNode.end = tp.s - 1;
insert(tp);
insert(new TimePair(tp.e + 1, tmp, currentNode.value, currentNode.opType));
}
}
return true;
case LFIRSTCROSS: // tp first cross | No need to modify currentNode value.
currentNode.start = tp.e + 1;
return update(tp, currentNode.left, currentNode, true, insertList);
case RFIRSTCROSS: // currentNode first cross | No need to modify currentNode value.
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
return update(new TimePair(currentNode.end + 1, tp.e, tp.v, tp.opType), currentNode.right, currentNode, false, insertList);
} else {
currentNode.end = tp.s - 1;
return update(tp, currentNode.right, currentNode, false, insertList);
}
case LFIRST: // tp first
return update(tp, currentNode.left, currentNode, true, insertList);
case RFIRST: // currentNode first
return update(tp, currentNode.right, currentNode, false, insertList);
default:
return true;
}
}
/**
* Put the IntervalTree mid order serialization into OutputStream.
*
* @param out OutputStream
*/
public void midOrderSerialize(OutputStream out) throws IOException {
midOrderSerialize(root, out);
root = null;
}
/**
* IntervalTree mid order serialization.
* <p>
* Notice that "String" data type adopts unsigned var int encoding.
*
* @param pNode - root node of IntervalTree
* @param out - output file stream
*/
private void midOrderSerialize(TreeNode pNode, OutputStream out) throws IOException {
if (pNode == null)
return;
midOrderSerialize(pNode.left, out);
if (pNode.opType.equals(OverflowOpType.DELETE)) {
out.write(BytesUtils.longToBytes(-pNode.start));
out.write(BytesUtils.longToBytes(-pNode.end));
} else if (pNode.opType.equals(OverflowOpType.UPDATE)) {
out.write(BytesUtils.longToBytes(pNode.start));
out.write(BytesUtils.longToBytes(pNode.end));
if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) {
ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out);
out.write(pNode.value);
} else {
out.write(pNode.value);
}
} else {
out.write(BytesUtils.longToBytes(pNode.start));
out.write(BytesUtils.longToBytes(-pNode.end));
if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) {
ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out);
out.write(pNode.value);
} else {
out.write(pNode.value);
}
}
midOrderSerialize(pNode.right, out);
}
/**
* given the query TimePair tp, return all covered time pair with tp.
* <p>
* for concurrency safety, the queryAns variable must be defined in
* the query method.
*
* @param tp - TimePair to query
* @return - all new TimePairs accord with the query tp.
*/
public List<TimePair> query(TimePair tp) {
List<TimePair> queryAns = new ArrayList<>();
query(tp, root, queryAns);
return queryAns;
}
private void query(TimePair tp, TreeNode pNode, List<TimePair> queryAns) {
if (pNode == null)
return;
CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(pNode.start, pNode.end));
switch (relation) {
case LCOVERSR: // tp covers currentNode
if (tp.s == pNode.start && pNode.end == tp.e) {
queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType));
} else if (tp.s == pNode.start) {
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
} else if (tp.e == pNode.end) {
query(new TimePair(tp.s, pNode.end - 1), pNode.left, queryAns);
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
} else {
query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns);
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
}
return;
case RCOVERSL: // currentNode covers tp
queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType));
return;
case LFIRSTCROSS: // tp first cross
query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns);
queryAns.add(new TimePair(pNode.start, tp.e, pNode.value, pNode.opType));
return;
case RFIRSTCROSS: // currentNode first cross
queryAns.add(new TimePair(tp.s, pNode.end, pNode.value, pNode.opType));
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
return;
case LFIRST: // tp first
query(tp, pNode.left, queryAns);
break;
case RFIRST: // currentNode first
query(tp, pNode.right, queryAns);
break;
}
}
/**
* Given time filter and value filter, return the answer List<DynamicOneColumn> in IntervalTree.</br>
* Could not use member variable. </br>
* Must notice thread safety! </br>
* Note that value filter is not given, apply value filter in this method would cause errors.</br>
* May make wrong intervals in old data valid in IntervalTreeOperation.queryFileBlock method.
*
* @param timeFilter filter for time
* @param dataType TSDataType
* @return DynamicOneColumnData
*/
public DynamicOneColumnData dynamicQuery(SingleSeriesFilterExpression timeFilter, TSDataType dataType) {
DynamicOneColumnData crudResult = new DynamicOneColumnData(dataType, true);
if (timeFilter == null) {
timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries("NULL", "NULL", FilterSeriesType.TIME_FILTER), 0L, true);
}
LongInterval val = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter);
dynamicQuery(crudResult, root, val, dataType);
return crudResult;
}
/**
* Implemented for above dynamicQuery method.
*
* @param crudResult DynamicOneColumnData
* @param pNode query root TreeNode
* @param val LongInterval
* @param dataType TSDataType
*/
private void dynamicQuery(DynamicOneColumnData crudResult, TreeNode pNode, LongInterval val, TSDataType dataType) {
if (pNode == null)
return;
// to examine whether left child node has answer
if (val.count != 0 && val.v[0] < pNode.start) {
dynamicQuery(crudResult, pNode.left, val, dataType);
}
for (int i = 0; i < val.count; i += 2) {
CrossRelation r = IntervalRelation.getRelation(new TimePair(val.v[i], val.v[i + 1]), new TimePair(pNode.start, pNode.end));
switch (r) {
case LFIRST:
continue;
case RFIRST:
continue;
case LFIRSTCROSS:
putTreeNodeValue(crudResult, pNode.opType, pNode.start, val.v[i + 1], dataType, pNode.value);
break;
case RFIRSTCROSS:
putTreeNodeValue(crudResult, pNode.opType, val.v[i], pNode.end, dataType, pNode.value);
break;
case LCOVERSR:
putTreeNodeValue(crudResult, pNode.opType, pNode.start, pNode.end, dataType, pNode.value);
break;
case RCOVERSL:
putTreeNodeValue(crudResult, pNode.opType, val.v[i], val.v[i + 1], dataType, pNode.value);
break;
default:
LOG.error("un defined CrossRelation");
}
}
// to examine whether right child node has answer
if (val.count != 0 && val.v[val.count - 1] > pNode.end) { // left node has answer
dynamicQuery(crudResult, pNode.right, val, dataType);
}
}
/**
* Put TreeNode information into DynamicColumnData crudResult.
*
* @param crudResult DynamicOneColumnData stores operations
* @param opType OverflowOpType
* @param s start time
* @param e end time
* @param dataType TSDataType
* @param value byte value
*/
private void putTreeNodeValue(DynamicOneColumnData crudResult, OverflowOpType opType, long s, long e, TSDataType dataType,
byte[] value) {
switch (dataType) {
case INT32:
switch (opType) {
case INSERT:
crudResult.putTimePair(s, -e);
crudResult.putInt(BytesUtils.bytesToInt(value));
break;
case DELETE:
crudResult.putTimePair(-s, -e);
crudResult.putInt(0);
break;
case UPDATE:
crudResult.putTimePair(s, e);
crudResult.putInt(BytesUtils.bytesToInt(value));
break;
}
break;
case INT64:
switch (opType) {
case INSERT:
crudResult.putTimePair(s, -e);
crudResult.putLong(BytesUtils.bytesToLong(value));
break;
case DELETE:
crudResult.putTimePair(-s, -e);
crudResult.putLong(0);
break;
case UPDATE:
crudResult.putTimePair(s, e);
crudResult.putLong(BytesUtils.bytesToLong(value));
}
break;
case FLOAT:
switch (opType) {
case INSERT:
crudResult.putTimePair(s, -e);
crudResult.putFloat(BytesUtils.bytesToFloat(value));
break;
case DELETE:
crudResult.putTimePair(-s, -e);
crudResult.putFloat(0);
break;
case UPDATE:
crudResult.putTimePair(s, e);
crudResult.putFloat(BytesUtils.bytesToFloat(value));
break;
}
break;
case DOUBLE:
switch (opType) {
case INSERT:
crudResult.putTimePair(s, -e);
crudResult.putDouble(BytesUtils.bytesToDouble(value));
break;
case DELETE:
crudResult.putTimePair(-s, -e);
crudResult.putDouble(0);
break;
case UPDATE:
crudResult.putTimePair(s, e);
crudResult.putDouble(BytesUtils.bytesToDouble(value));
break;
}
break;
case BYTE_ARRAY:
switch (opType) {
case INSERT:
crudResult.putTimePair(s, -e);
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
break;
case DELETE:
crudResult.putTimePair(-s, -e);
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
break;
case UPDATE:
crudResult.putTimePair(s, e);
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
break;
}
break;
default:
LOG.error("Unsupported tsfile data type.");
throw new UnSupportedDataTypeException("Unsupported tsfile data type.");
}
}
public int getNodeSize() {
return nodeSize;
}
public int getValueSize() {
return this.valueSize;
}
}

34
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/TreeNode.java Normal file
View File

@ -0,0 +1,34 @@
package cn.edu.thu.tsfiledb.engine.overflow.index;
import java.util.Random;
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
/**
* Used for IntervalTree (Treap Data Structure).
*
* @author CGF
*/
public class TreeNode {
private Random random = new Random();
public long start; // start time
public long end; // end time
public int fix; // priority for treap IntervalTree
public OverflowOpType opType; // overflow operation type
public byte[] value; // the value stored in this node
public TreeNode left;
public TreeNode right;
public TreeNode(long start, long end, byte[] value, OverflowOpType type) {
this.start = start;
this.end = end;
this.value = value;
this.opType = type;
left = null;
right = null;
fix = random.nextInt();
}
}

13
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/MergeStatus.java Normal file
View File

@ -0,0 +1,13 @@
package cn.edu.thu.tsfiledb.engine.overflow.utils;
/**
* Used for IntervalTreeOperation.queryMemory() and IntervalTreeOperation.queryFileBlock(); </br>
*
* DONE means that a time pair is not used or this time pair has been merged into a new DynamicOneColumn</br>
* MERGING means that a time pair is merging into a new DynamicOneColumn</br>
*
* @author CGF.
*/
public enum MergeStatus {
DONE, MERGING
}

16
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/OverflowOpType.java Normal file
View File

@ -0,0 +1,16 @@
package cn.edu.thu.tsfiledb.engine.overflow.utils;
/**
* Include three types: INSERT,UPDATE,DELETE;
*
* INSERT is an operation which inserts a time point.</br>
* UPDATE is an operation which updates a time range.</br>
* DELETE is an operation which deletes a time range. Note that DELETE operation could only
* delete a time which is less than given time T. </br>
*
* @author kangrong
*
*/
public enum OverflowOpType {
INSERT, UPDATE, DELETE
}

75
src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/TimePair.java Normal file
View File

@ -0,0 +1,75 @@
package cn.edu.thu.tsfiledb.engine.overflow.utils;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
/**
* TimePair represents an overflow operation.
*
* @author CGF
*/
public class TimePair {
public long s; // start time
public long e; // end time
public byte[] v; // value
public OverflowOpType opType = null;
public TSDataType dataType;
public MergeStatus mergestatus;
public TimePair(long s, long e) {
this.s = s;
this.e = e;
this.v = null;
}
public TimePair(long s, long e, byte[] v) {
this.s = s;
this.e = e;
this.v = v;
}
public TimePair(long s, long e, MergeStatus status) {
this(s, e);
this.mergestatus = status;
}
public TimePair(long s, long e, byte[] v, TSDataType dataType) {
this(s, e, v);
this.dataType = dataType;
}
public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType) {
this(s, e, v);
this.opType = overflowOpType;
}
public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType, TSDataType dataType) {
this(s, e, v, overflowOpType);
this.dataType = dataType;
}
public TimePair(long s, long e, byte[] v, OverflowOpType type, MergeStatus status) {
this(s, e, v);
this.opType = type;
this.mergestatus = status;
}
/**
* Set TimePair s = -1 and e = -1 means reset.
*/
public void reset() {
s = -1;
e = -1;
mergestatus = MergeStatus.DONE;
}
public String toString() {
StringBuffer sb = new StringBuffer().append(this.s).append(",").append(this.e);
if (this.opType != null)
sb.append(",").append(this.opType);
return sb.toString();
}
}

30
src/main/java/cn/edu/thu/tsfiledb/engine/utils/FlushState.java Normal file
View File

@ -0,0 +1,30 @@
package cn.edu.thu.tsfiledb.engine.utils;
/**
* This class is used to represent the state of flush. It's can be used in the
* bufferwrite flush{@code BufferWriteProcessor} and overflow
* flush{@code OverFlowProcessor}.
*
* @author liukun
*/
public class FlushState {
private boolean isFlushing;
public FlushState() {
this.isFlushing = false;
}
public boolean isFlushing() {
return isFlushing;
}
public void setFlushing() {
this.isFlushing = true;
}
public void setUnFlushing() {
this.isFlushing = false;
}
}

10
src/main/java/cn/edu/thu/tsfiledb/exception/ArgsErrorException.java Normal file
View File

@ -0,0 +1,10 @@
package cn.edu.thu.tsfiledb.exception;
public class ArgsErrorException extends Exception{
private static final long serialVersionUID = -3614543017182165265L;
public ArgsErrorException(String msg){
super(msg);
}
}

23
src/main/java/cn/edu/thu/tsfiledb/exception/DeltaEngineRunningException.java Normal file
View File

@ -0,0 +1,23 @@
package cn.edu.thu.tsfiledb.exception;
/**
*
* This Exception is the parent class for all delta engine runtime exceptions.<br>
* This Exception extends super class {@link java.lang.RuntimeException}
*
* @author CGF
*/
public abstract class DeltaEngineRunningException extends RuntimeException{
public DeltaEngineRunningException() { super();}
public DeltaEngineRunningException(String message, Throwable cause) { super(message, cause);}
public DeltaEngineRunningException(String message) {
super(message);
}
public DeltaEngineRunningException(Throwable cause) {
super(cause);
}
}

18
src/main/java/cn/edu/thu/tsfiledb/exception/ErrorDebugException.java Normal file
View File

@ -0,0 +1,18 @@
package cn.edu.thu.tsfiledb.exception;
/**
* @author kangrong
*
*/
public class ErrorDebugException extends RuntimeException {
private static final long serialVersionUID = -1123099620556170447L;
public ErrorDebugException(String msg) {
super(msg);
}
public ErrorDebugException(Exception e) {
super(e);
}
}

16
src/main/java/cn/edu/thu/tsfiledb/exception/FileNodeNotExistException.java Normal file
View File

@ -0,0 +1,16 @@
package cn.edu.thu.tsfiledb.exception;
/**
* Throw this exception when the file node processor is not exists
*
* @author kangrong
*
*/
public class FileNodeNotExistException extends RuntimeException {
private static final long serialVersionUID = -4334041411884083545L;
public FileNodeNotExistException(String msg) {
super(msg);
}
}

18
src/main/java/cn/edu/thu/tsfiledb/exception/MetadataArgsErrorException.java Normal file
View File

@ -0,0 +1,18 @@
package cn.edu.thu.tsfiledb.exception;
/**
* If query metadata constructs schema but passes illegal parameters to
* EncodingConvertor or DataTypeConvertor,this exception will be threw.
*
* @author kangrong
*
*/
public class MetadataArgsErrorException extends ArgsErrorException {
private static final long serialVersionUID = 3415275599091623570L;
public MetadataArgsErrorException(String msg) {
super(msg);
}
}

20
src/main/java/cn/edu/thu/tsfiledb/exception/OverflowWrongParameterException.java Normal file
View File

@ -0,0 +1,20 @@
package cn.edu.thu.tsfiledb.exception;
/**
* Used for IntervalTree pass wrong parameters. </br>
* e.g. TSDataType inconsistent;
* e.g. start time is greater than end time.
*
* @author CGF
*/
public class OverflowWrongParameterException extends DeltaEngineRunningException{
public OverflowWrongParameterException(String message, Throwable cause) { super(message, cause);}
public OverflowWrongParameterException(String message) {
super(message);
}
public OverflowWrongParameterException(Throwable cause) {
super(cause);
}
}

10
src/main/java/cn/edu/thu/tsfiledb/exception/PathErrorException.java Normal file
View File

@ -0,0 +1,10 @@
package cn.edu.thu.tsfiledb.exception;
public class PathErrorException extends Exception{
private static final long serialVersionUID = 2141197032898163234L;
public PathErrorException(String msg){
super(msg);
}
}

24
src/main/java/cn/edu/thu/tsfiledb/exception/ProcessorException.java Normal file
View File

@ -0,0 +1,24 @@
package cn.edu.thu.tsfiledb.exception;
/**
* This exception is used for some error in processor
*
* @author kangrong
*
*/
public class ProcessorException extends Exception {
private static final long serialVersionUID = 4137638418544201605L;
public ProcessorException(String msg) {
super(msg);
}
public ProcessorException(PathErrorException pathExcp) {
super("PathErrorException: " + pathExcp.getMessage());
}
public ProcessorException() {
super();
}
}

20
src/main/java/cn/edu/thu/tsfiledb/exception/UnSupportedOverflowOpTypeException.java Normal file
View File

@ -0,0 +1,20 @@
package cn.edu.thu.tsfiledb.exception;
/**
* OverflowOpType could only be INSERT, UPDATE, DELETE.
* The other situations would throw this exception.
*
* @author CGF
*/
public class UnSupportedOverflowOpTypeException extends DeltaEngineRunningException{
public UnSupportedOverflowOpTypeException(String message, Throwable cause) { super(message, cause);}
public UnSupportedOverflowOpTypeException(String message) {
super(message);
}
public UnSupportedOverflowOpTypeException(Throwable cause) {
super(cause);
}
}

102
src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSInputStream.java Normal file
View File

@ -0,0 +1,102 @@
package cn.edu.thu.tsfiledb.hadoop.io;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader;
import java.io.IOException;
/**
* This class is used to wrap the {@link}FSDataInputStream and implement the
* interface {@link}TSRandomAccessFileReader.
*
* @author liukun
*/
public class HDFSInputStream implements TSRandomAccessFileReader {
private FSDataInputStream fsDataInputStream;
private FileStatus fileStatus;
public HDFSInputStream(String filePath) throws IOException {
this(filePath, new Configuration());
}
public HDFSInputStream(String filePath, Configuration configuration) throws IOException {
this(new Path(filePath),configuration);
}
public HDFSInputStream(Path path, Configuration configuration) throws IOException {
FileSystem fs = FileSystem.get(configuration);
fsDataInputStream = fs.open(path);
fileStatus = fs.getFileStatus(path);
}
@Override
public void seek(long offset) throws IOException {
fsDataInputStream.seek(offset);
}
@Override
public int read() throws IOException {
return fsDataInputStream.read();
}
@Override
public long length() throws IOException {
return fileStatus.getLen();
}
@Override
public int readInt() throws IOException {
return fsDataInputStream.readInt();
}
public void close() throws IOException {
fsDataInputStream.close();
}
public long getPos() throws IOException {
return fsDataInputStream.getPos();
}
/**
* Read the data into b, and the check the length
*
* @param b
* read the data into
* @param off
* the begin offset to read into
* @param len
* the length to read into
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (len < 0) {
throw new IndexOutOfBoundsException();
}
int n = 0;
while (n < len) {
int count = fsDataInputStream.read(b, off + n, len - n);
if (count < 0) {
throw new IOException("The read length is out of the length of inputstream");
}
n += count;
}
return n;
}
}

75
src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSOutputStream.java Normal file
View File

@ -0,0 +1,75 @@
package cn.edu.thu.tsfiledb.hadoop.io;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
/**
* This class is used to wrap the {@link}FSDataOutputStream and implement the
* interface {@link}TSRandomAccessFileWriter
*
* @author liukun
*/
public class HDFSOutputStream implements TSRandomAccessFileWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(HDFSOutputStream.class);
private FSDataOutputStream fsDataOutputStream;
public HDFSOutputStream(String filePath, boolean overwriter) throws IOException {
this(filePath, new Configuration(), overwriter);
}
public HDFSOutputStream(String filePath, Configuration configuration, boolean overwriter) throws IOException {
this(new Path(filePath),configuration,overwriter);
}
public HDFSOutputStream(Path path,Configuration configuration,boolean overwriter) throws IOException{
FileSystem fsFileSystem = FileSystem.get(configuration);
fsDataOutputStream = fsFileSystem.create(path, overwriter);
}
@Override
public OutputStream getOutputStream() {
return fsDataOutputStream;
}
@Override
public long getPos() throws IOException {
return fsDataOutputStream.getPos();
}
@Override
public void write(int b) throws IOException {
fsDataOutputStream.write(b);
}
@Override
public void write(byte[] b) throws IOException {
fsDataOutputStream.write(b);
}
@Override
public void close() throws IOException {
fsDataOutputStream.close();
}
}

40
src/main/java/cn/edu/thu/tsfiledb/metadata/ColumnSchema.java Normal file
View File

@ -0,0 +1,40 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
public class ColumnSchema implements Serializable {
private static final long serialVersionUID = -8257474930341487207L;
public String name;
public TSDataType dataType;
public TSEncoding encoding;
private Map<String, String> args;
public ColumnSchema(String name, TSDataType dataType, TSEncoding encoding) {
this.name = name;
this.dataType = dataType;
this.encoding = encoding;
this.args = new HashMap<>();
}
public void putKeyValueToArgs(String key, String value) {
this.args.put(key, value);
}
public String getValueFromArgs(String key) {
return args.get(key);
}
public Map<String, String> getArgsMap() {
return args;
}
public void setArgsMap(Map<String, String> argsMap) {
this.args = argsMap;
}
}

250
src/main/java/cn/edu/thu/tsfiledb/metadata/MGraph.java Normal file
View File

@ -0,0 +1,250 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* Metadata Graph consists of one {@code MTree} and several {@code PTree}
*
* @author Jinrui Zhang
*
*/
public class MGraph implements Serializable {
private static final long serialVersionUID = 8214849219614352834L;
private MTree mTree;
private HashMap<String, PTree> pTreeMap;
public MGraph(String MTreeName) {
mTree = new MTree(MTreeName);
pTreeMap = new HashMap<>();
}
/**
* Add a {@code PTree} to current {@code MGraph}
* @throws MetadataArgsErrorException
*/
public void addAPTree(String pTreeRootName) throws MetadataArgsErrorException {
if(pTreeRootName.toLowerCase().equals("root")){
throw new MetadataArgsErrorException("Property Tree's root name should not be 'root'");
}
PTree pTree = new PTree(pTreeRootName, mTree);
pTreeMap.put(pTreeRootName, pTree);
}
/**
* Add a path to Metadata Tree
*
* @param path Format: root.node.(node)*
* @return The count of new nodes added
* @throws MetadataArgsErrorException
* @throws PathErrorException
*/
public int addPathToMTree(String path, String dataType, String encoding, String args[])
throws PathErrorException, MetadataArgsErrorException {
String nodes[] = path.trim().split("\\.");
if (nodes.length == 0) {
throw new PathErrorException("Path is null. Path: " + path);
}
return this.mTree.addPath(path, dataType, encoding, args);
}
/**
* Add a path to {@code PTree}
*/
public void addPathToPTree(String path) throws PathErrorException, MetadataArgsErrorException {
String nodes[] = path.trim().split("\\.");
if (nodes.length == 0) {
throw new PathErrorException("Path is null. Path: " + path);
}
String rootName = path.trim().split("\\.")[0];
if (pTreeMap.containsKey(rootName)) {
PTree pTree = pTreeMap.get(rootName);
pTree.addPath(path);
} else {
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
}
}
/**
* Delete path in current MGraph.
* @param path a path belongs to MTree or PTree
* @throws PathErrorException
*/
public void deletePath(String path) throws PathErrorException {
String nodes[] = path.trim().split("\\.");
if (nodes.length == 0) {
throw new PathErrorException("Path is null. Path: " + path);
}
String rootName = path.trim().split("\\.")[0];
if (mTree.getRoot().getName().equals(rootName)) {
mTree.deletePath(path);
} else if (pTreeMap.containsKey(rootName)) {
PTree pTree = pTreeMap.get(rootName);
pTree.deletePath(path);
} else {
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
}
}
/**
* Link a {@code MNode} to a {@code PNode} in current PTree
*/
public void linkMNodeToPTree(String path, String mPath) throws PathErrorException {
String pTreeName = path.trim().split("\\.")[0];
if (!pTreeMap.containsKey(pTreeName)) {
throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path);
} else {
pTreeMap.get(pTreeName).linkMNode(path, mPath);
}
}
/**
* Unlink a {@code MNode} from a {@code PNode} in current PTree
*/
public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException {
String pTreeName = path.trim().split("\\.")[0];
if (!pTreeMap.containsKey(pTreeName)) {
throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path);
} else {
pTreeMap.get(pTreeName).unlinkMNode(path, mPath);
}
}
/**
* Set storage level for current Metadata Tree.
* @param path Format: root.node.(node)*
* @throws PathErrorException
*/
public void setStorageLevel(String path) throws PathErrorException {
mTree.setStorageLevel(path);
}
/**
* Get all paths for given path regular expression if given path belongs to
* MTree, or get all linked path for given path if given path belongs to
* PTree Notice: Regular expression in this method is formed by the
* amalgamation of path and the character '*'
*
* @return A HashMap whose Keys are separated by the storage file name.
*/
public HashMap<String, ArrayList<String>> getAllPathGroupByFilename(String path) throws PathErrorException {
String rootName = path.trim().split("\\.")[0];
if (mTree.getRoot().getName().equals(rootName)) {
return mTree.getAllPath(path);
} else if (pTreeMap.containsKey(rootName)) {
PTree pTree = pTreeMap.get(rootName);
return pTree.getAllLinkedPath(path);
}
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
}
/**
* Get all DeltaObject type in current Metadata Tree
* @return a HashMap contains all distinct DeltaObject type separated by
* DeltaObject Type
*/
public Map<String, List<ColumnSchema>> getSchemaForAllType() throws PathErrorException {
Map<String, List<ColumnSchema>> res = new HashMap<>();
List<String> typeList = mTree.getAllType();
for (String type : typeList) {
res.put(type, getSchemaForOneType("root." + type));
}
return res;
}
private ArrayList<String> getDeltaObjectForOneType(String type) throws PathErrorException {
return mTree.getDeltaObjectForOneType(type);
}
/**
* Get all delta objects group by DeltaObject type
*/
public Map<String, List<String>> getDeltaObjectForAllType() throws PathErrorException {
Map<String, List<String>> res = new HashMap<>();
ArrayList<String> types = mTree.getAllType();
for (String type : types) {
res.put(type, getDeltaObjectForOneType(type));
}
return res;
}
/**
* Get the full Metadata info.
* @return A {@code Metadata} instance which stores all metadata info
*/
public Metadata getMetadata() throws PathErrorException {
Map<String, List<ColumnSchema>> seriesMap = getSchemaForAllType();
Map<String, List<String>> deltaObjectMap = getDeltaObjectForAllType();
Metadata metadata = new Metadata(seriesMap, deltaObjectMap);
return metadata;
}
/**
* Get all ColumnSchemas for given delta object type
* @param path A path represented one Delta object
* @return a list contains all column schema
*/
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
return mTree.getSchemaForOneType(path);
}
/**
* Calculate the count of storage-level nodes included in given path
* @return The total count of storage-level nodes.
*/
public int getFileCountForOneType(String path) throws PathErrorException {
return mTree.getFileCountForOneType(path);
}
/**
* Get the file name for given path Notice: This method could be called if
* and only if the path includes one node whose {@code isStorageLevel} is
* true
*/
public String getFileNameByPath(String path) throws PathErrorException {
return mTree.getFileNameByPath(path);
}
/**
* Check whether the path given exists
*/
public boolean pathExist(String path) {
return mTree.hasPath(path);
}
/**
* Extract the DeltaObjectId from given path
* @return String represents the DeltaObjectId
*/
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
return mTree.getDeltaObjectTypeByPath(path);
}
/**
* Get ColumnSchema for given path. Notice: Path must be a complete Path
* from root to leaf node.
*/
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
return mTree.getSchemaForOnePath(path);
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("=== MetaData Tree ===\n\n");
sb.append(mTree.toString());
sb.append("\n\n=== Properties Tree === Size : " + pTreeMap.size() + "\n\n");
for (String key : pTreeMap.keySet()) {
sb.append("--- name : " + key + "---\n");
sb.append(pTreeMap.get(key).toString());
sb.append("\n\n");
}
return sb.toString();
}
}

392
src/main/java/cn/edu/thu/tsfiledb/metadata/MManager.java Normal file
View File

@ -0,0 +1,392 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import cn.edu.thu.tsfile.common.conf.TSFileDescriptor;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.timeseries.read.qp.Path;
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* This class takes the responsibility of serialization of all the metadata info
* and persistent it into files. This class contains all the interfaces to
* modify the metadata for delta system. All the operations will be write into
* the logs temporary in case the downtime of the delta system.
*
* @author Jinrui Zhang
*
*/
public class MManager {
private static MManager manager = new MManager();
// The file storing the serialize info for metadata
private String datafilePath;
// log file path
private String logFilePath;
private MGraph mGraph;
private BufferedWriter bw;
private boolean writeToLog;
private MManager() {
writeToLog = false;
String folderPath = TSFileDescriptor.getInstance().getConfig().metadataDir;
datafilePath = folderPath + "/mdata.obj";
logFilePath = folderPath + "/mlog.txt";
init();
}
private void init() {
try {
File file = new File(datafilePath);
// inital MGraph from file
if (file.exists()) {
FileInputStream fis = new FileInputStream(file);
ObjectInputStream ois = new ObjectInputStream(fis);
mGraph = (MGraph) ois.readObject();
ois.close();
fis.close();
} else {
mGraph = new MGraph("root");
}
// recover operation from log file
File logFile = new File(logFilePath);
if (logFile.exists()) {
FileReader fr;
fr = new FileReader(logFile);
BufferedReader br = new BufferedReader(fr);
String cmd;
while ((cmd = br.readLine()) != null) {
operation(cmd);
}
br.close();
} else {
if (!logFile.getParentFile().exists()) {
logFile.getParentFile().mkdirs();
}
logFile.createNewFile();
}
FileWriter fw = new FileWriter(logFile, true);
bw = new BufferedWriter(fw);
writeToLog = true;
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (PathErrorException e) {
e.printStackTrace();
} catch (MetadataArgsErrorException e) {
e.printStackTrace();
}
}
/**
* Clear all metadata info
*/
public void clear() {
this.mGraph = new MGraph("root");
}
public void deleteLogAndDataFiles() {
File file = new File(logFilePath);
if (file.exists()) {
file.delete();
}
File dataFile = new File(datafilePath);
if (dataFile.exists()) {
dataFile.delete();
}
}
private void operation(String cmd) throws PathErrorException, IOException, MetadataArgsErrorException {
String args[] = cmd.trim().split(",");
if (args[0].equals("0")) {
String[] leftArgs;
if (args.length > 4) {
leftArgs = new String[args.length - 4];
for (int k = 4; k < args.length; k++) {
leftArgs[k - 4] = args[k];
}
} else {
leftArgs = new String[0];
}
addAPathToMTree(args[1], args[2], args[3], leftArgs);
} else if (args[0].equals(MetadataOperationType.ADD_PATH_TO_MTREE)) {
deletePathFromMTree(args[1]);
} else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_MTREE)) {
setStorageLevelToMTree(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE);
} else if (args[0].equals(MetadataOperationType.ADD_A_PTREE)) {
addAPTree(args[1]);
} else if (args[0].equals(MetadataOperationType.ADD_A_PATH_TO_PTREE)) {
addAPathToPTree(args[1]);
} else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_PTREE)) {
deletePathFromPTree(args[1]);
} else if (args[0].equals(MetadataOperationType.LINK_MNODE_TO_PTREE)) {
linkMNodeToPTree(args[1], args[2]);
} else if (args[0].equals(MetadataOperationType.UNLINK_MNODE_FROM_PTREE)) {
unlinkMNodeFromPTree(args[1], args[2]);
}
}
/**
* operation: Add a path to Metadata Tree
*/
public int addAPathToMTree(String path, String dataType, String encoding, String[] args)
throws PathErrorException, IOException, MetadataArgsErrorException {
int addCount = mGraph.addPathToMTree(path, dataType, encoding, args);
if (writeToLog) {
bw.write(MetadataOperationType.ADD_PATH_TO_MTREE + "," + path + "," + dataType + "," + encoding);
for (int i = 0; i < args.length; i++) {
bw.write("," + args[i]);
}
bw.newLine();
bw.flush();
}
return addCount;
}
public void deletePathFromMTree(String path) throws PathErrorException, IOException {
mGraph.deletePath(path);
if (writeToLog) {
bw.write(MetadataOperationType.DELETE_PATH_FROM_MTREE + "," + path);
bw.newLine();
bw.flush();
}
}
public void setStorageLevelToMTree(String path) throws PathErrorException, IOException {
mGraph.setStorageLevel(path);
if (writeToLog) {
bw.write(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE + "," + path);
bw.newLine();
bw.flush();
}
}
public void addAPTree(String pTreeRootName) throws IOException, MetadataArgsErrorException {
mGraph.addAPTree(pTreeRootName);
if (writeToLog) {
bw.write(MetadataOperationType.ADD_A_PTREE + "," + pTreeRootName);
bw.newLine();
bw.flush();
}
}
public void addAPathToPTree(String path) throws PathErrorException, IOException, MetadataArgsErrorException {
mGraph.addPathToPTree(path);
if (writeToLog) {
bw.write(MetadataOperationType.ADD_A_PATH_TO_PTREE + "," + path);
bw.newLine();
bw.flush();
}
}
public void deletePathFromPTree(String path) throws PathErrorException, IOException {
mGraph.deletePath(path);
if (writeToLog) {
bw.write(MetadataOperationType.DELETE_PATH_FROM_PTREE + "," + path);
bw.newLine();
bw.flush();
}
}
public void linkMNodeToPTree(String path, String mPath) throws PathErrorException, IOException {
mGraph.linkMNodeToPTree(path, mPath);
if (writeToLog) {
bw.write(MetadataOperationType.LINK_MNODE_TO_PTREE + "," + path + "," + mPath);
bw.newLine();
bw.flush();
}
}
public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException, IOException {
mGraph.unlinkMNodeFromPTree(path, mPath);
if (writeToLog) {
bw.write(MetadataOperationType.UNLINK_MNODE_FROM_PTREE + "," + path + "," + mPath);
bw.newLine();
bw.flush();
}
}
/**
* Extract the DeltaObjectId from given path
* @param path
* @return String represents the DeltaObjectId
*/
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
return mGraph.getDeltaObjectTypeByPath(path);
}
/**
* Get series type for given path
*
* @param fullPath
* @return TSDataType
* @throws PathErrorException
*/
public TSDataType getSeriesType(String fullPath) throws PathErrorException {
return getSchemaForOnePath(fullPath).dataType;
}
/**
* Get all DeltaObject type in current Metadata Tree
*
* @return a HashMap contains all distinct DeltaObject type separated by
* DeltaObject Type
* @throws PathErrorException
*/
public Map<String, List<ColumnSchema>> getSchemaForAllType() throws PathErrorException {
return mGraph.getSchemaForAllType();
}
/**
* Get the full Metadata info.
*
* @return A {@code Metadata} instance which stores all metadata info
* @throws PathErrorException
*/
public Metadata getMetadata() throws PathErrorException {
return mGraph.getMetadata();
}
/**
* Get all ColumnSchemas for given delta object type
*
* @param path A path represented one Delta object
* @return a list contains all column schema
* @throws PathErrorException
*/
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
return mGraph.getSchemaForOneType(path);
}
/**
* Calculate the count of storage-level nodes included in given path
* @param path
* @return The total count of storage-level nodes.
* @throws PathErrorException
*/
public int getFileCountForOneType(String path) throws PathErrorException {
return mGraph.getFileCountForOneType(path);
}
/**
* Get the file name for given path Notice: This method could be called if
* and only if the path includes one node whose {@code isStorageLevel} is
* true
*
* @param path
* @return A String represented the file name
* @throws PathErrorException
*/
public String getFileNameByPath(String path) throws PathErrorException {
return mGraph.getFileNameByPath(path);
}
/**
* return a HashMap contains all the paths separated by File Name
*/
public HashMap<String, ArrayList<String>> getAllPathGroupByFileName(String path) throws PathErrorException {
return mGraph.getAllPathGroupByFilename(path);
}
/**
* return all paths for given path if the path is abstract.Or return the
* path itself.
*/
public ArrayList<String> getPaths(String path) throws PathErrorException {
ArrayList<String> res = new ArrayList<>();
HashMap<String, ArrayList<String>> pathsGroupByFilename = getAllPathGroupByFileName(path);
for (ArrayList<String> ps : pathsGroupByFilename.values()) {
res.addAll(ps);
}
return res;
}
/**
* Check whether the path given exists
*/
public boolean pathExist(String path) {
return mGraph.pathExist(path);
}
/**
* Get ColumnSchema for given path. Notice: Path must be a complete Path
* from root to leaf node.
*/
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
return mGraph.getSchemaForOnePath(path);
}
/**
* Check whether given path contains a MNode whose
* {@code MNode.isStorageLevel} is true
*/
public boolean checkFileLevel(List<Path> path) throws PathErrorException {
for (Path p : path) {
getFileNameByPath(p.getFullPath());
}
return true;
}
/**
* Persistent the MGraph instance into file
*/
public void flushObjectToFile() throws IOException {
File newDataFile = new File(datafilePath + ".backup");
FileOutputStream fos = new FileOutputStream(newDataFile);
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(mGraph);
oos.close();
// delete log file
File logFile = new File(logFilePath);
logFile.delete();
// Copy New DataFile to OldDataFile
FileOutputStream oldFileFos = new FileOutputStream(datafilePath);
FileChannel outChannel = oldFileFos.getChannel();
FileInputStream newFileFis = new FileInputStream(datafilePath + ".backup");
FileChannel inChannel = newFileFis.getChannel();
inChannel.transferTo(0, inChannel.size(), outChannel);
outChannel.close();
inChannel.close();
oldFileFos.close();
newFileFis.close();
// delete DataFileBackUp
newDataFile.delete();
}
public String getMetadataInString() {
return mGraph.toString();
}
public static MManager getInstance() {
return manager;
}
}

149
src/main/java/cn/edu/thu/tsfiledb/metadata/MNode.java Normal file
View File

@ -0,0 +1,149 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.LinkedHashMap;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
/**
* This class is the implementation of Metadata Node where "MNode" is the
* shorthand of "Metadata Node". One MNode instance represents one node in the
* Metadata Tree
*
* @author Jinrui Zhang
*
*/
public class MNode implements Serializable {
private static final long serialVersionUID = -770028375899514063L;
// The name of the MNode
private String name;
// Whether current node is a leaf in the Metadata Tree
private boolean isLeaf;
// Whether current node is Storage Level in the Metadata Tree
private boolean isStorageLevel;
// Corresponding data file name for current node
private String dataFileName;
// Column's Schema for one timeseries represented by current node if current
// node is one leaf
private ColumnSchema schema;
private MNode parent;
private LinkedHashMap<String, MNode> children;
public MNode(String name, MNode parent, boolean isLeaf) {
this.setName(name);
this.parent = parent;
this.isLeaf = isLeaf;
this.isStorageLevel = false;
if (!isLeaf) {
children = new LinkedHashMap<>();
}
}
public MNode(String name, MNode parent, TSDataType dataType, TSEncoding encoding) {
this(name, parent, true);
this.schema = new ColumnSchema(name, dataType, encoding);
}
public boolean isStorageLevel() {
return isStorageLevel;
}
public void setStorageLevel(boolean b) {
this.isStorageLevel = b;
}
public boolean isLeaf() {
return isLeaf;
}
public void setLeaf(boolean isLeaf) {
this.isLeaf = isLeaf;
}
public boolean hasChild(String key) {
if (!isLeaf) {
return this.children.containsKey(key);
}
return false;
}
public void addChild(String key, MNode child) {
if (!isLeaf) {
this.children.put(key, child);
}
}
public void deleteChild(String key) {
children.remove(key);
}
public MNode getChild(String key) {
if (!isLeaf) {
return children.get(key);
}
return null;
}
/**
* @return the count of all leaves whose ancestor is current node
*/
public int getLeafCount() {
if (isLeaf) {
return 1;
} else {
int leafCount = 0;
for (MNode child : this.children.values()) {
leafCount += child.getLeafCount();
}
return leafCount;
}
}
public String getDataFileName() {
return dataFileName;
}
public void setDataFileName(String dataFileName) {
this.dataFileName = dataFileName;
}
public String toString() {
return this.getName();
}
public ColumnSchema getSchema() {
return schema;
}
public void setSchema(ColumnSchema schema) {
this.schema = schema;
}
public MNode getParent() {
return parent;
}
public void setParent(MNode parent) {
this.parent = parent;
}
public LinkedHashMap<String, MNode> getChildren() {
return children;
}
public void setChildren(LinkedHashMap<String, MNode> children) {
this.children = children;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

487
src/main/java/cn/edu/thu/tsfiledb/metadata/MTree.java Normal file
View File

@ -0,0 +1,487 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* The hierarchical struct of the Metadata Tree is implemented in this class.
*
* @author Jinrui Zhang
*
*/
public class MTree implements Serializable {
private static final long serialVersionUID = -4200394435237291964L;
private final String space = " ";
private MNode root;
public MTree(String rootName) {
this.setRoot(new MNode(rootName, null, false));
}
public MTree(MNode root) {
this.setRoot(root);
}
/**
* Add a path to current Metadata Tree
* @param path Format: root.node.(node)*
*/
public int addPath(String path, String dataType, String encoding, String[] args)
throws PathErrorException, MetadataArgsErrorException {
int addCount = 0;
if (getRoot() == null) {
throw new PathErrorException("Root Node is null, Please initialize root first");
}
String[] nodeNames = path.trim().split("\\.");
if (nodeNames.length <= 1 || !nodeNames[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path not exist. Path: " + path);
}
MNode cur = getRoot();
int i;
for (i = 1; i < nodeNames.length - 1; i++) {
if (!cur.hasChild(nodeNames[i])) {
cur.addChild(nodeNames[i], new MNode(nodeNames[i], cur, false));
addCount++;
}
cur = cur.getChild(nodeNames[i]);
}
if (cur.hasChild(nodeNames[i])) {
throw new PathErrorException("Path already exists. Path: " + path);
} else {
TSDataType dt = TSDataType.valueOf(dataType);
TSEncoding ed = TSEncoding.valueOf(encoding);
MNode leaf = new MNode(nodeNames[i], cur, dt, ed);
if (args.length > 0) {
for (int k = 0; k < args.length; k++) {
String[] arg = args[k].split("=");
leaf.getSchema().putKeyValueToArgs(arg[0], arg[1]);
}
}
cur.addChild(nodeNames[i], leaf);
addCount++;
}
return addCount;
}
/**
* Delete one path from current Metadata Tree
* @param path
* Format: root.node.(node)* Notice: Path must be a complete Path
* from root to leaf node.
*/
public void deletePath(String path) throws PathErrorException {
String[] nodes = path.split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Path not correct. Path:" + path);
}
MNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (!cur.hasChild(nodes[i])) {
throw new PathErrorException(
"Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]);
}
cur = cur.getChild(nodes[i]);
}
cur.getParent().deleteChild(cur.getName());
}
/**
* Set filename to one node in the Metadata Tree Notice: This method will set
* the {@code MNode.dataFileName} and {@code MNode.isStorageLevel} of current node
* called if and only if {@code MNode.isStorageLevel} is true.
* @param path Format: root.node.(node)*
*/
private void setFileNameToOnePath(String path) throws PathErrorException {
String nodes[] = path.trim().split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
MNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (cur.hasChild(nodes[i])) {
cur = cur.getChild(nodes[i]);
} else {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
}
cur.setStorageLevel(true);
setFileNameToMNode(cur, path);
}
/**
* Set storage level for current Metadata Tree.
* @param path Format: root.node.(node)*
*/
public void setStorageLevel(String path) throws PathErrorException {
String nodes[] = path.trim().split("\\.");
if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Root Name NOT correct. Root:" + path);
}
int level = 0;
MNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (cur.hasChild(nodes[i])) {
cur = cur.getChild(nodes[i]);
level++;
} else {
throw new PathErrorException("Input path NOT correct. Nodes[" + i + "] not correct :" + nodes[i]);
}
}
cur = getRoot().getChild(nodes[1]);
setFileName(1, level, cur, "root." + cur.getName());
}
private void setFileName(int curLevel, int level, MNode node, String path) throws PathErrorException {
if (curLevel == level) {
setFileNameToOnePath(path);
return;
}
for (MNode child : node.getChildren().values()) {
setFileName(curLevel + 1, level, child, path + "." + child.getName());
}
}
private void setFileNameToMNode(MNode node, String fileName) {
node.setDataFileName(fileName);
if (!node.isLeaf()) {
for (MNode child : node.getChildren().values()) {
setFileNameToMNode(child, fileName);
}
}
}
/**
* Check whether the path given exists
*/
public boolean hasPath(String path) {
String[] nodes = path.split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
return false;
}
return hasPath(getRoot(), nodes, 1);
}
private boolean hasPath(MNode node, String[] nodes, int idx) {
if (idx >= nodes.length) {
return true;
}
if (nodes[idx].equals("*")) {
boolean res = false;
for (MNode child : node.getChildren().values()) {
res |= hasPath(child, nodes, idx + 1);
}
return res;
} else {
if (node.hasChild(nodes[idx])) {
return hasPath(node.getChild(nodes[idx]), nodes, idx + 1);
}
return false;
}
}
/**
* Get ColumnSchema for given path. Notice: Path must be a complete Path
* from root to leaf node.
*/
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
MNode leaf = getLeafByPath(path);
return leaf.getSchema();
}
private MNode getLeafByPath(String path) throws PathErrorException {
checkPath(path);
String[] node = path.split("\\.");
MNode cur = getRoot();
for (int i = 1; i < node.length; i++) {
cur = cur.getChild(node[i]);
}
if (!cur.isLeaf()) {
throw new PathErrorException("Input path not end with a leaf node.");
}
return cur;
}
/**
* Extract the DeltaObjectType from given path
* @return String represents the DeltaObjectId
*/
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
checkPath(path);
String[] nodes = path.split("\\.");
if (nodes.length < 2) {
throw new PathErrorException("Input path must has two or more nodes");
}
return nodes[0] + "." + nodes[1];
}
/**
* Check whether a path is available
*/
private void checkPath(String path) throws PathErrorException {
String[] nodes = path.split("\\.");
if (nodes.length < 2 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
MNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (!cur.hasChild(nodes[i])) {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
cur = cur.getChild(nodes[i]);
}
}
/**
* Get the file name for given path Notice: This method could be called if
* and only if the path includes one node whose {@code isStorageLevel} is
* true
*/
public String getFileNameByPath(String path) throws PathErrorException {
String[] nodes = path.split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
MNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (!cur.hasChild(nodes[i])) {
throw new PathErrorException("Input path NOT correct. Path: " + path);
}
if (cur.getDataFileName() != null) {
return cur.getDataFileName();
}
cur = cur.getChild(nodes[i]);
}
if (cur.getDataFileName() != null) {
return cur.getDataFileName();
}
throw new PathErrorException("Do not set FileLevel for this path: " + path);
}
/**
* Get all paths for given path regular expression Regular expression in
* this method is formed by the amalgamation of path and the character '*'
* @return A HashMap whose Keys are separated by the storage file name.
*/
public HashMap<String, ArrayList<String>> getAllPath(String pathReg) throws PathErrorException {
HashMap<String, ArrayList<String>> paths = new HashMap<>();
String[] nodes = pathReg.split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path NOT correct. PathReg: " + pathReg);
}
findPath(getRoot(), nodes, 1, "", paths);
return paths;
}
public ArrayList<String> getAllPathInList(String path) throws PathErrorException{
ArrayList<String> res = new ArrayList<>();
HashMap<String, ArrayList<String>> mapRet = getAllPath(path);
for(ArrayList<String> value : mapRet.values()){
res.addAll(value);
}
return res;
}
/**
* Calculate the count of storage-level nodes included in given path
* @return The total count of storage-level nodes.
*/
public int getFileCountForOneType(String path) throws PathErrorException {
String nodes[] = path.split("\\.");
if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) {
throw new PathErrorException(
"Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)");
}
return getFileCountForOneNode(getRoot().getChild(nodes[1]));
}
private int getFileCountForOneNode(MNode node) {
if (node.isStorageLevel()) {
return 1;
}
int sum = 0;
if (!node.isLeaf()) {
for (MNode child : node.getChildren().values()) {
sum += getFileCountForOneNode(child);
}
}
return sum;
}
/**
* Get all DeltaObject type in current Metadata Tree
* @return a list contains all distinct DeltaObject type
*/
public ArrayList<String> getAllType() {
ArrayList<String> res = new ArrayList<>();
if (getRoot() != null) {
for (String type : getRoot().getChildren().keySet()) {
res.add(type);
}
}
return res;
}
/**
* Get all delta objects for given type
* @param type DeltaObject Type
* @return a list contains all delta objects for given type
* @throws PathErrorException
*/
public ArrayList<String> getDeltaObjectForOneType(String type) throws PathErrorException {
String path = getRoot().getName() + "." + type;
checkPath(path);
HashMap<String, Integer> deltaObjectMap = new HashMap<>();
MNode typeNode = getRoot().getChild(type);
putDeltaObjectToMap(getRoot().getName(), typeNode, deltaObjectMap);
ArrayList<String> res = new ArrayList<>();
res.addAll(deltaObjectMap.keySet());
return res;
}
private void putDeltaObjectToMap(String path, MNode node, HashMap<String, Integer> deltaObjectMap) {
if (node.isLeaf()) {
deltaObjectMap.put(path, 1);
} else {
for (String child : node.getChildren().keySet()) {
String newPath = path + "." + node.getName();
putDeltaObjectToMap(newPath, node.getChildren().get(child), deltaObjectMap);
}
}
}
/**
* Get all ColumnSchemas for given delta object type
* @param path A path represented one Delta object
* @return a list contains all column schema
* @throws PathErrorException
*/
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
String nodes[] = path.split("\\.");
if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) {
throw new PathErrorException(
"Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)");
}
HashMap<String, ColumnSchema> leafMap = new HashMap<>();
putLeafToLeafMap(getRoot().getChild(nodes[1]), leafMap);
ArrayList<ColumnSchema> res = new ArrayList<>();
res.addAll(leafMap.values());
return res;
}
private void putLeafToLeafMap(MNode node, HashMap<String, ColumnSchema> leafMap) {
if (node.isLeaf()) {
if (!leafMap.containsKey(node.getName())) {
leafMap.put(node.getName(), node.getSchema());
}
return;
}
for (MNode child : node.getChildren().values()) {
putLeafToLeafMap(child, leafMap);
}
}
private void findPath(MNode node, String[] nodes, int idx, String parent, HashMap<String, ArrayList<String>> paths)
throws PathErrorException {
if (node.isLeaf()) {
String fileName = node.getDataFileName();
String nodePath = parent + node;
putAPath(paths, fileName, nodePath);
return;
}
String nodeReg;
if (idx >= nodes.length) {
nodeReg = "*";
} else {
nodeReg = nodes[idx];
}
if (!nodeReg.equals("*")) {
if (!node.hasChild(nodeReg)) {
throw new PathErrorException("Path Error in node : " + nodeReg);
}
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths);
} else {
for (MNode child : node.getChildren().values()) {
findPath(child, nodes, idx + 1, parent + node.getName() + ".", paths);
}
}
}
private void putAPath(HashMap<String, ArrayList<String>> paths, String fileName, String nodePath) {
if (paths.containsKey(fileName)) {
paths.get(fileName).add(nodePath);
} else {
ArrayList<String> pathList = new ArrayList<>();
pathList.add(nodePath);
paths.put(fileName, pathList);
}
}
public String toString() {
return MNodeToString(getRoot(), 0);
}
private String MNodeToString(MNode node, int tab) {
String s = "";
for (int i = 0; i < tab; i++) {
s += space;
}
s += node.getName();
if (!node.isLeaf() && node.getChildren().size() > 0) {
s += ":{\n";
int first = 0;
for (MNode child : node.getChildren().values()) {
if (first == 0) {
first = 1;
} else {
s += ",\n";
}
s += MNodeToString(child, tab + 1);
}
s += "\n";
for (int i = 0; i < tab; i++) {
s += space;
}
s += "}";
} else if (node.isLeaf()) {
s += ":{\n";
s += getTabs(tab + 1) + "DataType :" + node.getSchema().dataType + ",\n";
s += getTabs(tab + 1) + "Encoding :" + node.getSchema().encoding + ",\n";
s += getTabs(tab + 1) + "args :" + node.getSchema().getArgsMap() + ",\n";
s += getTabs(tab + 1) + "FileName :" + node.getDataFileName() + "\n";
s += getTabs(tab) + "}";
}
return s;
}
private String getTabs(int count) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < count; i++) {
sb.append(space);
}
return sb.toString();
}
public MNode getRoot() {
return root;
}
public void setRoot(MNode root) {
this.root = root;
}
}

53
src/main/java/cn/edu/thu/tsfiledb/metadata/Metadata.java Normal file
View File

@ -0,0 +1,53 @@
package cn.edu.thu.tsfiledb.metadata;
import java.util.List;
import java.util.Map;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* This class stores all the metadata info for every deltaObject and every
* timeseries
*
* @author Jinrui Zhang
*
*/
public class Metadata {
private Map<String, List<ColumnSchema>> seriesMap;
private Map<String, List<String>> deltaObjectMap;
public Metadata(Map<String, List<ColumnSchema>> seriesMap, Map<String, List<String>> deltaObjectMap) {
this.seriesMap = seriesMap;
this.deltaObjectMap = deltaObjectMap;
}
public List<ColumnSchema> getSeriesForOneType(String type) throws PathErrorException {
if (this.seriesMap.containsKey(type)) {
return seriesMap.get(type);
} else {
throw new PathErrorException("Input DeltaObjectType is not exist. " + type);
}
}
public List<String> getDeltaObjectsForOneType(String type) throws PathErrorException {
if (this.seriesMap.containsKey(type)) {
return deltaObjectMap.get(type);
} else {
throw new PathErrorException("Input DeltaObjectType is not exist. " + type);
}
}
public Map<String, List<ColumnSchema>> getSeriesMap() {
return seriesMap;
}
public Map<String, List<String>> getDeltaObjectMap() {
return deltaObjectMap;
}
@Override
public String toString() {
return seriesMap.toString() + "\n" + deltaObjectMap.toString();
}
}

12
src/main/java/cn/edu/thu/tsfiledb/metadata/MetadataOperationType.java Normal file
View File

@ -0,0 +1,12 @@
package cn.edu.thu.tsfiledb.metadata;
public class MetadataOperationType {
public final static String ADD_PATH_TO_MTREE = "0";
public final static String DELETE_PATH_FROM_MTREE = "1";
public final static String SET_STORAGE_LEVEL_TO_MTREE = "2";
public final static String ADD_A_PTREE = "3";
public final static String ADD_A_PATH_TO_PTREE = "4";
public final static String DELETE_PATH_FROM_PTREE = "5";
public final static String LINK_MNODE_TO_PTREE = "6";
public final static String UNLINK_MNODE_FROM_PTREE = "7";
}

113
src/main/java/cn/edu/thu/tsfiledb/metadata/PNode.java Normal file
View File

@ -0,0 +1,113 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* PNode is the shorthand for "Property Node", which make up The {@code PTree}
* @author zhangjinrui
*
*/
public class PNode implements Serializable{
private static final long serialVersionUID = -7166236304286006338L;
private String name;
private PNode parent;
private HashMap<String, PNode> children;
private boolean isLeaf;
/**
* This HashMap contains all the {@code MNode} this {@code PNode} is responsible for
*/
private LinkedHashMap<String, Integer> linkedMTreePathMap;
public PNode(String name, PNode parent, boolean isLeaf) {
this.name = name;
this.parent = parent;
this.isLeaf = isLeaf;
if (!isLeaf) {
setChildren(new HashMap<>());
} else {
linkedMTreePathMap = new LinkedHashMap<>();
}
}
public void linkMPath(String mTreePath) throws PathErrorException{
if(!isLeaf){
throw new PathErrorException("Current PNode is NOT leaf node");
}
if(!linkedMTreePathMap.containsKey(mTreePath)){
linkedMTreePathMap.put(mTreePath, 1);
}
}
public void unlinkMPath(String mTreePath) throws PathErrorException{
if(!isLeaf){
throw new PathErrorException("Current PNode is NOT leaf node");
}
if(!linkedMTreePathMap.containsKey(mTreePath)){
return;
}
linkedMTreePathMap.remove(mTreePath);
}
public boolean hasChild(String key) {
return getChildren().containsKey(key);
}
public PNode getChild(String key) {
return getChildren().get(key);
}
public void addChild(String name, PNode node) {
this.getChildren().put(name, node);
}
public void deleteChild(String name){
this.getChildren().remove(name);
}
public boolean isLeaf() {
return isLeaf;
}
public void setLeaf(boolean isLeaf) {
this.isLeaf = isLeaf;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public PNode getParent() {
return parent;
}
public void setParent(PNode parent) {
this.parent = parent;
}
public HashMap<String, PNode> getChildren() {
return children;
}
public void setChildren(HashMap<String, PNode> children) {
this.children = children;
}
public HashMap<String, Integer> getLinkedMTreePathMap() {
return linkedMTreePathMap;
}
public void setLinkedMTreePathMap(LinkedHashMap<String, Integer> linkedMTreePathMap) {
this.linkedMTreePathMap = linkedMTreePathMap;
}
}

241
src/main/java/cn/edu/thu/tsfiledb/metadata/PTree.java Normal file
View File

@ -0,0 +1,241 @@
package cn.edu.thu.tsfiledb.metadata;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import cn.edu.thu.tsfiledb.exception.PathErrorException;
/**
* "PTree" is the shorthand for "Property Tree". One {@code PTree} consists
* several {@code PNode}
*
* @author Jinrui Zhang
*
*/
public class PTree implements Serializable {
private static final long serialVersionUID = 2642766399323283900L;
private PNode root;
private MTree mTree;
private String name;
private String space = " ";
public PTree(String name, MTree mTree) {
this.setRoot(new PNode(name, null, false));
this.setName(name);
this.setmTree(mTree);
}
public PTree(String name, PNode root, MTree mTree) {
this.setName(name);
this.setRoot(root);
this.setmTree(mTree);
}
/**
* Add a path to current PTree
* @return The count of new added {@code PNode}
* @throws PathErrorException
*/
public int addPath(String path) throws PathErrorException {
int addCount = 0;
if (getRoot() == null) {
throw new PathErrorException("Root Node is null, Please initialize root first");
}
String[] nodes = path.trim().split("\\.");
if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Input path not exist. Path: " + path);
}
PNode cur = getRoot();
int i;
for (i = 1; i < nodes.length - 1; i++) {
if (!cur.hasChild(nodes[i])) {
cur.addChild(nodes[i], new PNode(nodes[i], cur, false));
addCount++;
}
cur = cur.getChild(nodes[i]);
}
if (cur.hasChild(nodes[i])) {
throw new PathErrorException("Path already exists. Path: " + path);
} else {
PNode node = new PNode(nodes[i], cur, true);
cur.addChild(node.getName(), node);
addCount++;
}
return addCount;
}
/**
* Remove a path from current PTree
* @throws PathErrorException
*/
public void deletePath(String path) throws PathErrorException {
String[] nodes = path.split("\\.");
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
throw new PathErrorException("Path not correct. Path:" + path);
}
PNode cur = getRoot();
for (int i = 1; i < nodes.length; i++) {
if (!cur.hasChild(nodes[i])) {
throw new PathErrorException(
"Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]);
}
cur = cur.getChild(nodes[i]);
}
cur.getParent().deleteChild(cur.getName());
}
/**
* Link a {@code MNode} to a {@code PNode} in current PTree
* @throws PathErrorException
*/
public void linkMNode(String pTreePath, String mTreePath) throws PathErrorException {
List<String> paths = mTree.getAllPathInList(mTreePath);
String nodes[] = pTreePath.trim().split("\\.");
PNode leaf = getLeaf(getRoot(), nodes, 0);
for (String p : paths) {
leaf.linkMPath(p);
}
}
/**
* Unlink a {@code MNode} from a {@code PNode} in current PTree
* @throws PathErrorException
*/
public void unlinkMNode(String pTreePath, String mTreePath) throws PathErrorException {
List<String> paths = mTree.getAllPathInList(mTreePath);
String nodes[] = pTreePath.trim().split("\\.");
PNode leaf = getLeaf(getRoot(), nodes, 0);
for (String p : paths) {
leaf.unlinkMPath(p);
}
}
private PNode getLeaf(PNode node, String[] nodes, int idx) throws PathErrorException {
if (idx >= nodes.length) {
throw new PathErrorException("PTree path not exist. ");
}
if (node.isLeaf()) {
if (idx != nodes.length - 1 || !nodes[idx].equals(node.getName())) {
throw new PathErrorException("PTree path not exist. ");
}
return node;
}else{
if (idx >= nodes.length - 1 || !node.hasChild(nodes[idx + 1])) {
throw new PathErrorException("PTree path not exist. ");
}
return getLeaf(node.getChild(nodes[idx + 1]), nodes, idx + 1);
}
}
/**
*
* @param path
* a path in current {@code PTree} Get all linked path in MTree
* according to the given path in PTree
* @return Paths will be separated by the {@code MNode.dataFileName} in the
* HashMap
* @throws PathErrorException
*/
public HashMap<String, ArrayList<String>> getAllLinkedPath(String path) throws PathErrorException {
String[] nodes = path.trim().split("\\.");
PNode leaf = getLeaf(getRoot(), nodes, 0);
HashMap<String, ArrayList<String>> res = new HashMap<>();
for (String MPath : leaf.getLinkedMTreePathMap().keySet()) {
HashMap<String, ArrayList<String>> tr = getmTree().getAllPath(MPath);
mergePathRes(res, tr);
}
return res;
}
private void mergePathRes(HashMap<String, ArrayList<String>> res, HashMap<String, ArrayList<String>> tr) {
for (String key : tr.keySet()) {
if (!res.containsKey(key)) {
res.put(key, new ArrayList<String>());
}
for (String p : tr.get(key)) {
if (!res.get(key).contains(p)) {
res.get(key).add(p);
}
}
}
}
public String toString() {
return PNodeToString(getRoot(), 0);
}
private String PNodeToString(PNode node, int tab) {
String s = "";
for (int i = 0; i < tab; i++) {
s += space;
}
s += node.getName();
if (!node.isLeaf() && node.getChildren().size() > 0) {
s += ":{\n";
int first = 0;
for (PNode child : node.getChildren().values()) {
if (first == 0) {
first = 1;
} else {
s += ",\n";
}
s += PNodeToString(child, tab + 1);
}
s += "\n";
for (int i = 0; i < tab; i++) {
s += space;
}
s += "}";
} else if (node.isLeaf()) {
s += ":{\n";
String[] linkedPaths = new String[node.getLinkedMTreePathMap().values().size()];
linkedPaths = node.getLinkedMTreePathMap().values().toArray(linkedPaths);
for (int i = 0; i < linkedPaths.length; i++) {
if (i != linkedPaths.length - 1) {
s += getTabs(tab + 1) + linkedPaths[i] + ",\n";
} else {
s += getTabs(tab + 1) + linkedPaths[i] + "\n";
}
}
s += getTabs(tab) + "}";
}
return s;
}
private String getTabs(int count) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < count; i++) {
sb.append(space);
}
return sb.toString();
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public MTree getmTree() {
return mTree;
}
public void setmTree(MTree mTree) {
this.mTree = mTree;
}
public PNode getRoot() {
return root;
}
public void setRoot(PNode root) {
this.root = root;
}
}

38
src/test/java/cn/edu/thu/tsfiledb/AppTest.java Normal file
View File

@ -0,0 +1,38 @@
package cn.edu.thu.tsfiledb;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* Unit test for simple App.
*/
public class AppTest
extends TestCase
{
/**
* Create the test case
*
* @param testName name of the test case
*/
public AppTest( String testName )
{
super( testName );
}
/**
* @return the suite of tests being tested
*/
public static Test suite()
{
return new TestSuite( AppTest.class );
}
/**
* Rigourous Test :-)
*/
public void testApp()
{
assertTrue( true );
}
}