mirror of https://github.com/apache/iotdb
Transfer codes from pre-project
This commit is contained in:
commit
a3a5a392e3
|
@ -0,0 +1,30 @@
|
|||
# Eclipse IDE files
|
||||
**/.classpath
|
||||
**/.project
|
||||
**/.settings/
|
||||
|
||||
# intellij IDE files
|
||||
**/*.iml
|
||||
**/.idea/
|
||||
|
||||
# Apple OS X related
|
||||
**/.DS_Store
|
||||
|
||||
# build generated
|
||||
**/target/
|
||||
**/build/
|
||||
|
||||
# intermediately generated locally
|
||||
**/logs/
|
||||
|
||||
tsfile-timeseries/src/main/resources/logback.out.out.xml
|
||||
tsfile-timeseries/src/main/resources/logback.out.xml
|
||||
tsfile-service/derby-tsfile-db/
|
||||
tsfile-timeseries/src/test/resources/data
|
||||
src/main/resources/metadata/mlog.txt
|
||||
tsfile-jdbc/src/main/resources/output/queryRes.csv
|
||||
|
||||
*.jar
|
||||
*.gz
|
||||
*.tar.gz
|
||||
*.tar
|
|
@ -0,0 +1,72 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>cn.edu.thu</groupId>
|
||||
<artifactId>tsfiledb</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>tsfiledb</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<compiler.version>1.8</compiler.version>
|
||||
<hadoop.version>2.6.0</hadoop.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>cn.edu.thu</groupId>
|
||||
<artifactId>tsfile</artifactId>
|
||||
<version>0.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.3</version>
|
||||
<configuration>
|
||||
<source>${compiler.version}</source>
|
||||
<target>${compiler.version}</target>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
<!-- compile java files in src/it/java -->
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile-integration-test</id>
|
||||
<phase>pre-integration-test</phase>
|
||||
<goals>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<testIncludes>
|
||||
<testInclude>**/*.java</testInclude>
|
||||
</testIncludes>
|
||||
<outputDirectory>${project.build.directory}/it-classes</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
|
@ -0,0 +1,13 @@
|
|||
package cn.edu.thu.tsfiledb;
|
||||
|
||||
/**
|
||||
* Hello world!
|
||||
*
|
||||
*/
|
||||
public class App
|
||||
{
|
||||
public static void main( String[] args )
|
||||
{
|
||||
System.out.println( "Hello World!" );
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
/**
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface Action {
|
||||
void act();
|
||||
}
|
|
@ -0,0 +1,46 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
|
||||
import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData;
|
||||
import cn.edu.thu.tsfile.timeseries.write.io.TSFileIOWriter;
|
||||
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
|
||||
|
||||
/**
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public class BufferWriteIOWriter extends TSFileIOWriter {
|
||||
|
||||
/*
|
||||
* The backup list is used to store the rowgroup's metadata whose data has
|
||||
* been flushed into file.
|
||||
*/
|
||||
private final List<RowGroupMetaData> backUpList = new ArrayList<RowGroupMetaData>();
|
||||
|
||||
public BufferWriteIOWriter(FileSchema schema, TSRandomAccessFileWriter output) throws IOException {
|
||||
super(schema, output);
|
||||
}
|
||||
|
||||
/**
|
||||
* <b>Note that</b>,the method is not thread safe.
|
||||
*/
|
||||
public void addNewRowGroupMetaDataToBackUp() {
|
||||
backUpList.add(rowGroups.get(rowGroups.size() - 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* <b>Note that</b>, the method is not thread safe. You mustn't do any
|
||||
* change on the return.<br>
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public List<RowGroupMetaData> getCurrentRowGroupMetaList() {
|
||||
List<RowGroupMetaData> ret = new ArrayList<>();
|
||||
backUpList.forEach(ret::add);
|
||||
return ret;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
|
||||
|
||||
/**
|
||||
* The function of this interface is to store and index TSRecord in memory
|
||||
* temporarily
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public interface BufferWriteIndex {
|
||||
/**
|
||||
* insert a tsRecord
|
||||
*
|
||||
* @param tsRecord
|
||||
*/
|
||||
void insert(TSRecord tsRecord);
|
||||
|
||||
/**
|
||||
* Get the DynamicOneColumnData from the buffer index
|
||||
*
|
||||
* @param deltaObjectId
|
||||
* @param measurementId
|
||||
* @return
|
||||
*/
|
||||
public DynamicOneColumnData query(String deltaObjectId, String measurementId);
|
||||
|
||||
/**
|
||||
* clear all data written in the bufferindex structure which will be used
|
||||
* for next stage
|
||||
*/
|
||||
void clear();
|
||||
}
|
|
@ -0,0 +1,190 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.edu.thu.tsfile.common.conf.TSFileConfig;
|
||||
import cn.edu.thu.tsfile.common.conf.TSFileDescriptor;
|
||||
import cn.edu.thu.tsfile.common.constant.JsonFormatConstant;
|
||||
import cn.edu.thu.tsfile.common.utils.Pair;
|
||||
import cn.edu.thu.tsfile.common.utils.RandomAccessOutputStream;
|
||||
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
|
||||
import cn.edu.thu.tsfile.timeseries.write.TSRecordWriteSupport;
|
||||
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
|
||||
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
|
||||
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
import cn.edu.thu.tsfiledb.exception.ProcessorException;
|
||||
import cn.edu.thu.tsfiledb.metadata.ColumnSchema;
|
||||
import cn.edu.thu.tsfiledb.metadata.MManager;
|
||||
|
||||
/**
|
||||
* BufferWrite manager manage a list of bufferwrite processor
|
||||
*
|
||||
* @author kangrong
|
||||
* @author liukun
|
||||
*/
|
||||
public class BufferWriteManager extends LRUManager<BufferWriteProcessor> {
|
||||
private static Logger LOG = LoggerFactory.getLogger(BufferWriteManager.class);
|
||||
private static TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
|
||||
protected static BufferWriteManager instance = new BufferWriteManager(conf.maxBufferWriteNodeNum,
|
||||
MManager.getInstance(), conf.BufferWriteDir);
|
||||
|
||||
public static BufferWriteManager getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the instance of BufferWriteManager using special parameters
|
||||
*
|
||||
* @param maxLRUNumber
|
||||
* @param mManager
|
||||
* @param normalDataDir
|
||||
*/
|
||||
public synchronized static void init(int maxLRUNumber, MManager mManager, String normalDataDir) {
|
||||
instance = new BufferWriteManager(maxLRUNumber, mManager, normalDataDir);
|
||||
}
|
||||
|
||||
private BufferWriteManager(int maxLRUNumber, MManager mManager, String normalDataDir) {
|
||||
super(maxLRUNumber, mManager, normalDataDir);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the buffer write processor is not exist before, it will be constructed
|
||||
* using name space path and key-value information(args)
|
||||
*/
|
||||
@Override
|
||||
protected BufferWriteProcessor constructNewProcessor(String namespacePath, Map<String, Object> args)
|
||||
throws ProcessorException, IOException, WriteProcessException {
|
||||
long startTime = (Long) args.get(FileNodeConstants.TIMESTAMP_KEY);
|
||||
Pair<String, String> fileNamePair = prepareMeasurePathFileName.apply(namespacePath,
|
||||
Long.valueOf(startTime).toString());
|
||||
TSRecordWriterParameter param = prepareTSRecordWriterParameter.apply(namespacePath, mManager, fileNamePair);
|
||||
BufferWriteIOWriter bufferTSFileWriter = new BufferWriteIOWriter(param.fileSchema, param.outputStream);
|
||||
LOG.info("Construct a new buffer write processor, fileName:{}", fileNamePair);
|
||||
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
|
||||
return new BufferWriteProcessor(conf, bufferTSFileWriter, param.writeSupport, param.fileSchema, namespacePath,
|
||||
fileNamePair);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void initProcessor(BufferWriteProcessor processor, String namespacePath, Map<String, Object> args)
|
||||
throws ProcessorException {
|
||||
if (processor.getCloseAction() == null && args == null) {
|
||||
LOG.error(
|
||||
"The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args");
|
||||
throw new RuntimeException(
|
||||
"The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args");
|
||||
}
|
||||
if (args != null && args.containsKey(FileNodeConstants.CLOSE_ACTION))
|
||||
processor.setCloseAction((Action) args.get(FileNodeConstants.CLOSE_ACTION));
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param namespacePath
|
||||
* -this path is a concept in name space tree. like
|
||||
* root.laptop.xxx. The absolute file path is deltaDataDir +
|
||||
* namespacepath + number.
|
||||
* @param MManager
|
||||
* @param Pair<String,String>
|
||||
*
|
||||
* @return TSRecordWriterParameter
|
||||
*/
|
||||
public TriFunction<String, MManager, Pair<String, String>, TSRecordWriterParameter> prepareTSRecordWriterParameter = (
|
||||
namespacePath, inputMManager, filePathPair) -> {
|
||||
Optional<List<ColumnSchema>> meaSchema;
|
||||
|
||||
String deltaObjectType;
|
||||
try {
|
||||
deltaObjectType = mManager.getDeltaObjectTypeByPath(namespacePath);
|
||||
meaSchema = Optional.ofNullable(inputMManager.getSchemaForOneType(deltaObjectType));
|
||||
} catch (PathErrorException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
FileSchema fSchema = null;
|
||||
try {
|
||||
if (meaSchema == null) {
|
||||
throw new ProcessorException("Measures schema list is null");
|
||||
}
|
||||
fSchema = getFileSchemaFromColumnSchema(meaSchema.orElse(new ArrayList<ColumnSchema>()), deltaObjectType);
|
||||
} catch (ProcessorException e) {
|
||||
LOG.error("The measures schema list is {}", meaSchema);
|
||||
throw new RuntimeException("Get the File schema failed");
|
||||
}
|
||||
File file = new File(filePathPair.left);
|
||||
WriteSupport<TSRecord> writeSupport = new TSRecordWriteSupport();
|
||||
TSRandomAccessFileWriter outputStream;
|
||||
try {
|
||||
outputStream = new RandomAccessOutputStream(file);
|
||||
} catch (IOException e) {
|
||||
LOG.error("Initialized the data output stream failed, the file path is {}", filePathPair.left);
|
||||
throw new RuntimeException(new ProcessorException(
|
||||
"IOException: " + e.getMessage() + " create data output stream failed:" + file));
|
||||
}
|
||||
return new TSRecordWriterParameter(fSchema, outputStream, writeSupport);
|
||||
};
|
||||
|
||||
/**
|
||||
* @param namespacePath
|
||||
* -this path is a concept in name space tree. like
|
||||
* root.laptop.xxx. The absolute file path is deltaDataDir +
|
||||
* namespacepath + number.
|
||||
* @param filename
|
||||
* - start time for this file
|
||||
* @return Pair<String,String> left - data file right -error data file
|
||||
*/
|
||||
public BiFunction<String, String, Pair<String, String>> prepareMeasurePathFileName = (namespacePath, fileName) -> {
|
||||
String absolutePathDir = normalDataDir + namespacePath;
|
||||
File dir = new File(absolutePathDir);
|
||||
if (!dir.exists()) {
|
||||
dir.mkdirs();
|
||||
}
|
||||
String absolutePath = dir.getAbsolutePath() + File.separator + fileName;
|
||||
|
||||
return new Pair<>(absolutePath, "null");
|
||||
};
|
||||
|
||||
/**
|
||||
* Construct one FileSchema from a list of columnschema and the deltaobject
|
||||
* type
|
||||
*
|
||||
* @param schemaList
|
||||
* @param deltaObjectType
|
||||
* @return
|
||||
*/
|
||||
private FileSchema getFileSchemaFromColumnSchema(List<ColumnSchema> schemaList, String deltaObjectType) throws WriteProcessException {
|
||||
JSONArray rowGroup = new JSONArray();
|
||||
|
||||
for (ColumnSchema col : schemaList) {
|
||||
JSONObject measurement = new JSONObject();
|
||||
measurement.put(JsonFormatConstant.MEASUREMENT_UID, col.name);
|
||||
measurement.put(JsonFormatConstant.DATA_TYPE, col.dataType.toString());
|
||||
measurement.put(JsonFormatConstant.MEASUREMENT_ENCODING, col.encoding.toString());
|
||||
for (Entry<String, String> entry : col.getArgsMap().entrySet()) {
|
||||
if (JsonFormatConstant.ENUM_VALUES.equals(entry.getKey())) {
|
||||
String[] valueArray = entry.getValue().split(",");
|
||||
measurement.put(JsonFormatConstant.ENUM_VALUES, new JSONArray(valueArray));
|
||||
} else
|
||||
measurement.put(entry.getKey(), entry.getValue().toString());
|
||||
}
|
||||
rowGroup.put(measurement);
|
||||
}
|
||||
JSONObject jsonSchema = new JSONObject();
|
||||
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, rowGroup);
|
||||
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, deltaObjectType);
|
||||
return new FileSchema(jsonSchema);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,330 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.locks.ReadWriteLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.edu.thu.tsfile.common.conf.TSFileConfig;
|
||||
import cn.edu.thu.tsfile.common.utils.Pair;
|
||||
import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
import cn.edu.thu.tsfile.timeseries.write.TSRecordWriter;
|
||||
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
|
||||
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.DataPoint;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
|
||||
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
|
||||
import cn.edu.thu.tsfile.timeseries.write.series.IRowGroupWriter;
|
||||
import cn.edu.thu.tsfiledb.engine.utils.FlushState;
|
||||
|
||||
/**
|
||||
* BufferWriteProcessor is used to write one data point or one tsRecord whose
|
||||
* timestamp is greater than than the last update time in
|
||||
* {@code FileNodeProcessor}<br>
|
||||
*
|
||||
* @author kangrong
|
||||
* @author liukun
|
||||
*/
|
||||
public class BufferWriteProcessor extends LRUProcessor {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(BufferWriteProcessor.class);
|
||||
|
||||
private BufferWriteIndex workingBufferIndex;
|
||||
private BufferWriteIndex flushingBufferIndex;
|
||||
|
||||
private boolean isFlushingSync = false;
|
||||
private FlushState flushState = new FlushState();
|
||||
|
||||
private BufferWriteIOWriter bufferIOWriter;
|
||||
private ReadWriteLock convertBufferLock = new ReentrantReadWriteLock(false);
|
||||
private BufferWriteRecordWriter recordWriter = null;
|
||||
private Pair<String, String> fileNamePair;
|
||||
private boolean isNewProcessor = false;
|
||||
|
||||
private Action closeAction;
|
||||
|
||||
BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport<TSRecord> writeSupport,
|
||||
FileSchema schema, String nsPath) {
|
||||
super(nsPath);
|
||||
this.workingBufferIndex = new MemoryBufferWriteIndexImpl();
|
||||
this.flushingBufferIndex = new MemoryBufferWriteIndexImpl();
|
||||
this.bufferIOWriter = ioFileWriter;
|
||||
this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema);
|
||||
}
|
||||
|
||||
BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport<TSRecord> writeSupport,
|
||||
FileSchema schema, String nsPath, Pair<String, String> fileNamePair) {
|
||||
super(nsPath);
|
||||
this.workingBufferIndex = new MemoryBufferWriteIndexImpl();
|
||||
this.flushingBufferIndex = new MemoryBufferWriteIndexImpl();
|
||||
this.bufferIOWriter = ioFileWriter;
|
||||
this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema);
|
||||
this.fileNamePair = fileNamePair;
|
||||
this.isNewProcessor = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a data point
|
||||
*
|
||||
* @param deltaObjectId
|
||||
* @param measurementId
|
||||
* @param timestamp
|
||||
* @param deltaObjectType
|
||||
* @param dataType
|
||||
* @param value
|
||||
* @throws WriteProcessException
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(String deltaObjectId, String measurementId, long timestamp, String deltaObjectType,
|
||||
TSDataType dataType, String value) throws IOException, WriteProcessException {
|
||||
TSRecord record = new TSRecord(timestamp, deltaObjectId);
|
||||
DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementId, value);
|
||||
record.addTuple(dataPoint);
|
||||
write(record);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a tsRecord
|
||||
*
|
||||
* @param tsRecord
|
||||
* @throws WriteProcessException
|
||||
* @throws IOException
|
||||
*/
|
||||
public void write(TSRecord tsRecord) throws IOException, WriteProcessException {
|
||||
workingBufferIndex.insert(tsRecord);
|
||||
recordWriter.write(tsRecord);
|
||||
}
|
||||
|
||||
private void switchIndexFromWorkingToFlushing() {
|
||||
BufferWriteIndex temp = workingBufferIndex;
|
||||
workingBufferIndex = flushingBufferIndex;
|
||||
flushingBufferIndex = temp;
|
||||
|
||||
}
|
||||
|
||||
private void switchIndexFromFlushingToWorking() {
|
||||
flushingBufferIndex.clear();
|
||||
bufferIOWriter.addNewRowGroupMetaDataToBackUp();
|
||||
flushState.setUnFlushing();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the result of DynamicOneColumnData from work index and flush index
|
||||
*
|
||||
* @param deltaObjectId
|
||||
* @param measurementId
|
||||
* @return
|
||||
*/
|
||||
private DynamicOneColumnData mergeTwoDynamicColumnData(String deltaObjectId, String measurementId) {
|
||||
DynamicOneColumnData working = workingBufferIndex.query(deltaObjectId, measurementId);
|
||||
DynamicOneColumnData ret = (working == null || working.length == 0) ? new DynamicOneColumnData()
|
||||
: new DynamicOneColumnData(working.dataType, true);
|
||||
if (flushState.isFlushing()) {
|
||||
DynamicOneColumnData flushing = flushingBufferIndex.query(deltaObjectId, measurementId);
|
||||
ret.mergeRecord(flushing);
|
||||
}
|
||||
ret.mergeRecord(working);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
isFlushingSync = true;
|
||||
try {
|
||||
recordWriter.close();
|
||||
isFlushingSync = false;
|
||||
} catch (IOException e) {
|
||||
LOG.error("Close BufferWriteProcessor error in namespace {}. Message:{}", nameSpacePath, e.getMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
closeAction.act();
|
||||
|
||||
/*
|
||||
* try { WriteLogManager.getInstance().bufferFlush(); } catch (Exception
|
||||
* e) { LOG.error("Write log for bufferredWrite error", e);
|
||||
* System.exit(1); }
|
||||
*/
|
||||
}
|
||||
|
||||
public void setCloseAction(Action action) {
|
||||
this.closeAction = action;
|
||||
}
|
||||
|
||||
public Action getCloseAction() {
|
||||
return this.closeAction;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canBeClosed() {
|
||||
LOG.debug("{} can be closed-{}", nameSpacePath, !flushState.isFlushing());
|
||||
return !flushState.isFlushing();
|
||||
}
|
||||
|
||||
public Pair<DynamicOneColumnData, List<RowGroupMetaData>> getIndexAndRowGroupList(String deltaObjectId,
|
||||
String measurementId) {
|
||||
convertBufferLock.readLock().lock();
|
||||
DynamicOneColumnData index = null;
|
||||
List<RowGroupMetaData> list = null;
|
||||
try {
|
||||
index = mergeTwoDynamicColumnData(deltaObjectId, measurementId);
|
||||
list = bufferIOWriter.getCurrentRowGroupMetaList();
|
||||
} finally {
|
||||
convertBufferLock.readLock().unlock();
|
||||
}
|
||||
return new Pair<>(index, list);
|
||||
}
|
||||
|
||||
public Pair<String, String> getPair() {
|
||||
return fileNamePair;
|
||||
}
|
||||
|
||||
public void setIsNewProcessor(boolean isNewProcessor) {
|
||||
this.isNewProcessor = isNewProcessor;
|
||||
}
|
||||
|
||||
public boolean isNewProcessor() {
|
||||
return isNewProcessor;
|
||||
}
|
||||
|
||||
private class BufferWriteRecordWriter extends TSRecordWriter {
|
||||
|
||||
private Map<String, IRowGroupWriter> flushingRowGroupWriters;
|
||||
private Set<String> flushingRowGroupSet;
|
||||
private long flushingRecordCount;
|
||||
|
||||
BufferWriteRecordWriter(TSFileConfig conf, BufferWriteIOWriter ioFileWriter,
|
||||
WriteSupport<TSRecord> writeSupport, FileSchema schema) {
|
||||
super(conf, ioFileWriter, writeSupport, schema);
|
||||
}
|
||||
|
||||
/**
|
||||
* insert a list of data value in form of TimePair.
|
||||
*
|
||||
* @param record
|
||||
* - TSRecord to be written
|
||||
* @throws WriteProcessException
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public void write(TSRecord record) throws IOException, WriteProcessException {
|
||||
super.write(record);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void flushRowGroup(boolean isFillRowGroup) throws IOException {
|
||||
if (recordCount > 0) {
|
||||
synchronized (flushState) {
|
||||
// This thread wait until the subThread flush finished
|
||||
while (flushState.isFlushing()) {
|
||||
try {
|
||||
flushState.wait();
|
||||
} catch (InterruptedException e) {
|
||||
LOG.error("Interrupt error when waiting flush,processor:{}. Message: {}", nameSpacePath,
|
||||
e.getMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (isFlushingSync) {
|
||||
try {
|
||||
super.flushRowGroup(false);
|
||||
} catch (IOException e) {
|
||||
LOG.error("Flush row group to store failed, processor:{}. Message: {}", nameSpacePath,
|
||||
e.getMessage());
|
||||
/*
|
||||
* There should be added system log by CGF and throw
|
||||
* exception
|
||||
*/
|
||||
e.printStackTrace();
|
||||
System.exit(1);
|
||||
}
|
||||
} else {
|
||||
flushState.setFlushing();
|
||||
switchIndexFromWorkingToFlushing();
|
||||
switchRecordWriterFromWorkingToFlushing();
|
||||
Runnable flushThread;
|
||||
flushThread = () -> {
|
||||
LOG.info("Asynchronous flushing start,-Thread id {}", Thread.currentThread().getId());
|
||||
try {
|
||||
asyncFlushRowGroupToStore();
|
||||
} catch (IOException e) {
|
||||
/*
|
||||
* There should be added system log by CGF and throw
|
||||
* exception
|
||||
*/
|
||||
LOG.error("{} Asynchronous flush error, sleep this thread-{}. Message:{}", nameSpacePath,
|
||||
Thread.currentThread().getId(), e.getMessage());
|
||||
e.printStackTrace();
|
||||
while (true) {
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
} catch (InterruptedException e1) {
|
||||
e1.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
switchRecordWriterFromFlushingToWorking();
|
||||
convertBufferLock.writeLock().lock();
|
||||
// notify the thread which is waiting for asynchronous
|
||||
// flush
|
||||
synchronized (flushState) {
|
||||
switchIndexFromFlushingToWorking();
|
||||
flushState.notify();
|
||||
}
|
||||
convertBufferLock.writeLock().unlock();
|
||||
LOG.info("Asynchronous flushing end,-Thread is {}", Thread.currentThread().getId());
|
||||
};
|
||||
Thread flush = new Thread(flushThread);
|
||||
flush.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void asyncFlushRowGroupToStore() throws IOException {
|
||||
if (flushingRecordCount > 0) {
|
||||
String deltaObjectType = schema.getDeltaType();
|
||||
long totalMemStart = deltaFileWriter.getPos();
|
||||
for (String deltaObjectId : flushingRowGroupSet) {
|
||||
long rowGroupStart = deltaFileWriter.getPos();
|
||||
deltaFileWriter.startRowGroup(flushingRecordCount, deltaObjectId, deltaObjectType);
|
||||
IRowGroupWriter groupWriter = flushingRowGroupWriters.get(deltaObjectId);
|
||||
groupWriter.flushToFileWriter(deltaFileWriter);
|
||||
deltaFileWriter.endRowGroup(deltaFileWriter.getPos() - rowGroupStart);
|
||||
}
|
||||
long actualTotalRowGroupSize = deltaFileWriter.getPos() - totalMemStart;
|
||||
fillInRowGroupSize(actualTotalRowGroupSize);
|
||||
LOG.info("Asynchronous total row group size:{}, actual:{}, less:{}", primaryRowGroupSize,
|
||||
actualTotalRowGroupSize, primaryRowGroupSize - actualTotalRowGroupSize);
|
||||
LOG.info("Asynchronous write row group end");
|
||||
}
|
||||
}
|
||||
|
||||
private void switchRecordWriterFromWorkingToFlushing() {
|
||||
|
||||
flushingRowGroupWriters = groupWriters;
|
||||
flushingRowGroupSet = schema.getDeltaObjectAppearedSet();
|
||||
for (String DeltaObjectId : schema.getDeltaObjectAppearedSet()) {
|
||||
flushingRowGroupSet.add(DeltaObjectId);
|
||||
}
|
||||
flushingRecordCount = recordCount;
|
||||
// reset
|
||||
groupWriters = new HashMap<String, IRowGroupWriter>();
|
||||
schema.getDeltaObjectAppearedSet().clear();
|
||||
recordCount = 0;
|
||||
|
||||
}
|
||||
|
||||
private void switchRecordWriterFromFlushingToWorking() {
|
||||
flushingRowGroupSet = null;
|
||||
flushingRowGroupWriters = null;
|
||||
flushingRecordCount = -1;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
/**
|
||||
* Constants for using in bufferwrite,overflow and filenode
|
||||
*
|
||||
* @author liukun
|
||||
*
|
||||
*/
|
||||
public class FileNodeConstants {
|
||||
|
||||
public static final String FILE_NODE_OPERATOR_TYPE = "OPERATOR_TYPE";
|
||||
public static final String TIMESTAMP_KEY = "TIMESTAMP";
|
||||
public static final String FILE_NODE = "FILE_NODE";
|
||||
public static final String CLOSE_ACTION = "CLOSE_ACTION";
|
||||
|
||||
public static final String ERR_EXTENSION = "err";
|
||||
public static final String PATH_SEPARATOR = ".";
|
||||
|
||||
}
|
|
@ -0,0 +1,322 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
|
||||
import cn.edu.thu.tsfiledb.exception.ErrorDebugException;
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
import cn.edu.thu.tsfiledb.exception.ProcessorException;
|
||||
import cn.edu.thu.tsfiledb.metadata.MManager;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* LRUManager manage a list of processor: overflow/filenode/bufferwrite
|
||||
* processor<br>
|
||||
*
|
||||
* @author kangrong
|
||||
* @author liukun
|
||||
*/
|
||||
public abstract class LRUManager<T extends LRUProcessor> {
|
||||
private static Logger LOG = LoggerFactory.getLogger(LRUManager.class);
|
||||
private LinkedList<T> processorLRUList;
|
||||
private Map<String, T> processorMap;
|
||||
private final int maxLRUNodeNumber;
|
||||
protected final MManager mManager;
|
||||
protected final String normalDataDir;
|
||||
private static final long removeCheckInterval = 1000;
|
||||
|
||||
protected LRUManager(int maxLRUNumber, MManager mManager, String normalDataDir) {
|
||||
this.mManager = mManager;
|
||||
this.maxLRUNodeNumber = maxLRUNumber;
|
||||
LOG.info("The max of LRUProcessor number of {} is {}", this.getClass().getSimpleName(), maxLRUNumber);
|
||||
processorLRUList = new LinkedList<>();
|
||||
processorMap = new HashMap<>();
|
||||
|
||||
if (normalDataDir.charAt(normalDataDir.length() - 1) != File.separatorChar)
|
||||
normalDataDir += File.separatorChar;
|
||||
this.normalDataDir = normalDataDir;
|
||||
File dir = new File(normalDataDir);
|
||||
if (dir.mkdirs()) {
|
||||
LOG.info("{} dir home is not exists, create it", this.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Get the data directory
|
||||
*
|
||||
* @return data directory
|
||||
*/
|
||||
public String getNormalDataDir() {
|
||||
return normalDataDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Get processor just using the metadata path
|
||||
*
|
||||
* @param path
|
||||
* @param isWriteLock
|
||||
* @return LRUProcessor
|
||||
* @throws ProcessorException
|
||||
* @throws IOException
|
||||
*/
|
||||
public T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock)
|
||||
throws ProcessorException, IOException, WriteProcessException {
|
||||
return getProcessorWithDeltaObjectIdByLRU(path, isWriteLock, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* check whether the given path's exists. If it doesn't exist, add it to
|
||||
* list. If the list has been full(list.size == maxLRUNodeNumber), remove
|
||||
* the last one. If it exists, raise this processor to the first position
|
||||
*
|
||||
* @param path
|
||||
* - measurement path in name space.
|
||||
* @param isWriteLock
|
||||
* @param args
|
||||
* - save some key-value information used for constructing a
|
||||
* special processor
|
||||
* @return return processor which has the specified nsPath
|
||||
* @throws ProcessorException
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock, Map<String, Object> args)
|
||||
throws ProcessorException, IOException, WriteProcessException {
|
||||
String nsPath;
|
||||
try {
|
||||
nsPath = mManager.getFileNameByPath(path);
|
||||
} catch (PathErrorException e) {
|
||||
throw new ProcessorException(e);
|
||||
}
|
||||
return getProcessorByLRU(nsPath, isWriteLock, args);
|
||||
}
|
||||
|
||||
/**
|
||||
* check the processor exists or not by namespacepath
|
||||
*
|
||||
* @param namespacePath
|
||||
* @return
|
||||
*/
|
||||
public synchronized boolean containNamespacePath(String namespacePath) {
|
||||
return processorMap.containsKey(namespacePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get processor using namespacepath
|
||||
*
|
||||
* @param namespacePath
|
||||
* @param isWriteLock
|
||||
* true if add a write lock on return {@code T}
|
||||
* @return
|
||||
* @throws ProcessorException
|
||||
* @throws IOException
|
||||
*/
|
||||
public T getProcessorByLRU(String namespacePath, boolean isWriteLock) throws ProcessorException, IOException, WriteProcessException {
|
||||
return getProcessorByLRU(namespacePath, isWriteLock, null);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* check whether the given namespace path exists. If it doesn't exist, add
|
||||
* it to list. If the list has been full(list.size == maxLRUNodeNumber),
|
||||
* remove the last one. If it exists, raise this processor to the first
|
||||
* position
|
||||
*
|
||||
* @param namespacePath
|
||||
* @param isWriteLock
|
||||
* @param args
|
||||
* @return
|
||||
* @throws ProcessorException
|
||||
* throw it for meeting InterruptedException while removing
|
||||
* unused processor
|
||||
* @throws IOException
|
||||
*/
|
||||
public synchronized T getProcessorByLRU(String namespacePath, boolean isWriteLock, Map<String, Object> args)
|
||||
throws ProcessorException, IOException, WriteProcessException {
|
||||
LOG.debug("Try to getProcessorByLRU, {}, nsPath:{}-Thread id {}", this.getClass().getSimpleName(),
|
||||
namespacePath, Thread.currentThread().getId());
|
||||
T processor;
|
||||
if (processorMap.containsKey(namespacePath)) {
|
||||
LOG.debug("{} contains nsPath:{}", this.getClass().getSimpleName(), namespacePath);
|
||||
processor = processorMap.get(namespacePath);
|
||||
processor.lock(isWriteLock);
|
||||
initProcessor(processor, namespacePath, args);
|
||||
processorLRUList.remove(processor);
|
||||
processorLRUList.addFirst(processor);
|
||||
LOG.debug("Use {}:{}, raise it to top", processor.getClass().getSimpleName(), processor.getNameSpacePath());
|
||||
} else {
|
||||
if (processorLRUList.size() == maxLRUNodeNumber) {
|
||||
LOG.debug(
|
||||
"LRU list doesn't contain the processor and is full, remove the oldest unused processor firstly");
|
||||
try {
|
||||
removeLastUnusedProcessor();
|
||||
} catch (InterruptedException e) {
|
||||
throw new ProcessorException(
|
||||
"thread is interrupted. nsPath:" + namespacePath + " reason: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
LOG.debug("Construct and init a new processor in {}:{}", this.getClass().getSimpleName(), namespacePath);
|
||||
processor = constructNewProcessor(namespacePath, args);
|
||||
initProcessor(processor, namespacePath, args);
|
||||
processorLRUList.addFirst(processor);
|
||||
processorMap.put(namespacePath, processor);
|
||||
processor.lock(isWriteLock);
|
||||
LOG.debug("{} LUR list: size is {}, content:{}", this.getClass().getSimpleName(), processorLRUList.size(),
|
||||
processorLRUList);
|
||||
}
|
||||
return processor;
|
||||
}
|
||||
|
||||
/**
|
||||
* remove the last unused processor if the LRU list is full
|
||||
*
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
private void removeLastUnusedProcessor() throws InterruptedException {
|
||||
while (true) {
|
||||
int index = -1;
|
||||
for (int i = processorLRUList.size() - 1; i >= 0; i--) {
|
||||
T pro = processorLRUList.get(i);
|
||||
if (pro.tryWriteLock()) {
|
||||
try {
|
||||
LOG.debug("Try write lock successfully, class:{},namespace:{}", pro.getClass().getSimpleName(),
|
||||
pro.getNameSpacePath());
|
||||
if (pro.canBeClosed()) {
|
||||
index = i;
|
||||
LOG.debug("find a removable processor, index:{}, namespace:{} break loop", index,
|
||||
pro.getNameSpacePath());
|
||||
break;
|
||||
} else {
|
||||
LOG.debug("{}:{} cannot be removed", pro.getClass().getSimpleName(),
|
||||
pro.getNameSpacePath());
|
||||
}
|
||||
} finally {
|
||||
pro.writeUnlock();
|
||||
}
|
||||
} else {
|
||||
LOG.info("Can't remove the processor {} namespace path is {}", pro.getClass().getSimpleName(),
|
||||
pro.getNameSpacePath());
|
||||
}
|
||||
}
|
||||
if (index != -1) {
|
||||
T pro = processorLRUList.get(index);
|
||||
pro.close();
|
||||
processorLRUList.remove(index);
|
||||
String proNsPath = pro.getNameSpacePath();
|
||||
processorMap.remove(proNsPath);
|
||||
LOG.debug("Remove the last processor {}:{}", pro.getClass().getSimpleName(), proNsPath);
|
||||
break;
|
||||
} else {
|
||||
LOG.debug("All processors are occupied, sleep {} millisecond", removeCheckInterval);
|
||||
Thread.sleep(removeCheckInterval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Try to close the processor of the nsPath<br>
|
||||
*
|
||||
* @param nsPath
|
||||
* @param isSync
|
||||
* true must close this processor, false close this processor if
|
||||
* possible
|
||||
* @return
|
||||
*/
|
||||
public synchronized boolean close(String nsPath, boolean isSync) {
|
||||
if (!processorMap.containsKey(nsPath))
|
||||
return true;
|
||||
LRUProcessor pro = processorMap.get(nsPath);
|
||||
if (!isSync && !pro.tryWriteLock()) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
do {
|
||||
if (pro.canBeClosed()) {
|
||||
pro.close();
|
||||
processorMap.remove(nsPath);
|
||||
processorLRUList.remove(pro);
|
||||
LOG.info("Close {} file:{}", pro.getClass().getSimpleName(), nsPath);
|
||||
return true;
|
||||
} else if (isSync) {
|
||||
try {
|
||||
LOG.info("Can't close {} file:{}, sleep 1000ms", pro.getClass(), nsPath);
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
throw new ErrorDebugException(e);
|
||||
}
|
||||
} else
|
||||
return false;
|
||||
} while (true);
|
||||
} finally {
|
||||
pro.writeUnlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* close all processor still exists in memory<br>
|
||||
*/
|
||||
public synchronized void close() {
|
||||
List<String> cannotRemoveList = new ArrayList<>();
|
||||
|
||||
do {
|
||||
cannotRemoveList.clear();
|
||||
for (String nsPath : processorMap.keySet()) {
|
||||
cannotRemoveList.add(nsPath);
|
||||
}
|
||||
for (String nsPath : cannotRemoveList) {
|
||||
close(nsPath, false);
|
||||
}
|
||||
if (!processorMap.isEmpty()) {
|
||||
LOG.info("{}:{} Can't be remove, waiting 1000ms", processorMap.get(cannotRemoveList.get(0)),
|
||||
cannotRemoveList);
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
throw new ErrorDebugException(e);
|
||||
}
|
||||
} else {
|
||||
processorMap.clear();
|
||||
processorLRUList.clear();
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* construct processor using namespacepath and key-value object<br>
|
||||
*
|
||||
* @param namespacePath
|
||||
* @param args
|
||||
* @return
|
||||
* @throws ProcessorException
|
||||
* @throws IOException
|
||||
*/
|
||||
protected abstract T constructNewProcessor(String namespacePath, Map<String, Object> args)
|
||||
throws ProcessorException, IOException, WriteProcessException;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* initialize the processor with the key-value object<br>
|
||||
*
|
||||
* @param processor
|
||||
* @param namespacePath
|
||||
* @param args
|
||||
* @throws ProcessorException
|
||||
*/
|
||||
protected void initProcessor(T processor, String namespacePath, Map<String, Object> args)
|
||||
throws ProcessorException {
|
||||
}
|
||||
}
|
|
@ -0,0 +1,155 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.concurrent.locks.ReadWriteLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* LRUProcessor is used for implementing different processor with different
|
||||
* operation.<br>
|
||||
*
|
||||
* @see BufferWriteProcessor
|
||||
* @see OverflowProcessor
|
||||
* @see FileNodeProcessor
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public abstract class LRUProcessor {
|
||||
private final static Logger LOG = LoggerFactory.getLogger(LRUProcessor.class);
|
||||
protected String nameSpacePath;
|
||||
private final ReadWriteLock lock;
|
||||
|
||||
/**
|
||||
* Construct processor using name space path
|
||||
*
|
||||
* @param nameSpacePath
|
||||
*/
|
||||
public LRUProcessor(String nameSpacePath) {
|
||||
this.nameSpacePath = nameSpacePath;
|
||||
this.lock = new ReentrantReadWriteLock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the read lock
|
||||
*/
|
||||
public void readUnlock() {
|
||||
LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
|
||||
lock.readLock().unlock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the read lock
|
||||
*/
|
||||
public void readLock() {
|
||||
LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
|
||||
lock.readLock().lock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the write lock
|
||||
*/
|
||||
public void writeLock() {
|
||||
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
|
||||
lock.writeLock().lock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the write lock
|
||||
*/
|
||||
public void writeUnlock() {
|
||||
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId());
|
||||
lock.writeLock().unlock();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param isWriteLock
|
||||
* true acquire write lock, false acquire read lock
|
||||
*/
|
||||
public void lock(boolean isWriteLock) {
|
||||
if (isWriteLock) {
|
||||
LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(),
|
||||
Thread.currentThread().getId());
|
||||
lock.writeLock().lock();
|
||||
} else {
|
||||
LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(),
|
||||
Thread.currentThread().getId());
|
||||
lock.readLock().lock();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @param isWriteUnlock
|
||||
* true release write lock, false release read unlock
|
||||
*/
|
||||
public void unlock(boolean isWriteUnlock) {
|
||||
if (isWriteUnlock) {
|
||||
LOG.debug("{}: lru write unlock-Thread id {}", this.getClass().getSimpleName(),
|
||||
Thread.currentThread().getId());
|
||||
lock.writeLock().unlock();
|
||||
} else {
|
||||
LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(),
|
||||
Thread.currentThread().getId());
|
||||
lock.readLock().unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the name space path
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public String getNameSpacePath() {
|
||||
return nameSpacePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to get the write lock
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean tryWriteLock() {
|
||||
return lock.writeLock().tryLock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to get the read lock
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public boolean tryReadLock() {
|
||||
return lock.readLock().tryLock();
|
||||
}
|
||||
|
||||
/**
|
||||
* Judge whether this processor can be closed.
|
||||
*
|
||||
* @return true if subclass doesn't have other implementation.
|
||||
*/
|
||||
public abstract boolean canBeClosed();
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return nameSpacePath.hashCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object pro) {
|
||||
if (pro == null)
|
||||
return false;
|
||||
if (!pro.getClass().equals(this.getClass()))
|
||||
return false;
|
||||
LRUProcessor lruPro = (LRUProcessor) pro;
|
||||
return nameSpacePath.equals(lruPro.getNameSpacePath());
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the processor.<br>
|
||||
* Notice: Thread is not safe
|
||||
*/
|
||||
public abstract void close();
|
||||
}
|
|
@ -0,0 +1,77 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import cn.edu.thu.tsfile.common.utils.Binary;
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.DataPoint;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
|
||||
|
||||
/**
|
||||
* Implement the {@code BufferWriteIndex}<br>
|
||||
* This class is used to store bufferwrite data in memory, also index data
|
||||
* easily<br>
|
||||
*
|
||||
* @author kangrong
|
||||
* @author liukun
|
||||
*
|
||||
*/
|
||||
public class MemoryBufferWriteIndexImpl implements BufferWriteIndex {
|
||||
private Map<String, DynamicOneColumnData> indexMap;
|
||||
|
||||
public MemoryBufferWriteIndexImpl() {
|
||||
indexMap = new HashMap<String, DynamicOneColumnData>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicOneColumnData query(String deltaObjectId, String measurementId) {
|
||||
return indexMap.get(concatPath(deltaObjectId, measurementId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insert(TSRecord tsRecord) {
|
||||
String deltaObjectId = tsRecord.deltaObjectId;
|
||||
for (DataPoint dp : tsRecord.dataPointList) {
|
||||
String measurementId = dp.getMeasurementId();
|
||||
String path = concatPath(deltaObjectId, measurementId);
|
||||
if (!indexMap.containsKey(path))
|
||||
indexMap.put(path, new DynamicOneColumnData(dp.getType(), true));
|
||||
DynamicOneColumnData deltaMea = indexMap.get(path);
|
||||
Object value = dp.getValue();
|
||||
deltaMea.putTime(tsRecord.time);
|
||||
switch (dp.getType()) {
|
||||
case INT32:
|
||||
deltaMea.putInt((Integer) value);
|
||||
break;
|
||||
case INT64:
|
||||
deltaMea.putLong((Long) value);
|
||||
break;
|
||||
case FLOAT:
|
||||
deltaMea.putFloat((Float) value);
|
||||
break;
|
||||
case DOUBLE:
|
||||
deltaMea.putDouble((Double) value);
|
||||
break;
|
||||
case BYTE_ARRAY:
|
||||
deltaMea.putBinary((Binary) value);
|
||||
break;
|
||||
case BOOLEAN:
|
||||
deltaMea.putBoolean((boolean) value);
|
||||
break;
|
||||
default:
|
||||
throw new UnsupportedOperationException("no support type:" + dp.getType() + "record:" + tsRecord);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
indexMap.clear();
|
||||
}
|
||||
|
||||
private String concatPath(String deltaObjectId, String measurementId) {
|
||||
return deltaObjectId + FileNodeConstants.PATH_SEPARATOR + measurementId;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,25 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
|
||||
import cn.edu.thu.tsfile.timeseries.write.WriteSupport;
|
||||
import cn.edu.thu.tsfile.timeseries.write.record.TSRecord;
|
||||
import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema;
|
||||
|
||||
/**
|
||||
* @author kangrong
|
||||
* @author liukun
|
||||
*
|
||||
*/
|
||||
public class TSRecordWriterParameter {
|
||||
public final FileSchema fileSchema;
|
||||
public final TSRandomAccessFileWriter outputStream;
|
||||
public final WriteSupport<TSRecord> writeSupport;
|
||||
|
||||
public TSRecordWriterParameter(FileSchema fileSchema, TSRandomAccessFileWriter outputStream,
|
||||
WriteSupport<TSRecord> writeSupport) {
|
||||
this.fileSchema = fileSchema;
|
||||
this.outputStream = outputStream;
|
||||
this.writeSupport = writeSupport;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
package cn.edu.thu.tsfiledb.engine.bufferwrite;
|
||||
|
||||
import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException;
|
||||
|
||||
/**
|
||||
* @author kangrong
|
||||
*
|
||||
* @param <U>
|
||||
* @param <T>
|
||||
* @param <C>
|
||||
* @param <R>
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface TriFunction<U, T, C, R> {
|
||||
R apply(U u, T t, C c) throws WriteProcessException;
|
||||
}
|
|
@ -0,0 +1,109 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow;
|
||||
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* we advise the implementation class of this interface can be constructed by two ways:<br>
|
||||
* 1.construct a empty index without parameters. 2.construct a empty index with a input stream for
|
||||
* restoring.
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public interface IIntervalTreeOperator {
|
||||
/**
|
||||
* insert a value to a time point. Insert doesn't support time range insert
|
||||
*
|
||||
* @param t - time
|
||||
* @param value - value
|
||||
*/
|
||||
void insert(long t, byte[] value);
|
||||
|
||||
/**
|
||||
* update a value to a time point or a time range.
|
||||
*
|
||||
* @param s - start time.
|
||||
* @param e - end time.
|
||||
* @param value - value to be updated.
|
||||
*/
|
||||
void update(long s, long e, byte[] value);
|
||||
|
||||
/**
|
||||
* The implementation maintains an overflow index in memory. The data in the index is prior to
|
||||
* <em>newerMemoryData</em>, which means the overflow operators corresponding to the index are covered with
|
||||
* <em>newerMemoryData</em>. This function merges current index into <em>newerMemoryData</em> and return the
|
||||
* merged result.
|
||||
*
|
||||
* @param timeFilter - timeFilter is specified by user.
|
||||
* @param valueFilter - valueFilter is specified by user.
|
||||
* @param freqFilter - freqFilter is specified by user.
|
||||
* @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.)
|
||||
* @param dataType - data type of this series
|
||||
* @param newerMemoryData - newer overflow data.
|
||||
*
|
||||
* @return merged result.
|
||||
*/
|
||||
DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize,
|
||||
TSDataType dataType, DynamicOneColumnData newerMemoryData);
|
||||
|
||||
/**
|
||||
* This function merges the older data which deserialized from given parameter <em>in</em> into <em>newerData</em>
|
||||
* and return the merged result. The data in <em>in</em> is prior to <em>newerData</em>, which means the overflow
|
||||
* operators corresponding to <em>in</em> are covered with <em>newerData</em>.
|
||||
*
|
||||
* @param timeFilter - timeFilter is specified by user.
|
||||
* @param valueFilter - valueFilter is specified by user.
|
||||
* @param freqFilter - freqFilter is specified by user.
|
||||
* @param in - the inputstream to be merged into newerData which contains older overflow data .
|
||||
* @param newerData - newer overflow data.
|
||||
* @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.)
|
||||
* @param dataType - data type of this series
|
||||
*
|
||||
* @return merged result.
|
||||
*/
|
||||
DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream in,
|
||||
DynamicOneColumnData newerData, int valueSize, TSDataType dataType) throws IOException;
|
||||
|
||||
/**
|
||||
* Get List<Object>(insert operations, update operations and delete operations which meet the expression of time filter,
|
||||
* value filter and frequency filter in DynamicOneColumnData data.)
|
||||
*
|
||||
* @param timeFilter - timeFilter is specified by user.
|
||||
* @param valueFilter - valueFilter is specified by user.
|
||||
* @param freqFilter - freqFilter is specified by user.
|
||||
* @param data - a DynamicOneColumnData information.
|
||||
* @param dataType - TSDataType
|
||||
* @return - List<Object>
|
||||
*/
|
||||
List<Object> getDynamicList(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter,
|
||||
DynamicOneColumnData data, TSDataType dataType);
|
||||
|
||||
/**
|
||||
* delete all values earlier than timestamp.
|
||||
*
|
||||
* @param timestamp - delete timestamp
|
||||
*/
|
||||
void delete(long timestamp);
|
||||
|
||||
/**
|
||||
* given an outputstream, serialize the index into it.
|
||||
*
|
||||
* @param out - serialization output stream.
|
||||
*/
|
||||
void toBytes(OutputStream out) throws IOException;
|
||||
|
||||
/**
|
||||
* @return the memory size for this index
|
||||
*/
|
||||
long calcMemSize();
|
||||
}
|
|
@ -0,0 +1,991 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow;
|
||||
|
||||
import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException;
|
||||
import cn.edu.thu.tsfile.common.utils.Binary;
|
||||
import cn.edu.thu.tsfile.common.utils.BytesUtils;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.operators.And;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.operators.GtEq;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitor;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitorFactory;
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalRelation;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalTree;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.MergeStatus;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
|
||||
import cn.edu.thu.tsfiledb.exception.UnSupportedOverflowOpTypeException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils.readUnsignedVarInt;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* @author CGF
|
||||
*/
|
||||
public class IntervalTreeOperation implements IIntervalTreeOperator {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(IntervalTreeOperation.class);
|
||||
|
||||
private IntervalTree index = null;
|
||||
|
||||
private TSDataType dataType; // All operations data type in IntervalTreeOperation.
|
||||
|
||||
public IntervalTreeOperation(TSDataType dataType) {
|
||||
index = new IntervalTree(dataType);
|
||||
this.dataType = dataType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insert(long t, byte[] value) {
|
||||
index.update(new TimePair(t, t, value, OverflowOpType.INSERT, dataType));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void update(long s, long e, byte[] value) {
|
||||
// s must >= e !
|
||||
index.update(new TimePair(s, e, value, OverflowOpType.UPDATE, dataType));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(long timestamp) {
|
||||
// timestamp-1 for TSFile sql expression
|
||||
// in TSFile sql expression, there only exists "DELETE X where Y < Z"
|
||||
if (timestamp > 0)
|
||||
index.update(new TimePair(0, timestamp - 1, null, OverflowOpType.DELETE, dataType));
|
||||
else
|
||||
// for "DELETE X where Y = 0"
|
||||
index.update(new TimePair(0, timestamp, null, OverflowOpType.DELETE, dataType));
|
||||
}
|
||||
|
||||
/**
|
||||
* Notice that serialization file doesn't store the value when DELETE operation.
|
||||
*
|
||||
* @param out - serialization output stream.
|
||||
* @throws IOException - serialization problem
|
||||
*/
|
||||
@Override
|
||||
public void toBytes(OutputStream out) throws IOException {
|
||||
index.midOrderSerialize(out);
|
||||
index = new IntervalTree();
|
||||
}
|
||||
|
||||
/**
|
||||
* used for queryFileBlock()
|
||||
*
|
||||
* @param doc - DynamicOneColumn to be put
|
||||
* @param s - start time
|
||||
* @param e - end time
|
||||
* @param value - overflow value
|
||||
* @param overflowOpType - OverflowOpType
|
||||
* @param dataType - TSDataType
|
||||
*/
|
||||
private void putValueUsingFileData(DynamicOneColumnData doc, long s, long e,
|
||||
byte[] value, OverflowOpType overflowOpType, TSDataType dataType) {
|
||||
switch (overflowOpType) {
|
||||
case INSERT:
|
||||
doc.putTimePair(s, -e);
|
||||
break;
|
||||
case UPDATE:
|
||||
doc.putTimePair(s, e);
|
||||
break;
|
||||
case DELETE:
|
||||
doc.putTimePair(-s, -e);
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported Overflow operation type.");
|
||||
throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type.");
|
||||
}
|
||||
|
||||
switch (dataType) {
|
||||
case INT32:
|
||||
if (overflowOpType != OverflowOpType.DELETE)
|
||||
doc.putInt(BytesUtils.bytesToInt(value));
|
||||
else
|
||||
doc.putInt(0);
|
||||
break;
|
||||
case INT64:
|
||||
if (overflowOpType != OverflowOpType.DELETE)
|
||||
doc.putLong(BytesUtils.bytesToLong(value));
|
||||
else
|
||||
doc.putLong(0L);
|
||||
break;
|
||||
case FLOAT:
|
||||
if (overflowOpType != OverflowOpType.DELETE)
|
||||
doc.putFloat(BytesUtils.bytesToFloat(value));
|
||||
else
|
||||
doc.putFloat(0.0f);
|
||||
break;
|
||||
case DOUBLE:
|
||||
if (overflowOpType != OverflowOpType.DELETE)
|
||||
doc.putDouble(BytesUtils.bytesToDouble(value));
|
||||
else
|
||||
doc.putDouble(0);
|
||||
break;
|
||||
case BYTE_ARRAY:
|
||||
if (overflowOpType != OverflowOpType.DELETE)
|
||||
doc.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
|
||||
else
|
||||
doc.putDouble(0);
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported TSFile data type.");
|
||||
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put value into newer DynamicOneColumnData.
|
||||
*
|
||||
* @param ansData new DynamicOneColumn
|
||||
* @param startTime - start time
|
||||
* @param endTime - end time
|
||||
* @param doc - previous DynamicOneColumn
|
||||
* @param i - index
|
||||
* @param dataType - TSDataType
|
||||
*/
|
||||
private void putValueUseDynamic(DynamicOneColumnData ansData, long startTime, long endTime,
|
||||
DynamicOneColumnData doc, int i, OverflowOpType overflowOpType, TSDataType dataType) {
|
||||
|
||||
// don't care the plus or minus of the value of start time or end time.
|
||||
switch (overflowOpType) {
|
||||
case INSERT:
|
||||
ansData.putTimePair(Math.abs(startTime), -Math.abs(endTime));
|
||||
break;
|
||||
case UPDATE:
|
||||
ansData.putTimePair(Math.abs(startTime), Math.abs(endTime));
|
||||
break;
|
||||
case DELETE:
|
||||
ansData.putTimePair(-Math.abs(startTime), -Math.abs(endTime));
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported Overflow operation type.");
|
||||
throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type.");
|
||||
}
|
||||
|
||||
switch (dataType) {
|
||||
case INT32:
|
||||
ansData.putInt(doc.getInt(i));
|
||||
break;
|
||||
case INT64:
|
||||
ansData.putLong(doc.getLong(i));
|
||||
break;
|
||||
case FLOAT:
|
||||
ansData.putFloat(doc.getFloat(i));
|
||||
break;
|
||||
case DOUBLE:
|
||||
ansData.putDouble(doc.getDouble(i));
|
||||
break;
|
||||
case BYTE_ARRAY:
|
||||
ansData.putBinary(doc.getBinary(i));
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported TSFile data type.");
|
||||
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notice that both s and e are >= 0. </br>
|
||||
*
|
||||
* @param s - start time
|
||||
* @param e - end time
|
||||
* @param value - time pair value
|
||||
* @param status - time pair merge status
|
||||
* @return - TimePair tp has both positive start time and end time values.
|
||||
*/
|
||||
private TimePair constructTimePair(long s, long e, byte[] value, MergeStatus status) {
|
||||
if (s <= 0 && e <= 0) // s<=0 && e<=0 for delete operation
|
||||
return new TimePair(-s, -e, value, OverflowOpType.DELETE, status);
|
||||
else if (s > 0 && e < 0) {
|
||||
return new TimePair(s, -e, value, OverflowOpType.INSERT, status);
|
||||
} else {
|
||||
return new TimePair(s, e, value, OverflowOpType.UPDATE, status);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Notice that both s and e are >= 0. </br>
|
||||
* <p>
|
||||
* Return the time pair using given start time value and end time value.
|
||||
*
|
||||
* @param startTime - start time
|
||||
* @param endTime - end time
|
||||
* @param status - time pair merge status
|
||||
* @return - TimePair tp has both positive start time and end time values.
|
||||
*/
|
||||
private TimePair constructTimePair(long startTime, long endTime, MergeStatus status) {
|
||||
if (startTime <= 0 && endTime <= 0) // s<=0 && e<=0 for delete operation
|
||||
return new TimePair(-startTime, -endTime, null, OverflowOpType.DELETE, status);
|
||||
else if (startTime > 0 && endTime < 0) {
|
||||
return new TimePair(startTime, -endTime, null, OverflowOpType.INSERT, status);
|
||||
} else {
|
||||
return new TimePair(startTime, endTime, null, OverflowOpType.UPDATE, status);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read data from overflow file which has been serialized,
|
||||
* return the correspond time pair structure. </br>
|
||||
*
|
||||
* @param inputStream - InputStream
|
||||
* @param valueSize - value byte size
|
||||
* @return - TimePair
|
||||
* @throws IOException - IOException
|
||||
*/
|
||||
private TimePair readTimePairFromOldOverflow(InputStream inputStream, int valueSize) throws IOException {
|
||||
long s = BytesUtils.readLong(inputStream);
|
||||
long e = BytesUtils.readLong(inputStream);
|
||||
if (s <= 0 && e < 0) { // DELETE OPERATION. s may < 0.
|
||||
return constructTimePair(s, e, null, MergeStatus.MERGING);
|
||||
} else { // INSERT or UPDATE OPERATION
|
||||
if (valueSize == -1) { // var length read method
|
||||
int len = readUnsignedVarInt(inputStream);
|
||||
byte[] stringBytes = new byte[len];
|
||||
inputStream.read(stringBytes);
|
||||
return constructTimePair(s, e, stringBytes, MergeStatus.MERGING);
|
||||
}
|
||||
return constructTimePair(s, e, BytesUtils.safeReadInputStreamToBytes(valueSize, inputStream), MergeStatus.MERGING);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream inputStream,
|
||||
DynamicOneColumnData newData, int valueSize, TSDataType dataType) throws IOException {
|
||||
|
||||
DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true); // merge answer
|
||||
int i = 0;
|
||||
TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE);
|
||||
TimePair newTimePair = constructTimePair(-1, -1, MergeStatus.DONE);
|
||||
long L = Long.MIN_VALUE;
|
||||
long R = Long.MIN_VALUE;
|
||||
|
||||
while (i < newData.length) {
|
||||
|
||||
if (newTimePair.mergestatus == MergeStatus.DONE) {
|
||||
// (L,R) represent new time pair range.
|
||||
L = newData.getTime(i * 2);
|
||||
R = newData.getTime(i * 2 + 1);
|
||||
newTimePair = constructTimePair(newData.getTime(i * 2), newData.getTime(i * 2 + 1), MergeStatus.MERGING);
|
||||
}
|
||||
|
||||
if (inputStream.available() == 0 && oldTimePair.mergestatus == MergeStatus.DONE) { // old overflow file is empty, but newData is not empty
|
||||
putValueUseDynamic(ans, L, R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (L <= 0 && R < 0) { // DELETE OPERATION
|
||||
long rightTime = -R;
|
||||
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
|
||||
}
|
||||
if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10]
|
||||
oldTimePair.reset();
|
||||
} else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16]
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else { // e.g. [0, 12] [10, 20]
|
||||
oldTimePair.s = rightTime + 1;
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (L > 0 && R < 0) { // INSERT OPERATION
|
||||
R = -R;
|
||||
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
|
||||
}
|
||||
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
|
||||
if (relation == CrossRelation.RCOVERSL) {
|
||||
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (oldTimePair.e == R) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else {
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
|
||||
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
} else {
|
||||
// relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible
|
||||
// relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible
|
||||
// relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else { // UPDATE OPERATION
|
||||
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
|
||||
}
|
||||
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
|
||||
if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair
|
||||
if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
|
||||
L = oldTimePair.s + 1;
|
||||
oldTimePair.reset();
|
||||
} else if (oldTimePair.e == R) {
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
} else {
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
}
|
||||
} else if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.DELETE, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
} else if (oldTimePair.e == R) {
|
||||
// the start time of DELETE time pair > 0
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
} else {
|
||||
// the start time of DELETE time pair > 0
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
}
|
||||
} else {
|
||||
oldTimePair.reset();
|
||||
}
|
||||
} else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair
|
||||
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
i++;
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
putValueUseDynamic(ans, L, -R, newData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
}
|
||||
} else if (oldTimePair.s == L) {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, R, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.s = R + 1;
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else {
|
||||
// oldTimePair.opType == OverflowOpType.INSERT
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
} else if (oldTimePair.e == R) {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
} else {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross
|
||||
// old TimePair could not be INSERT, DELETE
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
// delete operation need to be added to ans
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ??
|
||||
putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
|
||||
putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
|
||||
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newData is empty, but overflow file still has data.
|
||||
while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize);
|
||||
}
|
||||
|
||||
putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
}
|
||||
|
||||
return ans;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize, TSDataType dataType
|
||||
, DynamicOneColumnData newerMemoryData) {
|
||||
if (newerMemoryData == null) {
|
||||
return index.dynamicQuery(timeFilter, dataType);
|
||||
}
|
||||
DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true);
|
||||
DynamicOneColumnData oldData = index.dynamicQuery(timeFilter, dataType);
|
||||
|
||||
int i = 0, j = 0; // i represents newMemoryData, j represents oldMemoryData
|
||||
TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE);
|
||||
TimePair newTimePair = new TimePair(-1, -1, MergeStatus.DONE);
|
||||
long L = Long.MIN_VALUE;
|
||||
long R = Long.MIN_VALUE;
|
||||
|
||||
while (i < newerMemoryData.length) {
|
||||
|
||||
if (newTimePair.mergestatus == MergeStatus.DONE) {
|
||||
// (L,R) represent new time pair range.
|
||||
L = newerMemoryData.getTime(i * 2);
|
||||
R = newerMemoryData.getTime(i * 2 + 1);
|
||||
newTimePair = constructTimePair(newerMemoryData.getTime(i * 2), newerMemoryData.getTime(i * 2 + 1), MergeStatus.MERGING);
|
||||
}
|
||||
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE && j >= oldData.length) { // old overflow file is empty, but newData is not empty
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, newTimePair.opType, dataType);
|
||||
newTimePair.reset();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (L <= 0 && R < 0) { // DELETE OPERATION
|
||||
long rightTime = -R;
|
||||
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
|
||||
}
|
||||
if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10]
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16]
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else { // e.g. [0, 12] [10, 20]
|
||||
oldTimePair.s = rightTime + 1;
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (L > 0 && R < 0) { // INSERT OPERATION
|
||||
R = -R;
|
||||
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
|
||||
}
|
||||
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
|
||||
if (relation == CrossRelation.RCOVERSL) {
|
||||
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
break;
|
||||
} else if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (oldTimePair.e == R) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j++, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
break;
|
||||
} else {
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
|
||||
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else {
|
||||
// relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible
|
||||
// relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible
|
||||
// relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
}
|
||||
}
|
||||
} else { // UPDATE OPERATION
|
||||
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
|
||||
}
|
||||
if (oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair);
|
||||
if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair
|
||||
if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
|
||||
L = oldTimePair.s + 1;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else if (oldTimePair.e == R) {
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else {
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
|
||||
putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
} else if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
if (oldTimePair.s == L) {
|
||||
putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.DELETE, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else if (oldTimePair.e == R) {
|
||||
// the start time of DELETE time pair > 0
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else {
|
||||
// the start time of DELETE time pair > 0
|
||||
putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
} else {
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
} else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair
|
||||
if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
|
||||
i++;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
break;
|
||||
}
|
||||
} else if (oldTimePair.s == L) {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, R, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.s = R + 1;
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else {
|
||||
// oldTimePair.opType == OverflowOpType.INSERT
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
} else if (oldTimePair.e == R) {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
newTimePair.reset();
|
||||
i++;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
i++;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
} else {
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
i++;
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) {
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (oldTimePair.opType == OverflowOpType.INSERT) {
|
||||
// oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible
|
||||
LOG.error("unreachable method");
|
||||
}
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross
|
||||
// old TimePair could not be INSERT, DELETE
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
oldTimePair.s = R + 1;
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross
|
||||
if (oldTimePair.opType == OverflowOpType.DELETE) {
|
||||
// delete operation need to be added to ans
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
|
||||
L = oldTimePair.e + 1;
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
} else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ??
|
||||
putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
} else if (relation == CrossRelation.LFIRST) { // newTimePair first
|
||||
putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType);
|
||||
newTimePair.reset();
|
||||
break;
|
||||
} else if (relation == CrossRelation.RFIRST) { // oldTimePair first
|
||||
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newData is empty, but old memory still has data.
|
||||
while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) {
|
||||
if (oldTimePair.mergestatus == MergeStatus.DONE) {
|
||||
oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING);
|
||||
}
|
||||
|
||||
putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType);
|
||||
oldTimePair.reset();
|
||||
j++;
|
||||
}
|
||||
|
||||
return ans;
|
||||
}
|
||||
|
||||
/**
|
||||
* To determine whether a time pair is satisfy the demand of the SingleSeriesFilterExpression.
|
||||
*
|
||||
* @param valueFilter - value filter
|
||||
* @param data - DynamicOneColumnData
|
||||
* @param i - index
|
||||
* @return - boolean
|
||||
*/
|
||||
private boolean isIntervalSatisfy(SingleSeriesFilterExpression valueFilter, DynamicOneColumnData data, int i) {
|
||||
if (valueFilter == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
SingleValueVisitor<?> visitor = new SingleValueVisitor(valueFilter);
|
||||
switch (valueFilter.getFilterSeries().getSeriesDataType()) {
|
||||
case INT32:
|
||||
return visitor.verify(data.getInt(i));
|
||||
case INT64:
|
||||
return visitor.verify(data.getLong(i));
|
||||
case FLOAT:
|
||||
return visitor.verify(data.getFloat(i));
|
||||
case DOUBLE:
|
||||
return visitor.verify(data.getDouble(i));
|
||||
case BYTE_ARRAY:
|
||||
return SingleValueVisitorFactory.getSingleValueVistor(TSDataType.BYTE_ARRAY).satisfyObject(data.getBinary(i).getStringValue(), valueFilter);
|
||||
default:
|
||||
LOG.error("Unsupported TSFile data type.");
|
||||
throw new UnSupportedDataTypeException("Unsupported TSFile data type.");
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* put data from DynamicOneColumnData[data] to DynamicOneColumnData[ope]
|
||||
* <p>
|
||||
* value in ope must > 0
|
||||
*/
|
||||
private void putDynamicValue(long s, long e, TSDataType dataType, DynamicOneColumnData ope
|
||||
, DynamicOneColumnData data, int i) {
|
||||
if (s > 0 && e < 0) { // INSERT OPERATION, storage single point
|
||||
ope.putTime(s < 0 ? -s : s);
|
||||
|
||||
} else if (s > 0 && e > 0) { // UPDATE OPERATION
|
||||
ope.putTime(s < 0 ? -s : s);
|
||||
ope.putTime(e < 0 ? -e : e);
|
||||
}
|
||||
|
||||
switch (dataType) {
|
||||
case INT32:
|
||||
ope.putInt(data.getInt(i));
|
||||
break;
|
||||
case INT64:
|
||||
ope.putLong(data.getLong(i));
|
||||
break;
|
||||
case FLOAT:
|
||||
ope.putFloat(data.getFloat(i));
|
||||
break;
|
||||
case DOUBLE:
|
||||
ope.putDouble(data.getDouble(i));
|
||||
break;
|
||||
case BYTE_ARRAY:
|
||||
ope.putBinary(data.getBinary(i));
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported tsfile data type.");
|
||||
throw new UnSupportedDataTypeException("Unsupported tsfile data type.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Given time filter, value filter and frequency filter,
|
||||
* return the correspond data which meet all the filters expression. </br>
|
||||
* List<Object> stores three DynamicOneColumnData structures, insertAdopt, updateAdopt
|
||||
* and updateNotAdopt.
|
||||
*
|
||||
* @param timeFilter - time filter specified by user
|
||||
* @param valueFilter - value filter specified by user
|
||||
* @param freqFilter - frequency filter specified by user
|
||||
* @param overflowData - overflow data
|
||||
* @param dataType - TSDataType
|
||||
* @return - List<Object>
|
||||
*/
|
||||
@Override
|
||||
public List<Object> getDynamicList(SingleSeriesFilterExpression timeFilter,
|
||||
SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, DynamicOneColumnData overflowData, TSDataType dataType) {
|
||||
|
||||
long deleteMaxLength = -1;
|
||||
|
||||
if (timeFilter == null) {
|
||||
timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries(
|
||||
"NoName", "NoName", FilterSeriesType.TIME_FILTER), 0L, true);
|
||||
}
|
||||
List<Object> ans = new ArrayList<>();
|
||||
DynamicOneColumnData insertAdopt = new DynamicOneColumnData(dataType, true);
|
||||
DynamicOneColumnData updateAdopt = new DynamicOneColumnData(dataType, true);
|
||||
DynamicOneColumnData updateNotAdopt = new DynamicOneColumnData(dataType, true);
|
||||
LongInterval filterInterval = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter);
|
||||
|
||||
for (int i = 0; i < overflowData.length; i++) {
|
||||
long L = overflowData.getTime(i * 2);
|
||||
long R = overflowData.getTime(i * 2 + 1);
|
||||
for (int j = 0; j < filterInterval.count; j += 2) {
|
||||
TimePair exist = constructTimePair(L, R, MergeStatus.MERGING);
|
||||
TimePair filterTimePair = constructTimePair(filterInterval.v[j], filterInterval.v[j + 1], MergeStatus.MERGING);
|
||||
CrossRelation crossRelation = IntervalRelation.getRelation(filterTimePair, exist);
|
||||
if (L > 0 && R < 0) { // INSERT
|
||||
switch (crossRelation) {
|
||||
case LCOVERSR:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(L, R, dataType, insertAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case RCOVERSL:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, insertAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case LFIRSTCROSS:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(exist.s, filterTimePair.e, dataType, insertAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case RFIRSTCROSS:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(filterTimePair.s, exist.e, dataType, insertAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
}
|
||||
} else if (L > 0 && R > 0) { // UPDATE
|
||||
switch (crossRelation) {
|
||||
case LCOVERSR:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(L, R, dataType, updateAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(L, R, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case RCOVERSL:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case LFIRSTCROSS:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(exist.s, filterTimePair.e, dataType, updateAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(exist.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
case RFIRSTCROSS:
|
||||
if (isIntervalSatisfy(valueFilter, overflowData, i)) {
|
||||
putDynamicValue(filterTimePair.s, exist.e, dataType, updateAdopt, overflowData, i);
|
||||
} else {
|
||||
putDynamicValue(filterTimePair.s, exist.e, dataType, updateNotAdopt, overflowData, i);
|
||||
}
|
||||
break;
|
||||
}
|
||||
} else { // DELETE
|
||||
// LOG.info("getDynamicList max length:{} delete length:{}", deleteMaxLength, endTime);
|
||||
deleteMaxLength = Math.max(deleteMaxLength, -R);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ans.add(insertAdopt);
|
||||
ans.add(updateAdopt);
|
||||
ans.add(updateNotAdopt);
|
||||
GtEq<Long> deleteFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries(
|
||||
"Any", "Any", FilterSeriesType.TIME_FILTER), deleteMaxLength, false);
|
||||
And and = (And) FilterFactory.and(timeFilter, deleteFilter);
|
||||
ans.add(and);
|
||||
return ans;
|
||||
}
|
||||
|
||||
/**
|
||||
* both start time and end time are "long" datatype and occupy 16 bytes.
|
||||
*
|
||||
* @return - memory occupation
|
||||
*/
|
||||
@Override
|
||||
public long calcMemSize() {
|
||||
if (index.getValueSize() == 0) {
|
||||
return 16;
|
||||
} else {
|
||||
return index.getNodeSize() * (16 + index.getValueSize());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* reset the status of IntervalTreeOperation.
|
||||
*/
|
||||
public void reset() {
|
||||
index = new IntervalTree();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.index;
|
||||
|
||||
/**
|
||||
* An enum represent the cross relation between two time pairs.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
public enum CrossRelation {
|
||||
LCOVERSR, // left time pair covers right time pair. e.g. [1, 10], [5, 8]
|
||||
RCOVERSL, // right time pair covers (or equals) left time pair. e.g. [5, 8], [1, 10]
|
||||
LFIRSTCROSS, // left time pair first cross right time pair. e.g. [1, 10], [8, 13]
|
||||
RFIRSTCROSS, // right time pair first covers left time pair. e.g. [8, 13], [1, 10]
|
||||
LFIRST, // left time pair is on the left of right time pair. e.g. [1, 10], [15, 18]
|
||||
RFIRST // right time pair is on the left of left time pair. e.g. [15, 18], [1, 10]
|
||||
}
|
|
@ -0,0 +1,69 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.index;
|
||||
|
||||
|
||||
import static cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation.*;
|
||||
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
|
||||
|
||||
/**
|
||||
* This class is used to determine the relation between two time pairs.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
|
||||
public class IntervalRelation {
|
||||
|
||||
/**
|
||||
* To determine the relation between the two time pairs.
|
||||
*
|
||||
* @param left - left time pair
|
||||
* @param right - right time pair
|
||||
*/
|
||||
private static CrossRelation getCrossRelation(TimePair left, TimePair right) {
|
||||
if (right.s <= left.s && right.e >= left.e) { // right covers left | right equals left
|
||||
return RCOVERSL;
|
||||
} else if (right.s >= left.s && right.e <= left.e) { // left covers right
|
||||
return LCOVERSR;
|
||||
} else if (right.s > left.s) { // left first cross
|
||||
return LFIRSTCROSS;
|
||||
} else { // right first cross
|
||||
return RFIRSTCROSS;
|
||||
}
|
||||
}
|
||||
|
||||
private static CrossRelation getCrossRelation(long s1, long e1, long s2, long e2) {
|
||||
if (s2 <= s1 && e2 >= e1) { // right covers left | right equals left
|
||||
return RCOVERSL;
|
||||
} else if (s2 >= s1 && e2 <= e1) { // left covers right
|
||||
return LCOVERSR;
|
||||
} else if (s2 > s1) { // left first cross
|
||||
return LFIRSTCROSS;
|
||||
} else { // right first cross
|
||||
return RFIRSTCROSS;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param left - left time pair
|
||||
* @param right - right time pair
|
||||
* @return CrossRelation
|
||||
*/
|
||||
|
||||
public static CrossRelation getRelation(TimePair left, TimePair right) {
|
||||
if (left.e < right.s) { // left first
|
||||
return LFIRST;
|
||||
} else if (right.e < left.s) { // right first
|
||||
return RFIRST;
|
||||
} else
|
||||
return getCrossRelation(left, right);
|
||||
}
|
||||
|
||||
public static CrossRelation getRelation(long s1, long e1, long s2, long e2) {
|
||||
if (e1 < s2) { // left first
|
||||
return LFIRST;
|
||||
} else if (e2 < s1) { // right first
|
||||
return RFIRST;
|
||||
} else
|
||||
return getCrossRelation(s1, e1, s2, e2);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,588 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.index;
|
||||
|
||||
|
||||
import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException;
|
||||
import cn.edu.thu.tsfile.common.utils.Binary;
|
||||
import cn.edu.thu.tsfile.common.utils.BytesUtils;
|
||||
import cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval;
|
||||
import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier;
|
||||
import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair;
|
||||
import cn.edu.thu.tsfiledb.exception.OverflowWrongParameterException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* IntervalTree is a data structure implemented used Treap. </br>
|
||||
* An IntervalTree stores many TreeNodes, a TreeNode saves a time range operation. </br>
|
||||
* A time range operation is explained in OverflowOpType. </br>
|
||||
* see https://en.wikipedia.org/wiki/Treap.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
public class IntervalTree {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(IntervalTree.class);
|
||||
|
||||
private int nodeSize = 0; // num of tree nodes
|
||||
private int valueSize = 0; // num of bytes of value
|
||||
private TreeNode root = null;
|
||||
private TSDataType dataType = null; // The data type of IntervalTree structure.
|
||||
|
||||
public IntervalTree() {
|
||||
|
||||
}
|
||||
|
||||
public IntervalTree(TSDataType dataType) {
|
||||
this.dataType = dataType;
|
||||
if (dataType == TSDataType.BYTE_ARRAY) {
|
||||
this.valueSize = -1; // -1 represents var length encoding.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* TimePair must be no conflict with current IntervalTree.
|
||||
*
|
||||
* @param tp TimePair to be inserted.
|
||||
*/
|
||||
private void insert(TimePair tp) {
|
||||
nodeSize++;
|
||||
if (root == null) {
|
||||
root = new TreeNode(tp.s, tp.e, tp.v, tp.opType);
|
||||
} else {
|
||||
insertNoConflict(tp, root, null, false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to insert the no conflicting TreeNode.
|
||||
*
|
||||
* @param timePair time pair to be inserted
|
||||
* @param currentNode current TreeNode
|
||||
* @param parentNode parent TreeNode
|
||||
* @param left whether the pNode is left child of paNode.
|
||||
*/
|
||||
private void insertNoConflict(TimePair timePair, TreeNode currentNode, TreeNode parentNode, boolean left) {
|
||||
if (currentNode.start > timePair.e) {
|
||||
if (currentNode.left == null) {
|
||||
currentNode.left = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType);
|
||||
} else {
|
||||
insertNoConflict(timePair, currentNode.left, currentNode, true);
|
||||
}
|
||||
if (currentNode.fix > currentNode.left.fix)
|
||||
rightTurn(currentNode, parentNode, left);
|
||||
} else if (currentNode.end < timePair.s) {
|
||||
if (currentNode.right == null) {
|
||||
currentNode.right = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType);
|
||||
} else {
|
||||
insertNoConflict(timePair, currentNode.right, currentNode, false);
|
||||
}
|
||||
if (currentNode.fix > currentNode.right.fix)
|
||||
leftTurn(currentNode, parentNode, left);
|
||||
}
|
||||
}
|
||||
|
||||
private void leftTurn(TreeNode currentNode, TreeNode parentNode, boolean left) {
|
||||
if (parentNode == null) {
|
||||
root = currentNode.right;
|
||||
} else if (left) {
|
||||
parentNode.left = currentNode.right;
|
||||
} else {
|
||||
parentNode.right = currentNode.right;
|
||||
}
|
||||
TreeNode tmpNode = currentNode.right.left;
|
||||
currentNode.right.left = currentNode;
|
||||
currentNode.right = tmpNode;
|
||||
}
|
||||
|
||||
private void rightTurn(TreeNode currentNode, TreeNode parentNode, boolean left) {
|
||||
if (parentNode == null) {
|
||||
root = currentNode.left;
|
||||
} else if (left) {
|
||||
parentNode.left = currentNode.left;
|
||||
} else {
|
||||
parentNode.right = currentNode.left;
|
||||
}
|
||||
TreeNode tmpNode = currentNode.left.right;
|
||||
currentNode.left.right = currentNode;
|
||||
currentNode.left = tmpNode;
|
||||
}
|
||||
|
||||
private void delete(TreeNode currentNode, TreeNode parentNode, boolean left) {
|
||||
|
||||
if (parentNode == null) {
|
||||
if (currentNode.left == null && currentNode.right == null) {
|
||||
root = null;
|
||||
nodeSize--;
|
||||
return;
|
||||
} else if (currentNode.left == null) {
|
||||
root = currentNode.right;
|
||||
nodeSize--;
|
||||
return;
|
||||
} else if (currentNode.right == null) {
|
||||
root = currentNode.left;
|
||||
nodeSize--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (left) {
|
||||
if (currentNode.left == null) {
|
||||
parentNode.left = currentNode.right;
|
||||
nodeSize--;
|
||||
} else if (currentNode.right == null) {
|
||||
parentNode.left = currentNode.left;
|
||||
nodeSize--;
|
||||
} else {
|
||||
if (currentNode.left.fix < currentNode.right.fix) {
|
||||
TreeNode tmpNode = currentNode.left;
|
||||
rightTurn(currentNode, parentNode, left);
|
||||
delete(currentNode, tmpNode, false);
|
||||
} else {
|
||||
TreeNode tmpNode = currentNode.right;
|
||||
leftTurn(currentNode, parentNode, left);
|
||||
delete(currentNode, tmpNode, true);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (currentNode.left == null) {
|
||||
parentNode.right = currentNode.right;
|
||||
nodeSize--;
|
||||
} else if (currentNode.right == null) {
|
||||
parentNode.right = currentNode.left;
|
||||
nodeSize--;
|
||||
} else {
|
||||
if (currentNode.left.fix < currentNode.right.fix) {
|
||||
TreeNode tmpNode = currentNode.left;
|
||||
rightTurn(currentNode, parentNode, left);
|
||||
delete(currentNode, tmpNode, false);
|
||||
} else {
|
||||
TreeNode tmpNode = currentNode.right;
|
||||
leftTurn(currentNode, parentNode, left);
|
||||
delete(currentNode, tmpNode, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update operation api.
|
||||
*
|
||||
* @param tp TimePair to be updated
|
||||
*/
|
||||
public void update(TimePair tp) {
|
||||
|
||||
|
||||
if (this.dataType == null) {
|
||||
if (tp.opType != OverflowOpType.DELETE) {
|
||||
this.valueSize = tp.v.length;
|
||||
}
|
||||
this.dataType = tp.dataType;
|
||||
} else if (tp.dataType != this.dataType){
|
||||
LOG.error("IntervalTree wrong time pair parameters");
|
||||
throw new OverflowWrongParameterException("IntervalTree wrong time pair parameters");
|
||||
}
|
||||
|
||||
// When an UPDATE operation covers root(INSERT) node,
|
||||
// we delete root(INSERT) node and stores it in insertList temporary.
|
||||
List<TimePair> insertList = new ArrayList<>();
|
||||
while (root != null) {
|
||||
if (update(tp, root, null, false, insertList)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (root == null) {
|
||||
insert(tp);
|
||||
}
|
||||
|
||||
for (TimePair t : insertList) {
|
||||
while (root != null) {
|
||||
if (update(t, root, null, false, null)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* IntervalTree update operation.
|
||||
*
|
||||
* @param tp TimePair to be updated
|
||||
* @param currentNode current TreeNode
|
||||
* @param parentNode parent node of pNode
|
||||
* @param left whether pNode is left child of paNode
|
||||
* @return whether update process is over
|
||||
*/
|
||||
private boolean update(TimePair tp, TreeNode currentNode, TreeNode parentNode, boolean left, List insertList) {
|
||||
if (currentNode == null) {
|
||||
insert(tp);
|
||||
return true;
|
||||
}
|
||||
|
||||
CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(currentNode.start, currentNode.end));
|
||||
switch (relation) {
|
||||
case LCOVERSR: // tp covers currentNode
|
||||
if (currentNode.opType == OverflowOpType.INSERT && tp.opType == OverflowOpType.UPDATE) {
|
||||
insertList.add(new TimePair(currentNode.start, currentNode.end, tp.v,
|
||||
OverflowOpType.INSERT));
|
||||
}
|
||||
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
|
||||
tp.s = currentNode.end + 1;
|
||||
} else {
|
||||
delete(currentNode, parentNode, left);
|
||||
}
|
||||
return false;
|
||||
case RCOVERSL: // currentNode covers tp
|
||||
if (tp.s == currentNode.start && currentNode.end == tp.e) {
|
||||
if (currentNode.opType == OverflowOpType.INSERT
|
||||
&& tp.opType == OverflowOpType.UPDATE) {
|
||||
currentNode.value = tp.v;
|
||||
} else if (currentNode.opType == OverflowOpType.DELETE
|
||||
&& tp.opType == OverflowOpType.UPDATE) {
|
||||
return true;
|
||||
} else {
|
||||
currentNode.opType = tp.opType;
|
||||
currentNode.value = tp.v;
|
||||
}
|
||||
} else if (tp.s == currentNode.start) {
|
||||
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
|
||||
return true;
|
||||
} else {
|
||||
currentNode.start = tp.e + 1;
|
||||
insert(tp);
|
||||
}
|
||||
} else if (tp.e == currentNode.end) {
|
||||
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
|
||||
return true;
|
||||
} else {
|
||||
currentNode.end = tp.s - 1;
|
||||
insert(tp);
|
||||
}
|
||||
} else { // (currentNode tp currentNode)
|
||||
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
|
||||
return true;
|
||||
} else {
|
||||
long tmp = currentNode.end;
|
||||
currentNode.end = tp.s - 1;
|
||||
insert(tp);
|
||||
insert(new TimePair(tp.e + 1, tmp, currentNode.value, currentNode.opType));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
case LFIRSTCROSS: // tp first cross | No need to modify currentNode value.
|
||||
currentNode.start = tp.e + 1;
|
||||
return update(tp, currentNode.left, currentNode, true, insertList);
|
||||
case RFIRSTCROSS: // currentNode first cross | No need to modify currentNode value.
|
||||
if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) {
|
||||
return update(new TimePair(currentNode.end + 1, tp.e, tp.v, tp.opType), currentNode.right, currentNode, false, insertList);
|
||||
} else {
|
||||
currentNode.end = tp.s - 1;
|
||||
return update(tp, currentNode.right, currentNode, false, insertList);
|
||||
}
|
||||
case LFIRST: // tp first
|
||||
return update(tp, currentNode.left, currentNode, true, insertList);
|
||||
case RFIRST: // currentNode first
|
||||
return update(tp, currentNode.right, currentNode, false, insertList);
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put the IntervalTree mid order serialization into OutputStream.
|
||||
*
|
||||
* @param out OutputStream
|
||||
*/
|
||||
public void midOrderSerialize(OutputStream out) throws IOException {
|
||||
midOrderSerialize(root, out);
|
||||
root = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* IntervalTree mid order serialization.
|
||||
* <p>
|
||||
* Notice that "String" data type adopts unsigned var int encoding.
|
||||
*
|
||||
* @param pNode - root node of IntervalTree
|
||||
* @param out - output file stream
|
||||
*/
|
||||
private void midOrderSerialize(TreeNode pNode, OutputStream out) throws IOException {
|
||||
if (pNode == null)
|
||||
return;
|
||||
midOrderSerialize(pNode.left, out);
|
||||
if (pNode.opType.equals(OverflowOpType.DELETE)) {
|
||||
out.write(BytesUtils.longToBytes(-pNode.start));
|
||||
out.write(BytesUtils.longToBytes(-pNode.end));
|
||||
} else if (pNode.opType.equals(OverflowOpType.UPDATE)) {
|
||||
out.write(BytesUtils.longToBytes(pNode.start));
|
||||
out.write(BytesUtils.longToBytes(pNode.end));
|
||||
if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) {
|
||||
ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out);
|
||||
out.write(pNode.value);
|
||||
} else {
|
||||
out.write(pNode.value);
|
||||
}
|
||||
} else {
|
||||
out.write(BytesUtils.longToBytes(pNode.start));
|
||||
out.write(BytesUtils.longToBytes(-pNode.end));
|
||||
if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) {
|
||||
ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out);
|
||||
out.write(pNode.value);
|
||||
} else {
|
||||
out.write(pNode.value);
|
||||
}
|
||||
|
||||
}
|
||||
midOrderSerialize(pNode.right, out);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* given the query TimePair tp, return all covered time pair with tp.
|
||||
* <p>
|
||||
* for concurrency safety, the queryAns variable must be defined in
|
||||
* the query method.
|
||||
*
|
||||
* @param tp - TimePair to query
|
||||
* @return - all new TimePairs accord with the query tp.
|
||||
*/
|
||||
public List<TimePair> query(TimePair tp) {
|
||||
List<TimePair> queryAns = new ArrayList<>();
|
||||
query(tp, root, queryAns);
|
||||
return queryAns;
|
||||
}
|
||||
|
||||
private void query(TimePair tp, TreeNode pNode, List<TimePair> queryAns) {
|
||||
if (pNode == null)
|
||||
return;
|
||||
CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(pNode.start, pNode.end));
|
||||
switch (relation) {
|
||||
case LCOVERSR: // tp covers currentNode
|
||||
if (tp.s == pNode.start && pNode.end == tp.e) {
|
||||
queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType));
|
||||
} else if (tp.s == pNode.start) {
|
||||
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
|
||||
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
|
||||
} else if (tp.e == pNode.end) {
|
||||
query(new TimePair(tp.s, pNode.end - 1), pNode.left, queryAns);
|
||||
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
|
||||
} else {
|
||||
query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns);
|
||||
queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType));
|
||||
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
|
||||
}
|
||||
return;
|
||||
case RCOVERSL: // currentNode covers tp
|
||||
queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType));
|
||||
return;
|
||||
case LFIRSTCROSS: // tp first cross
|
||||
query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns);
|
||||
queryAns.add(new TimePair(pNode.start, tp.e, pNode.value, pNode.opType));
|
||||
return;
|
||||
case RFIRSTCROSS: // currentNode first cross
|
||||
queryAns.add(new TimePair(tp.s, pNode.end, pNode.value, pNode.opType));
|
||||
query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns);
|
||||
return;
|
||||
case LFIRST: // tp first
|
||||
query(tp, pNode.left, queryAns);
|
||||
break;
|
||||
case RFIRST: // currentNode first
|
||||
query(tp, pNode.right, queryAns);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given time filter and value filter, return the answer List<DynamicOneColumn> in IntervalTree.</br>
|
||||
* Could not use member variable. </br>
|
||||
* Must notice thread safety! </br>
|
||||
* Note that value filter is not given, apply value filter in this method would cause errors.</br>
|
||||
* May make wrong intervals in old data valid in IntervalTreeOperation.queryFileBlock method.
|
||||
*
|
||||
* @param timeFilter filter for time
|
||||
* @param dataType TSDataType
|
||||
* @return DynamicOneColumnData
|
||||
*/
|
||||
public DynamicOneColumnData dynamicQuery(SingleSeriesFilterExpression timeFilter, TSDataType dataType) {
|
||||
|
||||
DynamicOneColumnData crudResult = new DynamicOneColumnData(dataType, true);
|
||||
|
||||
if (timeFilter == null) {
|
||||
timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries("NULL", "NULL", FilterSeriesType.TIME_FILTER), 0L, true);
|
||||
}
|
||||
|
||||
LongInterval val = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter);
|
||||
dynamicQuery(crudResult, root, val, dataType);
|
||||
|
||||
return crudResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Implemented for above dynamicQuery method.
|
||||
*
|
||||
* @param crudResult DynamicOneColumnData
|
||||
* @param pNode query root TreeNode
|
||||
* @param val LongInterval
|
||||
* @param dataType TSDataType
|
||||
*/
|
||||
private void dynamicQuery(DynamicOneColumnData crudResult, TreeNode pNode, LongInterval val, TSDataType dataType) {
|
||||
if (pNode == null)
|
||||
return;
|
||||
|
||||
// to examine whether left child node has answer
|
||||
if (val.count != 0 && val.v[0] < pNode.start) {
|
||||
dynamicQuery(crudResult, pNode.left, val, dataType);
|
||||
}
|
||||
|
||||
for (int i = 0; i < val.count; i += 2) {
|
||||
CrossRelation r = IntervalRelation.getRelation(new TimePair(val.v[i], val.v[i + 1]), new TimePair(pNode.start, pNode.end));
|
||||
switch (r) {
|
||||
case LFIRST:
|
||||
continue;
|
||||
case RFIRST:
|
||||
continue;
|
||||
case LFIRSTCROSS:
|
||||
putTreeNodeValue(crudResult, pNode.opType, pNode.start, val.v[i + 1], dataType, pNode.value);
|
||||
break;
|
||||
case RFIRSTCROSS:
|
||||
putTreeNodeValue(crudResult, pNode.opType, val.v[i], pNode.end, dataType, pNode.value);
|
||||
break;
|
||||
case LCOVERSR:
|
||||
putTreeNodeValue(crudResult, pNode.opType, pNode.start, pNode.end, dataType, pNode.value);
|
||||
break;
|
||||
case RCOVERSL:
|
||||
putTreeNodeValue(crudResult, pNode.opType, val.v[i], val.v[i + 1], dataType, pNode.value);
|
||||
break;
|
||||
default:
|
||||
LOG.error("un defined CrossRelation");
|
||||
}
|
||||
}
|
||||
|
||||
// to examine whether right child node has answer
|
||||
if (val.count != 0 && val.v[val.count - 1] > pNode.end) { // left node has answer
|
||||
dynamicQuery(crudResult, pNode.right, val, dataType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Put TreeNode information into DynamicColumnData crudResult.
|
||||
*
|
||||
* @param crudResult DynamicOneColumnData stores operations
|
||||
* @param opType OverflowOpType
|
||||
* @param s start time
|
||||
* @param e end time
|
||||
* @param dataType TSDataType
|
||||
* @param value byte value
|
||||
*/
|
||||
private void putTreeNodeValue(DynamicOneColumnData crudResult, OverflowOpType opType, long s, long e, TSDataType dataType,
|
||||
byte[] value) {
|
||||
switch (dataType) {
|
||||
case INT32:
|
||||
switch (opType) {
|
||||
case INSERT:
|
||||
crudResult.putTimePair(s, -e);
|
||||
crudResult.putInt(BytesUtils.bytesToInt(value));
|
||||
break;
|
||||
case DELETE:
|
||||
crudResult.putTimePair(-s, -e);
|
||||
crudResult.putInt(0);
|
||||
break;
|
||||
case UPDATE:
|
||||
crudResult.putTimePair(s, e);
|
||||
crudResult.putInt(BytesUtils.bytesToInt(value));
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case INT64:
|
||||
switch (opType) {
|
||||
case INSERT:
|
||||
crudResult.putTimePair(s, -e);
|
||||
crudResult.putLong(BytesUtils.bytesToLong(value));
|
||||
break;
|
||||
case DELETE:
|
||||
crudResult.putTimePair(-s, -e);
|
||||
crudResult.putLong(0);
|
||||
break;
|
||||
case UPDATE:
|
||||
crudResult.putTimePair(s, e);
|
||||
crudResult.putLong(BytesUtils.bytesToLong(value));
|
||||
}
|
||||
break;
|
||||
case FLOAT:
|
||||
switch (opType) {
|
||||
case INSERT:
|
||||
crudResult.putTimePair(s, -e);
|
||||
crudResult.putFloat(BytesUtils.bytesToFloat(value));
|
||||
break;
|
||||
case DELETE:
|
||||
crudResult.putTimePair(-s, -e);
|
||||
crudResult.putFloat(0);
|
||||
break;
|
||||
case UPDATE:
|
||||
crudResult.putTimePair(s, e);
|
||||
crudResult.putFloat(BytesUtils.bytesToFloat(value));
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case DOUBLE:
|
||||
switch (opType) {
|
||||
case INSERT:
|
||||
crudResult.putTimePair(s, -e);
|
||||
crudResult.putDouble(BytesUtils.bytesToDouble(value));
|
||||
break;
|
||||
case DELETE:
|
||||
crudResult.putTimePair(-s, -e);
|
||||
crudResult.putDouble(0);
|
||||
break;
|
||||
case UPDATE:
|
||||
crudResult.putTimePair(s, e);
|
||||
crudResult.putDouble(BytesUtils.bytesToDouble(value));
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case BYTE_ARRAY:
|
||||
switch (opType) {
|
||||
case INSERT:
|
||||
crudResult.putTimePair(s, -e);
|
||||
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
|
||||
break;
|
||||
case DELETE:
|
||||
crudResult.putTimePair(-s, -e);
|
||||
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
|
||||
break;
|
||||
case UPDATE:
|
||||
crudResult.putTimePair(s, e);
|
||||
crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value)));
|
||||
break;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
LOG.error("Unsupported tsfile data type.");
|
||||
throw new UnSupportedDataTypeException("Unsupported tsfile data type.");
|
||||
}
|
||||
}
|
||||
|
||||
public int getNodeSize() {
|
||||
return nodeSize;
|
||||
}
|
||||
|
||||
public int getValueSize() {
|
||||
return this.valueSize;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.index;
|
||||
|
||||
|
||||
import java.util.Random;
|
||||
|
||||
import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType;
|
||||
|
||||
/**
|
||||
* Used for IntervalTree (Treap Data Structure).
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
|
||||
public class TreeNode {
|
||||
|
||||
private Random random = new Random();
|
||||
public long start; // start time
|
||||
public long end; // end time
|
||||
public int fix; // priority for treap IntervalTree
|
||||
public OverflowOpType opType; // overflow operation type
|
||||
public byte[] value; // the value stored in this node
|
||||
public TreeNode left;
|
||||
public TreeNode right;
|
||||
|
||||
public TreeNode(long start, long end, byte[] value, OverflowOpType type) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.value = value;
|
||||
this.opType = type;
|
||||
left = null;
|
||||
right = null;
|
||||
fix = random.nextInt();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.utils;
|
||||
|
||||
/**
|
||||
* Used for IntervalTreeOperation.queryMemory() and IntervalTreeOperation.queryFileBlock(); </br>
|
||||
*
|
||||
* DONE means that a time pair is not used or this time pair has been merged into a new DynamicOneColumn</br>
|
||||
* MERGING means that a time pair is merging into a new DynamicOneColumn</br>
|
||||
*
|
||||
* @author CGF.
|
||||
*/
|
||||
public enum MergeStatus {
|
||||
DONE, MERGING
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.utils;
|
||||
|
||||
/**
|
||||
* Include three types: INSERT,UPDATE,DELETE;
|
||||
*
|
||||
* INSERT is an operation which inserts a time point.</br>
|
||||
* UPDATE is an operation which updates a time range.</br>
|
||||
* DELETE is an operation which deletes a time range. Note that DELETE operation could only
|
||||
* delete a time which is less than given time T. </br>
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public enum OverflowOpType {
|
||||
INSERT, UPDATE, DELETE
|
||||
}
|
|
@ -0,0 +1,75 @@
|
|||
package cn.edu.thu.tsfiledb.engine.overflow.utils;
|
||||
|
||||
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
|
||||
/**
|
||||
* TimePair represents an overflow operation.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
|
||||
public class TimePair {
|
||||
|
||||
public long s; // start time
|
||||
public long e; // end time
|
||||
public byte[] v; // value
|
||||
public OverflowOpType opType = null;
|
||||
public TSDataType dataType;
|
||||
public MergeStatus mergestatus;
|
||||
|
||||
public TimePair(long s, long e) {
|
||||
this.s = s;
|
||||
this.e = e;
|
||||
this.v = null;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, byte[] v) {
|
||||
this.s = s;
|
||||
this.e = e;
|
||||
this.v = v;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, MergeStatus status) {
|
||||
this(s, e);
|
||||
this.mergestatus = status;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, byte[] v, TSDataType dataType) {
|
||||
this(s, e, v);
|
||||
this.dataType = dataType;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType) {
|
||||
this(s, e, v);
|
||||
this.opType = overflowOpType;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType, TSDataType dataType) {
|
||||
this(s, e, v, overflowOpType);
|
||||
this.dataType = dataType;
|
||||
}
|
||||
|
||||
public TimePair(long s, long e, byte[] v, OverflowOpType type, MergeStatus status) {
|
||||
this(s, e, v);
|
||||
this.opType = type;
|
||||
this.mergestatus = status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set TimePair s = -1 and e = -1 means reset.
|
||||
*/
|
||||
public void reset() {
|
||||
s = -1;
|
||||
e = -1;
|
||||
mergestatus = MergeStatus.DONE;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuffer sb = new StringBuffer().append(this.s).append(",").append(this.e);
|
||||
if (this.opType != null)
|
||||
sb.append(",").append(this.opType);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package cn.edu.thu.tsfiledb.engine.utils;
|
||||
|
||||
/**
|
||||
* This class is used to represent the state of flush. It's can be used in the
|
||||
* bufferwrite flush{@code BufferWriteProcessor} and overflow
|
||||
* flush{@code OverFlowProcessor}.
|
||||
*
|
||||
* @author liukun
|
||||
*/
|
||||
public class FlushState {
|
||||
|
||||
private boolean isFlushing;
|
||||
|
||||
public FlushState() {
|
||||
this.isFlushing = false;
|
||||
}
|
||||
|
||||
public boolean isFlushing() {
|
||||
return isFlushing;
|
||||
}
|
||||
|
||||
public void setFlushing() {
|
||||
this.isFlushing = true;
|
||||
}
|
||||
|
||||
public void setUnFlushing() {
|
||||
this.isFlushing = false;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
public class ArgsErrorException extends Exception{
|
||||
|
||||
private static final long serialVersionUID = -3614543017182165265L;
|
||||
public ArgsErrorException(String msg){
|
||||
super(msg);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
*
|
||||
* This Exception is the parent class for all delta engine runtime exceptions.<br>
|
||||
* This Exception extends super class {@link java.lang.RuntimeException}
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
public abstract class DeltaEngineRunningException extends RuntimeException{
|
||||
public DeltaEngineRunningException() { super();}
|
||||
|
||||
public DeltaEngineRunningException(String message, Throwable cause) { super(message, cause);}
|
||||
|
||||
public DeltaEngineRunningException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public DeltaEngineRunningException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public class ErrorDebugException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = -1123099620556170447L;
|
||||
|
||||
public ErrorDebugException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public ErrorDebugException(Exception e) {
|
||||
super(e);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,16 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* Throw this exception when the file node processor is not exists
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public class FileNodeNotExistException extends RuntimeException {
|
||||
|
||||
private static final long serialVersionUID = -4334041411884083545L;
|
||||
|
||||
public FileNodeNotExistException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* If query metadata constructs schema but passes illegal parameters to
|
||||
* EncodingConvertor or DataTypeConvertor,this exception will be threw.
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public class MetadataArgsErrorException extends ArgsErrorException {
|
||||
|
||||
private static final long serialVersionUID = 3415275599091623570L;
|
||||
|
||||
public MetadataArgsErrorException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* Used for IntervalTree pass wrong parameters. </br>
|
||||
* e.g. TSDataType inconsistent;
|
||||
* e.g. start time is greater than end time.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
public class OverflowWrongParameterException extends DeltaEngineRunningException{
|
||||
public OverflowWrongParameterException(String message, Throwable cause) { super(message, cause);}
|
||||
|
||||
public OverflowWrongParameterException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public OverflowWrongParameterException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
public class PathErrorException extends Exception{
|
||||
|
||||
private static final long serialVersionUID = 2141197032898163234L;
|
||||
|
||||
public PathErrorException(String msg){
|
||||
super(msg);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,24 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* This exception is used for some error in processor
|
||||
*
|
||||
* @author kangrong
|
||||
*
|
||||
*/
|
||||
public class ProcessorException extends Exception {
|
||||
|
||||
private static final long serialVersionUID = 4137638418544201605L;
|
||||
|
||||
public ProcessorException(String msg) {
|
||||
super(msg);
|
||||
}
|
||||
|
||||
public ProcessorException(PathErrorException pathExcp) {
|
||||
super("PathErrorException: " + pathExcp.getMessage());
|
||||
}
|
||||
|
||||
public ProcessorException() {
|
||||
super();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
package cn.edu.thu.tsfiledb.exception;
|
||||
|
||||
/**
|
||||
* OverflowOpType could only be INSERT, UPDATE, DELETE.
|
||||
* The other situations would throw this exception.
|
||||
*
|
||||
* @author CGF
|
||||
*/
|
||||
public class UnSupportedOverflowOpTypeException extends DeltaEngineRunningException{
|
||||
|
||||
public UnSupportedOverflowOpTypeException(String message, Throwable cause) { super(message, cause);}
|
||||
|
||||
public UnSupportedOverflowOpTypeException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
public UnSupportedOverflowOpTypeException(Throwable cause) {
|
||||
super(cause);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,102 @@
|
|||
package cn.edu.thu.tsfiledb.hadoop.io;
|
||||
|
||||
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* This class is used to wrap the {@link}FSDataInputStream and implement the
|
||||
* interface {@link}TSRandomAccessFileReader.
|
||||
*
|
||||
* @author liukun
|
||||
*/
|
||||
public class HDFSInputStream implements TSRandomAccessFileReader {
|
||||
|
||||
private FSDataInputStream fsDataInputStream;
|
||||
private FileStatus fileStatus;
|
||||
|
||||
public HDFSInputStream(String filePath) throws IOException {
|
||||
|
||||
this(filePath, new Configuration());
|
||||
}
|
||||
|
||||
public HDFSInputStream(String filePath, Configuration configuration) throws IOException {
|
||||
|
||||
this(new Path(filePath),configuration);
|
||||
}
|
||||
|
||||
public HDFSInputStream(Path path, Configuration configuration) throws IOException {
|
||||
|
||||
FileSystem fs = FileSystem.get(configuration);
|
||||
fsDataInputStream = fs.open(path);
|
||||
fileStatus = fs.getFileStatus(path);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void seek(long offset) throws IOException {
|
||||
|
||||
fsDataInputStream.seek(offset);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
|
||||
return fsDataInputStream.read();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long length() throws IOException {
|
||||
|
||||
return fileStatus.getLen();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int readInt() throws IOException {
|
||||
|
||||
return fsDataInputStream.readInt();
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
|
||||
fsDataInputStream.close();
|
||||
}
|
||||
|
||||
public long getPos() throws IOException {
|
||||
|
||||
return fsDataInputStream.getPos();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the data into b, and the check the length
|
||||
*
|
||||
* @param b
|
||||
* read the data into
|
||||
* @param off
|
||||
* the begin offset to read into
|
||||
* @param len
|
||||
* the length to read into
|
||||
*/
|
||||
@Override
|
||||
public int read(byte[] b, int off, int len) throws IOException {
|
||||
if (len < 0) {
|
||||
throw new IndexOutOfBoundsException();
|
||||
}
|
||||
int n = 0;
|
||||
while (n < len) {
|
||||
int count = fsDataInputStream.read(b, off + n, len - n);
|
||||
if (count < 0) {
|
||||
throw new IOException("The read length is out of the length of inputstream");
|
||||
}
|
||||
n += count;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,75 @@
|
|||
package cn.edu.thu.tsfiledb.hadoop.io;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter;
|
||||
|
||||
|
||||
/**
|
||||
* This class is used to wrap the {@link}FSDataOutputStream and implement the
|
||||
* interface {@link}TSRandomAccessFileWriter
|
||||
*
|
||||
* @author liukun
|
||||
*/
|
||||
public class HDFSOutputStream implements TSRandomAccessFileWriter {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(HDFSOutputStream.class);
|
||||
|
||||
private FSDataOutputStream fsDataOutputStream;
|
||||
|
||||
public HDFSOutputStream(String filePath, boolean overwriter) throws IOException {
|
||||
|
||||
this(filePath, new Configuration(), overwriter);
|
||||
}
|
||||
|
||||
|
||||
public HDFSOutputStream(String filePath, Configuration configuration, boolean overwriter) throws IOException {
|
||||
|
||||
this(new Path(filePath),configuration,overwriter);
|
||||
}
|
||||
|
||||
public HDFSOutputStream(Path path,Configuration configuration,boolean overwriter) throws IOException{
|
||||
|
||||
FileSystem fsFileSystem = FileSystem.get(configuration);
|
||||
fsDataOutputStream = fsFileSystem.create(path, overwriter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OutputStream getOutputStream() {
|
||||
|
||||
return fsDataOutputStream;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getPos() throws IOException {
|
||||
|
||||
return fsDataOutputStream.getPos();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
|
||||
fsDataOutputStream.write(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(byte[] b) throws IOException {
|
||||
|
||||
fsDataOutputStream.write(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
|
||||
fsDataOutputStream.close();
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,40 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
|
||||
|
||||
public class ColumnSchema implements Serializable {
|
||||
private static final long serialVersionUID = -8257474930341487207L;
|
||||
|
||||
public String name;
|
||||
public TSDataType dataType;
|
||||
public TSEncoding encoding;
|
||||
private Map<String, String> args;
|
||||
|
||||
public ColumnSchema(String name, TSDataType dataType, TSEncoding encoding) {
|
||||
this.name = name;
|
||||
this.dataType = dataType;
|
||||
this.encoding = encoding;
|
||||
this.args = new HashMap<>();
|
||||
}
|
||||
|
||||
public void putKeyValueToArgs(String key, String value) {
|
||||
this.args.put(key, value);
|
||||
}
|
||||
|
||||
public String getValueFromArgs(String key) {
|
||||
return args.get(key);
|
||||
}
|
||||
|
||||
public Map<String, String> getArgsMap() {
|
||||
return args;
|
||||
}
|
||||
|
||||
public void setArgsMap(Map<String, String> argsMap) {
|
||||
this.args = argsMap;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,250 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* Metadata Graph consists of one {@code MTree} and several {@code PTree}
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class MGraph implements Serializable {
|
||||
private static final long serialVersionUID = 8214849219614352834L;
|
||||
|
||||
private MTree mTree;
|
||||
private HashMap<String, PTree> pTreeMap;
|
||||
|
||||
public MGraph(String MTreeName) {
|
||||
mTree = new MTree(MTreeName);
|
||||
pTreeMap = new HashMap<>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a {@code PTree} to current {@code MGraph}
|
||||
* @throws MetadataArgsErrorException
|
||||
*/
|
||||
public void addAPTree(String pTreeRootName) throws MetadataArgsErrorException {
|
||||
if(pTreeRootName.toLowerCase().equals("root")){
|
||||
throw new MetadataArgsErrorException("Property Tree's root name should not be 'root'");
|
||||
}
|
||||
PTree pTree = new PTree(pTreeRootName, mTree);
|
||||
pTreeMap.put(pTreeRootName, pTree);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a path to Metadata Tree
|
||||
*
|
||||
* @param path Format: root.node.(node)*
|
||||
* @return The count of new nodes added
|
||||
* @throws MetadataArgsErrorException
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public int addPathToMTree(String path, String dataType, String encoding, String args[])
|
||||
throws PathErrorException, MetadataArgsErrorException {
|
||||
String nodes[] = path.trim().split("\\.");
|
||||
if (nodes.length == 0) {
|
||||
throw new PathErrorException("Path is null. Path: " + path);
|
||||
}
|
||||
return this.mTree.addPath(path, dataType, encoding, args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a path to {@code PTree}
|
||||
*/
|
||||
public void addPathToPTree(String path) throws PathErrorException, MetadataArgsErrorException {
|
||||
String nodes[] = path.trim().split("\\.");
|
||||
if (nodes.length == 0) {
|
||||
throw new PathErrorException("Path is null. Path: " + path);
|
||||
}
|
||||
String rootName = path.trim().split("\\.")[0];
|
||||
if (pTreeMap.containsKey(rootName)) {
|
||||
PTree pTree = pTreeMap.get(rootName);
|
||||
pTree.addPath(path);
|
||||
} else {
|
||||
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete path in current MGraph.
|
||||
* @param path a path belongs to MTree or PTree
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public void deletePath(String path) throws PathErrorException {
|
||||
String nodes[] = path.trim().split("\\.");
|
||||
if (nodes.length == 0) {
|
||||
throw new PathErrorException("Path is null. Path: " + path);
|
||||
}
|
||||
String rootName = path.trim().split("\\.")[0];
|
||||
if (mTree.getRoot().getName().equals(rootName)) {
|
||||
mTree.deletePath(path);
|
||||
} else if (pTreeMap.containsKey(rootName)) {
|
||||
PTree pTree = pTreeMap.get(rootName);
|
||||
pTree.deletePath(path);
|
||||
} else {
|
||||
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Link a {@code MNode} to a {@code PNode} in current PTree
|
||||
*/
|
||||
public void linkMNodeToPTree(String path, String mPath) throws PathErrorException {
|
||||
String pTreeName = path.trim().split("\\.")[0];
|
||||
if (!pTreeMap.containsKey(pTreeName)) {
|
||||
throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path);
|
||||
} else {
|
||||
pTreeMap.get(pTreeName).linkMNode(path, mPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unlink a {@code MNode} from a {@code PNode} in current PTree
|
||||
*/
|
||||
public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException {
|
||||
String pTreeName = path.trim().split("\\.")[0];
|
||||
if (!pTreeMap.containsKey(pTreeName)) {
|
||||
throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path);
|
||||
} else {
|
||||
pTreeMap.get(pTreeName).unlinkMNode(path, mPath);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set storage level for current Metadata Tree.
|
||||
* @param path Format: root.node.(node)*
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public void setStorageLevel(String path) throws PathErrorException {
|
||||
mTree.setStorageLevel(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all paths for given path regular expression if given path belongs to
|
||||
* MTree, or get all linked path for given path if given path belongs to
|
||||
* PTree Notice: Regular expression in this method is formed by the
|
||||
* amalgamation of path and the character '*'
|
||||
*
|
||||
* @return A HashMap whose Keys are separated by the storage file name.
|
||||
*/
|
||||
public HashMap<String, ArrayList<String>> getAllPathGroupByFilename(String path) throws PathErrorException {
|
||||
String rootName = path.trim().split("\\.")[0];
|
||||
if (mTree.getRoot().getName().equals(rootName)) {
|
||||
return mTree.getAllPath(path);
|
||||
} else if (pTreeMap.containsKey(rootName)) {
|
||||
PTree pTree = pTreeMap.get(rootName);
|
||||
return pTree.getAllLinkedPath(path);
|
||||
}
|
||||
throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all DeltaObject type in current Metadata Tree
|
||||
* @return a HashMap contains all distinct DeltaObject type separated by
|
||||
* DeltaObject Type
|
||||
*/
|
||||
public Map<String, List<ColumnSchema>> getSchemaForAllType() throws PathErrorException {
|
||||
Map<String, List<ColumnSchema>> res = new HashMap<>();
|
||||
List<String> typeList = mTree.getAllType();
|
||||
for (String type : typeList) {
|
||||
res.put(type, getSchemaForOneType("root." + type));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private ArrayList<String> getDeltaObjectForOneType(String type) throws PathErrorException {
|
||||
return mTree.getDeltaObjectForOneType(type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all delta objects group by DeltaObject type
|
||||
*/
|
||||
public Map<String, List<String>> getDeltaObjectForAllType() throws PathErrorException {
|
||||
Map<String, List<String>> res = new HashMap<>();
|
||||
ArrayList<String> types = mTree.getAllType();
|
||||
for (String type : types) {
|
||||
res.put(type, getDeltaObjectForOneType(type));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full Metadata info.
|
||||
* @return A {@code Metadata} instance which stores all metadata info
|
||||
*/
|
||||
public Metadata getMetadata() throws PathErrorException {
|
||||
Map<String, List<ColumnSchema>> seriesMap = getSchemaForAllType();
|
||||
Map<String, List<String>> deltaObjectMap = getDeltaObjectForAllType();
|
||||
Metadata metadata = new Metadata(seriesMap, deltaObjectMap);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all ColumnSchemas for given delta object type
|
||||
* @param path A path represented one Delta object
|
||||
* @return a list contains all column schema
|
||||
*/
|
||||
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
|
||||
return mTree.getSchemaForOneType(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the count of storage-level nodes included in given path
|
||||
* @return The total count of storage-level nodes.
|
||||
*/
|
||||
public int getFileCountForOneType(String path) throws PathErrorException {
|
||||
return mTree.getFileCountForOneType(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file name for given path Notice: This method could be called if
|
||||
* and only if the path includes one node whose {@code isStorageLevel} is
|
||||
* true
|
||||
*/
|
||||
public String getFileNameByPath(String path) throws PathErrorException {
|
||||
return mTree.getFileNameByPath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the path given exists
|
||||
*/
|
||||
public boolean pathExist(String path) {
|
||||
return mTree.hasPath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the DeltaObjectId from given path
|
||||
* @return String represents the DeltaObjectId
|
||||
*/
|
||||
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
|
||||
return mTree.getDeltaObjectTypeByPath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ColumnSchema for given path. Notice: Path must be a complete Path
|
||||
* from root to leaf node.
|
||||
*/
|
||||
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
|
||||
return mTree.getSchemaForOnePath(path);
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("=== MetaData Tree ===\n\n");
|
||||
sb.append(mTree.toString());
|
||||
sb.append("\n\n=== Properties Tree === Size : " + pTreeMap.size() + "\n\n");
|
||||
for (String key : pTreeMap.keySet()) {
|
||||
sb.append("--- name : " + key + "---\n");
|
||||
sb.append(pTreeMap.get(key).toString());
|
||||
sb.append("\n\n");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,392 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import cn.edu.thu.tsfile.common.conf.TSFileDescriptor;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.timeseries.read.qp.Path;
|
||||
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* This class takes the responsibility of serialization of all the metadata info
|
||||
* and persistent it into files. This class contains all the interfaces to
|
||||
* modify the metadata for delta system. All the operations will be write into
|
||||
* the logs temporary in case the downtime of the delta system.
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class MManager {
|
||||
private static MManager manager = new MManager();
|
||||
|
||||
// The file storing the serialize info for metadata
|
||||
private String datafilePath;
|
||||
// log file path
|
||||
private String logFilePath;
|
||||
|
||||
private MGraph mGraph;
|
||||
private BufferedWriter bw;
|
||||
private boolean writeToLog;
|
||||
|
||||
private MManager() {
|
||||
writeToLog = false;
|
||||
String folderPath = TSFileDescriptor.getInstance().getConfig().metadataDir;
|
||||
datafilePath = folderPath + "/mdata.obj";
|
||||
logFilePath = folderPath + "/mlog.txt";
|
||||
init();
|
||||
}
|
||||
|
||||
private void init() {
|
||||
try {
|
||||
File file = new File(datafilePath);
|
||||
// inital MGraph from file
|
||||
if (file.exists()) {
|
||||
FileInputStream fis = new FileInputStream(file);
|
||||
ObjectInputStream ois = new ObjectInputStream(fis);
|
||||
mGraph = (MGraph) ois.readObject();
|
||||
ois.close();
|
||||
fis.close();
|
||||
|
||||
} else {
|
||||
mGraph = new MGraph("root");
|
||||
}
|
||||
|
||||
// recover operation from log file
|
||||
File logFile = new File(logFilePath);
|
||||
if (logFile.exists()) {
|
||||
FileReader fr;
|
||||
fr = new FileReader(logFile);
|
||||
BufferedReader br = new BufferedReader(fr);
|
||||
String cmd;
|
||||
while ((cmd = br.readLine()) != null) {
|
||||
operation(cmd);
|
||||
}
|
||||
br.close();
|
||||
} else {
|
||||
if (!logFile.getParentFile().exists()) {
|
||||
logFile.getParentFile().mkdirs();
|
||||
}
|
||||
logFile.createNewFile();
|
||||
}
|
||||
|
||||
FileWriter fw = new FileWriter(logFile, true);
|
||||
bw = new BufferedWriter(fw);
|
||||
writeToLog = true;
|
||||
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
} catch (PathErrorException e) {
|
||||
e.printStackTrace();
|
||||
} catch (MetadataArgsErrorException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all metadata info
|
||||
*/
|
||||
public void clear() {
|
||||
this.mGraph = new MGraph("root");
|
||||
}
|
||||
|
||||
public void deleteLogAndDataFiles() {
|
||||
File file = new File(logFilePath);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
File dataFile = new File(datafilePath);
|
||||
if (dataFile.exists()) {
|
||||
dataFile.delete();
|
||||
}
|
||||
}
|
||||
|
||||
private void operation(String cmd) throws PathErrorException, IOException, MetadataArgsErrorException {
|
||||
String args[] = cmd.trim().split(",");
|
||||
if (args[0].equals("0")) {
|
||||
String[] leftArgs;
|
||||
if (args.length > 4) {
|
||||
leftArgs = new String[args.length - 4];
|
||||
for (int k = 4; k < args.length; k++) {
|
||||
leftArgs[k - 4] = args[k];
|
||||
}
|
||||
} else {
|
||||
leftArgs = new String[0];
|
||||
}
|
||||
addAPathToMTree(args[1], args[2], args[3], leftArgs);
|
||||
} else if (args[0].equals(MetadataOperationType.ADD_PATH_TO_MTREE)) {
|
||||
deletePathFromMTree(args[1]);
|
||||
} else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_MTREE)) {
|
||||
setStorageLevelToMTree(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE);
|
||||
} else if (args[0].equals(MetadataOperationType.ADD_A_PTREE)) {
|
||||
addAPTree(args[1]);
|
||||
} else if (args[0].equals(MetadataOperationType.ADD_A_PATH_TO_PTREE)) {
|
||||
addAPathToPTree(args[1]);
|
||||
} else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_PTREE)) {
|
||||
deletePathFromPTree(args[1]);
|
||||
} else if (args[0].equals(MetadataOperationType.LINK_MNODE_TO_PTREE)) {
|
||||
linkMNodeToPTree(args[1], args[2]);
|
||||
} else if (args[0].equals(MetadataOperationType.UNLINK_MNODE_FROM_PTREE)) {
|
||||
unlinkMNodeFromPTree(args[1], args[2]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* operation: Add a path to Metadata Tree
|
||||
*/
|
||||
public int addAPathToMTree(String path, String dataType, String encoding, String[] args)
|
||||
throws PathErrorException, IOException, MetadataArgsErrorException {
|
||||
int addCount = mGraph.addPathToMTree(path, dataType, encoding, args);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.ADD_PATH_TO_MTREE + "," + path + "," + dataType + "," + encoding);
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
bw.write("," + args[i]);
|
||||
}
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
return addCount;
|
||||
}
|
||||
|
||||
public void deletePathFromMTree(String path) throws PathErrorException, IOException {
|
||||
mGraph.deletePath(path);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.DELETE_PATH_FROM_MTREE + "," + path);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
public void setStorageLevelToMTree(String path) throws PathErrorException, IOException {
|
||||
mGraph.setStorageLevel(path);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE + "," + path);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addAPTree(String pTreeRootName) throws IOException, MetadataArgsErrorException {
|
||||
mGraph.addAPTree(pTreeRootName);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.ADD_A_PTREE + "," + pTreeRootName);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
public void addAPathToPTree(String path) throws PathErrorException, IOException, MetadataArgsErrorException {
|
||||
mGraph.addPathToPTree(path);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.ADD_A_PATH_TO_PTREE + "," + path);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void deletePathFromPTree(String path) throws PathErrorException, IOException {
|
||||
mGraph.deletePath(path);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.DELETE_PATH_FROM_PTREE + "," + path);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
public void linkMNodeToPTree(String path, String mPath) throws PathErrorException, IOException {
|
||||
mGraph.linkMNodeToPTree(path, mPath);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.LINK_MNODE_TO_PTREE + "," + path + "," + mPath);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException, IOException {
|
||||
mGraph.unlinkMNodeFromPTree(path, mPath);
|
||||
if (writeToLog) {
|
||||
bw.write(MetadataOperationType.UNLINK_MNODE_FROM_PTREE + "," + path + "," + mPath);
|
||||
bw.newLine();
|
||||
bw.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the DeltaObjectId from given path
|
||||
* @param path
|
||||
* @return String represents the DeltaObjectId
|
||||
*/
|
||||
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
|
||||
return mGraph.getDeltaObjectTypeByPath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get series type for given path
|
||||
*
|
||||
* @param fullPath
|
||||
* @return TSDataType
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public TSDataType getSeriesType(String fullPath) throws PathErrorException {
|
||||
return getSchemaForOnePath(fullPath).dataType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all DeltaObject type in current Metadata Tree
|
||||
*
|
||||
* @return a HashMap contains all distinct DeltaObject type separated by
|
||||
* DeltaObject Type
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public Map<String, List<ColumnSchema>> getSchemaForAllType() throws PathErrorException {
|
||||
return mGraph.getSchemaForAllType();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full Metadata info.
|
||||
*
|
||||
* @return A {@code Metadata} instance which stores all metadata info
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public Metadata getMetadata() throws PathErrorException {
|
||||
return mGraph.getMetadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all ColumnSchemas for given delta object type
|
||||
*
|
||||
* @param path A path represented one Delta object
|
||||
* @return a list contains all column schema
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
|
||||
return mGraph.getSchemaForOneType(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the count of storage-level nodes included in given path
|
||||
* @param path
|
||||
* @return The total count of storage-level nodes.
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public int getFileCountForOneType(String path) throws PathErrorException {
|
||||
return mGraph.getFileCountForOneType(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file name for given path Notice: This method could be called if
|
||||
* and only if the path includes one node whose {@code isStorageLevel} is
|
||||
* true
|
||||
*
|
||||
* @param path
|
||||
* @return A String represented the file name
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public String getFileNameByPath(String path) throws PathErrorException {
|
||||
return mGraph.getFileNameByPath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* return a HashMap contains all the paths separated by File Name
|
||||
*/
|
||||
public HashMap<String, ArrayList<String>> getAllPathGroupByFileName(String path) throws PathErrorException {
|
||||
return mGraph.getAllPathGroupByFilename(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* return all paths for given path if the path is abstract.Or return the
|
||||
* path itself.
|
||||
*/
|
||||
public ArrayList<String> getPaths(String path) throws PathErrorException {
|
||||
ArrayList<String> res = new ArrayList<>();
|
||||
HashMap<String, ArrayList<String>> pathsGroupByFilename = getAllPathGroupByFileName(path);
|
||||
for (ArrayList<String> ps : pathsGroupByFilename.values()) {
|
||||
res.addAll(ps);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the path given exists
|
||||
*/
|
||||
public boolean pathExist(String path) {
|
||||
return mGraph.pathExist(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ColumnSchema for given path. Notice: Path must be a complete Path
|
||||
* from root to leaf node.
|
||||
*/
|
||||
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
|
||||
return mGraph.getSchemaForOnePath(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether given path contains a MNode whose
|
||||
* {@code MNode.isStorageLevel} is true
|
||||
*/
|
||||
public boolean checkFileLevel(List<Path> path) throws PathErrorException {
|
||||
for (Path p : path) {
|
||||
getFileNameByPath(p.getFullPath());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistent the MGraph instance into file
|
||||
*/
|
||||
public void flushObjectToFile() throws IOException {
|
||||
File newDataFile = new File(datafilePath + ".backup");
|
||||
FileOutputStream fos = new FileOutputStream(newDataFile);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(fos);
|
||||
oos.writeObject(mGraph);
|
||||
oos.close();
|
||||
|
||||
// delete log file
|
||||
File logFile = new File(logFilePath);
|
||||
logFile.delete();
|
||||
|
||||
// Copy New DataFile to OldDataFile
|
||||
FileOutputStream oldFileFos = new FileOutputStream(datafilePath);
|
||||
FileChannel outChannel = oldFileFos.getChannel();
|
||||
FileInputStream newFileFis = new FileInputStream(datafilePath + ".backup");
|
||||
FileChannel inChannel = newFileFis.getChannel();
|
||||
inChannel.transferTo(0, inChannel.size(), outChannel);
|
||||
outChannel.close();
|
||||
inChannel.close();
|
||||
oldFileFos.close();
|
||||
newFileFis.close();
|
||||
|
||||
// delete DataFileBackUp
|
||||
newDataFile.delete();
|
||||
}
|
||||
|
||||
public String getMetadataInString() {
|
||||
return mGraph.toString();
|
||||
}
|
||||
|
||||
public static MManager getInstance() {
|
||||
return manager;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,149 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.LinkedHashMap;
|
||||
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
|
||||
|
||||
/**
|
||||
* This class is the implementation of Metadata Node where "MNode" is the
|
||||
* shorthand of "Metadata Node". One MNode instance represents one node in the
|
||||
* Metadata Tree
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class MNode implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -770028375899514063L;
|
||||
|
||||
// The name of the MNode
|
||||
private String name;
|
||||
// Whether current node is a leaf in the Metadata Tree
|
||||
private boolean isLeaf;
|
||||
// Whether current node is Storage Level in the Metadata Tree
|
||||
private boolean isStorageLevel;
|
||||
// Corresponding data file name for current node
|
||||
private String dataFileName;
|
||||
// Column's Schema for one timeseries represented by current node if current
|
||||
// node is one leaf
|
||||
private ColumnSchema schema;
|
||||
private MNode parent;
|
||||
private LinkedHashMap<String, MNode> children;
|
||||
|
||||
public MNode(String name, MNode parent, boolean isLeaf) {
|
||||
this.setName(name);
|
||||
this.parent = parent;
|
||||
this.isLeaf = isLeaf;
|
||||
this.isStorageLevel = false;
|
||||
if (!isLeaf) {
|
||||
children = new LinkedHashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
public MNode(String name, MNode parent, TSDataType dataType, TSEncoding encoding) {
|
||||
this(name, parent, true);
|
||||
this.schema = new ColumnSchema(name, dataType, encoding);
|
||||
}
|
||||
|
||||
public boolean isStorageLevel() {
|
||||
return isStorageLevel;
|
||||
}
|
||||
|
||||
public void setStorageLevel(boolean b) {
|
||||
this.isStorageLevel = b;
|
||||
}
|
||||
|
||||
public boolean isLeaf() {
|
||||
return isLeaf;
|
||||
}
|
||||
|
||||
public void setLeaf(boolean isLeaf) {
|
||||
this.isLeaf = isLeaf;
|
||||
}
|
||||
|
||||
public boolean hasChild(String key) {
|
||||
if (!isLeaf) {
|
||||
return this.children.containsKey(key);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void addChild(String key, MNode child) {
|
||||
if (!isLeaf) {
|
||||
this.children.put(key, child);
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteChild(String key) {
|
||||
children.remove(key);
|
||||
}
|
||||
|
||||
public MNode getChild(String key) {
|
||||
if (!isLeaf) {
|
||||
return children.get(key);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the count of all leaves whose ancestor is current node
|
||||
*/
|
||||
public int getLeafCount() {
|
||||
if (isLeaf) {
|
||||
return 1;
|
||||
} else {
|
||||
int leafCount = 0;
|
||||
for (MNode child : this.children.values()) {
|
||||
leafCount += child.getLeafCount();
|
||||
}
|
||||
return leafCount;
|
||||
}
|
||||
}
|
||||
|
||||
public String getDataFileName() {
|
||||
return dataFileName;
|
||||
}
|
||||
|
||||
public void setDataFileName(String dataFileName) {
|
||||
this.dataFileName = dataFileName;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return this.getName();
|
||||
}
|
||||
|
||||
public ColumnSchema getSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
public void setSchema(ColumnSchema schema) {
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
public MNode getParent() {
|
||||
return parent;
|
||||
}
|
||||
|
||||
public void setParent(MNode parent) {
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
public LinkedHashMap<String, MNode> getChildren() {
|
||||
return children;
|
||||
}
|
||||
|
||||
public void setChildren(LinkedHashMap<String, MNode> children) {
|
||||
this.children = children;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,487 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSDataType;
|
||||
import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding;
|
||||
import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException;
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* The hierarchical struct of the Metadata Tree is implemented in this class.
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class MTree implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -4200394435237291964L;
|
||||
private final String space = " ";
|
||||
private MNode root;
|
||||
|
||||
public MTree(String rootName) {
|
||||
this.setRoot(new MNode(rootName, null, false));
|
||||
}
|
||||
|
||||
public MTree(MNode root) {
|
||||
this.setRoot(root);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a path to current Metadata Tree
|
||||
* @param path Format: root.node.(node)*
|
||||
*/
|
||||
public int addPath(String path, String dataType, String encoding, String[] args)
|
||||
throws PathErrorException, MetadataArgsErrorException {
|
||||
int addCount = 0;
|
||||
if (getRoot() == null) {
|
||||
throw new PathErrorException("Root Node is null, Please initialize root first");
|
||||
}
|
||||
String[] nodeNames = path.trim().split("\\.");
|
||||
if (nodeNames.length <= 1 || !nodeNames[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path not exist. Path: " + path);
|
||||
}
|
||||
|
||||
MNode cur = getRoot();
|
||||
int i;
|
||||
for (i = 1; i < nodeNames.length - 1; i++) {
|
||||
if (!cur.hasChild(nodeNames[i])) {
|
||||
cur.addChild(nodeNames[i], new MNode(nodeNames[i], cur, false));
|
||||
addCount++;
|
||||
}
|
||||
cur = cur.getChild(nodeNames[i]);
|
||||
}
|
||||
if (cur.hasChild(nodeNames[i])) {
|
||||
throw new PathErrorException("Path already exists. Path: " + path);
|
||||
} else {
|
||||
TSDataType dt = TSDataType.valueOf(dataType);
|
||||
TSEncoding ed = TSEncoding.valueOf(encoding);
|
||||
MNode leaf = new MNode(nodeNames[i], cur, dt, ed);
|
||||
if (args.length > 0) {
|
||||
for (int k = 0; k < args.length; k++) {
|
||||
String[] arg = args[k].split("=");
|
||||
leaf.getSchema().putKeyValueToArgs(arg[0], arg[1]);
|
||||
}
|
||||
}
|
||||
cur.addChild(nodeNames[i], leaf);
|
||||
addCount++;
|
||||
}
|
||||
return addCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete one path from current Metadata Tree
|
||||
* @param path
|
||||
* Format: root.node.(node)* Notice: Path must be a complete Path
|
||||
* from root to leaf node.
|
||||
*/
|
||||
public void deletePath(String path) throws PathErrorException {
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Path not correct. Path:" + path);
|
||||
}
|
||||
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (!cur.hasChild(nodes[i])) {
|
||||
throw new PathErrorException(
|
||||
"Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]);
|
||||
}
|
||||
cur = cur.getChild(nodes[i]);
|
||||
}
|
||||
cur.getParent().deleteChild(cur.getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Set filename to one node in the Metadata Tree Notice: This method will set
|
||||
* the {@code MNode.dataFileName} and {@code MNode.isStorageLevel} of current node
|
||||
* called if and only if {@code MNode.isStorageLevel} is true.
|
||||
* @param path Format: root.node.(node)*
|
||||
*/
|
||||
private void setFileNameToOnePath(String path) throws PathErrorException {
|
||||
String nodes[] = path.trim().split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (cur.hasChild(nodes[i])) {
|
||||
cur = cur.getChild(nodes[i]);
|
||||
} else {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
}
|
||||
|
||||
cur.setStorageLevel(true);
|
||||
setFileNameToMNode(cur, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set storage level for current Metadata Tree.
|
||||
* @param path Format: root.node.(node)*
|
||||
*/
|
||||
public void setStorageLevel(String path) throws PathErrorException {
|
||||
String nodes[] = path.trim().split("\\.");
|
||||
if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Root Name NOT correct. Root:" + path);
|
||||
}
|
||||
|
||||
int level = 0;
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (cur.hasChild(nodes[i])) {
|
||||
cur = cur.getChild(nodes[i]);
|
||||
level++;
|
||||
} else {
|
||||
throw new PathErrorException("Input path NOT correct. Nodes[" + i + "] not correct :" + nodes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
cur = getRoot().getChild(nodes[1]);
|
||||
setFileName(1, level, cur, "root." + cur.getName());
|
||||
}
|
||||
|
||||
private void setFileName(int curLevel, int level, MNode node, String path) throws PathErrorException {
|
||||
if (curLevel == level) {
|
||||
setFileNameToOnePath(path);
|
||||
return;
|
||||
}
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
setFileName(curLevel + 1, level, child, path + "." + child.getName());
|
||||
}
|
||||
}
|
||||
|
||||
private void setFileNameToMNode(MNode node, String fileName) {
|
||||
node.setDataFileName(fileName);
|
||||
if (!node.isLeaf()) {
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
setFileNameToMNode(child, fileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether the path given exists
|
||||
*/
|
||||
public boolean hasPath(String path) {
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
return false;
|
||||
}
|
||||
return hasPath(getRoot(), nodes, 1);
|
||||
}
|
||||
|
||||
private boolean hasPath(MNode node, String[] nodes, int idx) {
|
||||
if (idx >= nodes.length) {
|
||||
return true;
|
||||
}
|
||||
if (nodes[idx].equals("*")) {
|
||||
boolean res = false;
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
res |= hasPath(child, nodes, idx + 1);
|
||||
}
|
||||
return res;
|
||||
} else {
|
||||
if (node.hasChild(nodes[idx])) {
|
||||
return hasPath(node.getChild(nodes[idx]), nodes, idx + 1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ColumnSchema for given path. Notice: Path must be a complete Path
|
||||
* from root to leaf node.
|
||||
*/
|
||||
public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException {
|
||||
MNode leaf = getLeafByPath(path);
|
||||
return leaf.getSchema();
|
||||
}
|
||||
|
||||
private MNode getLeafByPath(String path) throws PathErrorException {
|
||||
checkPath(path);
|
||||
String[] node = path.split("\\.");
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < node.length; i++) {
|
||||
cur = cur.getChild(node[i]);
|
||||
}
|
||||
if (!cur.isLeaf()) {
|
||||
throw new PathErrorException("Input path not end with a leaf node.");
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the DeltaObjectType from given path
|
||||
* @return String represents the DeltaObjectId
|
||||
*/
|
||||
public String getDeltaObjectTypeByPath(String path) throws PathErrorException {
|
||||
checkPath(path);
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length < 2) {
|
||||
throw new PathErrorException("Input path must has two or more nodes");
|
||||
}
|
||||
return nodes[0] + "." + nodes[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether a path is available
|
||||
*/
|
||||
private void checkPath(String path) throws PathErrorException {
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length < 2 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (!cur.hasChild(nodes[i])) {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
cur = cur.getChild(nodes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file name for given path Notice: This method could be called if
|
||||
* and only if the path includes one node whose {@code isStorageLevel} is
|
||||
* true
|
||||
*/
|
||||
public String getFileNameByPath(String path) throws PathErrorException {
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
|
||||
MNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (!cur.hasChild(nodes[i])) {
|
||||
throw new PathErrorException("Input path NOT correct. Path: " + path);
|
||||
}
|
||||
if (cur.getDataFileName() != null) {
|
||||
return cur.getDataFileName();
|
||||
}
|
||||
cur = cur.getChild(nodes[i]);
|
||||
}
|
||||
if (cur.getDataFileName() != null) {
|
||||
return cur.getDataFileName();
|
||||
}
|
||||
throw new PathErrorException("Do not set FileLevel for this path: " + path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all paths for given path regular expression Regular expression in
|
||||
* this method is formed by the amalgamation of path and the character '*'
|
||||
* @return A HashMap whose Keys are separated by the storage file name.
|
||||
*/
|
||||
public HashMap<String, ArrayList<String>> getAllPath(String pathReg) throws PathErrorException {
|
||||
HashMap<String, ArrayList<String>> paths = new HashMap<>();
|
||||
String[] nodes = pathReg.split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path NOT correct. PathReg: " + pathReg);
|
||||
}
|
||||
findPath(getRoot(), nodes, 1, "", paths);
|
||||
return paths;
|
||||
}
|
||||
|
||||
|
||||
public ArrayList<String> getAllPathInList(String path) throws PathErrorException{
|
||||
ArrayList<String> res = new ArrayList<>();
|
||||
HashMap<String, ArrayList<String>> mapRet = getAllPath(path);
|
||||
for(ArrayList<String> value : mapRet.values()){
|
||||
res.addAll(value);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the count of storage-level nodes included in given path
|
||||
* @return The total count of storage-level nodes.
|
||||
*/
|
||||
public int getFileCountForOneType(String path) throws PathErrorException {
|
||||
String nodes[] = path.split("\\.");
|
||||
if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) {
|
||||
throw new PathErrorException(
|
||||
"Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)");
|
||||
}
|
||||
return getFileCountForOneNode(getRoot().getChild(nodes[1]));
|
||||
}
|
||||
|
||||
private int getFileCountForOneNode(MNode node) {
|
||||
|
||||
if (node.isStorageLevel()) {
|
||||
return 1;
|
||||
}
|
||||
int sum = 0;
|
||||
if (!node.isLeaf()) {
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
sum += getFileCountForOneNode(child);
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all DeltaObject type in current Metadata Tree
|
||||
* @return a list contains all distinct DeltaObject type
|
||||
*/
|
||||
public ArrayList<String> getAllType() {
|
||||
ArrayList<String> res = new ArrayList<>();
|
||||
if (getRoot() != null) {
|
||||
for (String type : getRoot().getChildren().keySet()) {
|
||||
res.add(type);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all delta objects for given type
|
||||
* @param type DeltaObject Type
|
||||
* @return a list contains all delta objects for given type
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public ArrayList<String> getDeltaObjectForOneType(String type) throws PathErrorException {
|
||||
String path = getRoot().getName() + "." + type;
|
||||
checkPath(path);
|
||||
HashMap<String, Integer> deltaObjectMap = new HashMap<>();
|
||||
MNode typeNode = getRoot().getChild(type);
|
||||
putDeltaObjectToMap(getRoot().getName(), typeNode, deltaObjectMap);
|
||||
ArrayList<String> res = new ArrayList<>();
|
||||
res.addAll(deltaObjectMap.keySet());
|
||||
return res;
|
||||
}
|
||||
|
||||
private void putDeltaObjectToMap(String path, MNode node, HashMap<String, Integer> deltaObjectMap) {
|
||||
if (node.isLeaf()) {
|
||||
deltaObjectMap.put(path, 1);
|
||||
} else {
|
||||
for (String child : node.getChildren().keySet()) {
|
||||
String newPath = path + "." + node.getName();
|
||||
putDeltaObjectToMap(newPath, node.getChildren().get(child), deltaObjectMap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all ColumnSchemas for given delta object type
|
||||
* @param path A path represented one Delta object
|
||||
* @return a list contains all column schema
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public ArrayList<ColumnSchema> getSchemaForOneType(String path) throws PathErrorException {
|
||||
String nodes[] = path.split("\\.");
|
||||
if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) {
|
||||
throw new PathErrorException(
|
||||
"Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)");
|
||||
}
|
||||
HashMap<String, ColumnSchema> leafMap = new HashMap<>();
|
||||
putLeafToLeafMap(getRoot().getChild(nodes[1]), leafMap);
|
||||
ArrayList<ColumnSchema> res = new ArrayList<>();
|
||||
res.addAll(leafMap.values());
|
||||
return res;
|
||||
}
|
||||
|
||||
private void putLeafToLeafMap(MNode node, HashMap<String, ColumnSchema> leafMap) {
|
||||
if (node.isLeaf()) {
|
||||
if (!leafMap.containsKey(node.getName())) {
|
||||
leafMap.put(node.getName(), node.getSchema());
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
putLeafToLeafMap(child, leafMap);
|
||||
}
|
||||
}
|
||||
|
||||
private void findPath(MNode node, String[] nodes, int idx, String parent, HashMap<String, ArrayList<String>> paths)
|
||||
throws PathErrorException {
|
||||
if (node.isLeaf()) {
|
||||
String fileName = node.getDataFileName();
|
||||
String nodePath = parent + node;
|
||||
putAPath(paths, fileName, nodePath);
|
||||
return;
|
||||
}
|
||||
String nodeReg;
|
||||
if (idx >= nodes.length) {
|
||||
nodeReg = "*";
|
||||
} else {
|
||||
nodeReg = nodes[idx];
|
||||
}
|
||||
|
||||
if (!nodeReg.equals("*")) {
|
||||
if (!node.hasChild(nodeReg)) {
|
||||
throw new PathErrorException("Path Error in node : " + nodeReg);
|
||||
}
|
||||
findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths);
|
||||
} else {
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
findPath(child, nodes, idx + 1, parent + node.getName() + ".", paths);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void putAPath(HashMap<String, ArrayList<String>> paths, String fileName, String nodePath) {
|
||||
if (paths.containsKey(fileName)) {
|
||||
paths.get(fileName).add(nodePath);
|
||||
} else {
|
||||
ArrayList<String> pathList = new ArrayList<>();
|
||||
pathList.add(nodePath);
|
||||
paths.put(fileName, pathList);
|
||||
}
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return MNodeToString(getRoot(), 0);
|
||||
}
|
||||
|
||||
private String MNodeToString(MNode node, int tab) {
|
||||
String s = "";
|
||||
for (int i = 0; i < tab; i++) {
|
||||
s += space;
|
||||
}
|
||||
s += node.getName();
|
||||
if (!node.isLeaf() && node.getChildren().size() > 0) {
|
||||
s += ":{\n";
|
||||
int first = 0;
|
||||
for (MNode child : node.getChildren().values()) {
|
||||
if (first == 0) {
|
||||
first = 1;
|
||||
} else {
|
||||
s += ",\n";
|
||||
}
|
||||
s += MNodeToString(child, tab + 1);
|
||||
}
|
||||
s += "\n";
|
||||
for (int i = 0; i < tab; i++) {
|
||||
s += space;
|
||||
}
|
||||
s += "}";
|
||||
} else if (node.isLeaf()) {
|
||||
s += ":{\n";
|
||||
s += getTabs(tab + 1) + "DataType :" + node.getSchema().dataType + ",\n";
|
||||
s += getTabs(tab + 1) + "Encoding :" + node.getSchema().encoding + ",\n";
|
||||
s += getTabs(tab + 1) + "args :" + node.getSchema().getArgsMap() + ",\n";
|
||||
s += getTabs(tab + 1) + "FileName :" + node.getDataFileName() + "\n";
|
||||
s += getTabs(tab) + "}";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
private String getTabs(int count) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < count; i++) {
|
||||
sb.append(space);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public MNode getRoot() {
|
||||
return root;
|
||||
}
|
||||
|
||||
public void setRoot(MNode root) {
|
||||
this.root = root;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,53 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* This class stores all the metadata info for every deltaObject and every
|
||||
* timeseries
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class Metadata {
|
||||
private Map<String, List<ColumnSchema>> seriesMap;
|
||||
private Map<String, List<String>> deltaObjectMap;
|
||||
|
||||
public Metadata(Map<String, List<ColumnSchema>> seriesMap, Map<String, List<String>> deltaObjectMap) {
|
||||
this.seriesMap = seriesMap;
|
||||
this.deltaObjectMap = deltaObjectMap;
|
||||
}
|
||||
|
||||
public List<ColumnSchema> getSeriesForOneType(String type) throws PathErrorException {
|
||||
if (this.seriesMap.containsKey(type)) {
|
||||
return seriesMap.get(type);
|
||||
} else {
|
||||
throw new PathErrorException("Input DeltaObjectType is not exist. " + type);
|
||||
}
|
||||
}
|
||||
|
||||
public List<String> getDeltaObjectsForOneType(String type) throws PathErrorException {
|
||||
if (this.seriesMap.containsKey(type)) {
|
||||
return deltaObjectMap.get(type);
|
||||
} else {
|
||||
throw new PathErrorException("Input DeltaObjectType is not exist. " + type);
|
||||
}
|
||||
}
|
||||
|
||||
public Map<String, List<ColumnSchema>> getSeriesMap() {
|
||||
return seriesMap;
|
||||
}
|
||||
|
||||
public Map<String, List<String>> getDeltaObjectMap() {
|
||||
return deltaObjectMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return seriesMap.toString() + "\n" + deltaObjectMap.toString();
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
public class MetadataOperationType {
|
||||
public final static String ADD_PATH_TO_MTREE = "0";
|
||||
public final static String DELETE_PATH_FROM_MTREE = "1";
|
||||
public final static String SET_STORAGE_LEVEL_TO_MTREE = "2";
|
||||
public final static String ADD_A_PTREE = "3";
|
||||
public final static String ADD_A_PATH_TO_PTREE = "4";
|
||||
public final static String DELETE_PATH_FROM_PTREE = "5";
|
||||
public final static String LINK_MNODE_TO_PTREE = "6";
|
||||
public final static String UNLINK_MNODE_FROM_PTREE = "7";
|
||||
}
|
|
@ -0,0 +1,113 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* PNode is the shorthand for "Property Node", which make up The {@code PTree}
|
||||
* @author zhangjinrui
|
||||
*
|
||||
*/
|
||||
public class PNode implements Serializable{
|
||||
|
||||
private static final long serialVersionUID = -7166236304286006338L;
|
||||
private String name;
|
||||
private PNode parent;
|
||||
private HashMap<String, PNode> children;
|
||||
private boolean isLeaf;
|
||||
|
||||
/**
|
||||
* This HashMap contains all the {@code MNode} this {@code PNode} is responsible for
|
||||
*/
|
||||
private LinkedHashMap<String, Integer> linkedMTreePathMap;
|
||||
|
||||
public PNode(String name, PNode parent, boolean isLeaf) {
|
||||
this.name = name;
|
||||
this.parent = parent;
|
||||
this.isLeaf = isLeaf;
|
||||
if (!isLeaf) {
|
||||
setChildren(new HashMap<>());
|
||||
} else {
|
||||
linkedMTreePathMap = new LinkedHashMap<>();
|
||||
}
|
||||
}
|
||||
|
||||
public void linkMPath(String mTreePath) throws PathErrorException{
|
||||
if(!isLeaf){
|
||||
throw new PathErrorException("Current PNode is NOT leaf node");
|
||||
}
|
||||
if(!linkedMTreePathMap.containsKey(mTreePath)){
|
||||
linkedMTreePathMap.put(mTreePath, 1);
|
||||
}
|
||||
}
|
||||
|
||||
public void unlinkMPath(String mTreePath) throws PathErrorException{
|
||||
if(!isLeaf){
|
||||
throw new PathErrorException("Current PNode is NOT leaf node");
|
||||
}
|
||||
if(!linkedMTreePathMap.containsKey(mTreePath)){
|
||||
return;
|
||||
}
|
||||
linkedMTreePathMap.remove(mTreePath);
|
||||
}
|
||||
|
||||
public boolean hasChild(String key) {
|
||||
return getChildren().containsKey(key);
|
||||
}
|
||||
|
||||
public PNode getChild(String key) {
|
||||
return getChildren().get(key);
|
||||
}
|
||||
|
||||
public void addChild(String name, PNode node) {
|
||||
this.getChildren().put(name, node);
|
||||
}
|
||||
|
||||
public void deleteChild(String name){
|
||||
this.getChildren().remove(name);
|
||||
}
|
||||
|
||||
public boolean isLeaf() {
|
||||
return isLeaf;
|
||||
}
|
||||
|
||||
public void setLeaf(boolean isLeaf) {
|
||||
this.isLeaf = isLeaf;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public PNode getParent() {
|
||||
return parent;
|
||||
}
|
||||
|
||||
public void setParent(PNode parent) {
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
public HashMap<String, PNode> getChildren() {
|
||||
return children;
|
||||
}
|
||||
|
||||
public void setChildren(HashMap<String, PNode> children) {
|
||||
this.children = children;
|
||||
}
|
||||
|
||||
public HashMap<String, Integer> getLinkedMTreePathMap() {
|
||||
return linkedMTreePathMap;
|
||||
}
|
||||
|
||||
public void setLinkedMTreePathMap(LinkedHashMap<String, Integer> linkedMTreePathMap) {
|
||||
this.linkedMTreePathMap = linkedMTreePathMap;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,241 @@
|
|||
package cn.edu.thu.tsfiledb.metadata;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import cn.edu.thu.tsfiledb.exception.PathErrorException;
|
||||
|
||||
/**
|
||||
* "PTree" is the shorthand for "Property Tree". One {@code PTree} consists
|
||||
* several {@code PNode}
|
||||
*
|
||||
* @author Jinrui Zhang
|
||||
*
|
||||
*/
|
||||
public class PTree implements Serializable {
|
||||
private static final long serialVersionUID = 2642766399323283900L;
|
||||
|
||||
private PNode root;
|
||||
private MTree mTree;
|
||||
private String name;
|
||||
private String space = " ";
|
||||
|
||||
public PTree(String name, MTree mTree) {
|
||||
this.setRoot(new PNode(name, null, false));
|
||||
this.setName(name);
|
||||
this.setmTree(mTree);
|
||||
}
|
||||
|
||||
public PTree(String name, PNode root, MTree mTree) {
|
||||
this.setName(name);
|
||||
this.setRoot(root);
|
||||
this.setmTree(mTree);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a path to current PTree
|
||||
* @return The count of new added {@code PNode}
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public int addPath(String path) throws PathErrorException {
|
||||
int addCount = 0;
|
||||
if (getRoot() == null) {
|
||||
throw new PathErrorException("Root Node is null, Please initialize root first");
|
||||
}
|
||||
String[] nodes = path.trim().split("\\.");
|
||||
if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Input path not exist. Path: " + path);
|
||||
}
|
||||
|
||||
PNode cur = getRoot();
|
||||
int i;
|
||||
for (i = 1; i < nodes.length - 1; i++) {
|
||||
if (!cur.hasChild(nodes[i])) {
|
||||
cur.addChild(nodes[i], new PNode(nodes[i], cur, false));
|
||||
addCount++;
|
||||
}
|
||||
cur = cur.getChild(nodes[i]);
|
||||
}
|
||||
if (cur.hasChild(nodes[i])) {
|
||||
throw new PathErrorException("Path already exists. Path: " + path);
|
||||
} else {
|
||||
PNode node = new PNode(nodes[i], cur, true);
|
||||
cur.addChild(node.getName(), node);
|
||||
addCount++;
|
||||
}
|
||||
return addCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a path from current PTree
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public void deletePath(String path) throws PathErrorException {
|
||||
String[] nodes = path.split("\\.");
|
||||
if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) {
|
||||
throw new PathErrorException("Path not correct. Path:" + path);
|
||||
}
|
||||
PNode cur = getRoot();
|
||||
for (int i = 1; i < nodes.length; i++) {
|
||||
if (!cur.hasChild(nodes[i])) {
|
||||
throw new PathErrorException(
|
||||
"Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]);
|
||||
}
|
||||
cur = cur.getChild(nodes[i]);
|
||||
}
|
||||
cur.getParent().deleteChild(cur.getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Link a {@code MNode} to a {@code PNode} in current PTree
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public void linkMNode(String pTreePath, String mTreePath) throws PathErrorException {
|
||||
List<String> paths = mTree.getAllPathInList(mTreePath);
|
||||
String nodes[] = pTreePath.trim().split("\\.");
|
||||
PNode leaf = getLeaf(getRoot(), nodes, 0);
|
||||
for (String p : paths) {
|
||||
leaf.linkMPath(p);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unlink a {@code MNode} from a {@code PNode} in current PTree
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public void unlinkMNode(String pTreePath, String mTreePath) throws PathErrorException {
|
||||
List<String> paths = mTree.getAllPathInList(mTreePath);
|
||||
String nodes[] = pTreePath.trim().split("\\.");
|
||||
PNode leaf = getLeaf(getRoot(), nodes, 0);
|
||||
for (String p : paths) {
|
||||
leaf.unlinkMPath(p);
|
||||
}
|
||||
}
|
||||
|
||||
private PNode getLeaf(PNode node, String[] nodes, int idx) throws PathErrorException {
|
||||
if (idx >= nodes.length) {
|
||||
throw new PathErrorException("PTree path not exist. ");
|
||||
}
|
||||
if (node.isLeaf()) {
|
||||
if (idx != nodes.length - 1 || !nodes[idx].equals(node.getName())) {
|
||||
throw new PathErrorException("PTree path not exist. ");
|
||||
}
|
||||
return node;
|
||||
}else{
|
||||
if (idx >= nodes.length - 1 || !node.hasChild(nodes[idx + 1])) {
|
||||
throw new PathErrorException("PTree path not exist. ");
|
||||
}
|
||||
return getLeaf(node.getChild(nodes[idx + 1]), nodes, idx + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param path
|
||||
* a path in current {@code PTree} Get all linked path in MTree
|
||||
* according to the given path in PTree
|
||||
* @return Paths will be separated by the {@code MNode.dataFileName} in the
|
||||
* HashMap
|
||||
* @throws PathErrorException
|
||||
*/
|
||||
public HashMap<String, ArrayList<String>> getAllLinkedPath(String path) throws PathErrorException {
|
||||
String[] nodes = path.trim().split("\\.");
|
||||
PNode leaf = getLeaf(getRoot(), nodes, 0);
|
||||
HashMap<String, ArrayList<String>> res = new HashMap<>();
|
||||
|
||||
for (String MPath : leaf.getLinkedMTreePathMap().keySet()) {
|
||||
HashMap<String, ArrayList<String>> tr = getmTree().getAllPath(MPath);
|
||||
mergePathRes(res, tr);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
private void mergePathRes(HashMap<String, ArrayList<String>> res, HashMap<String, ArrayList<String>> tr) {
|
||||
for (String key : tr.keySet()) {
|
||||
if (!res.containsKey(key)) {
|
||||
res.put(key, new ArrayList<String>());
|
||||
}
|
||||
for (String p : tr.get(key)) {
|
||||
if (!res.get(key).contains(p)) {
|
||||
res.get(key).add(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return PNodeToString(getRoot(), 0);
|
||||
}
|
||||
|
||||
private String PNodeToString(PNode node, int tab) {
|
||||
String s = "";
|
||||
for (int i = 0; i < tab; i++) {
|
||||
s += space;
|
||||
}
|
||||
s += node.getName();
|
||||
if (!node.isLeaf() && node.getChildren().size() > 0) {
|
||||
s += ":{\n";
|
||||
int first = 0;
|
||||
for (PNode child : node.getChildren().values()) {
|
||||
if (first == 0) {
|
||||
first = 1;
|
||||
} else {
|
||||
s += ",\n";
|
||||
}
|
||||
s += PNodeToString(child, tab + 1);
|
||||
}
|
||||
s += "\n";
|
||||
for (int i = 0; i < tab; i++) {
|
||||
s += space;
|
||||
}
|
||||
s += "}";
|
||||
} else if (node.isLeaf()) {
|
||||
s += ":{\n";
|
||||
String[] linkedPaths = new String[node.getLinkedMTreePathMap().values().size()];
|
||||
linkedPaths = node.getLinkedMTreePathMap().values().toArray(linkedPaths);
|
||||
for (int i = 0; i < linkedPaths.length; i++) {
|
||||
if (i != linkedPaths.length - 1) {
|
||||
s += getTabs(tab + 1) + linkedPaths[i] + ",\n";
|
||||
} else {
|
||||
s += getTabs(tab + 1) + linkedPaths[i] + "\n";
|
||||
}
|
||||
}
|
||||
s += getTabs(tab) + "}";
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
private String getTabs(int count) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < count; i++) {
|
||||
sb.append(space);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public MTree getmTree() {
|
||||
return mTree;
|
||||
}
|
||||
|
||||
public void setmTree(MTree mTree) {
|
||||
this.mTree = mTree;
|
||||
}
|
||||
|
||||
public PNode getRoot() {
|
||||
return root;
|
||||
}
|
||||
|
||||
public void setRoot(PNode root) {
|
||||
this.root = root;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,38 @@
|
|||
package cn.edu.thu.tsfiledb;
|
||||
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestCase;
|
||||
import junit.framework.TestSuite;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class AppTest
|
||||
extends TestCase
|
||||
{
|
||||
/**
|
||||
* Create the test case
|
||||
*
|
||||
* @param testName name of the test case
|
||||
*/
|
||||
public AppTest( String testName )
|
||||
{
|
||||
super( testName );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the suite of tests being tested
|
||||
*/
|
||||
public static Test suite()
|
||||
{
|
||||
return new TestSuite( AppTest.class );
|
||||
}
|
||||
|
||||
/**
|
||||
* Rigourous Test :-)
|
||||
*/
|
||||
public void testApp()
|
||||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue