From a3a5a392e3d4694ea721b51f1b7a50cb8e583898 Mon Sep 17 00:00:00 2001 From: xingtanzjr Date: Sun, 7 May 2017 14:50:07 +0800 Subject: [PATCH] Transfer codes from pre-project --- .gitignore | 30 + pom.xml | 72 ++ src/main/java/cn/edu/thu/tsfiledb/App.java | 13 + .../tsfiledb/engine/bufferwrite/Action.java | 10 + .../bufferwrite/BufferWriteIOWriter.java | 46 + .../engine/bufferwrite/BufferWriteIndex.java | 35 + .../bufferwrite/BufferWriteManager.java | 190 ++++ .../bufferwrite/BufferWriteProcessor.java | 330 ++++++ .../engine/bufferwrite/FileNodeConstants.java | 19 + .../engine/bufferwrite/LRUManager.java | 322 ++++++ .../engine/bufferwrite/LRUProcessor.java | 155 +++ .../MemoryBufferWriteIndexImpl.java | 77 ++ .../bufferwrite/TSRecordWriterParameter.java | 25 + .../engine/bufferwrite/TriFunction.java | 16 + .../overflow/IIntervalTreeOperator.java | 109 ++ .../overflow/IntervalTreeOperation.java | 991 ++++++++++++++++++ .../engine/overflow/index/CrossRelation.java | 15 + .../overflow/index/IntervalRelation.java | 69 ++ .../engine/overflow/index/IntervalTree.java | 588 +++++++++++ .../engine/overflow/index/TreeNode.java | 34 + .../engine/overflow/utils/MergeStatus.java | 13 + .../engine/overflow/utils/OverflowOpType.java | 16 + .../engine/overflow/utils/TimePair.java | 75 ++ .../thu/tsfiledb/engine/utils/FlushState.java | 30 + .../exception/ArgsErrorException.java | 10 + .../DeltaEngineRunningException.java | 23 + .../exception/ErrorDebugException.java | 18 + .../exception/FileNodeNotExistException.java | 16 + .../exception/MetadataArgsErrorException.java | 18 + .../OverflowWrongParameterException.java | 20 + .../exception/PathErrorException.java | 10 + .../exception/ProcessorException.java | 24 + .../UnSupportedOverflowOpTypeException.java | 20 + .../tsfiledb/hadoop/io/HDFSInputStream.java | 102 ++ .../tsfiledb/hadoop/io/HDFSOutputStream.java | 75 ++ .../thu/tsfiledb/metadata/ColumnSchema.java | 40 + .../cn/edu/thu/tsfiledb/metadata/MGraph.java | 250 +++++ .../edu/thu/tsfiledb/metadata/MManager.java | 392 +++++++ .../cn/edu/thu/tsfiledb/metadata/MNode.java | 149 +++ .../cn/edu/thu/tsfiledb/metadata/MTree.java | 487 +++++++++ .../edu/thu/tsfiledb/metadata/Metadata.java | 53 + .../metadata/MetadataOperationType.java | 12 + .../cn/edu/thu/tsfiledb/metadata/PNode.java | 113 ++ .../cn/edu/thu/tsfiledb/metadata/PTree.java | 241 +++++ .../java/cn/edu/thu/tsfiledb/AppTest.java | 38 + 45 files changed, 5391 insertions(+) create mode 100644 .gitignore create mode 100644 pom.xml create mode 100644 src/main/java/cn/edu/thu/tsfiledb/App.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/Action.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIOWriter.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIndex.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteManager.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteProcessor.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/FileNodeConstants.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUManager.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUProcessor.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/MemoryBufferWriteIndexImpl.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TSRecordWriterParameter.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TriFunction.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IIntervalTreeOperator.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IntervalTreeOperation.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/CrossRelation.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalRelation.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalTree.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/TreeNode.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/MergeStatus.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/OverflowOpType.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/TimePair.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/engine/utils/FlushState.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/ArgsErrorException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/DeltaEngineRunningException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/ErrorDebugException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/FileNodeNotExistException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/MetadataArgsErrorException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/OverflowWrongParameterException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/PathErrorException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/ProcessorException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/exception/UnSupportedOverflowOpTypeException.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSInputStream.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSOutputStream.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/ColumnSchema.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/MGraph.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/MManager.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/MNode.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/MTree.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/Metadata.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/MetadataOperationType.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/PNode.java create mode 100644 src/main/java/cn/edu/thu/tsfiledb/metadata/PTree.java create mode 100644 src/test/java/cn/edu/thu/tsfiledb/AppTest.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..95de56c3551 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Eclipse IDE files +**/.classpath +**/.project +**/.settings/ + +# intellij IDE files +**/*.iml +**/.idea/ + +# Apple OS X related +**/.DS_Store + +# build generated +**/target/ +**/build/ + +# intermediately generated locally +**/logs/ + +tsfile-timeseries/src/main/resources/logback.out.out.xml +tsfile-timeseries/src/main/resources/logback.out.xml +tsfile-service/derby-tsfile-db/ +tsfile-timeseries/src/test/resources/data +src/main/resources/metadata/mlog.txt +tsfile-jdbc/src/main/resources/output/queryRes.csv + +*.jar +*.gz +*.tar.gz +*.tar \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000000..9bc95f196cf --- /dev/null +++ b/pom.xml @@ -0,0 +1,72 @@ + + 4.0.0 + + cn.edu.thu + tsfiledb + 0.0.1-SNAPSHOT + jar + + tsfiledb + http://maven.apache.org + + + UTF-8 + 1.8 + 2.6.0 + + + + + + cn.edu.thu + tsfile + 0.1.0 + + + + junit + junit + 3.8.1 + test + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + ${compiler.version} + ${compiler.version} + UTF-8 + + + + + compile-integration-test + pre-integration-test + + testCompile + + + + **/*.java + + ${project.build.directory}/it-classes + + + + + + + diff --git a/src/main/java/cn/edu/thu/tsfiledb/App.java b/src/main/java/cn/edu/thu/tsfiledb/App.java new file mode 100644 index 00000000000..490878c7e48 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/App.java @@ -0,0 +1,13 @@ +package cn.edu.thu.tsfiledb; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/Action.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/Action.java new file mode 100644 index 00000000000..e0f793b9f72 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/Action.java @@ -0,0 +1,10 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +/** + * @author kangrong + * + */ +@FunctionalInterface +public interface Action { + void act(); +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIOWriter.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIOWriter.java new file mode 100644 index 00000000000..38a6f51aa7d --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIOWriter.java @@ -0,0 +1,46 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter; +import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData; +import cn.edu.thu.tsfile.timeseries.write.io.TSFileIOWriter; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; + +/** + * @author kangrong + * + */ +public class BufferWriteIOWriter extends TSFileIOWriter { + + /* + * The backup list is used to store the rowgroup's metadata whose data has + * been flushed into file. + */ + private final List backUpList = new ArrayList(); + + public BufferWriteIOWriter(FileSchema schema, TSRandomAccessFileWriter output) throws IOException { + super(schema, output); + } + + /** + * Note that,the method is not thread safe. + */ + public void addNewRowGroupMetaDataToBackUp() { + backUpList.add(rowGroups.get(rowGroups.size() - 1)); + } + + /** + * Note that, the method is not thread safe. You mustn't do any + * change on the return.
+ * + * @return + */ + public List getCurrentRowGroupMetaList() { + List ret = new ArrayList<>(); + backUpList.forEach(ret::add); + return ret; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIndex.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIndex.java new file mode 100644 index 00000000000..1ad51167d6e --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteIndex.java @@ -0,0 +1,35 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; + +/** + * The function of this interface is to store and index TSRecord in memory + * temporarily + * + * @author kangrong + * + */ +public interface BufferWriteIndex { + /** + * insert a tsRecord + * + * @param tsRecord + */ + void insert(TSRecord tsRecord); + + /** + * Get the DynamicOneColumnData from the buffer index + * + * @param deltaObjectId + * @param measurementId + * @return + */ + public DynamicOneColumnData query(String deltaObjectId, String measurementId); + + /** + * clear all data written in the bufferindex structure which will be used + * for next stage + */ + void clear(); +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteManager.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteManager.java new file mode 100644 index 00000000000..dd41876b450 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteManager.java @@ -0,0 +1,190 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.function.BiFunction; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.edu.thu.tsfile.common.conf.TSFileConfig; +import cn.edu.thu.tsfile.common.conf.TSFileDescriptor; +import cn.edu.thu.tsfile.common.constant.JsonFormatConstant; +import cn.edu.thu.tsfile.common.utils.Pair; +import cn.edu.thu.tsfile.common.utils.RandomAccessOutputStream; +import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter; +import cn.edu.thu.tsfile.timeseries.write.TSRecordWriteSupport; +import cn.edu.thu.tsfile.timeseries.write.WriteSupport; +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; +import cn.edu.thu.tsfiledb.exception.PathErrorException; +import cn.edu.thu.tsfiledb.exception.ProcessorException; +import cn.edu.thu.tsfiledb.metadata.ColumnSchema; +import cn.edu.thu.tsfiledb.metadata.MManager; + +/** + * BufferWrite manager manage a list of bufferwrite processor + * + * @author kangrong + * @author liukun + */ +public class BufferWriteManager extends LRUManager { + private static Logger LOG = LoggerFactory.getLogger(BufferWriteManager.class); + private static TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); + protected static BufferWriteManager instance = new BufferWriteManager(conf.maxBufferWriteNodeNum, + MManager.getInstance(), conf.BufferWriteDir); + + public static BufferWriteManager getInstance() { + return instance; + } + + /** + * Initialize the instance of BufferWriteManager using special parameters + * + * @param maxLRUNumber + * @param mManager + * @param normalDataDir + */ + public synchronized static void init(int maxLRUNumber, MManager mManager, String normalDataDir) { + instance = new BufferWriteManager(maxLRUNumber, mManager, normalDataDir); + } + + private BufferWriteManager(int maxLRUNumber, MManager mManager, String normalDataDir) { + super(maxLRUNumber, mManager, normalDataDir); + } + + /* + * If the buffer write processor is not exist before, it will be constructed + * using name space path and key-value information(args) + */ + @Override + protected BufferWriteProcessor constructNewProcessor(String namespacePath, Map args) + throws ProcessorException, IOException, WriteProcessException { + long startTime = (Long) args.get(FileNodeConstants.TIMESTAMP_KEY); + Pair fileNamePair = prepareMeasurePathFileName.apply(namespacePath, + Long.valueOf(startTime).toString()); + TSRecordWriterParameter param = prepareTSRecordWriterParameter.apply(namespacePath, mManager, fileNamePair); + BufferWriteIOWriter bufferTSFileWriter = new BufferWriteIOWriter(param.fileSchema, param.outputStream); + LOG.info("Construct a new buffer write processor, fileName:{}", fileNamePair); + TSFileConfig conf = TSFileDescriptor.getInstance().getConfig(); + return new BufferWriteProcessor(conf, bufferTSFileWriter, param.writeSupport, param.fileSchema, namespacePath, + fileNamePair); + } + + @Override + protected void initProcessor(BufferWriteProcessor processor, String namespacePath, Map args) + throws ProcessorException { + if (processor.getCloseAction() == null && args == null) { + LOG.error( + "The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args"); + throw new RuntimeException( + "The buffer write processor is not be initialized before, but also can't be initailized because of lacking of args"); + } + if (args != null && args.containsKey(FileNodeConstants.CLOSE_ACTION)) + processor.setCloseAction((Action) args.get(FileNodeConstants.CLOSE_ACTION)); + + } + + /** + * @param namespacePath + * -this path is a concept in name space tree. like + * root.laptop.xxx. The absolute file path is deltaDataDir + + * namespacepath + number. + * @param MManager + * @param Pair + * + * @return TSRecordWriterParameter + */ + public TriFunction, TSRecordWriterParameter> prepareTSRecordWriterParameter = ( + namespacePath, inputMManager, filePathPair) -> { + Optional> meaSchema; + + String deltaObjectType; + try { + deltaObjectType = mManager.getDeltaObjectTypeByPath(namespacePath); + meaSchema = Optional.ofNullable(inputMManager.getSchemaForOneType(deltaObjectType)); + } catch (PathErrorException e) { + throw new RuntimeException(e); + } + FileSchema fSchema = null; + try { + if (meaSchema == null) { + throw new ProcessorException("Measures schema list is null"); + } + fSchema = getFileSchemaFromColumnSchema(meaSchema.orElse(new ArrayList()), deltaObjectType); + } catch (ProcessorException e) { + LOG.error("The measures schema list is {}", meaSchema); + throw new RuntimeException("Get the File schema failed"); + } + File file = new File(filePathPair.left); + WriteSupport writeSupport = new TSRecordWriteSupport(); + TSRandomAccessFileWriter outputStream; + try { + outputStream = new RandomAccessOutputStream(file); + } catch (IOException e) { + LOG.error("Initialized the data output stream failed, the file path is {}", filePathPair.left); + throw new RuntimeException(new ProcessorException( + "IOException: " + e.getMessage() + " create data output stream failed:" + file)); + } + return new TSRecordWriterParameter(fSchema, outputStream, writeSupport); + }; + + /** + * @param namespacePath + * -this path is a concept in name space tree. like + * root.laptop.xxx. The absolute file path is deltaDataDir + + * namespacepath + number. + * @param filename + * - start time for this file + * @return Pair left - data file right -error data file + */ + public BiFunction> prepareMeasurePathFileName = (namespacePath, fileName) -> { + String absolutePathDir = normalDataDir + namespacePath; + File dir = new File(absolutePathDir); + if (!dir.exists()) { + dir.mkdirs(); + } + String absolutePath = dir.getAbsolutePath() + File.separator + fileName; + + return new Pair<>(absolutePath, "null"); + }; + + /** + * Construct one FileSchema from a list of columnschema and the deltaobject + * type + * + * @param schemaList + * @param deltaObjectType + * @return + */ + private FileSchema getFileSchemaFromColumnSchema(List schemaList, String deltaObjectType) throws WriteProcessException { + JSONArray rowGroup = new JSONArray(); + + for (ColumnSchema col : schemaList) { + JSONObject measurement = new JSONObject(); + measurement.put(JsonFormatConstant.MEASUREMENT_UID, col.name); + measurement.put(JsonFormatConstant.DATA_TYPE, col.dataType.toString()); + measurement.put(JsonFormatConstant.MEASUREMENT_ENCODING, col.encoding.toString()); + for (Entry entry : col.getArgsMap().entrySet()) { + if (JsonFormatConstant.ENUM_VALUES.equals(entry.getKey())) { + String[] valueArray = entry.getValue().split(","); + measurement.put(JsonFormatConstant.ENUM_VALUES, new JSONArray(valueArray)); + } else + measurement.put(entry.getKey(), entry.getValue().toString()); + } + rowGroup.put(measurement); + } + JSONObject jsonSchema = new JSONObject(); + jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, rowGroup); + jsonSchema.put(JsonFormatConstant.DELTA_TYPE, deltaObjectType); + return new FileSchema(jsonSchema); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteProcessor.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteProcessor.java new file mode 100644 index 00000000000..e7c988448e1 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/BufferWriteProcessor.java @@ -0,0 +1,330 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.edu.thu.tsfile.common.conf.TSFileConfig; +import cn.edu.thu.tsfile.common.utils.Pair; +import cn.edu.thu.tsfile.file.metadata.RowGroupMetaData; +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; +import cn.edu.thu.tsfile.timeseries.write.TSRecordWriter; +import cn.edu.thu.tsfile.timeseries.write.WriteSupport; +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; +import cn.edu.thu.tsfile.timeseries.write.record.DataPoint; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; +import cn.edu.thu.tsfile.timeseries.write.series.IRowGroupWriter; +import cn.edu.thu.tsfiledb.engine.utils.FlushState; + +/** + * BufferWriteProcessor is used to write one data point or one tsRecord whose + * timestamp is greater than than the last update time in + * {@code FileNodeProcessor}
+ * + * @author kangrong + * @author liukun + */ +public class BufferWriteProcessor extends LRUProcessor { + private static final Logger LOG = LoggerFactory.getLogger(BufferWriteProcessor.class); + + private BufferWriteIndex workingBufferIndex; + private BufferWriteIndex flushingBufferIndex; + + private boolean isFlushingSync = false; + private FlushState flushState = new FlushState(); + + private BufferWriteIOWriter bufferIOWriter; + private ReadWriteLock convertBufferLock = new ReentrantReadWriteLock(false); + private BufferWriteRecordWriter recordWriter = null; + private Pair fileNamePair; + private boolean isNewProcessor = false; + + private Action closeAction; + + BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport writeSupport, + FileSchema schema, String nsPath) { + super(nsPath); + this.workingBufferIndex = new MemoryBufferWriteIndexImpl(); + this.flushingBufferIndex = new MemoryBufferWriteIndexImpl(); + this.bufferIOWriter = ioFileWriter; + this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema); + } + + BufferWriteProcessor(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, WriteSupport writeSupport, + FileSchema schema, String nsPath, Pair fileNamePair) { + super(nsPath); + this.workingBufferIndex = new MemoryBufferWriteIndexImpl(); + this.flushingBufferIndex = new MemoryBufferWriteIndexImpl(); + this.bufferIOWriter = ioFileWriter; + this.recordWriter = new BufferWriteRecordWriter(conf, ioFileWriter, writeSupport, schema); + this.fileNamePair = fileNamePair; + this.isNewProcessor = true; + } + + /** + * Write a data point + * + * @param deltaObjectId + * @param measurementId + * @param timestamp + * @param deltaObjectType + * @param dataType + * @param value + * @throws WriteProcessException + * @throws IOException + */ + public void write(String deltaObjectId, String measurementId, long timestamp, String deltaObjectType, + TSDataType dataType, String value) throws IOException, WriteProcessException { + TSRecord record = new TSRecord(timestamp, deltaObjectId); + DataPoint dataPoint = DataPoint.getDataPoint(dataType, measurementId, value); + record.addTuple(dataPoint); + write(record); + } + + /** + * Write a tsRecord + * + * @param tsRecord + * @throws WriteProcessException + * @throws IOException + */ + public void write(TSRecord tsRecord) throws IOException, WriteProcessException { + workingBufferIndex.insert(tsRecord); + recordWriter.write(tsRecord); + } + + private void switchIndexFromWorkingToFlushing() { + BufferWriteIndex temp = workingBufferIndex; + workingBufferIndex = flushingBufferIndex; + flushingBufferIndex = temp; + + } + + private void switchIndexFromFlushingToWorking() { + flushingBufferIndex.clear(); + bufferIOWriter.addNewRowGroupMetaDataToBackUp(); + flushState.setUnFlushing(); + } + + /** + * Get the result of DynamicOneColumnData from work index and flush index + * + * @param deltaObjectId + * @param measurementId + * @return + */ + private DynamicOneColumnData mergeTwoDynamicColumnData(String deltaObjectId, String measurementId) { + DynamicOneColumnData working = workingBufferIndex.query(deltaObjectId, measurementId); + DynamicOneColumnData ret = (working == null || working.length == 0) ? new DynamicOneColumnData() + : new DynamicOneColumnData(working.dataType, true); + if (flushState.isFlushing()) { + DynamicOneColumnData flushing = flushingBufferIndex.query(deltaObjectId, measurementId); + ret.mergeRecord(flushing); + } + ret.mergeRecord(working); + return ret; + } + + @Override + public void close() { + isFlushingSync = true; + try { + recordWriter.close(); + isFlushingSync = false; + } catch (IOException e) { + LOG.error("Close BufferWriteProcessor error in namespace {}. Message:{}", nameSpacePath, e.getMessage()); + e.printStackTrace(); + } + closeAction.act(); + + /* + * try { WriteLogManager.getInstance().bufferFlush(); } catch (Exception + * e) { LOG.error("Write log for bufferredWrite error", e); + * System.exit(1); } + */ + } + + public void setCloseAction(Action action) { + this.closeAction = action; + } + + public Action getCloseAction() { + return this.closeAction; + } + + @Override + public boolean canBeClosed() { + LOG.debug("{} can be closed-{}", nameSpacePath, !flushState.isFlushing()); + return !flushState.isFlushing(); + } + + public Pair> getIndexAndRowGroupList(String deltaObjectId, + String measurementId) { + convertBufferLock.readLock().lock(); + DynamicOneColumnData index = null; + List list = null; + try { + index = mergeTwoDynamicColumnData(deltaObjectId, measurementId); + list = bufferIOWriter.getCurrentRowGroupMetaList(); + } finally { + convertBufferLock.readLock().unlock(); + } + return new Pair<>(index, list); + } + + public Pair getPair() { + return fileNamePair; + } + + public void setIsNewProcessor(boolean isNewProcessor) { + this.isNewProcessor = isNewProcessor; + } + + public boolean isNewProcessor() { + return isNewProcessor; + } + + private class BufferWriteRecordWriter extends TSRecordWriter { + + private Map flushingRowGroupWriters; + private Set flushingRowGroupSet; + private long flushingRecordCount; + + BufferWriteRecordWriter(TSFileConfig conf, BufferWriteIOWriter ioFileWriter, + WriteSupport writeSupport, FileSchema schema) { + super(conf, ioFileWriter, writeSupport, schema); + } + + /** + * insert a list of data value in form of TimePair. + * + * @param record + * - TSRecord to be written + * @throws WriteProcessException + * @throws IOException + */ + @Override + public void write(TSRecord record) throws IOException, WriteProcessException { + super.write(record); + } + + @Override + protected void flushRowGroup(boolean isFillRowGroup) throws IOException { + if (recordCount > 0) { + synchronized (flushState) { + // This thread wait until the subThread flush finished + while (flushState.isFlushing()) { + try { + flushState.wait(); + } catch (InterruptedException e) { + LOG.error("Interrupt error when waiting flush,processor:{}. Message: {}", nameSpacePath, + e.getMessage()); + e.printStackTrace(); + } + } + } + if (isFlushingSync) { + try { + super.flushRowGroup(false); + } catch (IOException e) { + LOG.error("Flush row group to store failed, processor:{}. Message: {}", nameSpacePath, + e.getMessage()); + /* + * There should be added system log by CGF and throw + * exception + */ + e.printStackTrace(); + System.exit(1); + } + } else { + flushState.setFlushing(); + switchIndexFromWorkingToFlushing(); + switchRecordWriterFromWorkingToFlushing(); + Runnable flushThread; + flushThread = () -> { + LOG.info("Asynchronous flushing start,-Thread id {}", Thread.currentThread().getId()); + try { + asyncFlushRowGroupToStore(); + } catch (IOException e) { + /* + * There should be added system log by CGF and throw + * exception + */ + LOG.error("{} Asynchronous flush error, sleep this thread-{}. Message:{}", nameSpacePath, + Thread.currentThread().getId(), e.getMessage()); + e.printStackTrace(); + while (true) { + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } + } + } + switchRecordWriterFromFlushingToWorking(); + convertBufferLock.writeLock().lock(); + // notify the thread which is waiting for asynchronous + // flush + synchronized (flushState) { + switchIndexFromFlushingToWorking(); + flushState.notify(); + } + convertBufferLock.writeLock().unlock(); + LOG.info("Asynchronous flushing end,-Thread is {}", Thread.currentThread().getId()); + }; + Thread flush = new Thread(flushThread); + flush.start(); + } + } + } + + private void asyncFlushRowGroupToStore() throws IOException { + if (flushingRecordCount > 0) { + String deltaObjectType = schema.getDeltaType(); + long totalMemStart = deltaFileWriter.getPos(); + for (String deltaObjectId : flushingRowGroupSet) { + long rowGroupStart = deltaFileWriter.getPos(); + deltaFileWriter.startRowGroup(flushingRecordCount, deltaObjectId, deltaObjectType); + IRowGroupWriter groupWriter = flushingRowGroupWriters.get(deltaObjectId); + groupWriter.flushToFileWriter(deltaFileWriter); + deltaFileWriter.endRowGroup(deltaFileWriter.getPos() - rowGroupStart); + } + long actualTotalRowGroupSize = deltaFileWriter.getPos() - totalMemStart; + fillInRowGroupSize(actualTotalRowGroupSize); + LOG.info("Asynchronous total row group size:{}, actual:{}, less:{}", primaryRowGroupSize, + actualTotalRowGroupSize, primaryRowGroupSize - actualTotalRowGroupSize); + LOG.info("Asynchronous write row group end"); + } + } + + private void switchRecordWriterFromWorkingToFlushing() { + + flushingRowGroupWriters = groupWriters; + flushingRowGroupSet = schema.getDeltaObjectAppearedSet(); + for (String DeltaObjectId : schema.getDeltaObjectAppearedSet()) { + flushingRowGroupSet.add(DeltaObjectId); + } + flushingRecordCount = recordCount; + // reset + groupWriters = new HashMap(); + schema.getDeltaObjectAppearedSet().clear(); + recordCount = 0; + + } + + private void switchRecordWriterFromFlushingToWorking() { + flushingRowGroupSet = null; + flushingRowGroupWriters = null; + flushingRecordCount = -1; + } + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/FileNodeConstants.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/FileNodeConstants.java new file mode 100644 index 00000000000..85da430352b --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/FileNodeConstants.java @@ -0,0 +1,19 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +/** + * Constants for using in bufferwrite，overflow and filenode + * + * @author liukun + * + */ +public class FileNodeConstants { + + public static final String FILE_NODE_OPERATOR_TYPE = "OPERATOR_TYPE"; + public static final String TIMESTAMP_KEY = "TIMESTAMP"; + public static final String FILE_NODE = "FILE_NODE"; + public static final String CLOSE_ACTION = "CLOSE_ACTION"; + + public static final String ERR_EXTENSION = "err"; + public static final String PATH_SEPARATOR = "."; + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUManager.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUManager.java new file mode 100644 index 00000000000..1d66938c199 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUManager.java @@ -0,0 +1,322 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; +import cn.edu.thu.tsfiledb.exception.ErrorDebugException; +import cn.edu.thu.tsfiledb.exception.PathErrorException; +import cn.edu.thu.tsfiledb.exception.ProcessorException; +import cn.edu.thu.tsfiledb.metadata.MManager; + +/** + *

+ * LRUManager manage a list of processor: overflow/filenode/bufferwrite + * processor
+ * + * @author kangrong + * @author liukun + */ +public abstract class LRUManager { + private static Logger LOG = LoggerFactory.getLogger(LRUManager.class); + private LinkedList processorLRUList; + private Map processorMap; + private final int maxLRUNodeNumber; + protected final MManager mManager; + protected final String normalDataDir; + private static final long removeCheckInterval = 1000; + + protected LRUManager(int maxLRUNumber, MManager mManager, String normalDataDir) { + this.mManager = mManager; + this.maxLRUNodeNumber = maxLRUNumber; + LOG.info("The max of LRUProcessor number of {} is {}", this.getClass().getSimpleName(), maxLRUNumber); + processorLRUList = new LinkedList<>(); + processorMap = new HashMap<>(); + + if (normalDataDir.charAt(normalDataDir.length() - 1) != File.separatorChar) + normalDataDir += File.separatorChar; + this.normalDataDir = normalDataDir; + File dir = new File(normalDataDir); + if (dir.mkdirs()) { + LOG.info("{} dir home is not exists, create it", this.getClass().getSimpleName()); + } + } + + /** + *

+ * Get the data directory + * + * @return data directory + */ + public String getNormalDataDir() { + return normalDataDir; + } + + /** + *

+ * Get processor just using the metadata path + * + * @param path + * @param isWriteLock + * @return LRUProcessor + * @throws ProcessorException + * @throws IOException + */ + public T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock) + throws ProcessorException, IOException, WriteProcessException { + return getProcessorWithDeltaObjectIdByLRU(path, isWriteLock, null); + } + + /** + * check whether the given path's exists. If it doesn't exist, add it to + * list. If the list has been full(list.size == maxLRUNodeNumber), remove + * the last one. If it exists, raise this processor to the first position + * + * @param path + * - measurement path in name space. + * @param isWriteLock + * @param args + * - save some key-value information used for constructing a + * special processor + * @return return processor which has the specified nsPath + * @throws ProcessorException + * @throws IOException + */ + public synchronized T getProcessorWithDeltaObjectIdByLRU(String path, boolean isWriteLock, Map args) + throws ProcessorException, IOException, WriteProcessException { + String nsPath; + try { + nsPath = mManager.getFileNameByPath(path); + } catch (PathErrorException e) { + throw new ProcessorException(e); + } + return getProcessorByLRU(nsPath, isWriteLock, args); + } + + /** + * check the processor exists or not by namespacepath + * + * @param namespacePath + * @return + */ + public synchronized boolean containNamespacePath(String namespacePath) { + return processorMap.containsKey(namespacePath); + } + + /** + * Get processor using namespacepath + * + * @param namespacePath + * @param isWriteLock + * true if add a write lock on return {@code T} + * @return + * @throws ProcessorException + * @throws IOException + */ + public T getProcessorByLRU(String namespacePath, boolean isWriteLock) throws ProcessorException, IOException, WriteProcessException { + return getProcessorByLRU(namespacePath, isWriteLock, null); + } + + /** + * + * check whether the given namespace path exists. If it doesn't exist, add + * it to list. If the list has been full(list.size == maxLRUNodeNumber), + * remove the last one. If it exists, raise this processor to the first + * position + * + * @param namespacePath + * @param isWriteLock + * @param args + * @return + * @throws ProcessorException + * throw it for meeting InterruptedException while removing + * unused processor + * @throws IOException + */ + public synchronized T getProcessorByLRU(String namespacePath, boolean isWriteLock, Map args) + throws ProcessorException, IOException, WriteProcessException { + LOG.debug("Try to getProcessorByLRU, {}, nsPath:{}-Thread id {}", this.getClass().getSimpleName(), + namespacePath, Thread.currentThread().getId()); + T processor; + if (processorMap.containsKey(namespacePath)) { + LOG.debug("{} contains nsPath:{}", this.getClass().getSimpleName(), namespacePath); + processor = processorMap.get(namespacePath); + processor.lock(isWriteLock); + initProcessor(processor, namespacePath, args); + processorLRUList.remove(processor); + processorLRUList.addFirst(processor); + LOG.debug("Use {}:{}, raise it to top", processor.getClass().getSimpleName(), processor.getNameSpacePath()); + } else { + if (processorLRUList.size() == maxLRUNodeNumber) { + LOG.debug( + "LRU list doesn't contain the processor and is full, remove the oldest unused processor firstly"); + try { + removeLastUnusedProcessor(); + } catch (InterruptedException e) { + throw new ProcessorException( + "thread is interrupted. nsPath:" + namespacePath + " reason: " + e.getMessage()); + } + } + LOG.debug("Construct and init a new processor in {}:{}", this.getClass().getSimpleName(), namespacePath); + processor = constructNewProcessor(namespacePath, args); + initProcessor(processor, namespacePath, args); + processorLRUList.addFirst(processor); + processorMap.put(namespacePath, processor); + processor.lock(isWriteLock); + LOG.debug("{} LUR list: size is {}, content:{}", this.getClass().getSimpleName(), processorLRUList.size(), + processorLRUList); + } + return processor; + } + + /** + * remove the last unused processor if the LRU list is full + * + * @throws InterruptedException + */ + private void removeLastUnusedProcessor() throws InterruptedException { + while (true) { + int index = -1; + for (int i = processorLRUList.size() - 1; i >= 0; i--) { + T pro = processorLRUList.get(i); + if (pro.tryWriteLock()) { + try { + LOG.debug("Try write lock successfully, class:{},namespace:{}", pro.getClass().getSimpleName(), + pro.getNameSpacePath()); + if (pro.canBeClosed()) { + index = i; + LOG.debug("find a removable processor, index:{}, namespace:{} break loop", index, + pro.getNameSpacePath()); + break; + } else { + LOG.debug("{}:{} cannot be removed", pro.getClass().getSimpleName(), + pro.getNameSpacePath()); + } + } finally { + pro.writeUnlock(); + } + } else { + LOG.info("Can't remove the processor {} namespace path is {}", pro.getClass().getSimpleName(), + pro.getNameSpacePath()); + } + } + if (index != -1) { + T pro = processorLRUList.get(index); + pro.close(); + processorLRUList.remove(index); + String proNsPath = pro.getNameSpacePath(); + processorMap.remove(proNsPath); + LOG.debug("Remove the last processor {}:{}", pro.getClass().getSimpleName(), proNsPath); + break; + } else { + LOG.debug("All processors are occupied, sleep {} millisecond", removeCheckInterval); + Thread.sleep(removeCheckInterval); + } + } + } + + /** + *

+ * Try to close the processor of the nsPath
+ * + * @param nsPath + * @param isSync + * true must close this processor, false close this processor if + * possible + * @return + */ + public synchronized boolean close(String nsPath, boolean isSync) { + if (!processorMap.containsKey(nsPath)) + return true; + LRUProcessor pro = processorMap.get(nsPath); + if (!isSync && !pro.tryWriteLock()) { + return false; + } + try { + do { + if (pro.canBeClosed()) { + pro.close(); + processorMap.remove(nsPath); + processorLRUList.remove(pro); + LOG.info("Close {} file:{}", pro.getClass().getSimpleName(), nsPath); + return true; + } else if (isSync) { + try { + LOG.info("Can't close {} file:{}, sleep 1000ms", pro.getClass(), nsPath); + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new ErrorDebugException(e); + } + } else + return false; + } while (true); + } finally { + pro.writeUnlock(); + } + } + + /** + *

+ * close all processor still exists in memory
+ */ + public synchronized void close() { + List cannotRemoveList = new ArrayList<>(); + + do { + cannotRemoveList.clear(); + for (String nsPath : processorMap.keySet()) { + cannotRemoveList.add(nsPath); + } + for (String nsPath : cannotRemoveList) { + close(nsPath, false); + } + if (!processorMap.isEmpty()) { + LOG.info("{}:{} Can't be remove, waiting 1000ms", processorMap.get(cannotRemoveList.get(0)), + cannotRemoveList); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new ErrorDebugException(e); + } + } else { + processorMap.clear(); + processorLRUList.clear(); + break; + } + } while (true); + } + + /** + *

+ * construct processor using namespacepath and key-value object
+ * + * @param namespacePath + * @param args + * @return + * @throws ProcessorException + * @throws IOException + */ + protected abstract T constructNewProcessor(String namespacePath, Map args) + throws ProcessorException, IOException, WriteProcessException; + + /** + *

+ * initialize the processor with the key-value object
+ * + * @param processor + * @param namespacePath + * @param args + * @throws ProcessorException + */ + protected void initProcessor(T processor, String namespacePath, Map args) + throws ProcessorException { + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUProcessor.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUProcessor.java new file mode 100644 index 00000000000..1bc73847dcf --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/LRUProcessor.java @@ -0,0 +1,155 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + *

+ * LRUProcessor is used for implementing different processor with different + * operation.
+ * + * @see BufferWriteProcessor + * @see OverflowProcessor + * @see FileNodeProcessor + * + * @author kangrong + * + */ +public abstract class LRUProcessor { + private final static Logger LOG = LoggerFactory.getLogger(LRUProcessor.class); + protected String nameSpacePath; + private final ReadWriteLock lock; + + /** + * Construct processor using name space path + * + * @param nameSpacePath + */ + public LRUProcessor(String nameSpacePath) { + this.nameSpacePath = nameSpacePath; + this.lock = new ReentrantReadWriteLock(); + } + + /** + * Release the read lock + */ + public void readUnlock() { + LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId()); + lock.readLock().unlock(); + } + + /** + * Acquire the read lock + */ + public void readLock() { + LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId()); + lock.readLock().lock(); + } + + /** + * Acquire the write lock + */ + public void writeLock() { + LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId()); + lock.writeLock().lock(); + } + + /** + * Release the write lock + */ + public void writeUnlock() { + LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), Thread.currentThread().getId()); + lock.writeLock().unlock(); + } + + /** + * @param isWriteLock + * true acquire write lock, false acquire read lock + */ + public void lock(boolean isWriteLock) { + if (isWriteLock) { + LOG.debug("{}: lru write lock-Thread id {}", this.getClass().getSimpleName(), + Thread.currentThread().getId()); + lock.writeLock().lock(); + } else { + LOG.debug("{}: lru read lock-Thread id {}", this.getClass().getSimpleName(), + Thread.currentThread().getId()); + lock.readLock().lock(); + } + + } + + /** + * @param isWriteUnlock + * true release write lock, false release read unlock + */ + public void unlock(boolean isWriteUnlock) { + if (isWriteUnlock) { + LOG.debug("{}: lru write unlock-Thread id {}", this.getClass().getSimpleName(), + Thread.currentThread().getId()); + lock.writeLock().unlock(); + } else { + LOG.debug("{}: lru read unlock-Thread id {}", this.getClass().getSimpleName(), + Thread.currentThread().getId()); + lock.readLock().unlock(); + } + } + + /** + * Get the name space path + * + * @return + */ + public String getNameSpacePath() { + return nameSpacePath; + } + + /** + * Try to get the write lock + * + * @return + */ + public boolean tryWriteLock() { + return lock.writeLock().tryLock(); + } + + /** + * Try to get the read lock + * + * @return + */ + public boolean tryReadLock() { + return lock.readLock().tryLock(); + } + + /** + * Judge whether this processor can be closed. + * + * @return true if subclass doesn't have other implementation. + */ + public abstract boolean canBeClosed(); + + @Override + public int hashCode() { + return nameSpacePath.hashCode(); + } + + @Override + public boolean equals(Object pro) { + if (pro == null) + return false; + if (!pro.getClass().equals(this.getClass())) + return false; + LRUProcessor lruPro = (LRUProcessor) pro; + return nameSpacePath.equals(lruPro.getNameSpacePath()); + } + + /** + * Close the processor.
+ * Notice: Thread is not safe + */ + public abstract void close(); +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/MemoryBufferWriteIndexImpl.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/MemoryBufferWriteIndexImpl.java new file mode 100644 index 00000000000..d203dbf770e --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/MemoryBufferWriteIndexImpl.java @@ -0,0 +1,77 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import java.util.HashMap; +import java.util.Map; + +import cn.edu.thu.tsfile.common.utils.Binary; +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; +import cn.edu.thu.tsfile.timeseries.write.record.DataPoint; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; + +/** + * Implement the {@code BufferWriteIndex}
+ * This class is used to store bufferwrite data in memory, also index data + * easily
+ * + * @author kangrong + * @author liukun + * + */ +public class MemoryBufferWriteIndexImpl implements BufferWriteIndex { + private Map indexMap; + + public MemoryBufferWriteIndexImpl() { + indexMap = new HashMap(); + } + + @Override + public DynamicOneColumnData query(String deltaObjectId, String measurementId) { + return indexMap.get(concatPath(deltaObjectId, measurementId)); + } + + @Override + public void insert(TSRecord tsRecord) { + String deltaObjectId = tsRecord.deltaObjectId; + for (DataPoint dp : tsRecord.dataPointList) { + String measurementId = dp.getMeasurementId(); + String path = concatPath(deltaObjectId, measurementId); + if (!indexMap.containsKey(path)) + indexMap.put(path, new DynamicOneColumnData(dp.getType(), true)); + DynamicOneColumnData deltaMea = indexMap.get(path); + Object value = dp.getValue(); + deltaMea.putTime(tsRecord.time); + switch (dp.getType()) { + case INT32: + deltaMea.putInt((Integer) value); + break; + case INT64: + deltaMea.putLong((Long) value); + break; + case FLOAT: + deltaMea.putFloat((Float) value); + break; + case DOUBLE: + deltaMea.putDouble((Double) value); + break; + case BYTE_ARRAY: + deltaMea.putBinary((Binary) value); + break; + case BOOLEAN: + deltaMea.putBoolean((boolean) value); + break; + default: + throw new UnsupportedOperationException("no support type:" + dp.getType() + "record:" + tsRecord); + } + } + } + + @Override + public void clear() { + indexMap.clear(); + } + + private String concatPath(String deltaObjectId, String measurementId) { + return deltaObjectId + FileNodeConstants.PATH_SEPARATOR + measurementId; + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TSRecordWriterParameter.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TSRecordWriterParameter.java new file mode 100644 index 00000000000..deefa244f23 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TSRecordWriterParameter.java @@ -0,0 +1,25 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter; +import cn.edu.thu.tsfile.timeseries.write.WriteSupport; +import cn.edu.thu.tsfile.timeseries.write.record.TSRecord; +import cn.edu.thu.tsfile.timeseries.write.schema.FileSchema; + +/** + * @author kangrong + * @author liukun + * + */ +public class TSRecordWriterParameter { + public final FileSchema fileSchema; + public final TSRandomAccessFileWriter outputStream; + public final WriteSupport writeSupport; + + public TSRecordWriterParameter(FileSchema fileSchema, TSRandomAccessFileWriter outputStream, + WriteSupport writeSupport) { + this.fileSchema = fileSchema; + this.outputStream = outputStream; + this.writeSupport = writeSupport; + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TriFunction.java b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TriFunction.java new file mode 100644 index 00000000000..6346eb820a1 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/bufferwrite/TriFunction.java @@ -0,0 +1,16 @@ +package cn.edu.thu.tsfiledb.engine.bufferwrite; + +import cn.edu.thu.tsfile.timeseries.write.exception.WriteProcessException; + +/** + * @author kangrong + * + * @param + * @param + * @param + * @param + */ +@FunctionalInterface +public interface TriFunction { + R apply(U u, T t, C c) throws WriteProcessException; +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IIntervalTreeOperator.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IIntervalTreeOperator.java new file mode 100644 index 00000000000..23768928be5 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IIntervalTreeOperator.java @@ -0,0 +1,109 @@ +package cn.edu.thu.tsfiledb.engine.overflow; + +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression; +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +/** + * we advise the implementation class of this interface can be constructed by two ways:
+ * 1.construct a empty index without parameters. 2.construct a empty index with a input stream for + * restoring. + * + * @author kangrong + * + */ +public interface IIntervalTreeOperator { + /** + * insert a value to a time point. Insert doesn't support time range insert + * + * @param t - time + * @param value - value + */ + void insert(long t, byte[] value); + + /** + * update a value to a time point or a time range. + * + * @param s - start time. + * @param e - end time. + * @param value - value to be updated. + */ + void update(long s, long e, byte[] value); + + /** + * The implementation maintains an overflow index in memory. The data in the index is prior to + * newerMemoryData, which means the overflow operators corresponding to the index are covered with + * newerMemoryData. This function merges current index into newerMemoryData and return the + * merged result. + * + * @param timeFilter - timeFilter is specified by user. + * @param valueFilter - valueFilter is specified by user. + * @param freqFilter - freqFilter is specified by user. + * @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.) + * @param dataType - data type of this series + * @param newerMemoryData - newer overflow data. + * + * @return merged result. + */ + DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize, + TSDataType dataType, DynamicOneColumnData newerMemoryData); + + /** + * This function merges the older data which deserialized from given parameter in into newerData + * and return the merged result. The data in in is prior to newerData, which means the overflow + * operators corresponding to in are covered with newerData. + * + * @param timeFilter - timeFilter is specified by user. + * @param valueFilter - valueFilter is specified by user. + * @param freqFilter - freqFilter is specified by user. + * @param in - the inputstream to be merged into newerData which contains older overflow data . + * @param newerData - newer overflow data. + * @param valueSize - byte size of one item of this series(e.g. Int32 is 4, Double is 8 etc.) + * @param dataType - data type of this series + * + * @return merged result. + */ + DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream in, + DynamicOneColumnData newerData, int valueSize, TSDataType dataType) throws IOException; + + /** + * Get List(insert operations, update operations and delete operations which meet the expression of time filter, + * value filter and frequency filter in DynamicOneColumnData data.) + * + * @param timeFilter - timeFilter is specified by user. + * @param valueFilter - valueFilter is specified by user. + * @param freqFilter - freqFilter is specified by user. + * @param data - a DynamicOneColumnData information. + * @param dataType - TSDataType + * @return - List + */ + List getDynamicList(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, + DynamicOneColumnData data, TSDataType dataType); + + /** + * delete all values earlier than timestamp. + * + * @param timestamp - delete timestamp + */ + void delete(long timestamp); + + /** + * given an outputstream, serialize the index into it. + * + * @param out - serialization output stream. + */ + void toBytes(OutputStream out) throws IOException; + + /** + * @return the memory size for this index + */ + long calcMemSize(); +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IntervalTreeOperation.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IntervalTreeOperation.java new file mode 100644 index 00000000000..6f11ff13574 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/IntervalTreeOperation.java @@ -0,0 +1,991 @@ +package cn.edu.thu.tsfiledb.engine.overflow; + +import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException; +import cn.edu.thu.tsfile.common.utils.Binary; +import cn.edu.thu.tsfile.common.utils.BytesUtils; +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory; +import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression; +import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType; +import cn.edu.thu.tsfile.timeseries.filter.definition.operators.And; +import cn.edu.thu.tsfile.timeseries.filter.definition.operators.GtEq; +import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval; +import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier; +import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitor; +import cn.edu.thu.tsfile.timeseries.filter.visitorImpl.SingleValueVisitorFactory; +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; +import cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation; +import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalRelation; +import cn.edu.thu.tsfiledb.engine.overflow.index.IntervalTree; +import cn.edu.thu.tsfiledb.engine.overflow.utils.MergeStatus; +import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType; +import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair; +import cn.edu.thu.tsfiledb.exception.UnSupportedOverflowOpTypeException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils.readUnsignedVarInt; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + + +/** + * @author CGF + */ +public class IntervalTreeOperation implements IIntervalTreeOperator { + + private static final Logger LOG = LoggerFactory.getLogger(IntervalTreeOperation.class); + + private IntervalTree index = null; + + private TSDataType dataType; // All operations data type in IntervalTreeOperation. + + public IntervalTreeOperation(TSDataType dataType) { + index = new IntervalTree(dataType); + this.dataType = dataType; + } + + @Override + public void insert(long t, byte[] value) { + index.update(new TimePair(t, t, value, OverflowOpType.INSERT, dataType)); + } + + @Override + public void update(long s, long e, byte[] value) { + // s must >= e ! + index.update(new TimePair(s, e, value, OverflowOpType.UPDATE, dataType)); + } + + @Override + public void delete(long timestamp) { + // timestamp-1 for TSFile sql expression + // in TSFile sql expression, there only exists "DELETE X where Y < Z" + if (timestamp > 0) + index.update(new TimePair(0, timestamp - 1, null, OverflowOpType.DELETE, dataType)); + else + // for "DELETE X where Y = 0" + index.update(new TimePair(0, timestamp, null, OverflowOpType.DELETE, dataType)); + } + + /** + * Notice that serialization file doesn't store the value when DELETE operation. + * + * @param out - serialization output stream. + * @throws IOException - serialization problem + */ + @Override + public void toBytes(OutputStream out) throws IOException { + index.midOrderSerialize(out); + index = new IntervalTree(); + } + + /** + * used for queryFileBlock() + * + * @param doc - DynamicOneColumn to be put + * @param s - start time + * @param e - end time + * @param value - overflow value + * @param overflowOpType - OverflowOpType + * @param dataType - TSDataType + */ + private void putValueUsingFileData(DynamicOneColumnData doc, long s, long e, + byte[] value, OverflowOpType overflowOpType, TSDataType dataType) { + switch (overflowOpType) { + case INSERT: + doc.putTimePair(s, -e); + break; + case UPDATE: + doc.putTimePair(s, e); + break; + case DELETE: + doc.putTimePair(-s, -e); + break; + default: + LOG.error("Unsupported Overflow operation type."); + throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type."); + } + + switch (dataType) { + case INT32: + if (overflowOpType != OverflowOpType.DELETE) + doc.putInt(BytesUtils.bytesToInt(value)); + else + doc.putInt(0); + break; + case INT64: + if (overflowOpType != OverflowOpType.DELETE) + doc.putLong(BytesUtils.bytesToLong(value)); + else + doc.putLong(0L); + break; + case FLOAT: + if (overflowOpType != OverflowOpType.DELETE) + doc.putFloat(BytesUtils.bytesToFloat(value)); + else + doc.putFloat(0.0f); + break; + case DOUBLE: + if (overflowOpType != OverflowOpType.DELETE) + doc.putDouble(BytesUtils.bytesToDouble(value)); + else + doc.putDouble(0); + break; + case BYTE_ARRAY: + if (overflowOpType != OverflowOpType.DELETE) + doc.putBinary(Binary.valueOf(BytesUtils.bytesToString(value))); + else + doc.putDouble(0); + break; + default: + LOG.error("Unsupported TSFile data type."); + throw new UnSupportedDataTypeException("Unsupported TSFile data type."); + } + } + + /** + * Put value into newer DynamicOneColumnData. + * + * @param ansData new DynamicOneColumn + * @param startTime - start time + * @param endTime - end time + * @param doc - previous DynamicOneColumn + * @param i - index + * @param dataType - TSDataType + */ + private void putValueUseDynamic(DynamicOneColumnData ansData, long startTime, long endTime, + DynamicOneColumnData doc, int i, OverflowOpType overflowOpType, TSDataType dataType) { + + // don't care the plus or minus of the value of start time or end time. + switch (overflowOpType) { + case INSERT: + ansData.putTimePair(Math.abs(startTime), -Math.abs(endTime)); + break; + case UPDATE: + ansData.putTimePair(Math.abs(startTime), Math.abs(endTime)); + break; + case DELETE: + ansData.putTimePair(-Math.abs(startTime), -Math.abs(endTime)); + break; + default: + LOG.error("Unsupported Overflow operation type."); + throw new UnSupportedOverflowOpTypeException("Unsupported Overflow operation type."); + } + + switch (dataType) { + case INT32: + ansData.putInt(doc.getInt(i)); + break; + case INT64: + ansData.putLong(doc.getLong(i)); + break; + case FLOAT: + ansData.putFloat(doc.getFloat(i)); + break; + case DOUBLE: + ansData.putDouble(doc.getDouble(i)); + break; + case BYTE_ARRAY: + ansData.putBinary(doc.getBinary(i)); + break; + default: + LOG.error("Unsupported TSFile data type."); + throw new UnSupportedDataTypeException("Unsupported TSFile data type."); + } + } + + /** + * Notice that both s and e are >= 0.
+ * + * @param s - start time + * @param e - end time + * @param value - time pair value + * @param status - time pair merge status + * @return - TimePair tp has both positive start time and end time values. + */ + private TimePair constructTimePair(long s, long e, byte[] value, MergeStatus status) { + if (s <= 0 && e <= 0) // s<=0 && e<=0 for delete operation + return new TimePair(-s, -e, value, OverflowOpType.DELETE, status); + else if (s > 0 && e < 0) { + return new TimePair(s, -e, value, OverflowOpType.INSERT, status); + } else { + return new TimePair(s, e, value, OverflowOpType.UPDATE, status); + } + } + + /** + * Notice that both s and e are >= 0.
+ *

+ * Return the time pair using given start time value and end time value. + * + * @param startTime - start time + * @param endTime - end time + * @param status - time pair merge status + * @return - TimePair tp has both positive start time and end time values. + */ + private TimePair constructTimePair(long startTime, long endTime, MergeStatus status) { + if (startTime <= 0 && endTime <= 0) // s<=0 && e<=0 for delete operation + return new TimePair(-startTime, -endTime, null, OverflowOpType.DELETE, status); + else if (startTime > 0 && endTime < 0) { + return new TimePair(startTime, -endTime, null, OverflowOpType.INSERT, status); + } else { + return new TimePair(startTime, endTime, null, OverflowOpType.UPDATE, status); + } + } + + /** + * Read data from overflow file which has been serialized, + * return the correspond time pair structure.
+ * + * @param inputStream - InputStream + * @param valueSize - value byte size + * @return - TimePair + * @throws IOException - IOException + */ + private TimePair readTimePairFromOldOverflow(InputStream inputStream, int valueSize) throws IOException { + long s = BytesUtils.readLong(inputStream); + long e = BytesUtils.readLong(inputStream); + if (s <= 0 && e < 0) { // DELETE OPERATION. s may < 0. + return constructTimePair(s, e, null, MergeStatus.MERGING); + } else { // INSERT or UPDATE OPERATION + if (valueSize == -1) { // var length read method + int len = readUnsignedVarInt(inputStream); + byte[] stringBytes = new byte[len]; + inputStream.read(stringBytes); + return constructTimePair(s, e, stringBytes, MergeStatus.MERGING); + } + return constructTimePair(s, e, BytesUtils.safeReadInputStreamToBytes(valueSize, inputStream), MergeStatus.MERGING); + } + } + + @Override + public DynamicOneColumnData queryFileBlock(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, InputStream inputStream, + DynamicOneColumnData newData, int valueSize, TSDataType dataType) throws IOException { + + DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true); // merge answer + int i = 0; + TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE); + TimePair newTimePair = constructTimePair(-1, -1, MergeStatus.DONE); + long L = Long.MIN_VALUE; + long R = Long.MIN_VALUE; + + while (i < newData.length) { + + if (newTimePair.mergestatus == MergeStatus.DONE) { + // (L,R) represent new time pair range. + L = newData.getTime(i * 2); + R = newData.getTime(i * 2 + 1); + newTimePair = constructTimePair(newData.getTime(i * 2), newData.getTime(i * 2 + 1), MergeStatus.MERGING); + } + + if (inputStream.available() == 0 && oldTimePair.mergestatus == MergeStatus.DONE) { // old overflow file is empty, but newData is not empty + putValueUseDynamic(ans, L, R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + continue; + } + + if (L <= 0 && R < 0) { // DELETE OPERATION + long rightTime = -R; + while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize); + } + if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10] + oldTimePair.reset(); + } else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16] + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType); + newTimePair.reset(); + break; + } else { // e.g. [0, 12] [10, 20] + oldTimePair.s = rightTime + 1; + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.DELETE, dataType); + newTimePair.reset(); + break; + } + } + } else if (L > 0 && R < 0) { // INSERT OPERATION + R = -R; + while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize); + } + if (oldTimePair.mergestatus == MergeStatus.MERGING) { + CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair); + if (relation == CrossRelation.RCOVERSL) { + if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair + putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } else if (oldTimePair.s == L) { + putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (oldTimePair.e == R) { + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } else { + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } + } else if (relation == CrossRelation.LFIRST) { // newTimePair first + putValueUseDynamic(ans, L, -R, newData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + break; + } else if (relation == CrossRelation.RFIRST) { // oldTimePair first + putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.reset(); + } else { + // relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible + // relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible + // relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible + LOG.error("unreachable method"); + } + } + } + } else { // UPDATE OPERATION + while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize); + } + if (oldTimePair.mergestatus == MergeStatus.MERGING) { + CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair); + if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair + if (oldTimePair.opType == OverflowOpType.INSERT) { + if (oldTimePair.s == L) { + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType); + L = oldTimePair.s + 1; + oldTimePair.reset(); + } else if (oldTimePair.e == R) { + putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType); + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType); + i++; + newTimePair.reset(); + oldTimePair.reset(); + } else { + putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType); + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.INSERT, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + } + } else if (oldTimePair.opType == OverflowOpType.DELETE) { + if (oldTimePair.s == L) { + putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newData, i, OverflowOpType.DELETE, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + } else if (oldTimePair.e == R) { + // the start time of DELETE time pair > 0 + putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType); + i++; + newTimePair.reset(); + oldTimePair.reset(); + } else { + // the start time of DELETE time pair > 0 + putValueUseDynamic(ans, L, oldTimePair.s - 1, newData, i, OverflowOpType.UPDATE, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + } + } else { + oldTimePair.reset(); + } + } else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair + if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType); + i++; + oldTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + putValueUseDynamic(ans, L, -R, newData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } + } else if (oldTimePair.s == L) { + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUsingFileData(ans, oldTimePair.s, R, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.s = R + 1; + i++; + newTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else { + // oldTimePair.opType == OverflowOpType.INSERT + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } else if (oldTimePair.e == R) { + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.reset(); + i++; + newTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } else { + if (oldTimePair.opType == OverflowOpType.DELETE) { + i++; + newTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } + } else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross + // old TimePair could not be INSERT, DELETE + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross + if (oldTimePair.opType == OverflowOpType.DELETE) { + // delete operation need to be added to ans + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ?? + putValueUsingFileData(ans, oldTimePair.s, L - 1, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.reset(); + } + } else if (relation == CrossRelation.LFIRST) { // newTimePair first + putValueUseDynamic(ans, L, R, newData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + break; + } else if (relation == CrossRelation.RFIRST) { // oldTimePair first + putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.reset(); + } + } + } + } + } + + // newData is empty, but overflow file still has data. + while (inputStream.available() != 0 || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = readTimePairFromOldOverflow(inputStream, valueSize); + } + + putValueUsingFileData(ans, oldTimePair.s, oldTimePair.e, oldTimePair.v, oldTimePair.opType, dataType); + oldTimePair.reset(); + } + + return ans; + } + + @Override + public DynamicOneColumnData queryMemory(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, int valueSize, TSDataType dataType + , DynamicOneColumnData newerMemoryData) { + if (newerMemoryData == null) { + return index.dynamicQuery(timeFilter, dataType); + } + DynamicOneColumnData ans = new DynamicOneColumnData(dataType, true); + DynamicOneColumnData oldData = index.dynamicQuery(timeFilter, dataType); + + int i = 0, j = 0; // i represents newMemoryData, j represents oldMemoryData + TimePair oldTimePair = new TimePair(-1, -1, MergeStatus.DONE); + TimePair newTimePair = new TimePair(-1, -1, MergeStatus.DONE); + long L = Long.MIN_VALUE; + long R = Long.MIN_VALUE; + + while (i < newerMemoryData.length) { + + if (newTimePair.mergestatus == MergeStatus.DONE) { + // (L,R) represent new time pair range. + L = newerMemoryData.getTime(i * 2); + R = newerMemoryData.getTime(i * 2 + 1); + newTimePair = constructTimePair(newerMemoryData.getTime(i * 2), newerMemoryData.getTime(i * 2 + 1), MergeStatus.MERGING); + } + + if (oldTimePair.mergestatus == MergeStatus.DONE && j >= oldData.length) { // old overflow file is empty, but newData is not empty + putValueUseDynamic(ans, L, R, newerMemoryData, i++, newTimePair.opType, dataType); + newTimePair.reset(); + continue; + } + + if (L <= 0 && R < 0) { // DELETE OPERATION + long rightTime = -R; + while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING); + } + if (rightTime >= oldTimePair.e) { // e.g. [0, 12] [3, 10] + oldTimePair.reset(); + j++; + } else if (rightTime < oldTimePair.s) { // e.g. [0, 12] [14, 16] + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType); + newTimePair.reset(); + break; + } else { // e.g. [0, 12] [10, 20] + oldTimePair.s = rightTime + 1; + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.DELETE, dataType); + newTimePair.reset(); + break; + } + } + } else if (L > 0 && R < 0) { // INSERT OPERATION + R = -R; + while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING); + } + if (oldTimePair.mergestatus == MergeStatus.MERGING) { + CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair); + if (relation == CrossRelation.RCOVERSL) { + if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.reset(); + j++; + break; + } else if (oldTimePair.s == L) { + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (oldTimePair.e == R) { + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j++, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.reset(); + break; + } else { + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } + } else if (relation == CrossRelation.LFIRST) { // newTimePair first + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + break; + } else if (relation == CrossRelation.RFIRST) { // oldTimePair first + putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType); + oldTimePair.reset(); + j++; + } else { + // relation == CrossRelation.LCOVERSR) : newerTimePair covers oldTimePair, newTimePair width must > 1, impossible + // relation == CrossRelation.LFIRSTCROSS) : newTimePair first cross, impossible + // relation == CrossRelation.RFIRSTCROSS) : oldTimePair first cross, impossible + LOG.error("unreachable method"); + } + } + } + } else { // UPDATE OPERATION + while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING); + } + if (oldTimePair.mergestatus == MergeStatus.MERGING) { + CrossRelation relation = IntervalRelation.getRelation(newTimePair, oldTimePair); + if (relation == CrossRelation.LCOVERSR) { // newTimePair covers oldTimePair + if (oldTimePair.opType == OverflowOpType.INSERT) { + if (oldTimePair.s == L) { + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType); + L = oldTimePair.s + 1; + oldTimePair.reset(); + j++; + } else if (oldTimePair.e == R) { + putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType); + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType); + i++; + newTimePair.reset(); + oldTimePair.reset(); + j++; + } else { + putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType); + putValueUseDynamic(ans, oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.INSERT, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + j++; + } + } else if (oldTimePair.opType == OverflowOpType.DELETE) { + if (oldTimePair.s == L) { + putValueUseDynamic(ans, -oldTimePair.s, -oldTimePair.e, newerMemoryData, i, OverflowOpType.DELETE, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + j++; + } else if (oldTimePair.e == R) { + // the start time of DELETE time pair > 0 + putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType); + i++; + newTimePair.reset(); + oldTimePair.reset(); + j++; + } else { + // the start time of DELETE time pair > 0 + putValueUseDynamic(ans, L, oldTimePair.s - 1, newerMemoryData, i, OverflowOpType.UPDATE, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + j++; + } + } else { + oldTimePair.reset(); + j++; + } + } else if (relation == CrossRelation.RCOVERSL) { // oldTimePair covers newTimePair + if (oldTimePair.s == L && oldTimePair.e == R) { // newTimePair equals oldTimePair + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType); + i++; + oldTimePair.reset(); + j++; + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.reset(); + j++; + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + putValueUseDynamic(ans, L, -R, newerMemoryData, i++, OverflowOpType.INSERT, dataType); + newTimePair.reset(); + oldTimePair.reset(); + j++; + break; + } + } else if (oldTimePair.s == L) { + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUseDynamic(ans, oldTimePair.s, R, oldData, j, oldTimePair.opType, dataType); + oldTimePair.s = R + 1; + i++; + newTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else { + // oldTimePair.opType == OverflowOpType.INSERT + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } else if (oldTimePair.e == R) { + if (oldTimePair.opType == OverflowOpType.DELETE) { + putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType); + oldTimePair.reset(); + j++; + newTimePair.reset(); + i++; + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + i++; + oldTimePair.reset(); + j++; + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } else { + if (oldTimePair.opType == OverflowOpType.DELETE) { + i++; + newTimePair.reset(); + break; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType); + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (oldTimePair.opType == OverflowOpType.INSERT) { + // oldTimePair covers newTimePair, but oldTimePair is INSERT operation. impossible + LOG.error("unreachable method"); + } + } + } else if (relation == CrossRelation.LFIRSTCROSS) { // newTimePair first cross + // old TimePair could not be INSERT, DELETE + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + oldTimePair.s = R + 1; + break; + } else if (relation == CrossRelation.RFIRSTCROSS) { // oldTimePair first cross + if (oldTimePair.opType == OverflowOpType.DELETE) { + // delete operation need to be added to ans + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType); + L = oldTimePair.e + 1; + oldTimePair.reset(); + j++; + } else if (oldTimePair.opType == OverflowOpType.UPDATE) { // ?? + putValueUseDynamic(ans, oldTimePair.s, L - 1, oldData, j, oldTimePair.opType, dataType); + oldTimePair.reset(); + j++; + } + } else if (relation == CrossRelation.LFIRST) { // newTimePair first + putValueUseDynamic(ans, L, R, newerMemoryData, i++, OverflowOpType.UPDATE, dataType); + newTimePair.reset(); + break; + } else if (relation == CrossRelation.RFIRST) { // oldTimePair first + putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType); + oldTimePair.reset(); + j++; + } + } + } + } + } + + // newData is empty, but old memory still has data. + while (j < oldData.length || oldTimePair.mergestatus == MergeStatus.MERGING) { + if (oldTimePair.mergestatus == MergeStatus.DONE) { + oldTimePair = constructTimePair(oldData.getTime(j * 2), oldData.getTime(j * 2 + 1), null, MergeStatus.MERGING); + } + + putValueUseDynamic(ans, oldTimePair.s, oldTimePair.e, oldData, j, oldTimePair.opType, dataType); + oldTimePair.reset(); + j++; + } + + return ans; + } + + /** + * To determine whether a time pair is satisfy the demand of the SingleSeriesFilterExpression. + * + * @param valueFilter - value filter + * @param data - DynamicOneColumnData + * @param i - index + * @return - boolean + */ + private boolean isIntervalSatisfy(SingleSeriesFilterExpression valueFilter, DynamicOneColumnData data, int i) { + if (valueFilter == null) { + return true; + } + + SingleValueVisitor visitor = new SingleValueVisitor(valueFilter); + switch (valueFilter.getFilterSeries().getSeriesDataType()) { + case INT32: + return visitor.verify(data.getInt(i)); + case INT64: + return visitor.verify(data.getLong(i)); + case FLOAT: + return visitor.verify(data.getFloat(i)); + case DOUBLE: + return visitor.verify(data.getDouble(i)); + case BYTE_ARRAY: + return SingleValueVisitorFactory.getSingleValueVistor(TSDataType.BYTE_ARRAY).satisfyObject(data.getBinary(i).getStringValue(), valueFilter); + default: + LOG.error("Unsupported TSFile data type."); + throw new UnSupportedDataTypeException("Unsupported TSFile data type."); + + } + } + + /** + * put data from DynamicOneColumnData[data] to DynamicOneColumnData[ope] + *

+ * value in ope must > 0 + */ + private void putDynamicValue(long s, long e, TSDataType dataType, DynamicOneColumnData ope + , DynamicOneColumnData data, int i) { + if (s > 0 && e < 0) { // INSERT OPERATION, storage single point + ope.putTime(s < 0 ? -s : s); + + } else if (s > 0 && e > 0) { // UPDATE OPERATION + ope.putTime(s < 0 ? -s : s); + ope.putTime(e < 0 ? -e : e); + } + + switch (dataType) { + case INT32: + ope.putInt(data.getInt(i)); + break; + case INT64: + ope.putLong(data.getLong(i)); + break; + case FLOAT: + ope.putFloat(data.getFloat(i)); + break; + case DOUBLE: + ope.putDouble(data.getDouble(i)); + break; + case BYTE_ARRAY: + ope.putBinary(data.getBinary(i)); + break; + default: + LOG.error("Unsupported tsfile data type."); + throw new UnSupportedDataTypeException("Unsupported tsfile data type."); + } + + } + + /** + * Given time filter, value filter and frequency filter, + * return the correspond data which meet all the filters expression.
+ * List stores three DynamicOneColumnData structures, insertAdopt, updateAdopt + * and updateNotAdopt. + * + * @param timeFilter - time filter specified by user + * @param valueFilter - value filter specified by user + * @param freqFilter - frequency filter specified by user + * @param overflowData - overflow data + * @param dataType - TSDataType + * @return - List + */ + @Override + public List getDynamicList(SingleSeriesFilterExpression timeFilter, + SingleSeriesFilterExpression valueFilter, SingleSeriesFilterExpression freqFilter, DynamicOneColumnData overflowData, TSDataType dataType) { + + long deleteMaxLength = -1; + + if (timeFilter == null) { + timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries( + "NoName", "NoName", FilterSeriesType.TIME_FILTER), 0L, true); + } + List ans = new ArrayList<>(); + DynamicOneColumnData insertAdopt = new DynamicOneColumnData(dataType, true); + DynamicOneColumnData updateAdopt = new DynamicOneColumnData(dataType, true); + DynamicOneColumnData updateNotAdopt = new DynamicOneColumnData(dataType, true); + LongInterval filterInterval = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter); + + for (int i = 0; i < overflowData.length; i++) { + long L = overflowData.getTime(i * 2); + long R = overflowData.getTime(i * 2 + 1); + for (int j = 0; j < filterInterval.count; j += 2) { + TimePair exist = constructTimePair(L, R, MergeStatus.MERGING); + TimePair filterTimePair = constructTimePair(filterInterval.v[j], filterInterval.v[j + 1], MergeStatus.MERGING); + CrossRelation crossRelation = IntervalRelation.getRelation(filterTimePair, exist); + if (L > 0 && R < 0) { // INSERT + switch (crossRelation) { + case LCOVERSR: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(L, R, dataType, insertAdopt, overflowData, i); + } else { + putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i); + } + break; + case RCOVERSL: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, insertAdopt, overflowData, i); + } else { + putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i); + } + break; + case LFIRSTCROSS: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(exist.s, filterTimePair.e, dataType, insertAdopt, overflowData, i); + } else { + putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i); + } + break; + case RFIRSTCROSS: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(filterTimePair.s, exist.e, dataType, insertAdopt, overflowData, i); + } else { + putDynamicValue(L, -R, dataType, updateNotAdopt, overflowData, i); + } + break; + } + } else if (L > 0 && R > 0) { // UPDATE + switch (crossRelation) { + case LCOVERSR: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(L, R, dataType, updateAdopt, overflowData, i); + } else { + putDynamicValue(L, R, dataType, updateNotAdopt, overflowData, i); + } + break; + case RCOVERSL: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateAdopt, overflowData, i); + } else { + putDynamicValue(filterTimePair.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i); + } + break; + case LFIRSTCROSS: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(exist.s, filterTimePair.e, dataType, updateAdopt, overflowData, i); + } else { + putDynamicValue(exist.s, filterTimePair.e, dataType, updateNotAdopt, overflowData, i); + } + break; + case RFIRSTCROSS: + if (isIntervalSatisfy(valueFilter, overflowData, i)) { + putDynamicValue(filterTimePair.s, exist.e, dataType, updateAdopt, overflowData, i); + } else { + putDynamicValue(filterTimePair.s, exist.e, dataType, updateNotAdopt, overflowData, i); + } + break; + } + } else { // DELETE + // LOG.info("getDynamicList max length:{} delete length:{}", deleteMaxLength, endTime); + deleteMaxLength = Math.max(deleteMaxLength, -R); + } + } + } + + ans.add(insertAdopt); + ans.add(updateAdopt); + ans.add(updateNotAdopt); + GtEq deleteFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries( + "Any", "Any", FilterSeriesType.TIME_FILTER), deleteMaxLength, false); + And and = (And) FilterFactory.and(timeFilter, deleteFilter); + ans.add(and); + return ans; + } + + /** + * both start time and end time are "long" datatype and occupy 16 bytes. + * + * @return - memory occupation + */ + @Override + public long calcMemSize() { + if (index.getValueSize() == 0) { + return 16; + } else { + return index.getNodeSize() * (16 + index.getValueSize()); + } + } + + /** + * reset the status of IntervalTreeOperation. + */ + public void reset() { + index = new IntervalTree(); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/CrossRelation.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/CrossRelation.java new file mode 100644 index 00000000000..e17a5e86261 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/CrossRelation.java @@ -0,0 +1,15 @@ +package cn.edu.thu.tsfiledb.engine.overflow.index; + +/** + * An enum represent the cross relation between two time pairs. + * + * @author CGF + */ +public enum CrossRelation { + LCOVERSR, // left time pair covers right time pair. e.g. [1, 10], [5, 8] + RCOVERSL, // right time pair covers (or equals) left time pair. e.g. [5, 8], [1, 10] + LFIRSTCROSS, // left time pair first cross right time pair. e.g. [1, 10], [8, 13] + RFIRSTCROSS, // right time pair first covers left time pair. e.g. [8, 13], [1, 10] + LFIRST, // left time pair is on the left of right time pair. e.g. [1, 10], [15, 18] + RFIRST // right time pair is on the left of left time pair. e.g. [15, 18], [1, 10] +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalRelation.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalRelation.java new file mode 100644 index 00000000000..460cca4be61 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalRelation.java @@ -0,0 +1,69 @@ +package cn.edu.thu.tsfiledb.engine.overflow.index; + + +import static cn.edu.thu.tsfiledb.engine.overflow.index.CrossRelation.*; + +import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair; + +/** + * This class is used to determine the relation between two time pairs. + * + * @author CGF + */ + +public class IntervalRelation { + + /** + * To determine the relation between the two time pairs. + * + * @param left - left time pair + * @param right - right time pair + */ + private static CrossRelation getCrossRelation(TimePair left, TimePair right) { + if (right.s <= left.s && right.e >= left.e) { // right covers left | right equals left + return RCOVERSL; + } else if (right.s >= left.s && right.e <= left.e) { // left covers right + return LCOVERSR; + } else if (right.s > left.s) { // left first cross + return LFIRSTCROSS; + } else { // right first cross + return RFIRSTCROSS; + } + } + + private static CrossRelation getCrossRelation(long s1, long e1, long s2, long e2) { + if (s2 <= s1 && e2 >= e1) { // right covers left | right equals left + return RCOVERSL; + } else if (s2 >= s1 && e2 <= e1) { // left covers right + return LCOVERSR; + } else if (s2 > s1) { // left first cross + return LFIRSTCROSS; + } else { // right first cross + return RFIRSTCROSS; + } + } + + /** + * @param left - left time pair + * @param right - right time pair + * @return CrossRelation + */ + + public static CrossRelation getRelation(TimePair left, TimePair right) { + if (left.e < right.s) { // left first + return LFIRST; + } else if (right.e < left.s) { // right first + return RFIRST; + } else + return getCrossRelation(left, right); + } + + public static CrossRelation getRelation(long s1, long e1, long s2, long e2) { + if (e1 < s2) { // left first + return LFIRST; + } else if (e2 < s1) { // right first + return RFIRST; + } else + return getCrossRelation(s1, e1, s2, e2); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalTree.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalTree.java new file mode 100644 index 00000000000..d8056b688cc --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/IntervalTree.java @@ -0,0 +1,588 @@ +package cn.edu.thu.tsfiledb.engine.overflow.index; + + +import cn.edu.thu.tsfile.common.exception.UnSupportedDataTypeException; +import cn.edu.thu.tsfile.common.utils.Binary; +import cn.edu.thu.tsfile.common.utils.BytesUtils; +import cn.edu.thu.tsfile.common.utils.ReadWriteStreamUtils; +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.timeseries.filter.definition.FilterFactory; +import cn.edu.thu.tsfile.timeseries.filter.definition.SingleSeriesFilterExpression; +import cn.edu.thu.tsfile.timeseries.filter.definition.filterseries.FilterSeriesType; +import cn.edu.thu.tsfile.timeseries.filter.utils.LongInterval; +import cn.edu.thu.tsfile.timeseries.filter.verifier.FilterVerifier; +import cn.edu.thu.tsfile.timeseries.read.query.DynamicOneColumnData; +import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType; +import cn.edu.thu.tsfiledb.engine.overflow.utils.TimePair; +import cn.edu.thu.tsfiledb.exception.OverflowWrongParameterException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * IntervalTree is a data structure implemented used Treap.
+ * An IntervalTree stores many TreeNodes, a TreeNode saves a time range operation.
+ * A time range operation is explained in OverflowOpType.
+ * see https://en.wikipedia.org/wiki/Treap. + * + * @author CGF + */ +public class IntervalTree { + + private static final Logger LOG = LoggerFactory.getLogger(IntervalTree.class); + + private int nodeSize = 0; // num of tree nodes + private int valueSize = 0; // num of bytes of value + private TreeNode root = null; + private TSDataType dataType = null; // The data type of IntervalTree structure. + + public IntervalTree() { + + } + + public IntervalTree(TSDataType dataType) { + this.dataType = dataType; + if (dataType == TSDataType.BYTE_ARRAY) { + this.valueSize = -1; // -1 represents var length encoding. + } + } + + + + /** + * TimePair must be no conflict with current IntervalTree. + * + * @param tp TimePair to be inserted. + */ + private void insert(TimePair tp) { + nodeSize++; + if (root == null) { + root = new TreeNode(tp.s, tp.e, tp.v, tp.opType); + } else { + insertNoConflict(tp, root, null, false); + } + } + + /** + * Used to insert the no conflicting TreeNode. + * + * @param timePair time pair to be inserted + * @param currentNode current TreeNode + * @param parentNode parent TreeNode + * @param left whether the pNode is left child of paNode. + */ + private void insertNoConflict(TimePair timePair, TreeNode currentNode, TreeNode parentNode, boolean left) { + if (currentNode.start > timePair.e) { + if (currentNode.left == null) { + currentNode.left = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType); + } else { + insertNoConflict(timePair, currentNode.left, currentNode, true); + } + if (currentNode.fix > currentNode.left.fix) + rightTurn(currentNode, parentNode, left); + } else if (currentNode.end < timePair.s) { + if (currentNode.right == null) { + currentNode.right = new TreeNode(timePair.s, timePair.e, timePair.v, timePair.opType); + } else { + insertNoConflict(timePair, currentNode.right, currentNode, false); + } + if (currentNode.fix > currentNode.right.fix) + leftTurn(currentNode, parentNode, left); + } + } + + private void leftTurn(TreeNode currentNode, TreeNode parentNode, boolean left) { + if (parentNode == null) { + root = currentNode.right; + } else if (left) { + parentNode.left = currentNode.right; + } else { + parentNode.right = currentNode.right; + } + TreeNode tmpNode = currentNode.right.left; + currentNode.right.left = currentNode; + currentNode.right = tmpNode; + } + + private void rightTurn(TreeNode currentNode, TreeNode parentNode, boolean left) { + if (parentNode == null) { + root = currentNode.left; + } else if (left) { + parentNode.left = currentNode.left; + } else { + parentNode.right = currentNode.left; + } + TreeNode tmpNode = currentNode.left.right; + currentNode.left.right = currentNode; + currentNode.left = tmpNode; + } + + private void delete(TreeNode currentNode, TreeNode parentNode, boolean left) { + + if (parentNode == null) { + if (currentNode.left == null && currentNode.right == null) { + root = null; + nodeSize--; + return; + } else if (currentNode.left == null) { + root = currentNode.right; + nodeSize--; + return; + } else if (currentNode.right == null) { + root = currentNode.left; + nodeSize--; + return; + } + } + + if (left) { + if (currentNode.left == null) { + parentNode.left = currentNode.right; + nodeSize--; + } else if (currentNode.right == null) { + parentNode.left = currentNode.left; + nodeSize--; + } else { + if (currentNode.left.fix < currentNode.right.fix) { + TreeNode tmpNode = currentNode.left; + rightTurn(currentNode, parentNode, left); + delete(currentNode, tmpNode, false); + } else { + TreeNode tmpNode = currentNode.right; + leftTurn(currentNode, parentNode, left); + delete(currentNode, tmpNode, true); + } + } + } else { + if (currentNode.left == null) { + parentNode.right = currentNode.right; + nodeSize--; + } else if (currentNode.right == null) { + parentNode.right = currentNode.left; + nodeSize--; + } else { + if (currentNode.left.fix < currentNode.right.fix) { + TreeNode tmpNode = currentNode.left; + rightTurn(currentNode, parentNode, left); + delete(currentNode, tmpNode, false); + } else { + TreeNode tmpNode = currentNode.right; + leftTurn(currentNode, parentNode, left); + delete(currentNode, tmpNode, true); + } + } + } + } + + /** + * Update operation api. + * + * @param tp TimePair to be updated + */ + public void update(TimePair tp) { + + + if (this.dataType == null) { + if (tp.opType != OverflowOpType.DELETE) { + this.valueSize = tp.v.length; + } + this.dataType = tp.dataType; + } else if (tp.dataType != this.dataType){ + LOG.error("IntervalTree wrong time pair parameters"); + throw new OverflowWrongParameterException("IntervalTree wrong time pair parameters"); + } + + // When an UPDATE operation covers root(INSERT) node, + // we delete root(INSERT) node and stores it in insertList temporary. + List insertList = new ArrayList<>(); + while (root != null) { + if (update(tp, root, null, false, insertList)) { + break; + } + } + + if (root == null) { + insert(tp); + } + + for (TimePair t : insertList) { + while (root != null) { + if (update(t, root, null, false, null)) { + break; + } + } + } + } + + /** + * IntervalTree update operation. + * + * @param tp TimePair to be updated + * @param currentNode current TreeNode + * @param parentNode parent node of pNode + * @param left whether pNode is left child of paNode + * @return whether update process is over + */ + private boolean update(TimePair tp, TreeNode currentNode, TreeNode parentNode, boolean left, List insertList) { + if (currentNode == null) { + insert(tp); + return true; + } + + CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(currentNode.start, currentNode.end)); + switch (relation) { + case LCOVERSR: // tp covers currentNode + if (currentNode.opType == OverflowOpType.INSERT && tp.opType == OverflowOpType.UPDATE) { + insertList.add(new TimePair(currentNode.start, currentNode.end, tp.v, + OverflowOpType.INSERT)); + } + if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) { + tp.s = currentNode.end + 1; + } else { + delete(currentNode, parentNode, left); + } + return false; + case RCOVERSL: // currentNode covers tp + if (tp.s == currentNode.start && currentNode.end == tp.e) { + if (currentNode.opType == OverflowOpType.INSERT + && tp.opType == OverflowOpType.UPDATE) { + currentNode.value = tp.v; + } else if (currentNode.opType == OverflowOpType.DELETE + && tp.opType == OverflowOpType.UPDATE) { + return true; + } else { + currentNode.opType = tp.opType; + currentNode.value = tp.v; + } + } else if (tp.s == currentNode.start) { + if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) { + return true; + } else { + currentNode.start = tp.e + 1; + insert(tp); + } + } else if (tp.e == currentNode.end) { + if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) { + return true; + } else { + currentNode.end = tp.s - 1; + insert(tp); + } + } else { // (currentNode tp currentNode) + if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) { + return true; + } else { + long tmp = currentNode.end; + currentNode.end = tp.s - 1; + insert(tp); + insert(new TimePair(tp.e + 1, tmp, currentNode.value, currentNode.opType)); + } + } + return true; + case LFIRSTCROSS: // tp first cross | No need to modify currentNode value. + currentNode.start = tp.e + 1; + return update(tp, currentNode.left, currentNode, true, insertList); + case RFIRSTCROSS: // currentNode first cross | No need to modify currentNode value. + if (currentNode.opType == OverflowOpType.DELETE && tp.opType == OverflowOpType.UPDATE) { + return update(new TimePair(currentNode.end + 1, tp.e, tp.v, tp.opType), currentNode.right, currentNode, false, insertList); + } else { + currentNode.end = tp.s - 1; + return update(tp, currentNode.right, currentNode, false, insertList); + } + case LFIRST: // tp first + return update(tp, currentNode.left, currentNode, true, insertList); + case RFIRST: // currentNode first + return update(tp, currentNode.right, currentNode, false, insertList); + default: + return true; + } + } + + /** + * Put the IntervalTree mid order serialization into OutputStream. + * + * @param out OutputStream + */ + public void midOrderSerialize(OutputStream out) throws IOException { + midOrderSerialize(root, out); + root = null; + } + + /** + * IntervalTree mid order serialization. + *

+ * Notice that "String" data type adopts unsigned var int encoding. + * + * @param pNode - root node of IntervalTree + * @param out - output file stream + */ + private void midOrderSerialize(TreeNode pNode, OutputStream out) throws IOException { + if (pNode == null) + return; + midOrderSerialize(pNode.left, out); + if (pNode.opType.equals(OverflowOpType.DELETE)) { + out.write(BytesUtils.longToBytes(-pNode.start)); + out.write(BytesUtils.longToBytes(-pNode.end)); + } else if (pNode.opType.equals(OverflowOpType.UPDATE)) { + out.write(BytesUtils.longToBytes(pNode.start)); + out.write(BytesUtils.longToBytes(pNode.end)); + if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) { + ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out); + out.write(pNode.value); + } else { + out.write(pNode.value); + } + } else { + out.write(BytesUtils.longToBytes(pNode.start)); + out.write(BytesUtils.longToBytes(-pNode.end)); + if (dataType == TSDataType.BYTE_ARRAY || valueSize == -1) { + ReadWriteStreamUtils.writeUnsignedVarInt(pNode.value.length, out); + out.write(pNode.value); + } else { + out.write(pNode.value); + } + + } + midOrderSerialize(pNode.right, out); + } + + + /** + * given the query TimePair tp, return all covered time pair with tp. + *

+ * for concurrency safety, the queryAns variable must be defined in + * the query method. + * + * @param tp - TimePair to query + * @return - all new TimePairs accord with the query tp. + */ + public List query(TimePair tp) { + List queryAns = new ArrayList<>(); + query(tp, root, queryAns); + return queryAns; + } + + private void query(TimePair tp, TreeNode pNode, List queryAns) { + if (pNode == null) + return; + CrossRelation relation = IntervalRelation.getRelation(tp, new TimePair(pNode.start, pNode.end)); + switch (relation) { + case LCOVERSR: // tp covers currentNode + if (tp.s == pNode.start && pNode.end == tp.e) { + queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType)); + } else if (tp.s == pNode.start) { + query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns); + queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType)); + } else if (tp.e == pNode.end) { + query(new TimePair(tp.s, pNode.end - 1), pNode.left, queryAns); + queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType)); + } else { + query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns); + queryAns.add(new TimePair(pNode.start, pNode.end, pNode.value, pNode.opType)); + query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns); + } + return; + case RCOVERSL: // currentNode covers tp + queryAns.add(new TimePair(tp.s, tp.e, pNode.value, pNode.opType)); + return; + case LFIRSTCROSS: // tp first cross + query(new TimePair(tp.s, pNode.start - 1), pNode.left, queryAns); + queryAns.add(new TimePair(pNode.start, tp.e, pNode.value, pNode.opType)); + return; + case RFIRSTCROSS: // currentNode first cross + queryAns.add(new TimePair(tp.s, pNode.end, pNode.value, pNode.opType)); + query(new TimePair(pNode.end + 1, tp.e), pNode.right, queryAns); + return; + case LFIRST: // tp first + query(tp, pNode.left, queryAns); + break; + case RFIRST: // currentNode first + query(tp, pNode.right, queryAns); + break; + } + } + + /** + * Given time filter and value filter, return the answer List in IntervalTree.
+ * Could not use member variable.
+ * Must notice thread safety!
+ * Note that value filter is not given, apply value filter in this method would cause errors.
+ * May make wrong intervals in old data valid in IntervalTreeOperation.queryFileBlock method. + * + * @param timeFilter filter for time + * @param dataType TSDataType + * @return DynamicOneColumnData + */ + public DynamicOneColumnData dynamicQuery(SingleSeriesFilterExpression timeFilter, TSDataType dataType) { + + DynamicOneColumnData crudResult = new DynamicOneColumnData(dataType, true); + + if (timeFilter == null) { + timeFilter = FilterFactory.gtEq(FilterFactory.longFilterSeries("NULL", "NULL", FilterSeriesType.TIME_FILTER), 0L, true); + } + + LongInterval val = (LongInterval) FilterVerifier.get(timeFilter).getInterval(timeFilter); + dynamicQuery(crudResult, root, val, dataType); + + return crudResult; + } + + /** + * Implemented for above dynamicQuery method. + * + * @param crudResult DynamicOneColumnData + * @param pNode query root TreeNode + * @param val LongInterval + * @param dataType TSDataType + */ + private void dynamicQuery(DynamicOneColumnData crudResult, TreeNode pNode, LongInterval val, TSDataType dataType) { + if (pNode == null) + return; + + // to examine whether left child node has answer + if (val.count != 0 && val.v[0] < pNode.start) { + dynamicQuery(crudResult, pNode.left, val, dataType); + } + + for (int i = 0; i < val.count; i += 2) { + CrossRelation r = IntervalRelation.getRelation(new TimePair(val.v[i], val.v[i + 1]), new TimePair(pNode.start, pNode.end)); + switch (r) { + case LFIRST: + continue; + case RFIRST: + continue; + case LFIRSTCROSS: + putTreeNodeValue(crudResult, pNode.opType, pNode.start, val.v[i + 1], dataType, pNode.value); + break; + case RFIRSTCROSS: + putTreeNodeValue(crudResult, pNode.opType, val.v[i], pNode.end, dataType, pNode.value); + break; + case LCOVERSR: + putTreeNodeValue(crudResult, pNode.opType, pNode.start, pNode.end, dataType, pNode.value); + break; + case RCOVERSL: + putTreeNodeValue(crudResult, pNode.opType, val.v[i], val.v[i + 1], dataType, pNode.value); + break; + default: + LOG.error("un defined CrossRelation"); + } + } + + // to examine whether right child node has answer + if (val.count != 0 && val.v[val.count - 1] > pNode.end) { // left node has answer + dynamicQuery(crudResult, pNode.right, val, dataType); + } + } + + /** + * Put TreeNode information into DynamicColumnData crudResult. + * + * @param crudResult DynamicOneColumnData stores operations + * @param opType OverflowOpType + * @param s start time + * @param e end time + * @param dataType TSDataType + * @param value byte value + */ + private void putTreeNodeValue(DynamicOneColumnData crudResult, OverflowOpType opType, long s, long e, TSDataType dataType, + byte[] value) { + switch (dataType) { + case INT32: + switch (opType) { + case INSERT: + crudResult.putTimePair(s, -e); + crudResult.putInt(BytesUtils.bytesToInt(value)); + break; + case DELETE: + crudResult.putTimePair(-s, -e); + crudResult.putInt(0); + break; + case UPDATE: + crudResult.putTimePair(s, e); + crudResult.putInt(BytesUtils.bytesToInt(value)); + break; + } + break; + case INT64: + switch (opType) { + case INSERT: + crudResult.putTimePair(s, -e); + crudResult.putLong(BytesUtils.bytesToLong(value)); + break; + case DELETE: + crudResult.putTimePair(-s, -e); + crudResult.putLong(0); + break; + case UPDATE: + crudResult.putTimePair(s, e); + crudResult.putLong(BytesUtils.bytesToLong(value)); + } + break; + case FLOAT: + switch (opType) { + case INSERT: + crudResult.putTimePair(s, -e); + crudResult.putFloat(BytesUtils.bytesToFloat(value)); + break; + case DELETE: + crudResult.putTimePair(-s, -e); + crudResult.putFloat(0); + break; + case UPDATE: + crudResult.putTimePair(s, e); + crudResult.putFloat(BytesUtils.bytesToFloat(value)); + break; + } + break; + case DOUBLE: + switch (opType) { + case INSERT: + crudResult.putTimePair(s, -e); + crudResult.putDouble(BytesUtils.bytesToDouble(value)); + break; + case DELETE: + crudResult.putTimePair(-s, -e); + crudResult.putDouble(0); + break; + case UPDATE: + crudResult.putTimePair(s, e); + crudResult.putDouble(BytesUtils.bytesToDouble(value)); + break; + } + break; + case BYTE_ARRAY: + switch (opType) { + case INSERT: + crudResult.putTimePair(s, -e); + crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value))); + break; + case DELETE: + crudResult.putTimePair(-s, -e); + crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value))); + break; + case UPDATE: + crudResult.putTimePair(s, e); + crudResult.putBinary(Binary.valueOf(BytesUtils.bytesToString(value))); + break; + } + break; + default: + LOG.error("Unsupported tsfile data type."); + throw new UnSupportedDataTypeException("Unsupported tsfile data type."); + } + } + + public int getNodeSize() { + return nodeSize; + } + + public int getValueSize() { + return this.valueSize; + } +} + diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/TreeNode.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/TreeNode.java new file mode 100644 index 00000000000..cc8cd3663c8 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/index/TreeNode.java @@ -0,0 +1,34 @@ +package cn.edu.thu.tsfiledb.engine.overflow.index; + + +import java.util.Random; + +import cn.edu.thu.tsfiledb.engine.overflow.utils.OverflowOpType; + +/** + * Used for IntervalTree (Treap Data Structure). + * + * @author CGF + */ + +public class TreeNode { + + private Random random = new Random(); + public long start; // start time + public long end; // end time + public int fix; // priority for treap IntervalTree + public OverflowOpType opType; // overflow operation type + public byte[] value; // the value stored in this node + public TreeNode left; + public TreeNode right; + + public TreeNode(long start, long end, byte[] value, OverflowOpType type) { + this.start = start; + this.end = end; + this.value = value; + this.opType = type; + left = null; + right = null; + fix = random.nextInt(); + } +} \ No newline at end of file diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/MergeStatus.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/MergeStatus.java new file mode 100644 index 00000000000..bc43e39cc8e --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/MergeStatus.java @@ -0,0 +1,13 @@ +package cn.edu.thu.tsfiledb.engine.overflow.utils; + +/** + * Used for IntervalTreeOperation.queryMemory() and IntervalTreeOperation.queryFileBlock();
+ * + * DONE means that a time pair is not used or this time pair has been merged into a new DynamicOneColumn
+ * MERGING means that a time pair is merging into a new DynamicOneColumn
+ * + * @author CGF. + */ +public enum MergeStatus { + DONE, MERGING +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/OverflowOpType.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/OverflowOpType.java new file mode 100644 index 00000000000..723c88378ee --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/OverflowOpType.java @@ -0,0 +1,16 @@ +package cn.edu.thu.tsfiledb.engine.overflow.utils; + +/** + * Include three types: INSERT,UPDATE,DELETE; + * + * INSERT is an operation which inserts a time point.
+ * UPDATE is an operation which updates a time range.
+ * DELETE is an operation which deletes a time range. Note that DELETE operation could only + * delete a time which is less than given time T.
+ * + * @author kangrong + * + */ +public enum OverflowOpType { + INSERT, UPDATE, DELETE +} \ No newline at end of file diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/TimePair.java b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/TimePair.java new file mode 100644 index 00000000000..2da4e911fb5 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/overflow/utils/TimePair.java @@ -0,0 +1,75 @@ +package cn.edu.thu.tsfiledb.engine.overflow.utils; + + +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; + +/** + * TimePair represents an overflow operation. + * + * @author CGF + */ + +public class TimePair { + + public long s; // start time + public long e; // end time + public byte[] v; // value + public OverflowOpType opType = null; + public TSDataType dataType; + public MergeStatus mergestatus; + + public TimePair(long s, long e) { + this.s = s; + this.e = e; + this.v = null; + } + + public TimePair(long s, long e, byte[] v) { + this.s = s; + this.e = e; + this.v = v; + } + + public TimePair(long s, long e, MergeStatus status) { + this(s, e); + this.mergestatus = status; + } + + public TimePair(long s, long e, byte[] v, TSDataType dataType) { + this(s, e, v); + this.dataType = dataType; + } + + public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType) { + this(s, e, v); + this.opType = overflowOpType; + } + + public TimePair(long s, long e, byte[] v, OverflowOpType overflowOpType, TSDataType dataType) { + this(s, e, v, overflowOpType); + this.dataType = dataType; + } + + public TimePair(long s, long e, byte[] v, OverflowOpType type, MergeStatus status) { + this(s, e, v); + this.opType = type; + this.mergestatus = status; + } + + /** + * Set TimePair s = -1 and e = -1 means reset. + */ + public void reset() { + s = -1; + e = -1; + mergestatus = MergeStatus.DONE; + } + + public String toString() { + StringBuffer sb = new StringBuffer().append(this.s).append(",").append(this.e); + if (this.opType != null) + sb.append(",").append(this.opType); + return sb.toString(); + } +} + diff --git a/src/main/java/cn/edu/thu/tsfiledb/engine/utils/FlushState.java b/src/main/java/cn/edu/thu/tsfiledb/engine/utils/FlushState.java new file mode 100644 index 00000000000..a97998d26c1 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/engine/utils/FlushState.java @@ -0,0 +1,30 @@ +package cn.edu.thu.tsfiledb.engine.utils; + +/** + * This class is used to represent the state of flush. It's can be used in the + * bufferwrite flush{@code BufferWriteProcessor} and overflow + * flush{@code OverFlowProcessor}. + * + * @author liukun + */ +public class FlushState { + + private boolean isFlushing; + + public FlushState() { + this.isFlushing = false; + } + + public boolean isFlushing() { + return isFlushing; + } + + public void setFlushing() { + this.isFlushing = true; + } + + public void setUnFlushing() { + this.isFlushing = false; + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/ArgsErrorException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/ArgsErrorException.java new file mode 100644 index 00000000000..4f3a885e6ea --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/ArgsErrorException.java @@ -0,0 +1,10 @@ +package cn.edu.thu.tsfiledb.exception; + +public class ArgsErrorException extends Exception{ + + private static final long serialVersionUID = -3614543017182165265L; + public ArgsErrorException(String msg){ + super(msg); + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/DeltaEngineRunningException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/DeltaEngineRunningException.java new file mode 100644 index 00000000000..59b82a94941 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/DeltaEngineRunningException.java @@ -0,0 +1,23 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * + * This Exception is the parent class for all delta engine runtime exceptions.
+ * This Exception extends super class {@link java.lang.RuntimeException} + * + * @author CGF + */ +public abstract class DeltaEngineRunningException extends RuntimeException{ + public DeltaEngineRunningException() { super();} + + public DeltaEngineRunningException(String message, Throwable cause) { super(message, cause);} + + public DeltaEngineRunningException(String message) { + super(message); + } + + public DeltaEngineRunningException(Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/ErrorDebugException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/ErrorDebugException.java new file mode 100644 index 00000000000..c520ff7c340 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/ErrorDebugException.java @@ -0,0 +1,18 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * @author kangrong + * + */ +public class ErrorDebugException extends RuntimeException { + + private static final long serialVersionUID = -1123099620556170447L; + + public ErrorDebugException(String msg) { + super(msg); + } + + public ErrorDebugException(Exception e) { + super(e); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/FileNodeNotExistException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/FileNodeNotExistException.java new file mode 100644 index 00000000000..7e4e62e9947 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/FileNodeNotExistException.java @@ -0,0 +1,16 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * Throw this exception when the file node processor is not exists + * + * @author kangrong + * + */ +public class FileNodeNotExistException extends RuntimeException { + + private static final long serialVersionUID = -4334041411884083545L; + + public FileNodeNotExistException(String msg) { + super(msg); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/MetadataArgsErrorException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/MetadataArgsErrorException.java new file mode 100644 index 00000000000..a72724f7f42 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/MetadataArgsErrorException.java @@ -0,0 +1,18 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * If query metadata constructs schema but passes illegal parameters to + * EncodingConvertor or DataTypeConvertor,this exception will be threw. + * + * @author kangrong + * + */ +public class MetadataArgsErrorException extends ArgsErrorException { + + private static final long serialVersionUID = 3415275599091623570L; + + public MetadataArgsErrorException(String msg) { + super(msg); + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/OverflowWrongParameterException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/OverflowWrongParameterException.java new file mode 100644 index 00000000000..6dcb54cad34 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/OverflowWrongParameterException.java @@ -0,0 +1,20 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * Used for IntervalTree pass wrong parameters.
+ * e.g. TSDataType inconsistent; + * e.g. start time is greater than end time. + * + * @author CGF + */ +public class OverflowWrongParameterException extends DeltaEngineRunningException{ + public OverflowWrongParameterException(String message, Throwable cause) { super(message, cause);} + + public OverflowWrongParameterException(String message) { + super(message); + } + + public OverflowWrongParameterException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/PathErrorException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/PathErrorException.java new file mode 100644 index 00000000000..c8fd6c65e05 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/PathErrorException.java @@ -0,0 +1,10 @@ +package cn.edu.thu.tsfiledb.exception; + +public class PathErrorException extends Exception{ + + private static final long serialVersionUID = 2141197032898163234L; + + public PathErrorException(String msg){ + super(msg); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/ProcessorException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/ProcessorException.java new file mode 100644 index 00000000000..322142ac325 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/ProcessorException.java @@ -0,0 +1,24 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * This exception is used for some error in processor + * + * @author kangrong + * + */ +public class ProcessorException extends Exception { + + private static final long serialVersionUID = 4137638418544201605L; + + public ProcessorException(String msg) { + super(msg); + } + + public ProcessorException(PathErrorException pathExcp) { + super("PathErrorException: " + pathExcp.getMessage()); + } + + public ProcessorException() { + super(); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/exception/UnSupportedOverflowOpTypeException.java b/src/main/java/cn/edu/thu/tsfiledb/exception/UnSupportedOverflowOpTypeException.java new file mode 100644 index 00000000000..53ddf10ff55 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/exception/UnSupportedOverflowOpTypeException.java @@ -0,0 +1,20 @@ +package cn.edu.thu.tsfiledb.exception; + +/** + * OverflowOpType could only be INSERT, UPDATE, DELETE. + * The other situations would throw this exception. + * + * @author CGF + */ +public class UnSupportedOverflowOpTypeException extends DeltaEngineRunningException{ + + public UnSupportedOverflowOpTypeException(String message, Throwable cause) { super(message, cause);} + + public UnSupportedOverflowOpTypeException(String message) { + super(message); + } + + public UnSupportedOverflowOpTypeException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSInputStream.java b/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSInputStream.java new file mode 100644 index 00000000000..092edb71421 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSInputStream.java @@ -0,0 +1,102 @@ +package cn.edu.thu.tsfiledb.hadoop.io; + + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileReader; + +import java.io.IOException; + +/** + * This class is used to wrap the {@link}FSDataInputStream and implement the + * interface {@link}TSRandomAccessFileReader. + * + * @author liukun + */ +public class HDFSInputStream implements TSRandomAccessFileReader { + + private FSDataInputStream fsDataInputStream; + private FileStatus fileStatus; + + public HDFSInputStream(String filePath) throws IOException { + + this(filePath, new Configuration()); + } + + public HDFSInputStream(String filePath, Configuration configuration) throws IOException { + + this(new Path(filePath),configuration); + } + + public HDFSInputStream(Path path, Configuration configuration) throws IOException { + + FileSystem fs = FileSystem.get(configuration); + fsDataInputStream = fs.open(path); + fileStatus = fs.getFileStatus(path); + } + + @Override + public void seek(long offset) throws IOException { + + fsDataInputStream.seek(offset); + } + + @Override + public int read() throws IOException { + + return fsDataInputStream.read(); + } + + @Override + public long length() throws IOException { + + return fileStatus.getLen(); + } + + @Override + public int readInt() throws IOException { + + return fsDataInputStream.readInt(); + } + + public void close() throws IOException { + + fsDataInputStream.close(); + } + + public long getPos() throws IOException { + + return fsDataInputStream.getPos(); + } + + /** + * Read the data into b, and the check the length + * + * @param b + * read the data into + * @param off + * the begin offset to read into + * @param len + * the length to read into + */ + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len < 0) { + throw new IndexOutOfBoundsException(); + } + int n = 0; + while (n < len) { + int count = fsDataInputStream.read(b, off + n, len - n); + if (count < 0) { + throw new IOException("The read length is out of the length of inputstream"); + } + n += count; + } + return n; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSOutputStream.java b/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSOutputStream.java new file mode 100644 index 00000000000..7b6aab267e7 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/hadoop/io/HDFSOutputStream.java @@ -0,0 +1,75 @@ +package cn.edu.thu.tsfiledb.hadoop.io; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.edu.thu.tsfile.common.utils.TSRandomAccessFileWriter; + + +/** + * This class is used to wrap the {@link}FSDataOutputStream and implement the + * interface {@link}TSRandomAccessFileWriter + * + * @author liukun + */ +public class HDFSOutputStream implements TSRandomAccessFileWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(HDFSOutputStream.class); + + private FSDataOutputStream fsDataOutputStream; + + public HDFSOutputStream(String filePath, boolean overwriter) throws IOException { + + this(filePath, new Configuration(), overwriter); + } + + + public HDFSOutputStream(String filePath, Configuration configuration, boolean overwriter) throws IOException { + + this(new Path(filePath),configuration,overwriter); + } + + public HDFSOutputStream(Path path,Configuration configuration,boolean overwriter) throws IOException{ + + FileSystem fsFileSystem = FileSystem.get(configuration); + fsDataOutputStream = fsFileSystem.create(path, overwriter); + } + + @Override + public OutputStream getOutputStream() { + + return fsDataOutputStream; + } + + @Override + public long getPos() throws IOException { + + return fsDataOutputStream.getPos(); + } + + @Override + public void write(int b) throws IOException { + + fsDataOutputStream.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + + fsDataOutputStream.write(b); + } + + @Override + public void close() throws IOException { + + fsDataOutputStream.close(); + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/ColumnSchema.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/ColumnSchema.java new file mode 100644 index 00000000000..d50fc1b5166 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/ColumnSchema.java @@ -0,0 +1,40 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding; + +public class ColumnSchema implements Serializable { + private static final long serialVersionUID = -8257474930341487207L; + + public String name; + public TSDataType dataType; + public TSEncoding encoding; + private Map args; + + public ColumnSchema(String name, TSDataType dataType, TSEncoding encoding) { + this.name = name; + this.dataType = dataType; + this.encoding = encoding; + this.args = new HashMap<>(); + } + + public void putKeyValueToArgs(String key, String value) { + this.args.put(key, value); + } + + public String getValueFromArgs(String key) { + return args.get(key); + } + + public Map getArgsMap() { + return args; + } + + public void setArgsMap(Map argsMap) { + this.args = argsMap; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/MGraph.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/MGraph.java new file mode 100644 index 00000000000..b1b8abe5838 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/MGraph.java @@ -0,0 +1,250 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException; +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * Metadata Graph consists of one {@code MTree} and several {@code PTree} + * + * @author Jinrui Zhang + * + */ +public class MGraph implements Serializable { + private static final long serialVersionUID = 8214849219614352834L; + + private MTree mTree; + private HashMap pTreeMap; + + public MGraph(String MTreeName) { + mTree = new MTree(MTreeName); + pTreeMap = new HashMap<>(); + } + + /** + * Add a {@code PTree} to current {@code MGraph} + * @throws MetadataArgsErrorException + */ + public void addAPTree(String pTreeRootName) throws MetadataArgsErrorException { + if(pTreeRootName.toLowerCase().equals("root")){ + throw new MetadataArgsErrorException("Property Tree's root name should not be 'root'"); + } + PTree pTree = new PTree(pTreeRootName, mTree); + pTreeMap.put(pTreeRootName, pTree); + } + + /** + * Add a path to Metadata Tree + * + * @param path Format: root.node.(node)* + * @return The count of new nodes added + * @throws MetadataArgsErrorException + * @throws PathErrorException + */ + public int addPathToMTree(String path, String dataType, String encoding, String args[]) + throws PathErrorException, MetadataArgsErrorException { + String nodes[] = path.trim().split("\\."); + if (nodes.length == 0) { + throw new PathErrorException("Path is null. Path: " + path); + } + return this.mTree.addPath(path, dataType, encoding, args); + } + + /** + * Add a path to {@code PTree} + */ + public void addPathToPTree(String path) throws PathErrorException, MetadataArgsErrorException { + String nodes[] = path.trim().split("\\."); + if (nodes.length == 0) { + throw new PathErrorException("Path is null. Path: " + path); + } + String rootName = path.trim().split("\\.")[0]; + if (pTreeMap.containsKey(rootName)) { + PTree pTree = pTreeMap.get(rootName); + pTree.addPath(path); + } else { + throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName); + } + } + + /** + * Delete path in current MGraph. + * @param path a path belongs to MTree or PTree + * @throws PathErrorException + */ + public void deletePath(String path) throws PathErrorException { + String nodes[] = path.trim().split("\\."); + if (nodes.length == 0) { + throw new PathErrorException("Path is null. Path: " + path); + } + String rootName = path.trim().split("\\.")[0]; + if (mTree.getRoot().getName().equals(rootName)) { + mTree.deletePath(path); + } else if (pTreeMap.containsKey(rootName)) { + PTree pTree = pTreeMap.get(rootName); + pTree.deletePath(path); + } else { + throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName); + } + } + + /** + * Link a {@code MNode} to a {@code PNode} in current PTree + */ + public void linkMNodeToPTree(String path, String mPath) throws PathErrorException { + String pTreeName = path.trim().split("\\.")[0]; + if (!pTreeMap.containsKey(pTreeName)) { + throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path); + } else { + pTreeMap.get(pTreeName).linkMNode(path, mPath); + } + } + + /** + * Unlink a {@code MNode} from a {@code PNode} in current PTree + */ + public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException { + String pTreeName = path.trim().split("\\.")[0]; + if (!pTreeMap.containsKey(pTreeName)) { + throw new PathErrorException("Error: PTree Path Not Correct. Path: " + path); + } else { + pTreeMap.get(pTreeName).unlinkMNode(path, mPath); + } + } + + /** + * Set storage level for current Metadata Tree. + * @param path Format: root.node.(node)* + * @throws PathErrorException + */ + public void setStorageLevel(String path) throws PathErrorException { + mTree.setStorageLevel(path); + } + + /** + * Get all paths for given path regular expression if given path belongs to + * MTree, or get all linked path for given path if given path belongs to + * PTree Notice: Regular expression in this method is formed by the + * amalgamation of path and the character '*' + * + * @return A HashMap whose Keys are separated by the storage file name. + */ + public HashMap> getAllPathGroupByFilename(String path) throws PathErrorException { + String rootName = path.trim().split("\\.")[0]; + if (mTree.getRoot().getName().equals(rootName)) { + return mTree.getAllPath(path); + } else if (pTreeMap.containsKey(rootName)) { + PTree pTree = pTreeMap.get(rootName); + return pTree.getAllLinkedPath(path); + } + throw new PathErrorException("Path Root is Not Correct. RootName: " + rootName); + } + + /** + * Get all DeltaObject type in current Metadata Tree + * @return a HashMap contains all distinct DeltaObject type separated by + * DeltaObject Type + */ + public Map> getSchemaForAllType() throws PathErrorException { + Map> res = new HashMap<>(); + List typeList = mTree.getAllType(); + for (String type : typeList) { + res.put(type, getSchemaForOneType("root." + type)); + } + return res; + } + + private ArrayList getDeltaObjectForOneType(String type) throws PathErrorException { + return mTree.getDeltaObjectForOneType(type); + } + + /** + * Get all delta objects group by DeltaObject type + */ + public Map> getDeltaObjectForAllType() throws PathErrorException { + Map> res = new HashMap<>(); + ArrayList types = mTree.getAllType(); + for (String type : types) { + res.put(type, getDeltaObjectForOneType(type)); + } + return res; + } + + /** + * Get the full Metadata info. + * @return A {@code Metadata} instance which stores all metadata info + */ + public Metadata getMetadata() throws PathErrorException { + Map> seriesMap = getSchemaForAllType(); + Map> deltaObjectMap = getDeltaObjectForAllType(); + Metadata metadata = new Metadata(seriesMap, deltaObjectMap); + return metadata; + } + + /** + * Get all ColumnSchemas for given delta object type + * @param path A path represented one Delta object + * @return a list contains all column schema + */ + public ArrayList getSchemaForOneType(String path) throws PathErrorException { + return mTree.getSchemaForOneType(path); + } + + /** + * Calculate the count of storage-level nodes included in given path + * @return The total count of storage-level nodes. + */ + public int getFileCountForOneType(String path) throws PathErrorException { + return mTree.getFileCountForOneType(path); + } + + /** + * Get the file name for given path Notice: This method could be called if + * and only if the path includes one node whose {@code isStorageLevel} is + * true + */ + public String getFileNameByPath(String path) throws PathErrorException { + return mTree.getFileNameByPath(path); + } + + /** + * Check whether the path given exists + */ + public boolean pathExist(String path) { + return mTree.hasPath(path); + } + + /** + * Extract the DeltaObjectId from given path + * @return String represents the DeltaObjectId + */ + public String getDeltaObjectTypeByPath(String path) throws PathErrorException { + return mTree.getDeltaObjectTypeByPath(path); + } + + /** + * Get ColumnSchema for given path. Notice: Path must be a complete Path + * from root to leaf node. + */ + public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException { + return mTree.getSchemaForOnePath(path); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("=== MetaData Tree ===\n\n"); + sb.append(mTree.toString()); + sb.append("\n\n=== Properties Tree === Size : " + pTreeMap.size() + "\n\n"); + for (String key : pTreeMap.keySet()) { + sb.append("--- name : " + key + "---\n"); + sb.append(pTreeMap.get(key).toString()); + sb.append("\n\n"); + } + return sb.toString(); + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/MManager.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/MManager.java new file mode 100644 index 00000000000..49bdfc2a62f --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/MManager.java @@ -0,0 +1,392 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import cn.edu.thu.tsfile.common.conf.TSFileDescriptor; +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.timeseries.read.qp.Path; +import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException; +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * This class takes the responsibility of serialization of all the metadata info + * and persistent it into files. This class contains all the interfaces to + * modify the metadata for delta system. All the operations will be write into + * the logs temporary in case the downtime of the delta system. + * + * @author Jinrui Zhang + * + */ +public class MManager { + private static MManager manager = new MManager(); + + // The file storing the serialize info for metadata + private String datafilePath; + // log file path + private String logFilePath; + + private MGraph mGraph; + private BufferedWriter bw; + private boolean writeToLog; + + private MManager() { + writeToLog = false; + String folderPath = TSFileDescriptor.getInstance().getConfig().metadataDir; + datafilePath = folderPath + "/mdata.obj"; + logFilePath = folderPath + "/mlog.txt"; + init(); + } + + private void init() { + try { + File file = new File(datafilePath); + // inital MGraph from file + if (file.exists()) { + FileInputStream fis = new FileInputStream(file); + ObjectInputStream ois = new ObjectInputStream(fis); + mGraph = (MGraph) ois.readObject(); + ois.close(); + fis.close(); + + } else { + mGraph = new MGraph("root"); + } + + // recover operation from log file + File logFile = new File(logFilePath); + if (logFile.exists()) { + FileReader fr; + fr = new FileReader(logFile); + BufferedReader br = new BufferedReader(fr); + String cmd; + while ((cmd = br.readLine()) != null) { + operation(cmd); + } + br.close(); + } else { + if (!logFile.getParentFile().exists()) { + logFile.getParentFile().mkdirs(); + } + logFile.createNewFile(); + } + + FileWriter fw = new FileWriter(logFile, true); + bw = new BufferedWriter(fw); + writeToLog = true; + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (PathErrorException e) { + e.printStackTrace(); + } catch (MetadataArgsErrorException e) { + e.printStackTrace(); + } + } + + /** + * Clear all metadata info + */ + public void clear() { + this.mGraph = new MGraph("root"); + } + + public void deleteLogAndDataFiles() { + File file = new File(logFilePath); + if (file.exists()) { + file.delete(); + } + File dataFile = new File(datafilePath); + if (dataFile.exists()) { + dataFile.delete(); + } + } + + private void operation(String cmd) throws PathErrorException, IOException, MetadataArgsErrorException { + String args[] = cmd.trim().split(","); + if (args[0].equals("0")) { + String[] leftArgs; + if (args.length > 4) { + leftArgs = new String[args.length - 4]; + for (int k = 4; k < args.length; k++) { + leftArgs[k - 4] = args[k]; + } + } else { + leftArgs = new String[0]; + } + addAPathToMTree(args[1], args[2], args[3], leftArgs); + } else if (args[0].equals(MetadataOperationType.ADD_PATH_TO_MTREE)) { + deletePathFromMTree(args[1]); + } else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_MTREE)) { + setStorageLevelToMTree(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE); + } else if (args[0].equals(MetadataOperationType.ADD_A_PTREE)) { + addAPTree(args[1]); + } else if (args[0].equals(MetadataOperationType.ADD_A_PATH_TO_PTREE)) { + addAPathToPTree(args[1]); + } else if (args[0].equals(MetadataOperationType.DELETE_PATH_FROM_PTREE)) { + deletePathFromPTree(args[1]); + } else if (args[0].equals(MetadataOperationType.LINK_MNODE_TO_PTREE)) { + linkMNodeToPTree(args[1], args[2]); + } else if (args[0].equals(MetadataOperationType.UNLINK_MNODE_FROM_PTREE)) { + unlinkMNodeFromPTree(args[1], args[2]); + } + } + + /** + * operation: Add a path to Metadata Tree + */ + public int addAPathToMTree(String path, String dataType, String encoding, String[] args) + throws PathErrorException, IOException, MetadataArgsErrorException { + int addCount = mGraph.addPathToMTree(path, dataType, encoding, args); + if (writeToLog) { + bw.write(MetadataOperationType.ADD_PATH_TO_MTREE + "," + path + "," + dataType + "," + encoding); + for (int i = 0; i < args.length; i++) { + bw.write("," + args[i]); + } + bw.newLine(); + bw.flush(); + } + return addCount; + } + + public void deletePathFromMTree(String path) throws PathErrorException, IOException { + mGraph.deletePath(path); + if (writeToLog) { + bw.write(MetadataOperationType.DELETE_PATH_FROM_MTREE + "," + path); + bw.newLine(); + bw.flush(); + } + } + + public void setStorageLevelToMTree(String path) throws PathErrorException, IOException { + mGraph.setStorageLevel(path); + if (writeToLog) { + bw.write(MetadataOperationType.SET_STORAGE_LEVEL_TO_MTREE + "," + path); + bw.newLine(); + bw.flush(); + } + } + + + public void addAPTree(String pTreeRootName) throws IOException, MetadataArgsErrorException { + mGraph.addAPTree(pTreeRootName); + if (writeToLog) { + bw.write(MetadataOperationType.ADD_A_PTREE + "," + pTreeRootName); + bw.newLine(); + bw.flush(); + } + } + + public void addAPathToPTree(String path) throws PathErrorException, IOException, MetadataArgsErrorException { + mGraph.addPathToPTree(path); + if (writeToLog) { + bw.write(MetadataOperationType.ADD_A_PATH_TO_PTREE + "," + path); + bw.newLine(); + bw.flush(); + } + } + + + public void deletePathFromPTree(String path) throws PathErrorException, IOException { + mGraph.deletePath(path); + if (writeToLog) { + bw.write(MetadataOperationType.DELETE_PATH_FROM_PTREE + "," + path); + bw.newLine(); + bw.flush(); + } + } + + public void linkMNodeToPTree(String path, String mPath) throws PathErrorException, IOException { + mGraph.linkMNodeToPTree(path, mPath); + if (writeToLog) { + bw.write(MetadataOperationType.LINK_MNODE_TO_PTREE + "," + path + "," + mPath); + bw.newLine(); + bw.flush(); + } + } + + public void unlinkMNodeFromPTree(String path, String mPath) throws PathErrorException, IOException { + mGraph.unlinkMNodeFromPTree(path, mPath); + if (writeToLog) { + bw.write(MetadataOperationType.UNLINK_MNODE_FROM_PTREE + "," + path + "," + mPath); + bw.newLine(); + bw.flush(); + } + } + + /** + * Extract the DeltaObjectId from given path + * @param path + * @return String represents the DeltaObjectId + */ + public String getDeltaObjectTypeByPath(String path) throws PathErrorException { + return mGraph.getDeltaObjectTypeByPath(path); + } + + /** + * Get series type for given path + * + * @param fullPath + * @return TSDataType + * @throws PathErrorException + */ + public TSDataType getSeriesType(String fullPath) throws PathErrorException { + return getSchemaForOnePath(fullPath).dataType; + } + + /** + * Get all DeltaObject type in current Metadata Tree + * + * @return a HashMap contains all distinct DeltaObject type separated by + * DeltaObject Type + * @throws PathErrorException + */ + public Map> getSchemaForAllType() throws PathErrorException { + return mGraph.getSchemaForAllType(); + } + + /** + * Get the full Metadata info. + * + * @return A {@code Metadata} instance which stores all metadata info + * @throws PathErrorException + */ + public Metadata getMetadata() throws PathErrorException { + return mGraph.getMetadata(); + } + + /** + * Get all ColumnSchemas for given delta object type + * + * @param path A path represented one Delta object + * @return a list contains all column schema + * @throws PathErrorException + */ + public ArrayList getSchemaForOneType(String path) throws PathErrorException { + return mGraph.getSchemaForOneType(path); + } + + /** + * Calculate the count of storage-level nodes included in given path + * @param path + * @return The total count of storage-level nodes. + * @throws PathErrorException + */ + public int getFileCountForOneType(String path) throws PathErrorException { + return mGraph.getFileCountForOneType(path); + } + + /** + * Get the file name for given path Notice: This method could be called if + * and only if the path includes one node whose {@code isStorageLevel} is + * true + * + * @param path + * @return A String represented the file name + * @throws PathErrorException + */ + public String getFileNameByPath(String path) throws PathErrorException { + return mGraph.getFileNameByPath(path); + } + + /** + * return a HashMap contains all the paths separated by File Name + */ + public HashMap> getAllPathGroupByFileName(String path) throws PathErrorException { + return mGraph.getAllPathGroupByFilename(path); + } + + /** + * return all paths for given path if the path is abstract.Or return the + * path itself. + */ + public ArrayList getPaths(String path) throws PathErrorException { + ArrayList res = new ArrayList<>(); + HashMap> pathsGroupByFilename = getAllPathGroupByFileName(path); + for (ArrayList ps : pathsGroupByFilename.values()) { + res.addAll(ps); + } + return res; + } + + /** + * Check whether the path given exists + */ + public boolean pathExist(String path) { + return mGraph.pathExist(path); + } + + /** + * Get ColumnSchema for given path. Notice: Path must be a complete Path + * from root to leaf node. + */ + public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException { + return mGraph.getSchemaForOnePath(path); + } + + /** + * Check whether given path contains a MNode whose + * {@code MNode.isStorageLevel} is true + */ + public boolean checkFileLevel(List path) throws PathErrorException { + for (Path p : path) { + getFileNameByPath(p.getFullPath()); + } + return true; + } + + /** + * Persistent the MGraph instance into file + */ + public void flushObjectToFile() throws IOException { + File newDataFile = new File(datafilePath + ".backup"); + FileOutputStream fos = new FileOutputStream(newDataFile); + ObjectOutputStream oos = new ObjectOutputStream(fos); + oos.writeObject(mGraph); + oos.close(); + + // delete log file + File logFile = new File(logFilePath); + logFile.delete(); + + // Copy New DataFile to OldDataFile + FileOutputStream oldFileFos = new FileOutputStream(datafilePath); + FileChannel outChannel = oldFileFos.getChannel(); + FileInputStream newFileFis = new FileInputStream(datafilePath + ".backup"); + FileChannel inChannel = newFileFis.getChannel(); + inChannel.transferTo(0, inChannel.size(), outChannel); + outChannel.close(); + inChannel.close(); + oldFileFos.close(); + newFileFis.close(); + + // delete DataFileBackUp + newDataFile.delete(); + } + + public String getMetadataInString() { + return mGraph.toString(); + } + + public static MManager getInstance() { + return manager; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/MNode.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/MNode.java new file mode 100644 index 00000000000..ffef9a51255 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/MNode.java @@ -0,0 +1,149 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.LinkedHashMap; + +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding; + +/** + * This class is the implementation of Metadata Node where "MNode" is the + * shorthand of "Metadata Node". One MNode instance represents one node in the + * Metadata Tree + * + * @author Jinrui Zhang + * + */ +public class MNode implements Serializable { + + private static final long serialVersionUID = -770028375899514063L; + + // The name of the MNode + private String name; + // Whether current node is a leaf in the Metadata Tree + private boolean isLeaf; + // Whether current node is Storage Level in the Metadata Tree + private boolean isStorageLevel; + // Corresponding data file name for current node + private String dataFileName; + // Column's Schema for one timeseries represented by current node if current + // node is one leaf + private ColumnSchema schema; + private MNode parent; + private LinkedHashMap children; + + public MNode(String name, MNode parent, boolean isLeaf) { + this.setName(name); + this.parent = parent; + this.isLeaf = isLeaf; + this.isStorageLevel = false; + if (!isLeaf) { + children = new LinkedHashMap<>(); + } + } + + public MNode(String name, MNode parent, TSDataType dataType, TSEncoding encoding) { + this(name, parent, true); + this.schema = new ColumnSchema(name, dataType, encoding); + } + + public boolean isStorageLevel() { + return isStorageLevel; + } + + public void setStorageLevel(boolean b) { + this.isStorageLevel = b; + } + + public boolean isLeaf() { + return isLeaf; + } + + public void setLeaf(boolean isLeaf) { + this.isLeaf = isLeaf; + } + + public boolean hasChild(String key) { + if (!isLeaf) { + return this.children.containsKey(key); + } + return false; + } + + public void addChild(String key, MNode child) { + if (!isLeaf) { + this.children.put(key, child); + } + } + + public void deleteChild(String key) { + children.remove(key); + } + + public MNode getChild(String key) { + if (!isLeaf) { + return children.get(key); + } + return null; + } + + /** + * @return the count of all leaves whose ancestor is current node + */ + public int getLeafCount() { + if (isLeaf) { + return 1; + } else { + int leafCount = 0; + for (MNode child : this.children.values()) { + leafCount += child.getLeafCount(); + } + return leafCount; + } + } + + public String getDataFileName() { + return dataFileName; + } + + public void setDataFileName(String dataFileName) { + this.dataFileName = dataFileName; + } + + public String toString() { + return this.getName(); + } + + public ColumnSchema getSchema() { + return schema; + } + + public void setSchema(ColumnSchema schema) { + this.schema = schema; + } + + public MNode getParent() { + return parent; + } + + public void setParent(MNode parent) { + this.parent = parent; + } + + public LinkedHashMap getChildren() { + return children; + } + + public void setChildren(LinkedHashMap children) { + this.children = children; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/MTree.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/MTree.java new file mode 100644 index 00000000000..fc5a0589ee4 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/MTree.java @@ -0,0 +1,487 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; + +import cn.edu.thu.tsfile.file.metadata.enums.TSDataType; +import cn.edu.thu.tsfile.file.metadata.enums.TSEncoding; +import cn.edu.thu.tsfiledb.exception.MetadataArgsErrorException; +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * The hierarchical struct of the Metadata Tree is implemented in this class. + * + * @author Jinrui Zhang + * + */ +public class MTree implements Serializable { + + private static final long serialVersionUID = -4200394435237291964L; + private final String space = " "; + private MNode root; + + public MTree(String rootName) { + this.setRoot(new MNode(rootName, null, false)); + } + + public MTree(MNode root) { + this.setRoot(root); + } + + /** + * Add a path to current Metadata Tree + * @param path Format: root.node.(node)* + */ + public int addPath(String path, String dataType, String encoding, String[] args) + throws PathErrorException, MetadataArgsErrorException { + int addCount = 0; + if (getRoot() == null) { + throw new PathErrorException("Root Node is null, Please initialize root first"); + } + String[] nodeNames = path.trim().split("\\."); + if (nodeNames.length <= 1 || !nodeNames[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path not exist. Path: " + path); + } + + MNode cur = getRoot(); + int i; + for (i = 1; i < nodeNames.length - 1; i++) { + if (!cur.hasChild(nodeNames[i])) { + cur.addChild(nodeNames[i], new MNode(nodeNames[i], cur, false)); + addCount++; + } + cur = cur.getChild(nodeNames[i]); + } + if (cur.hasChild(nodeNames[i])) { + throw new PathErrorException("Path already exists. Path: " + path); + } else { + TSDataType dt = TSDataType.valueOf(dataType); + TSEncoding ed = TSEncoding.valueOf(encoding); + MNode leaf = new MNode(nodeNames[i], cur, dt, ed); + if (args.length > 0) { + for (int k = 0; k < args.length; k++) { + String[] arg = args[k].split("="); + leaf.getSchema().putKeyValueToArgs(arg[0], arg[1]); + } + } + cur.addChild(nodeNames[i], leaf); + addCount++; + } + return addCount; + } + + /** + * Delete one path from current Metadata Tree + * @param path + * Format: root.node.(node)* Notice: Path must be a complete Path + * from root to leaf node. + */ + public void deletePath(String path) throws PathErrorException { + String[] nodes = path.split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Path not correct. Path:" + path); + } + + MNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (!cur.hasChild(nodes[i])) { + throw new PathErrorException( + "Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]); + } + cur = cur.getChild(nodes[i]); + } + cur.getParent().deleteChild(cur.getName()); + } + + /** + * Set filename to one node in the Metadata Tree Notice: This method will set + * the {@code MNode.dataFileName} and {@code MNode.isStorageLevel} of current node + * called if and only if {@code MNode.isStorageLevel} is true. + * @param path Format: root.node.(node)* + */ + private void setFileNameToOnePath(String path) throws PathErrorException { + String nodes[] = path.trim().split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + MNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (cur.hasChild(nodes[i])) { + cur = cur.getChild(nodes[i]); + } else { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + } + + cur.setStorageLevel(true); + setFileNameToMNode(cur, path); + } + + /** + * Set storage level for current Metadata Tree. + * @param path Format: root.node.(node)* + */ + public void setStorageLevel(String path) throws PathErrorException { + String nodes[] = path.trim().split("\\."); + if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Root Name NOT correct. Root:" + path); + } + + int level = 0; + MNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (cur.hasChild(nodes[i])) { + cur = cur.getChild(nodes[i]); + level++; + } else { + throw new PathErrorException("Input path NOT correct. Nodes[" + i + "] not correct :" + nodes[i]); + } + } + + cur = getRoot().getChild(nodes[1]); + setFileName(1, level, cur, "root." + cur.getName()); + } + + private void setFileName(int curLevel, int level, MNode node, String path) throws PathErrorException { + if (curLevel == level) { + setFileNameToOnePath(path); + return; + } + for (MNode child : node.getChildren().values()) { + setFileName(curLevel + 1, level, child, path + "." + child.getName()); + } + } + + private void setFileNameToMNode(MNode node, String fileName) { + node.setDataFileName(fileName); + if (!node.isLeaf()) { + for (MNode child : node.getChildren().values()) { + setFileNameToMNode(child, fileName); + } + } + } + + /** + * Check whether the path given exists + */ + public boolean hasPath(String path) { + String[] nodes = path.split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + return false; + } + return hasPath(getRoot(), nodes, 1); + } + + private boolean hasPath(MNode node, String[] nodes, int idx) { + if (idx >= nodes.length) { + return true; + } + if (nodes[idx].equals("*")) { + boolean res = false; + for (MNode child : node.getChildren().values()) { + res |= hasPath(child, nodes, idx + 1); + } + return res; + } else { + if (node.hasChild(nodes[idx])) { + return hasPath(node.getChild(nodes[idx]), nodes, idx + 1); + } + return false; + } + } + + /** + * Get ColumnSchema for given path. Notice: Path must be a complete Path + * from root to leaf node. + */ + public ColumnSchema getSchemaForOnePath(String path) throws PathErrorException { + MNode leaf = getLeafByPath(path); + return leaf.getSchema(); + } + + private MNode getLeafByPath(String path) throws PathErrorException { + checkPath(path); + String[] node = path.split("\\."); + MNode cur = getRoot(); + for (int i = 1; i < node.length; i++) { + cur = cur.getChild(node[i]); + } + if (!cur.isLeaf()) { + throw new PathErrorException("Input path not end with a leaf node."); + } + return cur; + } + + /** + * Extract the DeltaObjectType from given path + * @return String represents the DeltaObjectId + */ + public String getDeltaObjectTypeByPath(String path) throws PathErrorException { + checkPath(path); + String[] nodes = path.split("\\."); + if (nodes.length < 2) { + throw new PathErrorException("Input path must has two or more nodes"); + } + return nodes[0] + "." + nodes[1]; + } + + /** + * Check whether a path is available + */ + private void checkPath(String path) throws PathErrorException { + String[] nodes = path.split("\\."); + if (nodes.length < 2 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + MNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (!cur.hasChild(nodes[i])) { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + cur = cur.getChild(nodes[i]); + } + } + + /** + * Get the file name for given path Notice: This method could be called if + * and only if the path includes one node whose {@code isStorageLevel} is + * true + */ + public String getFileNameByPath(String path) throws PathErrorException { + String[] nodes = path.split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + + MNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (!cur.hasChild(nodes[i])) { + throw new PathErrorException("Input path NOT correct. Path: " + path); + } + if (cur.getDataFileName() != null) { + return cur.getDataFileName(); + } + cur = cur.getChild(nodes[i]); + } + if (cur.getDataFileName() != null) { + return cur.getDataFileName(); + } + throw new PathErrorException("Do not set FileLevel for this path: " + path); + } + + /** + * Get all paths for given path regular expression Regular expression in + * this method is formed by the amalgamation of path and the character '*' + * @return A HashMap whose Keys are separated by the storage file name. + */ + public HashMap> getAllPath(String pathReg) throws PathErrorException { + HashMap> paths = new HashMap<>(); + String[] nodes = pathReg.split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path NOT correct. PathReg: " + pathReg); + } + findPath(getRoot(), nodes, 1, "", paths); + return paths; + } + + + public ArrayList getAllPathInList(String path) throws PathErrorException{ + ArrayList res = new ArrayList<>(); + HashMap> mapRet = getAllPath(path); + for(ArrayList value : mapRet.values()){ + res.addAll(value); + } + return res; + } + + /** + * Calculate the count of storage-level nodes included in given path + * @return The total count of storage-level nodes. + */ + public int getFileCountForOneType(String path) throws PathErrorException { + String nodes[] = path.split("\\."); + if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) { + throw new PathErrorException( + "Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)"); + } + return getFileCountForOneNode(getRoot().getChild(nodes[1])); + } + + private int getFileCountForOneNode(MNode node) { + + if (node.isStorageLevel()) { + return 1; + } + int sum = 0; + if (!node.isLeaf()) { + for (MNode child : node.getChildren().values()) { + sum += getFileCountForOneNode(child); + } + } + return sum; + } + + /** + * Get all DeltaObject type in current Metadata Tree + * @return a list contains all distinct DeltaObject type + */ + public ArrayList getAllType() { + ArrayList res = new ArrayList<>(); + if (getRoot() != null) { + for (String type : getRoot().getChildren().keySet()) { + res.add(type); + } + } + return res; + } + + /** + * Get all delta objects for given type + * @param type DeltaObject Type + * @return a list contains all delta objects for given type + * @throws PathErrorException + */ + public ArrayList getDeltaObjectForOneType(String type) throws PathErrorException { + String path = getRoot().getName() + "." + type; + checkPath(path); + HashMap deltaObjectMap = new HashMap<>(); + MNode typeNode = getRoot().getChild(type); + putDeltaObjectToMap(getRoot().getName(), typeNode, deltaObjectMap); + ArrayList res = new ArrayList<>(); + res.addAll(deltaObjectMap.keySet()); + return res; + } + + private void putDeltaObjectToMap(String path, MNode node, HashMap deltaObjectMap) { + if (node.isLeaf()) { + deltaObjectMap.put(path, 1); + } else { + for (String child : node.getChildren().keySet()) { + String newPath = path + "." + node.getName(); + putDeltaObjectToMap(newPath, node.getChildren().get(child), deltaObjectMap); + } + } + } + + /** + * Get all ColumnSchemas for given delta object type + * @param path A path represented one Delta object + * @return a list contains all column schema + * @throws PathErrorException + */ + public ArrayList getSchemaForOneType(String path) throws PathErrorException { + String nodes[] = path.split("\\."); + if (nodes.length != 2 || !nodes[0].equals(getRoot().getName()) || !getRoot().hasChild(nodes[1])) { + throw new PathErrorException( + "Path must be " + getRoot().getName() + ".X (X is one of the nodes of root's children)"); + } + HashMap leafMap = new HashMap<>(); + putLeafToLeafMap(getRoot().getChild(nodes[1]), leafMap); + ArrayList res = new ArrayList<>(); + res.addAll(leafMap.values()); + return res; + } + + private void putLeafToLeafMap(MNode node, HashMap leafMap) { + if (node.isLeaf()) { + if (!leafMap.containsKey(node.getName())) { + leafMap.put(node.getName(), node.getSchema()); + } + return; + } + for (MNode child : node.getChildren().values()) { + putLeafToLeafMap(child, leafMap); + } + } + + private void findPath(MNode node, String[] nodes, int idx, String parent, HashMap> paths) + throws PathErrorException { + if (node.isLeaf()) { + String fileName = node.getDataFileName(); + String nodePath = parent + node; + putAPath(paths, fileName, nodePath); + return; + } + String nodeReg; + if (idx >= nodes.length) { + nodeReg = "*"; + } else { + nodeReg = nodes[idx]; + } + + if (!nodeReg.equals("*")) { + if (!node.hasChild(nodeReg)) { + throw new PathErrorException("Path Error in node : " + nodeReg); + } + findPath(node.getChild(nodeReg), nodes, idx + 1, parent + node.getName() + ".", paths); + } else { + for (MNode child : node.getChildren().values()) { + findPath(child, nodes, idx + 1, parent + node.getName() + ".", paths); + } + } + } + + private void putAPath(HashMap> paths, String fileName, String nodePath) { + if (paths.containsKey(fileName)) { + paths.get(fileName).add(nodePath); + } else { + ArrayList pathList = new ArrayList<>(); + pathList.add(nodePath); + paths.put(fileName, pathList); + } + } + + public String toString() { + return MNodeToString(getRoot(), 0); + } + + private String MNodeToString(MNode node, int tab) { + String s = ""; + for (int i = 0; i < tab; i++) { + s += space; + } + s += node.getName(); + if (!node.isLeaf() && node.getChildren().size() > 0) { + s += ":{\n"; + int first = 0; + for (MNode child : node.getChildren().values()) { + if (first == 0) { + first = 1; + } else { + s += ",\n"; + } + s += MNodeToString(child, tab + 1); + } + s += "\n"; + for (int i = 0; i < tab; i++) { + s += space; + } + s += "}"; + } else if (node.isLeaf()) { + s += ":{\n"; + s += getTabs(tab + 1) + "DataType :" + node.getSchema().dataType + ",\n"; + s += getTabs(tab + 1) + "Encoding :" + node.getSchema().encoding + ",\n"; + s += getTabs(tab + 1) + "args :" + node.getSchema().getArgsMap() + ",\n"; + s += getTabs(tab + 1) + "FileName :" + node.getDataFileName() + "\n"; + s += getTabs(tab) + "}"; + } + return s; + } + + private String getTabs(int count) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < count; i++) { + sb.append(space); + } + return sb.toString(); + } + + public MNode getRoot() { + return root; + } + + public void setRoot(MNode root) { + this.root = root; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/Metadata.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/Metadata.java new file mode 100644 index 00000000000..a25e3394e08 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/Metadata.java @@ -0,0 +1,53 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.util.List; +import java.util.Map; + +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * This class stores all the metadata info for every deltaObject and every + * timeseries + * + * @author Jinrui Zhang + * + */ +public class Metadata { + private Map> seriesMap; + private Map> deltaObjectMap; + + public Metadata(Map> seriesMap, Map> deltaObjectMap) { + this.seriesMap = seriesMap; + this.deltaObjectMap = deltaObjectMap; + } + + public List getSeriesForOneType(String type) throws PathErrorException { + if (this.seriesMap.containsKey(type)) { + return seriesMap.get(type); + } else { + throw new PathErrorException("Input DeltaObjectType is not exist. " + type); + } + } + + public List getDeltaObjectsForOneType(String type) throws PathErrorException { + if (this.seriesMap.containsKey(type)) { + return deltaObjectMap.get(type); + } else { + throw new PathErrorException("Input DeltaObjectType is not exist. " + type); + } + } + + public Map> getSeriesMap() { + return seriesMap; + } + + public Map> getDeltaObjectMap() { + return deltaObjectMap; + } + + @Override + public String toString() { + return seriesMap.toString() + "\n" + deltaObjectMap.toString(); + } + +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/MetadataOperationType.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/MetadataOperationType.java new file mode 100644 index 00000000000..8ce475af1ab --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/MetadataOperationType.java @@ -0,0 +1,12 @@ +package cn.edu.thu.tsfiledb.metadata; + +public class MetadataOperationType { + public final static String ADD_PATH_TO_MTREE = "0"; + public final static String DELETE_PATH_FROM_MTREE = "1"; + public final static String SET_STORAGE_LEVEL_TO_MTREE = "2"; + public final static String ADD_A_PTREE = "3"; + public final static String ADD_A_PATH_TO_PTREE = "4"; + public final static String DELETE_PATH_FROM_PTREE = "5"; + public final static String LINK_MNODE_TO_PTREE = "6"; + public final static String UNLINK_MNODE_FROM_PTREE = "7"; +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/PNode.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/PNode.java new file mode 100644 index 00000000000..c2f7b2f3a56 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/PNode.java @@ -0,0 +1,113 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; + +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * PNode is the shorthand for "Property Node", which make up The {@code PTree} + * @author zhangjinrui + * + */ +public class PNode implements Serializable{ + + private static final long serialVersionUID = -7166236304286006338L; + private String name; + private PNode parent; + private HashMap children; + private boolean isLeaf; + + /** + * This HashMap contains all the {@code MNode} this {@code PNode} is responsible for + */ + private LinkedHashMap linkedMTreePathMap; + + public PNode(String name, PNode parent, boolean isLeaf) { + this.name = name; + this.parent = parent; + this.isLeaf = isLeaf; + if (!isLeaf) { + setChildren(new HashMap<>()); + } else { + linkedMTreePathMap = new LinkedHashMap<>(); + } + } + + public void linkMPath(String mTreePath) throws PathErrorException{ + if(!isLeaf){ + throw new PathErrorException("Current PNode is NOT leaf node"); + } + if(!linkedMTreePathMap.containsKey(mTreePath)){ + linkedMTreePathMap.put(mTreePath, 1); + } + } + + public void unlinkMPath(String mTreePath) throws PathErrorException{ + if(!isLeaf){ + throw new PathErrorException("Current PNode is NOT leaf node"); + } + if(!linkedMTreePathMap.containsKey(mTreePath)){ + return; + } + linkedMTreePathMap.remove(mTreePath); + } + + public boolean hasChild(String key) { + return getChildren().containsKey(key); + } + + public PNode getChild(String key) { + return getChildren().get(key); + } + + public void addChild(String name, PNode node) { + this.getChildren().put(name, node); + } + + public void deleteChild(String name){ + this.getChildren().remove(name); + } + + public boolean isLeaf() { + return isLeaf; + } + + public void setLeaf(boolean isLeaf) { + this.isLeaf = isLeaf; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public PNode getParent() { + return parent; + } + + public void setParent(PNode parent) { + this.parent = parent; + } + + public HashMap getChildren() { + return children; + } + + public void setChildren(HashMap children) { + this.children = children; + } + + public HashMap getLinkedMTreePathMap() { + return linkedMTreePathMap; + } + + public void setLinkedMTreePathMap(LinkedHashMap linkedMTreePathMap) { + this.linkedMTreePathMap = linkedMTreePathMap; + } +} diff --git a/src/main/java/cn/edu/thu/tsfiledb/metadata/PTree.java b/src/main/java/cn/edu/thu/tsfiledb/metadata/PTree.java new file mode 100644 index 00000000000..155eab17fd8 --- /dev/null +++ b/src/main/java/cn/edu/thu/tsfiledb/metadata/PTree.java @@ -0,0 +1,241 @@ +package cn.edu.thu.tsfiledb.metadata; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import cn.edu.thu.tsfiledb.exception.PathErrorException; + +/** + * "PTree" is the shorthand for "Property Tree". One {@code PTree} consists + * several {@code PNode} + * + * @author Jinrui Zhang + * + */ +public class PTree implements Serializable { + private static final long serialVersionUID = 2642766399323283900L; + + private PNode root; + private MTree mTree; + private String name; + private String space = " "; + + public PTree(String name, MTree mTree) { + this.setRoot(new PNode(name, null, false)); + this.setName(name); + this.setmTree(mTree); + } + + public PTree(String name, PNode root, MTree mTree) { + this.setName(name); + this.setRoot(root); + this.setmTree(mTree); + } + + /** + * Add a path to current PTree + * @return The count of new added {@code PNode} + * @throws PathErrorException + */ + public int addPath(String path) throws PathErrorException { + int addCount = 0; + if (getRoot() == null) { + throw new PathErrorException("Root Node is null, Please initialize root first"); + } + String[] nodes = path.trim().split("\\."); + if (nodes.length <= 1 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Input path not exist. Path: " + path); + } + + PNode cur = getRoot(); + int i; + for (i = 1; i < nodes.length - 1; i++) { + if (!cur.hasChild(nodes[i])) { + cur.addChild(nodes[i], new PNode(nodes[i], cur, false)); + addCount++; + } + cur = cur.getChild(nodes[i]); + } + if (cur.hasChild(nodes[i])) { + throw new PathErrorException("Path already exists. Path: " + path); + } else { + PNode node = new PNode(nodes[i], cur, true); + cur.addChild(node.getName(), node); + addCount++; + } + return addCount; + } + + /** + * Remove a path from current PTree + * @throws PathErrorException + */ + public void deletePath(String path) throws PathErrorException { + String[] nodes = path.split("\\."); + if (nodes.length == 0 || !nodes[0].equals(getRoot().getName())) { + throw new PathErrorException("Path not correct. Path:" + path); + } + PNode cur = getRoot(); + for (int i = 1; i < nodes.length; i++) { + if (!cur.hasChild(nodes[i])) { + throw new PathErrorException( + "Path not correct. Node[" + cur.getName() + "] doesn't have child named:" + nodes[i]); + } + cur = cur.getChild(nodes[i]); + } + cur.getParent().deleteChild(cur.getName()); + } + + /** + * Link a {@code MNode} to a {@code PNode} in current PTree + * @throws PathErrorException + */ + public void linkMNode(String pTreePath, String mTreePath) throws PathErrorException { + List paths = mTree.getAllPathInList(mTreePath); + String nodes[] = pTreePath.trim().split("\\."); + PNode leaf = getLeaf(getRoot(), nodes, 0); + for (String p : paths) { + leaf.linkMPath(p); + } + } + + /** + * Unlink a {@code MNode} from a {@code PNode} in current PTree + * @throws PathErrorException + */ + public void unlinkMNode(String pTreePath, String mTreePath) throws PathErrorException { + List paths = mTree.getAllPathInList(mTreePath); + String nodes[] = pTreePath.trim().split("\\."); + PNode leaf = getLeaf(getRoot(), nodes, 0); + for (String p : paths) { + leaf.unlinkMPath(p); + } + } + + private PNode getLeaf(PNode node, String[] nodes, int idx) throws PathErrorException { + if (idx >= nodes.length) { + throw new PathErrorException("PTree path not exist. "); + } + if (node.isLeaf()) { + if (idx != nodes.length - 1 || !nodes[idx].equals(node.getName())) { + throw new PathErrorException("PTree path not exist. "); + } + return node; + }else{ + if (idx >= nodes.length - 1 || !node.hasChild(nodes[idx + 1])) { + throw new PathErrorException("PTree path not exist. "); + } + return getLeaf(node.getChild(nodes[idx + 1]), nodes, idx + 1); + } + } + + /** + * + * @param path + * a path in current {@code PTree} Get all linked path in MTree + * according to the given path in PTree + * @return Paths will be separated by the {@code MNode.dataFileName} in the + * HashMap + * @throws PathErrorException + */ + public HashMap> getAllLinkedPath(String path) throws PathErrorException { + String[] nodes = path.trim().split("\\."); + PNode leaf = getLeaf(getRoot(), nodes, 0); + HashMap> res = new HashMap<>(); + + for (String MPath : leaf.getLinkedMTreePathMap().keySet()) { + HashMap> tr = getmTree().getAllPath(MPath); + mergePathRes(res, tr); + } + return res; + } + + private void mergePathRes(HashMap> res, HashMap> tr) { + for (String key : tr.keySet()) { + if (!res.containsKey(key)) { + res.put(key, new ArrayList()); + } + for (String p : tr.get(key)) { + if (!res.get(key).contains(p)) { + res.get(key).add(p); + } + } + } + } + + public String toString() { + return PNodeToString(getRoot(), 0); + } + + private String PNodeToString(PNode node, int tab) { + String s = ""; + for (int i = 0; i < tab; i++) { + s += space; + } + s += node.getName(); + if (!node.isLeaf() && node.getChildren().size() > 0) { + s += ":{\n"; + int first = 0; + for (PNode child : node.getChildren().values()) { + if (first == 0) { + first = 1; + } else { + s += ",\n"; + } + s += PNodeToString(child, tab + 1); + } + s += "\n"; + for (int i = 0; i < tab; i++) { + s += space; + } + s += "}"; + } else if (node.isLeaf()) { + s += ":{\n"; + String[] linkedPaths = new String[node.getLinkedMTreePathMap().values().size()]; + linkedPaths = node.getLinkedMTreePathMap().values().toArray(linkedPaths); + for (int i = 0; i < linkedPaths.length; i++) { + if (i != linkedPaths.length - 1) { + s += getTabs(tab + 1) + linkedPaths[i] + ",\n"; + } else { + s += getTabs(tab + 1) + linkedPaths[i] + "\n"; + } + } + s += getTabs(tab) + "}"; + } + return s; + } + + private String getTabs(int count) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < count; i++) { + sb.append(space); + } + return sb.toString(); + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public MTree getmTree() { + return mTree; + } + + public void setmTree(MTree mTree) { + this.mTree = mTree; + } + + public PNode getRoot() { + return root; + } + + public void setRoot(PNode root) { + this.root = root; + } +} diff --git a/src/test/java/cn/edu/thu/tsfiledb/AppTest.java b/src/test/java/cn/edu/thu/tsfiledb/AppTest.java new file mode 100644 index 00000000000..56ebbf4911e --- /dev/null +++ b/src/test/java/cn/edu/thu/tsfiledb/AppTest.java @@ -0,0 +1,38 @@ +package cn.edu.thu.tsfiledb; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +}