fix sonars and add memory estimation method

iotdb/iotdb/bin/memory-tool.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${IOTDB_HOME}" ]; then
+  export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
+fi
+
+
+MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+
+
+CLASSPATH=""
+for f in ${IOTDB_HOME}/lib_cluster/*.jar; do
+  CLASSPATH=${CLASSPATH}":"$f
+done
+
+
+if [ -n "$JAVA_HOME" ]; then
+    for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
+        if [ -x "$java" ]; then
+            JAVA="$java"
+            break
+        fi
+    done
+else
+    JAVA=java
+fi
+
+exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+
+exit $?

iotdb/iotdb/bin/momory-tool.bat

@@ -0,0 +1,58 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM     http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM
+
+if "%OS%" == "Windows_NT" setlocal
+
+pushd %~dp0..
+if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
+popd
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+if NOT DEFINED JAVA_HOME goto :err
+
+@REM -----------------------------------------------------------------------------
+@REM JVM Opts we'll use in legacy run or installation
+set JAVA_OPTS=-ea^
+ -DIOTDB_HOME=%IOTDB_HOME%
+
+REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib_cluster\*.jar") do call :append "%%i"
+goto okClasspath
+
+:append
+set CLASSPATH=%CLASSPATH%;%1
+goto :eof
+
+REM -----------------------------------------------------------------------------
+:okClasspath
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+
+goto finally
+
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+
+@REM -----------------------------------------------------------------------------
+:finally
+
+ENDLOCAL

iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java

@@ -26,7 +26,6 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;

iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -211,7 +211,7 @@ public class IoTDBConfig {
   /**
    * The memory used for stat performance.
    */
-  private int performance_stat_memory_in_kb = 20;
+  private int performanceStatMemoryInKB = 20;
   /**
    * whether use chunkBufferPool.
    */
@@ -565,12 +565,12 @@ public class IoTDBConfig {
     this.performanceStatDisplayInterval = performanceStatDisplayInterval;
   }
 
-  public int getPerformance_stat_memory_in_kb() {
+  public int getPerformanceStatMemoryInKB() {
-    return performance_stat_memory_in_kb;
+    return performanceStatMemoryInKB;
   }
 
-  public void setPerformance_stat_memory_in_kb(int performance_stat_memory_in_kb) {
+  public void setPerformanceStatMemoryInKB(int performanceStatMemoryInKB) {
-    this.performance_stat_memory_in_kb = performance_stat_memory_in_kb;
+    this.performanceStatMemoryInKB = performanceStatMemoryInKB;
   }
 
   public long getMemtableSizeThreshold() {

iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -226,9 +226,9 @@ public class IoTDBDescriptor {
       conf.setPerformanceStatDisplayInterval(Long
           .parseLong(properties.getProperty("performance_stat_display_interval",
               Long.toString(conf.getPerformanceStatDisplayInterval())).trim()));
-      conf.setPerformance_stat_memory_in_kb(Integer
+      conf.setPerformanceStatMemoryInKB(Integer
           .parseInt(properties.getProperty("performance_stat_memory_in_kb",
-              Integer.toString(conf.getPerformance_stat_memory_in_kb())).trim()));
+              Integer.toString(conf.getPerformanceStatMemoryInKB())).trim()));
     } catch (IOException e) {
       logger.warn("Cannot load config file because, use default configuration", e);
     } catch (Exception e) {

iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java

@@ -20,7 +20,6 @@ package org.apache.iotdb.db.conf.adapter;
 
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.StorageEngine;
 import org.apache.iotdb.db.exception.ConfigAdjusterException;
 import org.apache.iotdb.db.metadata.MManager;
 import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
@@ -37,10 +36,11 @@ import org.slf4j.LoggerFactory;
  *
  * 1. maxMemTableNum. This parameter represents the size of the MemTable available in the MemTable
  * pool, which is closely related to the number of storage groups. When adding or deleting a storage
- * group, the parameter also adds or deletes two MemTables. The reason why adding or deleting two
+ * group, the parameter also adds or deletes four MemTables. The reason why adding or deleting four
  * MemTables is that when the system is running stably, the speed of the flush operation is faster
  * than that of data writing, so one is used for the Flush process and the other is used for data
- * writing. Otherwise, the system should limit the speed of data writing to maintain stability.
+ * writing. Otherwise, the system should limit the speed of data writing to maintain stability. And
+ * two for sequence data, two for unsequence data.
  *
  * 2. memtableSize. This parameter determines the threshold value for the MemTable in memory to be
  * flushed into disk. When the system load increases, the parameter should be set smaller so that
@@ -75,16 +75,10 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
 
   // static parameter section
 
-  /**
-   * When the size of the adjusted MemTable decreases more than this parameter, trigger the global
-   * flush operation and flush all MemTable that meets the flush condition to disk.
-   */
-  private static final double FLUSH_THRESHOLD = 0.2;
-
   /**
    * Maximum amount of memory allocated for write process.
    */
-  private static final long ALLOCATE_MEMORY_FOR_WRITE = CONFIG.getAllocateMemoryForWrite();
+  private static long allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
 
   /**
    * Metadata size of per timeseries, the default value is 2KB.
@@ -168,7 +162,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
     // when unit is byte, it's likely to cause Long type overflow.
     // so when b is larger than Integer.MAC_VALUE use the unit KB.
     double a = ratio * maxMemTableNum;
-    double b = (ALLOCATE_MEMORY_FOR_WRITE - staticMemory) * ratio;
+    double b = (allocateMemoryForWrite - staticMemory) * ratio;
     int magnification = b > Integer.MAX_VALUE ? 1024 : 1;
     b /= magnification;
     double c = (double) CONFIG.getTsFileSizeThreshold() * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE
@@ -187,7 +181,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
    * @return Tsfile byte threshold
    */
   private long calcTsFileSize(long memTableSize) {
-    return (long) ((ALLOCATE_MEMORY_FOR_WRITE - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
+    return (long) ((allocateMemoryForWrite - maxMemTableNum * memTableSize - staticMemory) * CompressionRatio
             .getInstance().getRatio()
             * memTableSize / (maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE * MManager.getInstance()
             .getMaximalSeriesNumberAmongStorageGroups()));
@@ -253,6 +247,7 @@ public class IoTDBConfigDynamicAdapter implements IDynamicAdapter {
     totalTimeseries = 0;
     staticMemory = 0;
     maxMemTableNum = MEM_TABLE_AVERAGE_QUEUE_LEN;
+    allocateMemoryForWrite = CONFIG.getAllocateMemoryForWrite();
     initialized = false;
   }
 

iotdb/src/main/java/org/apache/iotdb/db/cost/statistic/Measurement.java

@@ -120,7 +120,7 @@ public class Measurement implements MeasurementMBean, IService {
     IoTDBConfig tdbConfig = IoTDBDescriptor.getInstance().getConfig();
     isEnableStat = tdbConfig.isEnablePerformanceStat();
     displayIntervalInMs = tdbConfig.getPerformanceStatDisplayInterval();
-    int memoryInKb = tdbConfig.getPerformance_stat_memory_in_kb();
+    int memoryInKb = tdbConfig.getPerformanceStatMemoryInKB();
 
     queueSize = memoryInKb * 1000 / Operation.values().length / 8;
     operationLatenciesQueue = new ConcurrentCircularArray[Operation.values().length];

iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java

@@ -180,54 +180,6 @@ public abstract class AbstractMemTable implements IMemTable {
     this.modifications.add(deletion);
   }
 
-  /**
-   * If chunk contains data with timestamp less than 'timestamp', create a copy and delete all those
-   * data. Otherwise return null.
-   *
-   * @param chunk the source chunk.
-   * @param timestamp the upper-bound of deletion time.
-   * @return A reduced copy of chunk if chunk contains data with timestamp less than 'timestamp', of
-   * null.
-   */
-  private IWritableMemChunk filterChunk(IWritableMemChunk chunk, long timestamp) {
-
-    if (!chunk.isEmpty() && chunk.getMinTime() <= timestamp) {
-      //TODO we can avoid sorting data here by scanning data once.
-      List<TimeValuePair> timeValuePairs = chunk.getSortedTimeValuePairList();
-      TSDataType dataType = chunk.getType();
-      IWritableMemChunk newChunk = genMemSeries(dataType);
-      for (TimeValuePair pair : timeValuePairs) {
-        if (pair.getTimestamp() > timestamp) {
-          switch (dataType) {
-            case BOOLEAN:
-              newChunk.putBoolean(pair.getTimestamp(), pair.getValue().getBoolean());
-              break;
-            case DOUBLE:
-              newChunk.putDouble(pair.getTimestamp(), pair.getValue().getDouble());
-              break;
-            case INT64:
-              newChunk.putLong(pair.getTimestamp(), pair.getValue().getLong());
-              break;
-            case INT32:
-              newChunk.putInt(pair.getTimestamp(), pair.getValue().getInt());
-              break;
-            case FLOAT:
-              newChunk.putFloat(pair.getTimestamp(), pair.getValue().getFloat());
-              break;
-            case TEXT:
-              newChunk.putBinary(pair.getTimestamp(), pair.getValue().getBinary());
-              break;
-            default:
-                throw new UnsupportedOperationException("Unknown datatype: " + dataType);
-          }
-        }
-      }
-      TVListAllocator.getInstance().release(dataType, chunk.getTVList());
-      return newChunk;
-    }
-    return null;
-  }
-
   public void setVersion(long version) {
     this.version = version;
   }

iotdb/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java

@@ -25,14 +25,11 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
 import java.util.concurrent.ExecutionException;
-import java.util.stream.Collectors;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.engine.memtable.IMemTable;
 import org.apache.iotdb.db.engine.memtable.MemTableFlushTask;
 import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.engine.version.VersionController;
 import org.apache.iotdb.db.exception.ProcessorException;
 import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;

iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java

@@ -131,6 +131,7 @@ public class IoTDBConfigDynamicAdapterTest {
         MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 30 + 1);
       }
     } catch (ConfigAdjusterException e) {
+      System.out.println(i);
       assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
     }
     int j =0;
@@ -145,4 +146,44 @@ public class IoTDBConfigDynamicAdapterTest {
       assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
     }
   }
+
+  @Test
+  public void addOrDeleteTimeSeriesSyso2() {
+    int sgNum = 1;
+    IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+    long memTableSize = config.getMemtableSizeThreshold();
+    int maxMemtableNumber = config.getMaxMemtableNumber();
+    long tsFileSize = config.getTsFileSizeThreshold();
+    long memory = 1024 * 1024 * 1024L;
+    while (true) {
+      config.setAllocateMemoryForWrite(memory);
+      config.setMemtableSizeThreshold(memTableSize);
+      config.setMaxMemtableNumber(maxMemtableNumber);
+      config.setTsFileSizeThreshold(tsFileSize);
+      IoTDBConfigDynamicAdapter.getInstance().reset();
+      IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
+      MManager.getInstance().clear();
+      for (int i = 1; i <= 50 ; i++) {
+        try {
+          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum);
+        } catch (ConfigAdjusterException e) {
+          e.printStackTrace();
+        }
+      }
+      int i = 1;
+      try {
+        for (; i <= 10000000; i++) {
+          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
+          MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 50 + 1);
+        }
+      } catch (ConfigAdjusterException e) {
+//        System.out.println(i);
+        memory += 1024 * 1024 * 1024L;
+//        System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
+        continue;
+      }
+      break;
+    }
+    System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
+  }
 }

pom.xml

@@ -126,6 +126,11 @@
             <artifactId>jaxb-runtime</artifactId>
             <version>2.4.0-b180725.0644</version>
         </dependency>
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>airline</artifactId>
+            <version>0.8</version>
+        </dependency>
     </dependencies>
     <modules>
         <module>tsfile</module>