jixuan1989
/
IoTDB
mirror of https://github.com/apache/iotdb
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Check style and license-format (#590) 

* format Almost for obeying Google Java Style.

* fix the problem of checkstyle

* fix license-format template file location
This commit is contained in:
Rui Liu 2019-01-18 01:31:48 +08:00 committed by Xiangdong Huang
parent 1da230749c
commit 93727d8e34
802 changed files with 87300 additions and 80456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
.mvn/wrapper/MavenWrapperDownloader.java vendored
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -13,94 +17,92 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL =
"https://repo1.maven.org/maven2/io/takari/maven-wrapper/0.2.1/maven-wrapper-0.2.1.jar";
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL =
"https://repo1.maven.org/maven2/io/takari/maven-wrapper/0.2.1/maven-wrapper-0.2.1.jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: : " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if (mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
if (mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: : " + url);
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if (!outputFile.getParentFile().exists()) {
if (!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath()
+ "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

11
.mvn/wrapper/maven-wrapper.properties vendored
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -15,5 +19,4 @@
#
distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip
#wrapperUrl=https://repo1.maven.org/maven2/io/takari/maven-wrapper/0.2.1/maven-wrapper-0.2.1.jar

2
.travis.yml
View File

@ -35,7 +35,7 @@ before_install: true
# skip `install` stage
install: true
script:
script:
- mvn clean package -Dmaven.test.skip=true
# For each test, travis-ci limits its output log. So we cannot test some sub-modules like tsfile, iotdb.
# We only test jdbc to ensure maven commands work correctly on travis-ci.

17
asf.header Normal file
View File

@ -0,0 +1,17 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

106
checkstyle.xml
View File

@ -1,12 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN" "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE module PUBLIC "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" "https://checkstyle.org/dtds/configuration_1_3.dtd">
<!--
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
@ -18,25 +22,17 @@
-->
<!--
Checkstyle configurartion that checks the Google coding conventions from:
- Google Java Style
https://google-styleguide.googlecode.com/svn-history/r130/trunk/javaguide.html
Checkstyle configuration that checks the Google coding conventions from Google Java Style
that can be found at https://google.github.io/styleguide/javaguide.html.
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
Most Checks are configurable, be sure to consult the documentation.
To completely disable a check, just comment it out or delete it from the file.
Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
-->
<module name="Checker">
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
<property name="fileExtensions" value="java, properties, xml"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter">
@ -46,8 +42,8 @@
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
<property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
<property name="message" value="Avoid using corresponding octal or Unicode escape."/>
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
</module>
<module name="AvoidEscapedUnicodeCharacters">
<property name="allowEscapesForControlCharacters" value="true"/>
@ -63,23 +59,25 @@
<module name="NoLineWrap"/>
<module name="EmptyBlock">
<property name="option" value="TEXT"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
</module>
<module name="NeedBraces"/>
<module name="LeftCurly">
<property name="maxLineLength" value="100"/>
</module>
<module name="RightCurly"/>
<module name="LeftCurly"/>
<module name="RightCurly">
<property name="id" value="RightCurlySame"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/>
</module>
<module name="RightCurly">
<property name="id" value="RightCurlyAlone"/>
<property name="option" value="alone"/>
<property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT"/>
<property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
</module>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyTypes" value="true"/>
<property name="allowEmptyLoops" value="true"/>
<message key="ws.notFollowed" value="WhitespaceAround: ''{0}'' is not followed by whitespace."/>
<message key="ws.notFollowed" value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
<message key="ws.notPreceded" value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
</module>
<module name="OneStatementPerLine"/>
@ -93,13 +91,32 @@
<property name="allowNoEmptyLineBetweenFields" value="true"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapDot"/>
<property name="tokens" value="DOT"/>
<property name="option" value="nl"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapComma"/>
<property name="tokens" value="COMMA"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
<property name="id" value="SeparatorWrapEllipsis"/>
<property name="tokens" value="ELLIPSIS"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
<property name="id" value="SeparatorWrapArrayDeclarator"/>
<property name="tokens" value="ARRAY_DECLARATOR"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapMethodRef"/>
<property name="tokens" value="METHOD_REF"/>
<property name="option" value="nl"/>
</module>
<module name="PackageName">
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
<message key="name.invalidPattern" value="Package name ''{0}'' must match pattern ''{1}''."/>
@ -112,13 +129,20 @@
<message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ParameterName">
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="LambdaParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Lambda parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="CatchParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="LocalVariableName">
<property name="tokens" value="VARIABLE_DEF"/>
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
<property name="allowOneCharVarInForLoop" value="true"/>
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ClassTypeParameterName">
@ -129,6 +153,10 @@
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="InterfaceTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="NoFinalizer"/>
<module name="GenericWhitespace">
<message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
@ -151,26 +179,32 @@
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="thirdPartyPackageRegExp" value=".*"/>
<property name="specialImportsRegExp" value="com.google"/>
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="customImportOrderRules" value="STATIC###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE"/>
<property name="separateLineBetweenGroups" value="true"/>
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
</module>
<module name="MethodParamPad"/>
<module name="NoWhitespaceBefore">
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
<property name="allowLineBreaks" value="true"/>
</module>
<module name="ParenPad"/>
<module name="OperatorWrap">
<property name="option" value="NL"/>
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR "/>
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationMostCases"/>
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationVariables"/>
<property name="tokens" value="VARIABLE_DEF"/>
<property name="allowSamelineMultipleAnnotations" value="true"/>
</module>
<module name="NonEmptyAtclauseDescription"/>
<module name="JavadocTagContinuationIndentation"/>
<module name="SummaryJavadocCheck">
<module name="SummaryJavadoc">
<property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
</module>
<module name="JavadocParagraph"/>
@ -191,6 +225,12 @@
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
<message key="name.invalidPattern" value="Method name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="SingleLineJavadoc"/>
<module name="SingleLineJavadoc">
<property name="ignoreInlineTags" value="false"/>
</module>
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
</module>
</module>

12
codecov.yml
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -25,7 +29,7 @@ coverage:
changes: no
ignore:
# - "*/interface/thrift/**/*.java" # ignore thrift-gen folders and all its contents
# - "*/interface/thrift/**/*.java" # ignore thrift-gen folders and all its contents
- "**/generated-classes/**/*.java"
- "**/generated-sources/**/*.java"
- "grafana/**/*.java"

19
grafana/conf/application.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -14,9 +18,8 @@
# limitations under the License.
#
spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/
spring.datasource.username = root
spring.datasource.password = root
spring.datasource.url=jdbc:iotdb://127.0.0.1:6667/
spring.datasource.username=root
spring.datasource.password=root
spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver
server.port = 8888
server.port=8888

20
grafana/pom.xml
View File

@ -3,9 +3,13 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
@ -25,11 +29,11 @@
<relativePath>../pom.xml</relativePath>
</parent>
<!--parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.4.RELEASE</version>
<relativePath />
</parent-->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.4.RELEASE</version>
<relativePath />
</parent-->
<artifactId>iotdb-grafana</artifactId>
<packaging>war</packaging>
<name>IoTDB Grafana</name>

2
grafana/readme.md
View File

@ -44,7 +44,7 @@ Edit `application.properties`
spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/
spring.datasource.username = root
spring.datasource.password = root
spring.datasource.driver-class-name=org.apache.iotdb.jdbc.TsfileDriver
spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver
server.port = 8888
```

2
grafana/readme_zh.md
View File

@ -33,7 +33,7 @@ cd grafana
spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/
spring.datasource.username = root
spring.datasource.password = root
spring.datasource.driver-class-name=org.apache.iotdb.jdbc.TsfileDriver
spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver
server.port = 8888
```

18
grafana/src/main/java/org/apache/iotdb/web/grafana/TsfileWebDemoApplication.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -21,8 +25,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class TsfileWebDemoApplication {
public static void main(String[] args)
throws ClassNotFoundException, IllegalAccessException, InstantiationException {
SpringApplication.run(TsfileWebDemoApplication.class, args);
}
public static void main(String[] args)
throws ClassNotFoundException, IllegalAccessException, InstantiationException {
SpringApplication.run(TsfileWebDemoApplication.class, args);
}
}

46
grafana/src/main/java/org/apache/iotdb/web/grafana/bean/TimeValues.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -20,27 +24,27 @@ package org.apache.iotdb.web.grafana.bean;
*/
public class TimeValues {
private long time;
private float value;
private long time;
private float value;
@Override
public String toString() {
return "TimeValues{" + "time=" + time + ", values=" + value + '}';
}
@Override
public String toString() {
return "TimeValues{" + "time=" + time + ", values=" + value + '}';
}
public long getTime() {
return time;
}
public long getTime() {
return time;
}
public void setTime(long time) {
this.time = time;
}
public void setTime(long time) {
this.time = time;
}
public float getValue() {
return value;
}
public float getValue() {
return value;
}
public void setValue(float value) {
this.value = value;
}
public void setValue(float value) {
this.value = value;
}
}

31
grafana/src/main/java/org/apache/iotdb/web/grafana/conf/MyConfiguration.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -27,13 +31,16 @@ import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter
@Configuration
public class MyConfiguration {
@Bean
public WebMvcConfigurer corsConfigurer() {
return new WebMvcConfigurerAdapter() {
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**");
}
};
}
/**
* return WebMvcConfigurer.
*/
@Bean
public WebMvcConfigurer corsConfigurer() {
return new WebMvcConfigurerAdapter() {
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**");
}
};
}
}

182
grafana/src/main/java/org/apache/iotdb/web/grafana/controller/DBConnectController.java
View File

@ -1,182 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.web.grafana.controller;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.BufferedReader;
import java.io.IOException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Created by dell on 2017/7/17.
*/
@CrossOrigin
@Controller
public class DBConnectController {
private static final Logger logger = LoggerFactory.getLogger(DBConnectController.class);
@Autowired
private org.apache.iotdb.web.grafana.service.DBConnectService DBConnectService;
@RequestMapping(value = "/", method = RequestMethod.GET)
@ResponseStatus(value = HttpStatus.OK)
public void testDataConnection(HttpServletResponse response) throws IOException {
logger.info("Connection is ok now!");
response.getWriter().print("I have sent a message.");
}
@RequestMapping(value = "/search")
@ResponseBody
public String metricFindQuery(HttpServletRequest request, HttpServletResponse response) {
Map<Integer, String> target = new HashMap<>();
response.setStatus(200);
List<String> columnsName = new ArrayList<>();
try {
columnsName = DBConnectService.getMetaData();
} catch (Exception e) {
logger.error("Failed to get metadata", e);
}
Collections.sort(columnsName);
int cnt = 0;
for (String columnName : columnsName) {
target.put(cnt++, columnName);
}
JSONObject ojb = new JSONObject(target);
return ojb.toString();
}
@RequestMapping(value = "/query")
@ResponseBody
public String query(HttpServletRequest request, HttpServletResponse response) {
response.setStatus(200);
try {
JSONObject jsonObject = getRequestBodyJSON(request);
Pair<ZonedDateTime, ZonedDateTime> timeRange = getTimeFromAndTo(jsonObject);
JSONArray array = (JSONArray) jsonObject.get("targets"); // []
JSONArray result = new JSONArray();
for (int i = 0; i < array.length(); i++) {
JSONObject object = (JSONObject) array.get(i); // {}
if (object.isNull("target"))
return "[]";
String target = (String) object.get("target");
String type = getJSONType(jsonObject);
JSONObject obj = new JSONObject();
obj.put("target", target);
if (type.equals("table")) {
setJSONTable(obj, target, timeRange);
} else if (type.equals("timeserie")) {
setJSONTimeseries(obj, target, timeRange);
}
result.put(i, obj);
}
logger.info("query finished");
return result.toString();
} catch (Exception e) {
logger.error("/query failed", e);
}
return null;
}
private Pair<ZonedDateTime, ZonedDateTime> getTimeFromAndTo(JSONObject jsonObject) throws JSONException {
JSONObject obj = (JSONObject) jsonObject.get("range");
Instant from = Instant.parse((String) obj.get("from"));
Instant to = Instant.parse((String) obj.get("to"));
return new Pair<>(from.atZone(ZoneId.of("Asia/Shanghai")), to.atZone(ZoneId.of("Asia/Shanghai")));
}
private void setJSONTable(JSONObject obj, String target, Pair<ZonedDateTime, ZonedDateTime> timeRange)
throws JSONException {
List<TimeValues> timeValues = DBConnectService.querySeries(target, timeRange);
JSONArray columns = new JSONArray();
JSONObject column = new JSONObject();
column.put("text", "Time");
column.put("type", "time");
columns.put(column);
column = new JSONObject();
column.put("text", "Number");
column.put("type", "number");
columns.put(column);
obj.put("columns", columns);
JSONArray values = new JSONArray();
for (TimeValues tv : timeValues) {
JSONArray value = new JSONArray();
value.put(tv.getTime());
value.put(tv.getValue());
values.put(value);
}
obj.put("values", values);
}
private void setJSONTimeseries(JSONObject obj, String target, Pair<ZonedDateTime, ZonedDateTime> timeRange)
throws JSONException {
List<TimeValues> timeValues = DBConnectService.querySeries(target, timeRange);
logger.info("query size: {}", timeValues.size());
JSONArray dataPoints = new JSONArray();
for (TimeValues tv : timeValues) {
long time = tv.getTime();
float value = tv.getValue();
JSONArray jsonArray = new JSONArray();
jsonArray.put(value);
jsonArray.put(time);
dataPoints.put(jsonArray);
}
obj.put("datapoints", dataPoints);
}
public JSONObject getRequestBodyJSON(HttpServletRequest request) throws JSONException {
try {
BufferedReader br = request.getReader();
StringBuilder sb = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
}
return new JSONObject(sb.toString());
} catch (IOException e) {
logger.error("getRequestBodyJSON failed", e);
}
return null;
}
public String getJSONType(JSONObject jsonObject) throws JSONException {
JSONArray array = (JSONArray) jsonObject.get("targets"); // []
JSONObject object = (JSONObject) array.get(0); // {}
return (String) object.get("type");
}
}

219
grafana/src/main/java/org/apache/iotdb/web/grafana/controller/DatabaseConnectController.java Normal file
View File

@ -0,0 +1,219 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.web.grafana.controller;
import java.io.BufferedReader;
import java.io.IOException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import org.apache.iotdb.web.grafana.service.DatabaseConnectService;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.CrossOrigin;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
@CrossOrigin
@Controller
public class DatabaseConnectController {
private static final Logger logger = LoggerFactory.getLogger(DatabaseConnectController.class);
@Autowired
private DatabaseConnectService databaseConnectService;
@RequestMapping(value = "/", method = RequestMethod.GET)
@ResponseStatus(value = HttpStatus.OK)
public void testDataConnection(HttpServletResponse response) throws IOException {
logger.info("Connection is ok now!");
response.getWriter().print("I have sent a message.");
}
/**
* get metrics numbers in JSON string structure.
*
* @param request http request
* @param response http response
* @return metrics numbers in JSON string structure
*/
@RequestMapping(value = "/search")
@ResponseBody
public String metricFindQuery(HttpServletRequest request, HttpServletResponse response) {
Map<Integer, String> target = new HashMap<>();
response.setStatus(200);
List<String> columnsName = new ArrayList<>();
try {
columnsName = databaseConnectService.getMetaData();
} catch (Exception e) {
logger.error("Failed to get metadata", e);
}
Collections.sort(columnsName);
int cnt = 0;
for (String columnName : columnsName) {
target.put(cnt++, columnName);
}
JSONObject ojb = new JSONObject(target);
return ojb.toString();
}
/**
* convert query result data to JSON format.
*
* @param request http request
* @param response http response
* @return data in JSON format
*/
@RequestMapping(value = "/query")
@ResponseBody
public String query(HttpServletRequest request, HttpServletResponse response) {
response.setStatus(200);
try {
JSONObject jsonObject = getRequestBodyJson(request);
Pair<ZonedDateTime, ZonedDateTime> timeRange = getTimeFromAndTo(jsonObject);
JSONArray array = (JSONArray) jsonObject.get("targets"); // []
JSONArray result = new JSONArray();
for (int i = 0; i < array.length(); i++) {
JSONObject object = (JSONObject) array.get(i); // {}
if (object.isNull("target")) {
return "[]";
}
String target = (String) object.get("target");
String type = getJsonType(jsonObject);
JSONObject obj = new JSONObject();
obj.put("target", target);
if (type.equals("table")) {
setJsonTable(obj, target, timeRange);
} else if (type.equals("timeserie")) {
setJsonTimeseries(obj, target, timeRange);
}
result.put(i, obj);
}
logger.info("query finished");
return result.toString();
} catch (Exception e) {
logger.error("/query failed", e);
}
return null;
}
private Pair<ZonedDateTime, ZonedDateTime> getTimeFromAndTo(JSONObject jsonObject)
throws JSONException {
JSONObject obj = (JSONObject) jsonObject.get("range");
Instant from = Instant.parse((String) obj.get("from"));
Instant to = Instant.parse((String) obj.get("to"));
return new Pair<>(from.atZone(ZoneId.of("Asia/Shanghai")),
to.atZone(ZoneId.of("Asia/Shanghai")));
}
private void setJsonTable(JSONObject obj, String target,
Pair<ZonedDateTime, ZonedDateTime> timeRange)
throws JSONException {
List<TimeValues> timeValues = databaseConnectService.querySeries(target, timeRange);
JSONArray columns = new JSONArray();
JSONObject column = new JSONObject();
column.put("text", "Time");
column.put("type", "time");
columns.put(column);
column = new JSONObject();
column.put("text", "Number");
column.put("type", "number");
columns.put(column);
obj.put("columns", columns);
JSONArray values = new JSONArray();
for (TimeValues tv : timeValues) {
JSONArray value = new JSONArray();
value.put(tv.getTime());
value.put(tv.getValue());
values.put(value);
}
obj.put("values", values);
}
private void setJsonTimeseries(JSONObject obj, String target,
Pair<ZonedDateTime, ZonedDateTime> timeRange)
throws JSONException {
List<TimeValues> timeValues = databaseConnectService.querySeries(target, timeRange);
logger.info("query size: {}", timeValues.size());
JSONArray dataPoints = new JSONArray();
for (TimeValues tv : timeValues) {
long time = tv.getTime();
float value = tv.getValue();
JSONArray jsonArray = new JSONArray();
jsonArray.put(value);
jsonArray.put(time);
dataPoints.put(jsonArray);
}
obj.put("datapoints", dataPoints);
}
/**
* get request body JSON.
*
* @param request http request
* @return request JSON
* @throws JSONException JSONException
*/
public JSONObject getRequestBodyJson(HttpServletRequest request) throws JSONException {
try {
BufferedReader br = request.getReader();
StringBuilder sb = new StringBuilder();
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
}
return new JSONObject(sb.toString());
} catch (IOException e) {
logger.error("getRequestBodyJson failed", e);
}
return null;
}
/**
* get JSON type of input JSON object.
*
* @param jsonObject JSON Object
* @return type (string)
* @throws JSONException JSONException
*/
public String getJsonType(JSONObject jsonObject) throws JSONException {
JSONArray array = (JSONArray) jsonObject.get("targets"); // []
JSONObject object = (JSONObject) array.get(0); // {}
return (String) object.get("type");
}
}

19
grafana/src/main/java/org/apache/iotdb/web/grafana/dao/BasicDao.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,19 +19,18 @@
*/
package org.apache.iotdb.web.grafana.dao;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import java.time.ZonedDateTime;
import java.util.List;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
/**
* Created by dell on 2017/7/17.
*/
public interface BasicDao {
List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange);
List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange);
List<String> getMetaData();
List<String> getMetaData();
}

167
grafana/src/main/java/org/apache/iotdb/web/grafana/dao/impl/BasicDaoImpl.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,10 +19,16 @@
*/
package org.apache.iotdb.web.grafana.dao.impl;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import org.apache.iotdb.web.grafana.dao.BasicDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -27,96 +37,89 @@ import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.stereotype.Repository;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* Created by dell on 2017/7/17.
*/
@Repository
public class BasicDaoImpl implements BasicDao {
private static final Logger logger = LoggerFactory.getLogger(BasicDaoImpl.class);
private static final Logger logger = LoggerFactory.getLogger(BasicDaoImpl.class);
private final JdbcTemplate jdbcTemplate;
private final JdbcTemplate jdbcTemplate;
@Autowired
public BasicDaoImpl(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
@Autowired
public BasicDaoImpl(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public List<String> getMetaData() {
ConnectionCallback<Object> connectionCallback = new ConnectionCallback<Object>() {
public Object doInConnection(Connection connection) throws SQLException {
DatabaseMetaData databaseMetaData = connection.getMetaData();
ResultSet resultSet = databaseMetaData.getColumns(null, null, "root.*", null);
System.out.println("Start to get timeseries");
List<String> columnsName = new ArrayList<>();
while (resultSet.next()) {
String timeseries = resultSet.getString(1);
columnsName.add(timeseries.substring(5));
}
return columnsName;
}
};
return (List<String>) jdbcTemplate.execute(connectionCallback);
}
@Override
public List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange) {
Long from = zonedCovertToLong(timeRange.left);
Long to = zonedCovertToLong(timeRange.right);
String sql = "SELECT " + s.substring(s.lastIndexOf('.') + 1) + " FROM root."
+ s.substring(0, s.lastIndexOf('.')) + " WHERE time > " + from + " and time < " + to;
logger.info(sql);
List<TimeValues> rows = null;
try {
rows = jdbcTemplate.query(sql, new TimeValuesRowMapper("root." + s));
} catch (Exception e) {
logger.error(e.getMessage());
}
return rows;
}
private Long zonedCovertToLong(ZonedDateTime time) {
return time.toInstant().toEpochMilli();
}
static class TimeValuesRowMapper implements RowMapper<TimeValues> {
static final String TRUE_STR = "true";
static final String FALSE_STR = "false";
String columnName;
TimeValuesRowMapper(String columnName) {
this.columnName = columnName;
}
@Override
public List<String> getMetaData() {
ConnectionCallback<Object> connectionCallback = new ConnectionCallback<Object>() {
public Object doInConnection(Connection connection) throws SQLException {
DatabaseMetaData databaseMetaData = connection.getMetaData();
ResultSet resultSet = databaseMetaData.getColumns(null, null, "root.*", null);
System.out.println("Start to get timeseries");
List<String> columnsName = new ArrayList<>();
while (resultSet.next()) {
String timeseries = resultSet.getString(1);
columnsName.add(timeseries.substring(5));
}
return columnsName;
}
};
return (List<String>) jdbcTemplate.execute(connectionCallback);
}
@Override
public List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange) {
Long from = zonedCovertToLong(timeRange.left);
Long to = zonedCovertToLong(timeRange.right);
String sql = "SELECT " + s.substring(s.lastIndexOf('.') + 1) + " FROM root."
+ s.substring(0, s.lastIndexOf('.')) + " WHERE time > " + from + " and time < " + to;
logger.info(sql);
List<TimeValues> rows = null;
try {
rows = jdbcTemplate.query(sql, new TimeValuesRowMapper("root." + s));
} catch (Exception e) {
logger.error(e.getMessage());
}
return rows;
}
private Long zonedCovertToLong(ZonedDateTime time) {
return time.toInstant().toEpochMilli();
}
static class TimeValuesRowMapper implements RowMapper<TimeValues> {
String columnName;
final String TRUE_STR = "true";
final String FALSE_STR = "false";
TimeValuesRowMapper(String columnName) {
this.columnName = columnName;
}
@Override
public TimeValues mapRow(ResultSet resultSet, int i) throws SQLException {
TimeValues tv = new TimeValues();
tv.setTime(resultSet.getLong("Time"));
String vString = resultSet.getString(columnName);
if (vString != null) {
if (TRUE_STR.equals(vString.toLowerCase())) {
tv.setValue(1);
} else if (FALSE_STR.equals(vString.toLowerCase())) {
tv.setValue(0);
} else {
try {
tv.setValue(resultSet.getFloat(columnName));
} catch (Exception e) {
tv.setValue(0);
}
}
}
return tv;
public TimeValues mapRow(ResultSet resultSet, int i) throws SQLException {
TimeValues tv = new TimeValues();
tv.setTime(resultSet.getLong("Time"));
String valueString = resultSet.getString(columnName);
if (valueString != null) {
if (TRUE_STR.equals(valueString.toLowerCase())) {
tv.setValue(1);
} else if (FALSE_STR.equals(valueString.toLowerCase())) {
tv.setValue(0);
} else {
try {
tv.setValue(resultSet.getFloat(columnName));
} catch (Exception e) {
tv.setValue(0);
}
}
}
return tv;
}
}
}

26
grafana/src/main/java/org/apache/iotdb/web/grafana/service/DBConnectService.java → grafana/src/main/java/org/apache/iotdb/web/grafana/service/DatabaseConnectService.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,21 +19,17 @@
*/
package org.apache.iotdb.web.grafana.service;
import java.time.ZonedDateTime;
import java.util.List;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import java.time.ZonedDateTime;
import java.util.List;
public interface DatabaseConnectService {
/**
* Created by dell on 2017/7/17.
*/
public interface DBConnectService {
int testConnection();
int testConnection();
List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange);
List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange);
List<String> getMetaData();
List<String> getMetaData();
}

53
grafana/src/main/java/org/apache/iotdb/web/grafana/service/impl/DBConnectServiceImpl.java
View File

@ -1,53 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.web.grafana.service.impl;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import org.apache.iotdb.web.grafana.dao.BasicDao;
import org.apache.iotdb.web.grafana.service.DBConnectService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.time.ZonedDateTime;
import java.util.List;
/**
* Created by dell on 2017/7/17.
*/
@Service
public class DBConnectServiceImpl implements DBConnectService {
@Autowired
BasicDao basicDao;
@Override
public int testConnection() {
return 0;
}
@Override
public List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange) {
return basicDao.querySeries(s, timeRange);
}
@Override
public List<String> getMetaData() {
return basicDao.getMetaData();
}
}

52
grafana/src/main/java/org/apache/iotdb/web/grafana/service/impl/DatabaseConnectServiceImpl.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.web.grafana.service.impl;
import java.time.ZonedDateTime;
import java.util.List;
import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.web.grafana.bean.TimeValues;
import org.apache.iotdb.web.grafana.dao.BasicDao;
import org.apache.iotdb.web.grafana.service.DatabaseConnectService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class DatabaseConnectServiceImpl implements DatabaseConnectService {
@Autowired
BasicDao basicDao;
@Override
public int testConnection() {
return 0;
}
@Override
public List<TimeValues> querySeries(String s, Pair<ZonedDateTime, ZonedDateTime> timeRange) {
return basicDao.querySeries(s, timeRange);
}
@Override
public List<String> getMetaData() {
return basicDao.getMetaData();
}
}

10
hadoop/pom.xml
View File

@ -3,9 +3,13 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0

43
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFHadoopException.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -21,26 +25,27 @@ package org.apache.iotdb.tsfile.hadoop;
public class TSFHadoopException extends Exception {
private static final long serialVersionUID = 9206686224701568169L;
private static final long serialVersionUID = 9206686224701568169L;
public TSFHadoopException() {
super();
}
public TSFHadoopException() {
super();
}
public TSFHadoopException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public TSFHadoopException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public TSFHadoopException(String message, Throwable cause) {
super(message, cause);
}
public TSFHadoopException(String message, Throwable cause) {
super(message, cause);
}
public TSFHadoopException(String message) {
super(message);
}
public TSFHadoopException(String message) {
super(message);
}
public TSFHadoopException(Throwable cause) {
super(cause);
}
public TSFHadoopException(Throwable cause) {
super(cause);
}
}

617
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFInputFormat.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -20,7 +24,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
@ -35,343 +38,343 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.iotdb.tsfile.file.metadata.RowGroupMetaData;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.read.FileReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.file.metadata.RowGroupMetaData;
import org.apache.iotdb.tsfile.read.FileReader;
/**
* @author liukun
*/
public class TSFInputFormat extends FileInputFormat<NullWritable, ArrayWritable> {
private static final Logger LOGGER = LoggerFactory.getLogger(TSFInputFormat.class);
/**
* key to configure whether reading time enable
*/
public static final String READ_TIME_ENABLE = "tsfile.read.time.enable";
/**
* key to configure whether reading deltaObjectId enable
*/
public static final String READ_DELTAOBJECT_ENABLE = "tsfile.read.deltaObjectId.enable";
/**
* key to configure the type of filter
*/
@Deprecated
public static final String FILTER_TYPE = "tsfile.filter.type";
/**
* key to configure the filter
*/
@Deprecated
public static final String FILTER_EXPRESSION = "tsfile.filter.expression";
/**
* key to configure whether filtering is enable
*/
public static final String FILTER_EXIST = "tsfile.filter.exist";
/**
* key to configure the reading deltaObjectIds
*/
public static final String READ_DELTAOBJECTS = "tsfile.read.deltaobject";
/**
* key to configure the reading measurementIds
*/
public static final String READ_MEASUREMENTID = "tsfile.read.measurement";
private static final Logger LOGGER = LoggerFactory.getLogger(TSFInputFormat.class);
private static final String SPERATOR = ",";
/**
* key to configure whether reading time enable
*/
public static final String READ_TIME_ENABLE = "tsfile.read.time.enable";
/**
* key to configure whether reading deltaObjectId enable
*/
public static final String READ_DELTAOBJECT_ENABLE = "tsfile.read.deltaObjectId.enable";
/**
* key to configure the type of filter
*/
@Deprecated
public static final String FILTER_TYPE = "tsfile.filter.type";
/**
* key to configure the filter
*/
@Deprecated
public static final String FILTER_EXPRESSION = "tsfile.filter.expression";
/**
* key to configure whether filtering is enable
*/
public static final String FILTER_EXIST = "tsfile.filter.exist";
/**
* key to configure the reading deltaObjectIds
*/
public static final String READ_DELTAOBJECTS = "tsfile.read.deltaobject";
/**
* key to configure the reading measurementIds
*/
public static final String READ_MEASUREMENTID = "tsfile.read.measurement";
/**
* Set the deltaObjectIds which want to be read
*
* @param job hadoop job
* @param value the deltaObjectIds will be read
* @throws TSFHadoopException
*/
public static void setReadDeltaObjectIds(Job job, String[] value) throws TSFHadoopException {
if (value == null || value.length < 1) {
throw new TSFHadoopException("The devices selected is null or empty");
} else {
String deltaObjectIds = "";
for (String deltaObjectId : value) {
deltaObjectIds = deltaObjectIds + deltaObjectId + SPERATOR;
}
job.getConfiguration().set(READ_DELTAOBJECTS,
(String) deltaObjectIds.subSequence(0, deltaObjectIds.length() - 1));
}
}
private static final String SPERATOR = ",";
/**
* Get the deltaObjectIds which want to be read
*
* @param configuration
* @return List of deltaObject, if configuration has been set the deltaObjectIds.
* null, if configuration has not been set the deltaObjectIds.
*/
public static List<String> getReadDeltaObjectIds(Configuration configuration) {
String deltaObjectIds = configuration.get(READ_DELTAOBJECTS);
if (deltaObjectIds == null || deltaObjectIds.length() < 1) {
return null;
} else {
List<String> deltaObjectIdsList = Arrays.asList(deltaObjectIds.split(SPERATOR));
return deltaObjectIdsList;
}
}
/**
* Set the deltaObjectIds which want to be read
*
* @param job hadoop job
* @param value the deltaObjectIds will be read
* @throws TSFHadoopException
*/
public static void setReadDeltaObjectIds(Job job, String[] value) throws TSFHadoopException {
if (value == null || value.length < 1) {
throw new TSFHadoopException("The devices selected is null or empty");
} else {
String deltaObjectIds = "";
for (String deltaObjectId : value) {
deltaObjectIds = deltaObjectIds + deltaObjectId + SPERATOR;
}
job.getConfiguration().set(READ_DELTAOBJECTS, (String) deltaObjectIds.subSequence(0, deltaObjectIds.length() - 1));
}
}
/**
* Set the measurementIds which want to be read
*
* @param job hadoop job
* @param value the measurementIds will be read
* @throws TSFHadoopException
*/
public static void setReadMeasurementIds(Job job, String[] value) throws TSFHadoopException {
if (value == null || value.length < 1) {
throw new TSFHadoopException("The sensors selected is null or empty");
} else {
String measurementIds = "";
for (String measurementId : value) {
measurementIds = measurementIds + measurementId + SPERATOR;
}
// Get conf type
job.getConfiguration().set(READ_MEASUREMENTID,
(String) measurementIds.subSequence(0, measurementIds.length() - 1));
}
}
/**
* Get the deltaObjectIds which want to be read
*
* @param configuration
* @return List of deltaObject, if configuration has been set the deltaObjectIds.
* null, if configuration has not been set the deltaObjectIds.
*/
public static List<String> getReadDeltaObjectIds(Configuration configuration) {
String deltaObjectIds = configuration.get(READ_DELTAOBJECTS);
if (deltaObjectIds == null || deltaObjectIds.length() < 1) {
return null;
} else {
List<String> deltaObjectIdsList = Arrays.asList(deltaObjectIds.split(SPERATOR));
return deltaObjectIdsList;
}
}
/**
* Get the measurementIds which want to be read
*
* @param configuration hadoop configuration
* @return if not set the measurementIds, return null
*/
public static List<String> getReadMeasurementIds(Configuration configuration) {
String measurementIds = configuration.get(READ_MEASUREMENTID);
if (measurementIds == null || measurementIds.length() < 1) {
return null;
} else {
List<String> measurementIdsList = Arrays.asList(measurementIds.split(SPERATOR));
return measurementIdsList;
}
}
/**
* Set the measurementIds which want to be read
*
* @param job hadoop job
* @param value the measurementIds will be read
* @throws TSFHadoopException
*/
public static void setReadMeasurementIds(Job job, String[] value) throws TSFHadoopException {
if (value == null || value.length < 1) {
throw new TSFHadoopException("The sensors selected is null or empty");
} else {
String measurementIds = "";
for (String measurementId : value) {
measurementIds = measurementIds + measurementId + SPERATOR;
}
// Get conf type
job.getConfiguration().set(READ_MEASUREMENTID, (String) measurementIds.subSequence(0, measurementIds.length() - 1));
}
}
/**
* @param job
* @param value
*/
public static void setReadDeltaObjectId(Job job, boolean value) {
job.getConfiguration().setBoolean(READ_DELTAOBJECT_ENABLE, value);
}
/**
* Get the measurementIds which want to be read
*
* @param configuration hadoop configuration
* @return if not set the measurementIds, return null
*/
public static List<String> getReadMeasurementIds(Configuration configuration) {
String measurementIds = configuration.get(READ_MEASUREMENTID);
if (measurementIds == null || measurementIds.length() < 1) {
return null;
} else {
List<String> measurementIdsList = Arrays.asList(measurementIds.split(SPERATOR));
return measurementIdsList;
}
}
/**
* @param configuration
* @return
*/
public static boolean getReadDeltaObject(Configuration configuration) {
return configuration.getBoolean(READ_DELTAOBJECT_ENABLE, false);
}
/**
* @param job
* @param value
*/
public static void setReadDeltaObjectId(Job job, boolean value) {
job.getConfiguration().setBoolean(READ_DELTAOBJECT_ENABLE, value);
}
/**
* @param job
* @param value
*/
public static void setReadTime(Job job, boolean value) {
job.getConfiguration().setBoolean(READ_TIME_ENABLE, value);
}
/**
* @param configuration
* @return
*/
public static boolean getReadDeltaObject(Configuration configuration) {
return configuration.getBoolean(READ_DELTAOBJECT_ENABLE, false);
}
public static boolean getReadTime(Configuration configuration) {
return configuration.getBoolean(READ_TIME_ENABLE, false);
}
/**
* @param job
* @param value
*/
public static void setReadTime(Job job, boolean value) {
job.getConfiguration().setBoolean(READ_TIME_ENABLE, value);
}
/**
* Set filter exist or not
*
* @param job
* @param value
*/
@Deprecated
public static void setHasFilter(Job job, boolean value) {
job.getConfiguration().setBoolean(FILTER_EXIST, value);
}
public static boolean getReadTime(Configuration configuration) {
return configuration.getBoolean(READ_TIME_ENABLE, false);
}
// check is we didn't set this key, the value will be null or empty
/**
* Set filter exist or not
*
* @param job
* @param value
*/
@Deprecated
public static void setHasFilter(Job job, boolean value) {
job.getConfiguration().setBoolean(FILTER_EXIST, value);
}
/**
* Get filter exist or not
*
* @param configuration
* @return
*/
@Deprecated
public static boolean getHasFilter(Configuration configuration) {
return configuration.getBoolean(FILTER_EXIST, false);
}
// check is we didn't set this key, the value will be null or empty
/**
* @param job
* @param value
*/
@Deprecated
public static void setFilterType(Job job, String value) {
job.getConfiguration().set(FILTER_TYPE, value);
}
/**
* Get filter exist or not
*
* @param configuration
* @return
*/
@Deprecated
public static boolean getHasFilter(Configuration configuration) {
return configuration.getBoolean(FILTER_EXIST, false);
}
/**
* Get the filter type
*
* @param configuration
* @return
*/
// check if not set the filter type, the result will null or empty
@Deprecated
public static String getFilterType(Configuration configuration) {
return configuration.get(FILTER_TYPE);
}
/**
* @param job
* @param value
*/
@Deprecated
public static void setFilterType(Job job, String value) {
job.getConfiguration().set(FILTER_TYPE, value);
}
@Deprecated
public static void setFilterExp(Job job, String value) {
job.getConfiguration().set(FILTER_EXPRESSION, value);
}
/**
* Get the filter type
*
* @param configuration
* @return
*/
// check if not set the filter type, the result will null or empty
@Deprecated
public static String getFilterType(Configuration configuration) {
return configuration.get(FILTER_TYPE);
}
@Deprecated
public static void setFilterExp(Job job, String value) {
job.getConfiguration().set(FILTER_EXPRESSION, value);
}
@Deprecated
public static String getFilterExp(Configuration configuration) {
return configuration.get(FILTER_EXPRESSION);
}
@Deprecated
public static String getFilterExp(Configuration configuration) {
return configuration.get(FILTER_EXPRESSION);
}
@Override
public RecordReader<NullWritable, ArrayWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
return new TSFRecordReader();
}
@Override
public RecordReader<NullWritable, ArrayWritable> createRecordReader(InputSplit split,
TaskAttemptContext context)
throws IOException, InterruptedException {
return new TSFRecordReader();
}
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
Configuration configuration = job.getConfiguration();
BlockLocation[] blockLocations;
List<InputSplit> splits = new ArrayList<>();
// get the all file in the directory
List<FileStatus> listFileStatus = super.listStatus(job);
LOGGER.info("The number of this job file is {}", listFileStatus.size());
// For each file
for (FileStatus fileStatus : listFileStatus) {
LOGGER.info("The file path is {}", fileStatus.getPath());
// Get the file path
Path path = fileStatus.getPath();
// Get the file length
long length = fileStatus.getLen();
// Check the file length. if the length is less than 0, return the
// empty splits
if (length > 0) {
// Get block information in the local file system or hdfs
if (fileStatus instanceof LocatedFileStatus) {
LOGGER.info("The file status is {}", LocatedFileStatus.class.getName());
blockLocations = ((LocatedFileStatus) fileStatus).getBlockLocations();
} else {
FileSystem fileSystem = path.getFileSystem(configuration);
LOGGER.info("The file status is {}", fileStatus.getClass().getName());
System.out.println("The file status is " + fileStatus.getClass().getName());
System.out.println("The file system is " + fileSystem.getClass());
blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, length);
}
LOGGER.info("The block location information is {}", Arrays.toString(blockLocations));
HDFSInputStream hdfsInputStream = new HDFSInputStream(path, configuration);
FileReader fileReader = new FileReader(hdfsInputStream);
// Get the timeserise to test
splits.addAll(generateSplits(path, fileReader, blockLocations));
fileReader.close();
} else {
LOGGER.warn("The file length is " + length);
}
}
configuration.setLong(NUM_INPUT_FILES, listFileStatus.size());
LOGGER.info("The number of splits is " + splits.size());
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
Configuration configuration = job.getConfiguration();
BlockLocation[] blockLocations;
List<InputSplit> splits = new ArrayList<>();
// get the all file in the directory
List<FileStatus> listFileStatus = super.listStatus(job);
LOGGER.info("The number of this job file is {}", listFileStatus.size());
// For each file
for (FileStatus fileStatus : listFileStatus) {
LOGGER.info("The file path is {}", fileStatus.getPath());
// Get the file path
Path path = fileStatus.getPath();
// Get the file length
long length = fileStatus.getLen();
// Check the file length. if the length is less than 0, return the
// empty splits
if (length > 0) {
// Get block information in the local file system or hdfs
if (fileStatus instanceof LocatedFileStatus) {
LOGGER.info("The file status is {}", LocatedFileStatus.class.getName());
blockLocations = ((LocatedFileStatus) fileStatus).getBlockLocations();
} else {
FileSystem fileSystem = path.getFileSystem(configuration);
LOGGER.info("The file status is {}", fileStatus.getClass().getName());
System.out.println("The file status is " + fileStatus.getClass().getName());
System.out.println("The file system is " + fileSystem.getClass());
blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, length);
}
LOGGER.info("The block location information is {}", Arrays.toString(blockLocations));
HDFSInputStream hdfsInputStream = new HDFSInputStream(path, configuration);
FileReader fileReader = new FileReader(hdfsInputStream);
// Get the timeserise to test
splits.addAll(generateSplits(path, fileReader, blockLocations));
fileReader.close();
} else {
LOGGER.warn("The file length is " + length);
}
}
configuration.setLong(NUM_INPUT_FILES, listFileStatus.size());
LOGGER.info("The number of splits is " + splits.size());
return splits;
}
return splits;
}
/**
* get the TSFInputSplit from tsfMetaData and hdfs block location
* information with the filter
*
* @param path
* @param fileReader
* @param blockLocations
* @return
* @throws IOException
*/
private List<TSFInputSplit> generateSplits(Path path, FileReader fileReader, BlockLocation[] blockLocations)
throws IOException {
List<TSFInputSplit> splits = new ArrayList<TSFInputSplit>();
Comparator<BlockLocation> comparator = new Comparator<BlockLocation>() {
@Override
public int compare(BlockLocation o1, BlockLocation o2) {
/**
* get the TSFInputSplit from tsfMetaData and hdfs block location
* information with the filter
*
* @param path
* @param fileReader
* @param blockLocations
* @return
* @throws IOException
*/
private List<TSFInputSplit> generateSplits(Path path, FileReader fileReader,
BlockLocation[] blockLocations)
throws IOException {
List<TSFInputSplit> splits = new ArrayList<TSFInputSplit>();
Comparator<BlockLocation> comparator = new Comparator<BlockLocation>() {
@Override
public int compare(BlockLocation o1, BlockLocation o2) {
return Long.signum(o1.getOffset() - o2.getOffset());
}
return Long.signum(o1.getOffset() - o2.getOffset());
}
};
Arrays.sort(blockLocations, comparator);
};
Arrays.sort(blockLocations, comparator);
List<RowGroupMetaData> rowGroupMetaDataList = new ArrayList<>();
int currentBlockIndex = 0;
long splitSize = 0;
long splitStart = 0;
List<String> hosts = new ArrayList<>();
for (RowGroupMetaData rowGroupMetaData : fileReader.getSortedRowGroupMetaDataList()) {
LOGGER.info("The rowGroupMetaData information is {}", rowGroupMetaData);
List<RowGroupMetaData> rowGroupMetaDataList = new ArrayList<>();
int currentBlockIndex = 0;
long splitSize = 0;
long splitStart = 0;
List<String> hosts = new ArrayList<>();
for (RowGroupMetaData rowGroupMetaData : fileReader.getSortedRowGroupMetaDataList()) {
LOGGER.info("The rowGroupMetaData information is {}", rowGroupMetaData);
long start = getRowGroupStart(rowGroupMetaData);
int blkIndex = getBlockLocationIndex(blockLocations, start);
if(hosts.size() == 0)
{
hosts.addAll(Arrays.asList(blockLocations[blkIndex].getHosts()));
splitStart = start;
}
long start = getRowGroupStart(rowGroupMetaData);
int blkIndex = getBlockLocationIndex(blockLocations, start);
if (hosts.size() == 0) {
hosts.addAll(Arrays.asList(blockLocations[blkIndex].getHosts()));
splitStart = start;
}
if(blkIndex != currentBlockIndex)
{
TSFInputSplit tsfInputSplit = makeSplit(path, rowGroupMetaDataList, splitStart,
splitSize, hosts);
LOGGER.info("The tsfile inputsplit information is {}", tsfInputSplit);
splits.add(tsfInputSplit);
if (blkIndex != currentBlockIndex) {
TSFInputSplit tsfInputSplit = makeSplit(path, rowGroupMetaDataList, splitStart,
splitSize, hosts);
LOGGER.info("The tsfile inputsplit information is {}", tsfInputSplit);
splits.add(tsfInputSplit);
currentBlockIndex = blkIndex;
rowGroupMetaDataList.clear();
rowGroupMetaDataList.add(rowGroupMetaData);
splitStart = start;
splitSize = rowGroupMetaData.getTotalByteSize();
hosts.clear();
}
else
{
rowGroupMetaDataList.add(rowGroupMetaData);
splitSize += rowGroupMetaData.getTotalByteSize();
}
}
TSFInputSplit tsfInputSplit = makeSplit(path, rowGroupMetaDataList, splitStart,
splitSize, hosts);
LOGGER.info("The tsfile inputsplit information is {}", tsfInputSplit);
splits.add(tsfInputSplit);
return splits;
}
currentBlockIndex = blkIndex;
rowGroupMetaDataList.clear();
rowGroupMetaDataList.add(rowGroupMetaData);
splitStart = start;
splitSize = rowGroupMetaData.getTotalByteSize();
hosts.clear();
} else {
rowGroupMetaDataList.add(rowGroupMetaData);
splitSize += rowGroupMetaData.getTotalByteSize();
}
}
TSFInputSplit tsfInputSplit = makeSplit(path, rowGroupMetaDataList, splitStart,
splitSize, hosts);
LOGGER.info("The tsfile inputsplit information is {}", tsfInputSplit);
splits.add(tsfInputSplit);
return splits;
}
private long getRowGroupStart(RowGroupMetaData rowGroupMetaData) {
return rowGroupMetaData.getMetaDatas().get(0).getProperties().getFileOffset();
}
private long getRowGroupStart(RowGroupMetaData rowGroupMetaData) {
return rowGroupMetaData.getMetaDatas().get(0).getProperties().getFileOffset();
}
private int getBlockLocationIndex(BlockLocation[] blockLocations, long start) {
for (int i = 0; i < blockLocations.length; i++) {
if (blockLocations[i].getOffset() <= start
&& start < blockLocations[i].getOffset() + blockLocations[i].getLength()) {
return i;
}
}
LOGGER.warn(String.format("Can't find the block. The start is:%d. the last block is", start),
blockLocations[blockLocations.length - 1].getOffset()
+ blockLocations[blockLocations.length - 1].getLength());
return -1;
}
private int getBlockLocationIndex(BlockLocation[] blockLocations, long start) {
for (int i = 0; i < blockLocations.length; i++) {
if (blockLocations[i].getOffset() <= start
&& start < blockLocations[i].getOffset() + blockLocations[i].getLength()) {
return i;
}
}
LOGGER.warn(String.format("Can't find the block. The start is:%d. the last block is", start),
blockLocations[blockLocations.length - 1].getOffset()
+ blockLocations[blockLocations.length - 1].getLength());
return -1;
}
private TSFInputSplit makeSplit(Path path, List<RowGroupMetaData> rowGroupMataDataList, long start, long length,
List<String> hosts) {
String[] hosts_str = hosts.toArray(new String[hosts.size()]);
return new TSFInputSplit(path, rowGroupMataDataList, start, length, hosts_str);
}
private TSFInputSplit makeSplit(Path path, List<RowGroupMetaData> rowGroupMataDataList,
long start, long length,
List<String> hosts) {
String[] hosts_str = hosts.toArray(new String[hosts.size()]);
return new TSFInputSplit(path, rowGroupMataDataList, start, length, hosts_str);
}
}

259
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFInputSplit.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -22,11 +26,9 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.iotdb.tsfile.file.metadata.RowGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.TsRowGroupBlockMetaData;
import org.apache.iotdb.tsfile.file.utils.ReadWriteThriftFormatUtils;
@ -36,146 +38,151 @@ import org.apache.iotdb.tsfile.format.RowGroupBlockMetaData;
* This is tsfile <code>InputSplit</code>.<br>
* Each <code>InputSplit</code> will be processed by individual
* <code>Mapper</code> task.
*
*
* @author liukun
*/
public class TSFInputSplit extends InputSplit implements Writable {
private Path path;
private int numOfDeviceRowGroup;
private List<RowGroupMetaData> deviceRowGroupMetaDataList;
private long start;
private long length;
private String[] hosts;
private Path path;
private int numOfDeviceRowGroup;
private List<RowGroupMetaData> deviceRowGroupMetaDataList;
private long start;
private long length;
private String[] hosts;
public TSFInputSplit() {
public TSFInputSplit() {
}
}
/**
* @param path
* @param deviceRowGroupMetaDataList
* @param start
* @param length
* @param hosts
*/
public TSFInputSplit(Path path, List<RowGroupMetaData> deviceRowGroupMetaDataList, long start, long length,
String[] hosts) {
this.path = path;
this.deviceRowGroupMetaDataList = deviceRowGroupMetaDataList;
this.numOfDeviceRowGroup = deviceRowGroupMetaDataList.size();
this.start = start;
this.length = length;
this.hosts = hosts;
}
/**
* @param path
* @param deviceRowGroupMetaDataList
* @param start
* @param length
* @param hosts
*/
public TSFInputSplit(Path path, List<RowGroupMetaData> deviceRowGroupMetaDataList, long start,
long length,
String[] hosts) {
this.path = path;
this.deviceRowGroupMetaDataList = deviceRowGroupMetaDataList;
this.numOfDeviceRowGroup = deviceRowGroupMetaDataList.size();
this.start = start;
this.length = length;
this.hosts = hosts;
}
/**
* @return the path
*/
public Path getPath() {
return path;
}
/**
* @return the path
*/
public Path getPath() {
return path;
}
/**
* @param path
* the path to set
*/
public void setPath(Path path) {
this.path = path;
}
/**
* @param path
* the path to set
*/
public void setPath(Path path) {
this.path = path;
}
/**
* @return the numOfDeviceRowGroup
*/
public int getNumOfDeviceRowGroup() {
return numOfDeviceRowGroup;
}
/**
* @return the numOfDeviceRowGroup
*/
public int getNumOfDeviceRowGroup() {
return numOfDeviceRowGroup;
}
/**
* @param numOfDeviceRowGroup
* the numOfDeviceRowGroup to set
*/
public void setNumOfDeviceRowGroup(int numOfDeviceRowGroup) {
this.numOfDeviceRowGroup = numOfDeviceRowGroup;
}
/**
* @param numOfDeviceRowGroup
* the numOfDeviceRowGroup to set
*/
public void setNumOfDeviceRowGroup(int numOfDeviceRowGroup) {
this.numOfDeviceRowGroup = numOfDeviceRowGroup;
}
/**
* @return the deviceRowGroupMetaDataList
*/
public List<RowGroupMetaData> getDeviceRowGroupMetaDataList() {
return deviceRowGroupMetaDataList;
}
/**
* @return the deviceRowGroupMetaDataList
*/
public List<RowGroupMetaData> getDeviceRowGroupMetaDataList() {
return deviceRowGroupMetaDataList;
}
/**
* @param deviceRowGroupMetaDataList
* the deviceRowGroupMetaDataList to set
*/
public void setDeviceRowGroupMetaDataList(List<RowGroupMetaData> deviceRowGroupMetaDataList) {
this.deviceRowGroupMetaDataList = deviceRowGroupMetaDataList;
}
/**
* @param deviceRowGroupMetaDataList
* the deviceRowGroupMetaDataList to set
*/
public void setDeviceRowGroupMetaDataList(List<RowGroupMetaData> deviceRowGroupMetaDataList) {
this.deviceRowGroupMetaDataList = deviceRowGroupMetaDataList;
}
/**
* @return the start
*/
public long getStart() {
return start;
}
/**
* @return the start
*/
public long getStart() {
return start;
}
/**
* @param start
* the start to set
*/
public void setStart(long start) {
this.start = start;
}
/**
* @param start
* the start to set
*/
public void setStart(long start) {
this.start = start;
}
@Override
public long getLength() throws IOException, InterruptedException {
return this.length;
}
@Override
public long getLength() throws IOException, InterruptedException {
return this.length;
}
@Override
public String[] getLocations() throws IOException, InterruptedException {
return this.hosts;
}
@Override
public String[] getLocations() throws IOException, InterruptedException {
return this.hosts;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(path.toString());
out.writeLong(start);
out.writeLong(length);
out.writeInt(hosts.length);
for (int i = 0; i < hosts.length; i++) {
String string = hosts[i];
out.writeUTF(string);
}
out.writeInt(numOfDeviceRowGroup);
RowGroupBlockMetaData rowGroupBlockMetaData = new TsRowGroupBlockMetaData(deviceRowGroupMetaDataList)
.convertToThrift();
ReadWriteThriftFormatUtils.writeRowGroupBlockMetadata(rowGroupBlockMetaData, (OutputStream) out);
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(path.toString());
out.writeLong(start);
out.writeLong(length);
out.writeInt(hosts.length);
for (int i = 0; i < hosts.length; i++) {
String string = hosts[i];
out.writeUTF(string);
}
out.writeInt(numOfDeviceRowGroup);
RowGroupBlockMetaData rowGroupBlockMetaData = new TsRowGroupBlockMetaData(
deviceRowGroupMetaDataList)
.convertToThrift();
ReadWriteThriftFormatUtils
.writeRowGroupBlockMetadata(rowGroupBlockMetaData, (OutputStream) out);
}
@Override
public void readFields(DataInput in) throws IOException {
path = new Path(in.readUTF());
this.start = in.readLong();
this.length = in.readLong();
int len = in.readInt();
this.hosts = new String[len];
for (int i = 0; i < len; i++) {
hosts[i] = in.readUTF();
}
this.numOfDeviceRowGroup = in.readInt();
TsRowGroupBlockMetaData tsRowGroupBlockMetaData = new TsRowGroupBlockMetaData();
tsRowGroupBlockMetaData.convertToTSF(ReadWriteThriftFormatUtils.readRowGroupBlockMetaData((InputStream) in));
deviceRowGroupMetaDataList = tsRowGroupBlockMetaData.getRowGroups();
}
@Override
public void readFields(DataInput in) throws IOException {
path = new Path(in.readUTF());
this.start = in.readLong();
this.length = in.readLong();
int len = in.readInt();
this.hosts = new String[len];
for (int i = 0; i < len; i++) {
hosts[i] = in.readUTF();
}
this.numOfDeviceRowGroup = in.readInt();
TsRowGroupBlockMetaData tsRowGroupBlockMetaData = new TsRowGroupBlockMetaData();
tsRowGroupBlockMetaData
.convertToTSF(ReadWriteThriftFormatUtils.readRowGroupBlockMetaData((InputStream) in));
deviceRowGroupMetaDataList = tsRowGroupBlockMetaData.getRowGroups();
}
@Override
public String toString() {
return "TSFInputSplit [path=" + path + ", numOfDeviceGroup=" + numOfDeviceRowGroup
+ ", deviceRowGroupMetaDataList=" + deviceRowGroupMetaDataList + ", start=" + start + ", length="
+ length + ", hosts=" + Arrays.toString(hosts) + "]";
}
@Override
public String toString() {
return "TSFInputSplit [path=" + path + ", numOfDeviceGroup=" + numOfDeviceRowGroup
+ ", deviceRowGroupMetaDataList=" + deviceRowGroupMetaDataList + ", start=" + start
+ ", length="
+ length + ", hosts=" + Arrays.toString(hosts) + "]";
}
}

68
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFOutputFormat.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -16,7 +20,6 @@
package org.apache.iotdb.tsfile.hadoop;
import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
@ -30,44 +33,43 @@ import org.slf4j.LoggerFactory;
public class TSFOutputFormat extends FileOutputFormat<NullWritable, TSRow> {
private static final Logger LOGGER = LoggerFactory.getLogger(TSFOutputFormat.class);
public static final String FILE_SCHEMA = "tsfile.schema";
private static final Logger LOGGER = LoggerFactory.getLogger(TSFOutputFormat.class);
private static final String extension = "tsfile";
public static final String FILE_SCHEMA = "tsfile.schema";
public static void setWriterSchema(Job job, JSONObject schema) {
private static final String extension = "tsfile";
LOGGER.info("Set the write schema - {}", schema.toString());
public static void setWriterSchema(Job job, JSONObject schema) {
job.getConfiguration().set(FILE_SCHEMA, schema.toString());
}
LOGGER.info("Set the write schema - {}", schema.toString());
public static void setWriterSchema(Job job, String schema) {
job.getConfiguration().set(FILE_SCHEMA, schema.toString());
}
LOGGER.info("Set the write schema - {}", schema);
public static void setWriterSchema(Job job, String schema) {
job.getConfiguration().set(FILE_SCHEMA, schema);
}
LOGGER.info("Set the write schema - {}", schema);
public static JSONObject getWriterSchema(JobContext jobContext) throws InterruptedException {
job.getConfiguration().set(FILE_SCHEMA, schema);
}
String schema = jobContext.getConfiguration().get(FILE_SCHEMA);
if (schema == null || schema == "") {
throw new InterruptedException("The tsfile schema is null or empty");
}
JSONObject jsonSchema = new JSONObject(schema);
public static JSONObject getWriterSchema(JobContext jobContext) throws InterruptedException {
return jsonSchema;
}
String schema = jobContext.getConfiguration().get(FILE_SCHEMA);
if (schema == null || schema == "") {
throw new InterruptedException("The tsfile schema is null or empty");
}
JSONObject jsonSchema = new JSONObject(schema);
@Override
public RecordWriter<NullWritable, TSRow> getRecordWriter(TaskAttemptContext job)
throws IOException, InterruptedException {
return jsonSchema;
}
@Override
public RecordWriter<NullWritable, TSRow> getRecordWriter(TaskAttemptContext job)
throws IOException, InterruptedException {
Path outputPath = getDefaultWorkFile(job, extension);
LOGGER.info("The task attempt id is {}, the output path is {}", job.getTaskAttemptID(), outputPath);
JSONObject schema = getWriterSchema(job);
return new TSFRecordWriter(outputPath, schema);
}
Path outputPath = getDefaultWorkFile(job, extension);
LOGGER.info("The task attempt id is {}, the output path is {}", job.getTaskAttemptID(),
outputPath);
JSONObject schema = getWriterSchema(job);
return new TSFRecordWriter(outputPath, schema);
}
}

319
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordReader.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -20,7 +24,6 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
@ -35,185 +38,191 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.file.metadata.RowGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.TimeSeriesChunkMetaData;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.read.query.HadoopQueryEngine;
import org.apache.iotdb.tsfile.read.query.QueryDataSet;
import org.apache.iotdb.tsfile.read.support.Field;
import org.apache.iotdb.tsfile.read.support.RowRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author liukun
*/
public class TSFRecordReader extends RecordReader<NullWritable, ArrayWritable> {
private static final Logger LOGGER = LoggerFactory.getLogger(TSFRecordReader.class);
private static final Logger LOGGER = LoggerFactory.getLogger(TSFRecordReader.class);
private QueryDataSet dataSet = null;
private List<Field> fields = null;
private long timestamp = 0;
private String deviceId;
private int sensorNum = 0;
private int sensorIndex = 0;
private boolean isReadDeviceId = false;
private boolean isReadTime = false;
private int arraySize = 0;
private HDFSInputStream hdfsInputStream;
private QueryDataSet dataSet = null;
private List<Field> fields = null;
private long timestamp = 0;
private String deviceId;
private int sensorNum = 0;
private int sensorIndex = 0;
private boolean isReadDeviceId = false;
private boolean isReadTime = false;
private int arraySize = 0;
private HDFSInputStream hdfsInputStream;
@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
if (split instanceof TSFInputSplit) {
TSFInputSplit tsfInputSplit = (TSFInputSplit) split;
Path path = tsfInputSplit.getPath();
List<RowGroupMetaData> rowGroupMetaDataList = tsfInputSplit.getDeviceRowGroupMetaDataList();
Configuration configuration = context.getConfiguration();
hdfsInputStream = new HDFSInputStream(path, configuration);
// Get the read columns and filter information
List<String> deltaObjectIdsList = TSFInputFormat.getReadDeltaObjectIds(configuration);
if(deltaObjectIdsList == null)deltaObjectIdsList = initDeviceIdList(rowGroupMetaDataList);
List<String> measurementIdsList = TSFInputFormat.getReadMeasurementIds(configuration);
if(measurementIdsList == null)measurementIdsList = initSensorIdList(rowGroupMetaDataList);
LOGGER.info("deltaObjectIds:" + deltaObjectIdsList);
LOGGER.info("Sensors:" + measurementIdsList);
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
if (split instanceof TSFInputSplit) {
TSFInputSplit tsfInputSplit = (TSFInputSplit) split;
Path path = tsfInputSplit.getPath();
List<RowGroupMetaData> rowGroupMetaDataList = tsfInputSplit.getDeviceRowGroupMetaDataList();
Configuration configuration = context.getConfiguration();
hdfsInputStream = new HDFSInputStream(path, configuration);
// Get the read columns and filter information
List<String> deltaObjectIdsList = TSFInputFormat.getReadDeltaObjectIds(configuration);
if (deltaObjectIdsList == null) {
deltaObjectIdsList = initDeviceIdList(rowGroupMetaDataList);
}
List<String> measurementIdsList = TSFInputFormat.getReadMeasurementIds(configuration);
if (measurementIdsList == null) {
measurementIdsList = initSensorIdList(rowGroupMetaDataList);
}
LOGGER.info("deltaObjectIds:" + deltaObjectIdsList);
LOGGER.info("Sensors:" + measurementIdsList);
this.sensorNum = measurementIdsList.size();
isReadDeviceId = TSFInputFormat.getReadDeltaObject(configuration);
isReadTime = TSFInputFormat.getReadTime(configuration);
if (isReadDeviceId) {
arraySize++;
}
if (isReadTime) {
arraySize++;
}
arraySize += sensorNum;
this.sensorNum = measurementIdsList.size();
isReadDeviceId = TSFInputFormat.getReadDeltaObject(configuration);
isReadTime = TSFInputFormat.getReadTime(configuration);
if (isReadDeviceId) {
arraySize++;
}
if (isReadTime) {
arraySize++;
}
arraySize += sensorNum;
HadoopQueryEngine queryEngine = new HadoopQueryEngine(hdfsInputStream, rowGroupMetaDataList);
dataSet = queryEngine.queryWithSpecificRowGroups(deltaObjectIdsList, measurementIdsList, null, null, null);
} else {
LOGGER.error("The InputSplit class is not {}, the class is {}", TSFInputSplit.class.getName(),
split.getClass().getName());
throw new InternalError(String.format("The InputSplit class is not %s, the class is %s",
TSFInputSplit.class.getName(), split.getClass().getName()));
}
}
HadoopQueryEngine queryEngine = new HadoopQueryEngine(hdfsInputStream, rowGroupMetaDataList);
dataSet = queryEngine
.queryWithSpecificRowGroups(deltaObjectIdsList, measurementIdsList, null, null, null);
} else {
LOGGER.error("The InputSplit class is not {}, the class is {}", TSFInputSplit.class.getName(),
split.getClass().getName());
throw new InternalError(String.format("The InputSplit class is not %s, the class is %s",
TSFInputSplit.class.getName(), split.getClass().getName()));
}
}
private List<String> initDeviceIdList(List<RowGroupMetaData> rowGroupMetaDataList) {
Set<String> deviceIdSet = new HashSet<>();
for (RowGroupMetaData rowGroupMetaData : rowGroupMetaDataList) {
deviceIdSet.add(rowGroupMetaData.getDeltaObjectID());
}
return new ArrayList<>(deviceIdSet);
}
private List<String> initDeviceIdList(List<RowGroupMetaData> rowGroupMetaDataList) {
Set<String> deviceIdSet = new HashSet<>();
for (RowGroupMetaData rowGroupMetaData : rowGroupMetaDataList) {
deviceIdSet.add(rowGroupMetaData.getDeltaObjectID());
}
return new ArrayList<>(deviceIdSet);
}
private List<String> initSensorIdList(List<RowGroupMetaData> rowGroupMetaDataList){
Set<String> sensorIdSet = new HashSet<>();
for(RowGroupMetaData rowGroupMetaData : rowGroupMetaDataList) {
for(TimeSeriesChunkMetaData timeSeriesChunkMetaData : rowGroupMetaData.getTimeSeriesChunkMetaDataList()){
sensorIdSet.add(timeSeriesChunkMetaData.getProperties().getMeasurementUID());
}
}
return new ArrayList<>(sensorIdSet);
}
private List<String> initSensorIdList(List<RowGroupMetaData> rowGroupMetaDataList) {
Set<String> sensorIdSet = new HashSet<>();
for (RowGroupMetaData rowGroupMetaData : rowGroupMetaDataList) {
for (TimeSeriesChunkMetaData timeSeriesChunkMetaData : rowGroupMetaData
.getTimeSeriesChunkMetaDataList()) {
sensorIdSet.add(timeSeriesChunkMetaData.getProperties().getMeasurementUID());
}
}
return new ArrayList<>(sensorIdSet);
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
sensorIndex += sensorNum;
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
sensorIndex += sensorNum;
if(fields == null || sensorIndex >= fields.size()){
LOGGER.info("Start another row~");
if(!dataSet.next()){
LOGGER.info("Finish all rows~");
return false;
}
if (fields == null || sensorIndex >= fields.size()) {
LOGGER.info("Start another row~");
if (!dataSet.next()) {
LOGGER.info("Finish all rows~");
return false;
}
RowRecord rowRecord = dataSet.getCurrentRecord();
fields = rowRecord.getFields();
timestamp = rowRecord.getTime();
sensorIndex = 0;
}
deviceId = fields.get(sensorIndex).deltaObjectId;
RowRecord rowRecord = dataSet.getCurrentRecord();
fields = rowRecord.getFields();
timestamp = rowRecord.getTime();
sensorIndex = 0;
}
deviceId = fields.get(sensorIndex).deltaObjectId;
return true;
}
return true;
}
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
@Override
public NullWritable getCurrentKey() throws IOException, InterruptedException {
return NullWritable.get();
}
@Override
public ArrayWritable getCurrentValue() throws IOException, InterruptedException {
@Override
public ArrayWritable getCurrentValue() throws IOException, InterruptedException {
Writable[] writables = getEmptyWritables();
Text deviceid = new Text(deviceId);
LongWritable time = new LongWritable(timestamp);
int index = 0;
if (isReadTime && isReadDeviceId) {
writables[0] = time;
writables[1] = deviceid;
index = 2;
} else if (isReadTime && !isReadDeviceId) {
writables[0] = time;
index = 1;
} else if (!isReadTime && isReadDeviceId) {
writables[0] = deviceid;
index = 1;
}
Writable[] writables = getEmptyWritables();
Text deviceid = new Text(deviceId);
LongWritable time = new LongWritable(timestamp);
int index = 0;
if (isReadTime && isReadDeviceId) {
writables[0] = time;
writables[1] = deviceid;
index = 2;
} else if (isReadTime && !isReadDeviceId) {
writables[0] = time;
index = 1;
} else if (!isReadTime && isReadDeviceId) {
writables[0] = deviceid;
index = 1;
}
for(int i = 0;i < sensorNum;i++)
{
Field field = fields.get(sensorIndex + i);
if (field.isNull()) {
LOGGER.info("Current value is null");
writables[index] = NullWritable.get();
} else {
switch (field.dataType) {
case INT32:
writables[index] = new IntWritable(field.getIntV());
break;
case INT64:
writables[index] = new LongWritable(field.getLongV());
break;
case FLOAT:
writables[index] = new FloatWritable(field.getFloatV());
break;
case DOUBLE:
writables[index] = new DoubleWritable(field.getDoubleV());
break;
case BOOLEAN:
writables[index] = new BooleanWritable(field.getBoolV());
break;
case TEXT:
writables[index] = new Text(field.getBinaryV().getStringValue());
break;
default:
LOGGER.error("The data type is not support {}", field.dataType);
throw new InterruptedException(String.format("The data type %s is not support ", field.dataType));
}
}
index++;
}
return new ArrayWritable(Writable.class, writables);
}
for (int i = 0; i < sensorNum; i++) {
Field field = fields.get(sensorIndex + i);
if (field.isNull()) {
LOGGER.info("Current value is null");
writables[index] = NullWritable.get();
} else {
switch (field.dataType) {
case INT32:
writables[index] = new IntWritable(field.getIntV());
break;
case INT64:
writables[index] = new LongWritable(field.getLongV());
break;
case FLOAT:
writables[index] = new FloatWritable(field.getFloatV());
break;
case DOUBLE:
writables[index] = new DoubleWritable(field.getDoubleV());
break;
case BOOLEAN:
writables[index] = new BooleanWritable(field.getBoolV());
break;
case TEXT:
writables[index] = new Text(field.getBinaryV().getStringValue());
break;
default:
LOGGER.error("The data type is not support {}", field.dataType);
throw new InterruptedException(
String.format("The data type %s is not support ", field.dataType));
}
}
index++;
}
return new ArrayWritable(Writable.class, writables);
}
@Override
public float getProgress() throws IOException, InterruptedException {
return 0;
}
@Override
public float getProgress() throws IOException, InterruptedException {
return 0;
}
@Override
public void close() throws IOException {
dataSet = null;
hdfsInputStream.close();
}
@Override
public void close() throws IOException {
dataSet = null;
hdfsInputStream.close();
}
private Writable[] getEmptyWritables() {
Writable[] writables = new Writable[arraySize];
return writables;
}
private Writable[] getEmptyWritables() {
Writable[] writables = new Writable[arraySize];
return writables;
}
}

88
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSFRecordWriter.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -16,65 +20,63 @@
package org.apache.iotdb.tsfile.hadoop;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.iotdb.tsfile.hadoop.io.HDFSOutputStream;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.write.exception.InvalidJsonSchemaException;
import org.apache.iotdb.tsfile.write.exception.WriteProcessException;
import org.apache.iotdb.tsfile.write.schema.FileSchema;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TSFRecordWriter extends RecordWriter<NullWritable, TSRow> {
private static final Logger LOGGER = LoggerFactory.getLogger(TSFRecordWriter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(TSFRecordWriter.class);
private TsFile write = null;
private TsFile write = null;
public TSFRecordWriter(Path path, JSONObject schema) throws InterruptedException, IOException {
// construct the internalrecordwriter
FileSchema fileSchema = null;
try {
fileSchema = new FileSchema(schema);
} catch (InvalidJsonSchemaException e) {
e.printStackTrace();
LOGGER.error("Construct the tsfile schema failed, the reason is {}", e.getMessage());
throw new InterruptedException(e.getMessage());
}
public TSFRecordWriter(Path path, JSONObject schema) throws InterruptedException, IOException {
// construct the internalrecordwriter
FileSchema fileSchema = null;
try {
fileSchema = new FileSchema(schema);
} catch (InvalidJsonSchemaException e) {
e.printStackTrace();
LOGGER.error("Construct the tsfile schema failed, the reason is {}", e.getMessage());
throw new InterruptedException(e.getMessage());
}
HDFSOutputStream hdfsOutputStream = new HDFSOutputStream(path, new Configuration(), false);
try {
write = new TsFile(hdfsOutputStream, fileSchema);
} catch (WriteProcessException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
HDFSOutputStream hdfsOutputStream = new HDFSOutputStream(path, new Configuration(), false);
try {
write = new TsFile(hdfsOutputStream, fileSchema);
} catch (WriteProcessException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
@Override
public void write(NullWritable key, TSRow value) throws IOException, InterruptedException {
@Override
public void write(NullWritable key, TSRow value) throws IOException, InterruptedException {
try {
write.writeRecord(value.getRow());
} catch (WriteProcessException e) {
e.printStackTrace();
LOGGER.error("Write tsfile record error, the error message is {}", e.getMessage());
throw new InterruptedException(e.getMessage());
}
}
try {
write.writeRecord(value.getRow());
} catch (WriteProcessException e) {
e.printStackTrace();
LOGGER.error("Write tsfile record error, the error message is {}", e.getMessage());
throw new InterruptedException(e.getMessage());
}
}
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
LOGGER.info("Close the recordwriter, the task attempt id is {}", context.getTaskAttemptID());
write.close();
}
LOGGER.info("Close the recordwriter, the task attempt id is {}", context.getTaskAttemptID());
write.close();
}
}

50
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/TSRow.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -18,33 +22,31 @@ package org.apache.iotdb.tsfile.hadoop;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.iotdb.tsfile.write.record.TSRecord;
public class TSRow implements Writable {
private TSRecord row;
public TSRow(TSRecord row){
this.row = row;
}
public TSRecord getRow(){
return row;
}
@Override
public void write(DataOutput out) throws IOException {
private TSRecord row;
throw new IOException("Not support");
}
public TSRow(TSRecord row) {
@Override
public void readFields(DataInput in) throws IOException {
this.row = row;
}
throw new IOException("Not support");
}
public TSRecord getRow() {
return row;
}
@Override
public void write(DataOutput out) throws IOException {
throw new IOException("Not support");
}
@Override
public void readFields(DataInput in) throws IOException {
throw new IOException("Not support");
}
}

158
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/example/TSFMRReadExample.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -17,7 +21,6 @@ package org.apache.iotdb.tsfile.hadoop.example;
import java.io.IOException;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
@ -27,7 +30,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.iotdb.tsfile.hadoop.TSFHadoopException;
import org.apache.iotdb.tsfile.hadoop.TSFInputFormat;
import org.apache.iotdb.tsfile.hadoop.TSFOutputFormat;
@ -41,86 +43,86 @@ import org.apache.iotdb.tsfile.hadoop.TSFOutputFormat;
*/
public class TSFMRReadExample {
public static class TSMapper extends Mapper<NullWritable, ArrayWritable, Text, IntWritable> {
public static void main(String[] args)
throws IOException, ClassNotFoundException, TSFHadoopException, URISyntaxException {
private static final IntWritable one = new IntWritable(1);
if (args.length != 3) {
System.out.println("Please give hdfs url, input path, output path");
return;
}
String HDFSURL = args[0];
Path inputPath = new Path(args[1]);
Path outputPath = new Path(args[2]);
@Override
protected void map(NullWritable key, ArrayWritable value,
Mapper<NullWritable, ArrayWritable, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
Configuration configuration = new Configuration();
// set file system configuration
//configuration.set("fs.defaultFS", HDFSURL);
Job job = Job.getInstance(configuration);
job.setJobName("TsFile read jar");
job.setJarByClass(TSFMRReadExample.class);
// set mapper and reducer
job.setMapperClass(TSMapper.class);
job.setReducerClass(TSReducer.class);
// set inputformat and outputformat
job.setInputFormatClass(TSFInputFormat.class);
// set mapper output key and value
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// set reducer output key and value
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// set input file path
TSFInputFormat.setInputPaths(job, inputPath);
// set output file path
TSFOutputFormat.setOutputPath(job, outputPath);
/**
* special configuration for reading tsfile with TSFInputFormat
*/
TSFInputFormat.setReadTime(job, true); // configure reading time enable
TSFInputFormat.setReadDeltaObjectId(job, true); // configure reading deltaObjectId enable
String[] deltaObjectIds = {"device_1"};// configure reading which deltaObjectIds
TSFInputFormat.setReadDeltaObjectIds(job, deltaObjectIds);
String[] measurementIds = {"sensor_1",};// configure reading which measurementIds
TSFInputFormat.setReadMeasurementIds(job, measurementIds);
boolean isSuccess = false;
try {
isSuccess = job.waitForCompletion(true);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (isSuccess) {
System.out.println("Execute successfully");
} else {
System.out.println("Execute unsuccessfully");
}
}
Text deltaObjectId = (Text) value.get()[1];
context.write(deltaObjectId, one);
}
}
public static class TSMapper extends Mapper<NullWritable, ArrayWritable, Text, IntWritable> {
public static class TSReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private static final IntWritable one = new IntWritable(1);
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
@Override
protected void map(NullWritable key, ArrayWritable value,
Mapper<NullWritable, ArrayWritable, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable intWritable : values) {
sum = sum + intWritable.get();
}
context.write(key, new IntWritable(sum));
}
}
Text deltaObjectId = (Text) value.get()[1];
context.write(deltaObjectId, one);
}
}
public static void main(String[] args)
throws IOException, ClassNotFoundException, TSFHadoopException, URISyntaxException {
public static class TSReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
if (args.length != 3) {
System.out.println("Please give hdfs url, input path, output path");
return;
}
String HDFSURL = args[0];
Path inputPath = new Path(args[1]);
Path outputPath = new Path(args[2]);
@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
Configuration configuration = new Configuration();
// set file system configuration
//configuration.set("fs.defaultFS", HDFSURL);
Job job = Job.getInstance(configuration);
job.setJobName("TsFile read jar");
job.setJarByClass(TSFMRReadExample.class);
// set mapper and reducer
job.setMapperClass(TSMapper.class);
job.setReducerClass(TSReducer.class);
// set inputformat and outputformat
job.setInputFormatClass(TSFInputFormat.class);
// set mapper output key and value
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
// set reducer output key and value
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// set input file path
TSFInputFormat.setInputPaths(job, inputPath);
// set output file path
TSFOutputFormat.setOutputPath(job, outputPath);
/**
* special configuration for reading tsfile with TSFInputFormat
*/
TSFInputFormat.setReadTime(job, true); // configure reading time enable
TSFInputFormat.setReadDeltaObjectId(job, true); // configure reading deltaObjectId enable
String[] deltaObjectIds = { "device_1" };// configure reading which deltaObjectIds
TSFInputFormat.setReadDeltaObjectIds(job, deltaObjectIds);
String[] measurementIds = { "sensor_1", };// configure reading which measurementIds
TSFInputFormat.setReadMeasurementIds(job, measurementIds);
boolean isSuccess = false;
try {
isSuccess = job.waitForCompletion(true);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (isSuccess) {
System.out.println("Execute successfully");
} else {
System.out.println("Execute unsuccessfully");
}
}
int sum = 0;
for (IntWritable intWritable : values) {
sum = sum + intWritable.get();
}
context.write(key, new IntWritable(sum));
}
}
}

182
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/example/TsFileHelper.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -18,111 +22,109 @@ package org.apache.iotdb.tsfile.hadoop.example;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileWriter;
import org.apache.iotdb.tsfile.common.utils.TsRandomAccessFileWriter;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.read.TsRandomAccessLocalFileReader;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.write.exception.WriteProcessException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TsFileHelper {
private static final Logger LOGGER = LoggerFactory.getLogger(TsFileHelper.class);
public static void deleteTsFile(String filePath){
File file = new File(filePath);
file.delete();
}
public static void writeTsFile(String filePath) {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
conf.pageSizeInByte=100;
conf.groupSizeInByte = 2000;
conf.pageCheckSizeThreshold = 1;
conf.maxStringLength = 2;
private static final Logger LOGGER = LoggerFactory.getLogger(TsFileHelper.class);
File file = new File(filePath);
public static void deleteTsFile(String filePath) {
File file = new File(filePath);
file.delete();
}
if (file.exists())
file.delete();
public static void writeTsFile(String filePath) {
JSONObject jsonSchema = getJsonSchema();
try {
ITsRandomAccessFileWriter output = new TsRandomAccessFileWriter(new File(filePath));
TsFile tsFile = new TsFile(output, jsonSchema);
String line = "";
for(int i = 1;i<1000;i++){
line = "root.car.d1,"+i+",s1," + i + ",s2,1,s3,0.1,s4,0.1";
tsFile.writeLine(line);
}
tsFile.writeLine("root.car.d2,5, s1, 5, s2, 50, s3, 200.5, s4, 0.5");
tsFile.writeLine("root.car.d2,6, s1, 6, s2, 60, s3, 200.6, s4, 0.6");
tsFile.writeLine("root.car.d2,7, s1, 7, s2, 70, s3, 200.7, s4, 0.7");
tsFile.writeLine("root.car.d2,8, s1, 8, s2, 80, s3, 200.8, s4, 0.8");
tsFile.close();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
} catch (WriteProcessException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
conf.pageSizeInByte = 100;
conf.groupSizeInByte = 2000;
conf.pageCheckSizeThreshold = 1;
conf.maxStringLength = 2;
private static JSONObject getJsonSchema() {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
JSONObject s1 = new JSONObject();
s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
File file = new File(filePath);
JSONObject s2 = new JSONObject();
s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
if (file.exists()) {
file.delete();
}
JSONObject s3 = new JSONObject();
s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject jsonSchema = getJsonSchema();
JSONObject s4 = new JSONObject();
s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
try {
ITsRandomAccessFileWriter output = new TsRandomAccessFileWriter(new File(filePath));
TsFile tsFile = new TsFile(output, jsonSchema);
String line = "";
for (int i = 1; i < 1000; i++) {
line = "root.car.d1," + i + ",s1," + i + ",s2,1,s3,0.1,s4,0.1";
tsFile.writeLine(line);
}
tsFile.writeLine("root.car.d2,5, s1, 5, s2, 50, s3, 200.5, s4, 0.5");
tsFile.writeLine("root.car.d2,6, s1, 6, s2, 60, s3, 200.6, s4, 0.6");
tsFile.writeLine("root.car.d2,7, s1, 7, s2, 70, s3, 200.7, s4, 0.7");
tsFile.writeLine("root.car.d2,8, s1, 8, s2, 80, s3, 200.8, s4, 0.8");
tsFile.close();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
} catch (WriteProcessException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
JSONArray measureGroup = new JSONArray();
measureGroup.put(s1);
measureGroup.put(s2);
measureGroup.put(s3);
measureGroup.put(s4);
private static JSONObject getJsonSchema() {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
JSONObject s1 = new JSONObject();
s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject jsonSchema = new JSONObject();
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup);
return jsonSchema;
}
public static void main(String[] args) throws FileNotFoundException, IOException{
String filePath = "example_mr.tsfile";
File file = new File(filePath);
file.delete();
writeTsFile(filePath);
TsFile tsFile = new TsFile(new TsRandomAccessLocalFileReader(filePath));
LOGGER.info("Get columns information: {}",tsFile.getAllColumns());
LOGGER.info("Get all deltaObjectId: {}",tsFile.getAllDeltaObject());
tsFile.close();
}
JSONObject s2 = new JSONObject();
s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s3 = new JSONObject();
s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s4 = new JSONObject();
s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONArray measureGroup = new JSONArray();
measureGroup.put(s1);
measureGroup.put(s2);
measureGroup.put(s3);
measureGroup.put(s4);
JSONObject jsonSchema = new JSONObject();
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup);
return jsonSchema;
}
public static void main(String[] args) throws FileNotFoundException, IOException {
String filePath = "example_mr.tsfile";
File file = new File(filePath);
file.delete();
writeTsFile(filePath);
TsFile tsFile = new TsFile(new TsRandomAccessLocalFileReader(filePath));
LOGGER.info("Get columns information: {}", tsFile.getAllColumns());
LOGGER.info("Get all deltaObjectId: {}", tsFile.getAllDeltaObject());
tsFile.close();
}
}

122
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/io/HDFSInputStream.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -16,13 +20,11 @@
package org.apache.iotdb.tsfile.hadoop.io;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileReader;
/**
@ -33,78 +35,78 @@ import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileReader;
*/
public class HDFSInputStream implements ITsRandomAccessFileReader {
private FSDataInputStream fsDataInputStream;
private FileStatus fileStatus;
private FSDataInputStream fsDataInputStream;
private FileStatus fileStatus;
public HDFSInputStream(String filePath) throws IOException {
public HDFSInputStream(String filePath) throws IOException {
this(filePath, new Configuration());
}
this(filePath, new Configuration());
}
public HDFSInputStream(String filePath, Configuration configuration) throws IOException {
public HDFSInputStream(String filePath, Configuration configuration) throws IOException {
this(new Path(filePath), configuration);
}
this(new Path(filePath), configuration);
}
public HDFSInputStream(Path path, Configuration configuration) throws IOException {
public HDFSInputStream(Path path, Configuration configuration) throws IOException {
FileSystem fs = FileSystem.get(configuration);
fsDataInputStream = fs.open(path);
fileStatus = fs.getFileStatus(path);
}
FileSystem fs = FileSystem.get(configuration);
fsDataInputStream = fs.open(path);
fileStatus = fs.getFileStatus(path);
}
public void seek(long offset) throws IOException {
public void seek(long offset) throws IOException {
fsDataInputStream.seek(offset);
}
fsDataInputStream.seek(offset);
}
public int read() throws IOException {
public int read() throws IOException {
return fsDataInputStream.read();
}
return fsDataInputStream.read();
}
public long length() throws IOException {
public long length() throws IOException {
return fileStatus.getLen();
}
return fileStatus.getLen();
}
public int readInt() throws IOException {
public int readInt() throws IOException {
return fsDataInputStream.readInt();
}
return fsDataInputStream.readInt();
}
public void close() throws IOException {
public void close() throws IOException {
fsDataInputStream.close();
}
fsDataInputStream.close();
}
public long getPos() throws IOException {
public long getPos() throws IOException {
return fsDataInputStream.getPos();
}
return fsDataInputStream.getPos();
}
/**
* Read the data into b, and the check the length
*
* @param b
* read the data into
* @param off
* the begin offset to read into
* @param len
* the length to read into
*/
public int read(byte[] b, int off, int len) throws IOException {
if (len < 0) {
throw new IndexOutOfBoundsException();
}
int n = 0;
while (n < len) {
int count = fsDataInputStream.read(b, off + n, len - n);
if (count < 0) {
throw new IOException("The read length is out of the length of inputstream");
}
n += count;
}
return n;
}
/**
* Read the data into b, and the check the length
*
* @param b
* read the data into
* @param off
* the begin offset to read into
* @param len
* the length to read into
*/
public int read(byte[] b, int off, int len) throws IOException {
if (len < 0) {
throw new IndexOutOfBoundsException();
}
int n = 0;
while (n < len) {
int count = fsDataInputStream.read(b, off + n, len - n);
if (count < 0) {
throw new IOException("The read length is out of the length of inputstream");
}
n += count;
}
return n;
}
}

84
hadoop/src/main/java/org/apache/iotdb/tsfile/hadoop/io/HDFSOutputStream.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -17,12 +21,10 @@ package org.apache.iotdb.tsfile.hadoop.io;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileWriter;
/**
@ -33,57 +35,59 @@ import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileWriter;
*/
public class HDFSOutputStream implements ITsRandomAccessFileWriter {
private FSDataOutputStream fsDataOutputStream;
private FSDataOutputStream fsDataOutputStream;
public HDFSOutputStream(String filePath, boolean overwriter) throws IOException {
public HDFSOutputStream(String filePath, boolean overwriter) throws IOException {
this(filePath, new Configuration(), overwriter);
}
this(filePath, new Configuration(), overwriter);
}
public HDFSOutputStream(String filePath, Configuration configuration, boolean overwriter) throws IOException {
public HDFSOutputStream(String filePath, Configuration configuration, boolean overwriter)
throws IOException {
this(new Path(filePath), configuration, overwriter);
}
this(new Path(filePath), configuration, overwriter);
}
public HDFSOutputStream(Path path, Configuration configuration, boolean overwriter) throws IOException {
public HDFSOutputStream(Path path, Configuration configuration, boolean overwriter)
throws IOException {
FileSystem fsFileSystem = FileSystem.get(configuration);
fsDataOutputStream = fsFileSystem.create(path, overwriter);
}
FileSystem fsFileSystem = FileSystem.get(configuration);
fsDataOutputStream = fsFileSystem.create(path, overwriter);
}
@Override
public OutputStream getOutputStream() {
@Override
public OutputStream getOutputStream() {
return fsDataOutputStream;
}
return fsDataOutputStream;
}
@Override
public long getPos() throws IOException {
@Override
public long getPos() throws IOException {
return fsDataOutputStream.getPos();
}
return fsDataOutputStream.getPos();
}
@Override
public void seek(long offset) throws IOException {
throw new IOException("Not support");
}
@Override
public void seek(long offset) throws IOException {
throw new IOException("Not support");
}
@Override
public void write(int b) throws IOException {
@Override
public void write(int b) throws IOException {
fsDataOutputStream.write(b);
}
fsDataOutputStream.write(b);
}
@Override
public void write(byte[] b) throws IOException {
@Override
public void write(byte[] b) throws IOException {
fsDataOutputStream.write(b);
}
fsDataOutputStream.write(b);
}
@Override
public void close() throws IOException {
@Override
public void close() throws IOException {
fsDataOutputStream.close();
}
fsDataOutputStream.close();
}
}

127
hadoop/src/test/java/cn/edu/thu/tsfile/hadoop/InputOutputStreamTest.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,77 +19,76 @@
*/
package org.apache.iotdb.tsfile.hadoop;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.hadoop.io.HDFSOutputStream;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.apache.iotdb.tsfile.hadoop.io.HDFSInputStream;
import org.apache.iotdb.tsfile.hadoop.io.HDFSOutputStream;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
public class InputOutputStreamTest {
private HDFSInputStream hdfsInputStream = null;
private HDFSOutputStream hdfsOutputStream = null;
private int lenOfBytes = 50;
private byte b = 10;
private byte[] bs = new byte[lenOfBytes];
private byte[] rbs = new byte[lenOfBytes];
private String filename = "testinputandoutputstream.file";
private Path path;
private FileSystem fileSystem;
private HDFSInputStream hdfsInputStream = null;
private HDFSOutputStream hdfsOutputStream = null;
private int lenOfBytes = 50;
private byte b = 10;
private byte[] bs = new byte[lenOfBytes];
private byte[] rbs = new byte[lenOfBytes];
private String filename = "testinputandoutputstream.file";
private Path path;
private FileSystem fileSystem;
@Before
public void setUp() throws Exception {
fileSystem = FileSystem.get(new Configuration());
path = new Path(filename);
fileSystem.delete(path,true);
}
@Before
public void setUp() throws Exception {
@After
public void tearDown() throws Exception {
if (fileSystem.exists(path)) {
fileSystem.delete(path, true);
}
}
fileSystem = FileSystem.get(new Configuration());
path = new Path(filename);
fileSystem.delete(path, true);
}
@Test
public void test() throws Exception {
// write one byte
hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true);
hdfsOutputStream.write(b);
assertEquals(1, hdfsOutputStream.getPos());
hdfsOutputStream.close();
assertEquals(true, fileSystem.exists(path));
fileSystem.delete(path, true);
assertEquals(false, fileSystem.exists(path));
// write bytes
hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true);
hdfsOutputStream.write(bs);
assertEquals(bs.length, hdfsOutputStream.getPos());
hdfsOutputStream.close();
assertEquals(true, fileSystem.exists(path));
// read bytes using hdfs inputstream
hdfsInputStream = new HDFSInputStream(filename);
assertEquals(0, hdfsInputStream.getPos());
assertEquals(lenOfBytes, hdfsInputStream.length());
hdfsInputStream.seek(10);
assertEquals(10, hdfsInputStream.getPos());
hdfsInputStream.seek(0);
hdfsInputStream.read(rbs, 0, rbs.length);
assertEquals(lenOfBytes, hdfsInputStream.getPos());
assertArrayEquals(bs, rbs);
hdfsInputStream.close();
assertEquals(true, fileSystem.exists(path));
fileSystem.delete(path, true);
assertEquals(false, fileSystem.exists(path));
}
@After
public void tearDown() throws Exception {
if (fileSystem.exists(path)) {
fileSystem.delete(path, true);
}
}
@Test
public void test() throws Exception {
// write one byte
hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true);
hdfsOutputStream.write(b);
assertEquals(1, hdfsOutputStream.getPos());
hdfsOutputStream.close();
assertEquals(true, fileSystem.exists(path));
fileSystem.delete(path, true);
assertEquals(false, fileSystem.exists(path));
// write bytes
hdfsOutputStream = new HDFSOutputStream(filename, new Configuration(), true);
hdfsOutputStream.write(bs);
assertEquals(bs.length, hdfsOutputStream.getPos());
hdfsOutputStream.close();
assertEquals(true, fileSystem.exists(path));
// read bytes using hdfs inputstream
hdfsInputStream = new HDFSInputStream(filename);
assertEquals(0, hdfsInputStream.getPos());
assertEquals(lenOfBytes, hdfsInputStream.length());
hdfsInputStream.seek(10);
assertEquals(10, hdfsInputStream.getPos());
hdfsInputStream.seek(0);
hdfsInputStream.read(rbs, 0, rbs.length);
assertEquals(lenOfBytes, hdfsInputStream.getPos());
assertArrayEquals(bs, rbs);
hdfsInputStream.close();
assertEquals(true, fileSystem.exists(path));
fileSystem.delete(path, true);
assertEquals(false, fileSystem.exists(path));
}
}

293
hadoop/src/test/java/cn/edu/thu/tsfile/hadoop/TSFHadoopTest.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -21,7 +25,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.FloatWritable;
@ -33,165 +36,165 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileReader;
import org.apache.iotdb.tsfile.read.TsRandomAccessLocalFileReader;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileReader;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.read.TsRandomAccessLocalFileReader;
public class TSFHadoopTest {
private TSFInputFormat inputformat = null;
private TSFInputFormat inputformat = null;
private String tsfilePath = "tsfile";
private String tsfilePath = "tsfile";
@Before
public void setUp() throws Exception {
@Before
public void setUp() throws Exception {
TsFileTestHelper.deleteTsFile(tsfilePath);
inputformat = new TSFInputFormat();
}
TsFileTestHelper.deleteTsFile(tsfilePath);
inputformat = new TSFInputFormat();
}
@After
public void tearDown() throws Exception {
@After
public void tearDown() throws Exception {
TsFileTestHelper.deleteTsFile(tsfilePath);
}
TsFileTestHelper.deleteTsFile(tsfilePath);
}
@Test
public void staticMethodTest() {
Job job = null;
try {
job = Job.getInstance();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
//
// columns
//
String[] value = { "s1", "s2", "s3" };
try {
TSFInputFormat.setReadMeasurementIds(job, value);
String[] getValue = (String[])TSFInputFormat.getReadMeasurementIds(job.getConfiguration()).toArray();
assertArrayEquals(value, getValue);
} catch (TSFHadoopException e) {
e.printStackTrace();
fail(e.getMessage());
}
//
// deviceid
//
TSFInputFormat.setReadDeltaObjectId(job, true);
assertEquals(true, TSFInputFormat.getReadDeltaObject(job.getConfiguration()));
@Test
public void staticMethodTest() {
Job job = null;
try {
job = Job.getInstance();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
//
// columns
//
String[] value = {"s1", "s2", "s3"};
try {
TSFInputFormat.setReadMeasurementIds(job, value);
String[] getValue = (String[]) TSFInputFormat.getReadMeasurementIds(job.getConfiguration())
.toArray();
assertArrayEquals(value, getValue);
} catch (TSFHadoopException e) {
e.printStackTrace();
fail(e.getMessage());
}
//
// deviceid
//
TSFInputFormat.setReadDeltaObjectId(job, true);
assertEquals(true, TSFInputFormat.getReadDeltaObject(job.getConfiguration()));
//
// time
//
//
// time
//
TSFInputFormat.setReadTime(job, true);
assertEquals(true, TSFInputFormat.getReadTime(job.getConfiguration()));
TSFInputFormat.setReadTime(job, true);
assertEquals(true, TSFInputFormat.getReadTime(job.getConfiguration()));
//
// filter
//
TSFInputFormat.setHasFilter(job, true);
assertEquals(true, TSFInputFormat.getHasFilter(job.getConfiguration()));
//
// filter
//
TSFInputFormat.setHasFilter(job, true);
assertEquals(true, TSFInputFormat.getHasFilter(job.getConfiguration()));
String filterType = "singleFilter";
TSFInputFormat.setFilterType(job, filterType);
assertEquals(filterType, TSFInputFormat.getFilterType(job.getConfiguration()));
String filterType = "singleFilter";
TSFInputFormat.setFilterType(job, filterType);
assertEquals(filterType, TSFInputFormat.getFilterType(job.getConfiguration()));
String filterExpr = "s1>100";
TSFInputFormat.setFilterExp(job, filterExpr);
assertEquals(filterExpr, TSFInputFormat.getFilterExp(job.getConfiguration()));
}
String filterExpr = "s1>100";
TSFInputFormat.setFilterExp(job, filterExpr);
assertEquals(filterExpr, TSFInputFormat.getFilterExp(job.getConfiguration()));
}
@Test
public void InputFormatTest() {
@Test
public void InputFormatTest() {
//
// test getinputsplit method
//
TsFileTestHelper.writeTsFile(tsfilePath);
try {
Job job = Job.getInstance();
// set input path to the job
TSFInputFormat.setInputPaths(job, tsfilePath);
List<InputSplit> inputSplits = inputformat.getSplits(job);
ITsRandomAccessFileReader reader = new TsRandomAccessLocalFileReader(tsfilePath);
TsFile tsFile = new TsFile(reader);
System.out.println(tsFile.getDeltaObjectRowGroupCount());
//assertEquals(tsFile.getRowGroupPosList().size(), inputSplits.size());
for (InputSplit inputSplit : inputSplits) {
System.out.println(inputSplit);
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
//
// test getinputsplit method
//
TsFileTestHelper.writeTsFile(tsfilePath);
try {
Job job = Job.getInstance();
// set input path to the job
TSFInputFormat.setInputPaths(job, tsfilePath);
List<InputSplit> inputSplits = inputformat.getSplits(job);
ITsRandomAccessFileReader reader = new TsRandomAccessLocalFileReader(tsfilePath);
TsFile tsFile = new TsFile(reader);
System.out.println(tsFile.getDeltaObjectRowGroupCount());
//assertEquals(tsFile.getRowGroupPosList().size(), inputSplits.size());
for (InputSplit inputSplit : inputSplits) {
System.out.println(inputSplit);
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
public void RecordReaderTest() {
TsFileTestHelper.writeTsFile(tsfilePath);
try {
Job job = Job.getInstance();
// set input path to the job
TSFInputFormat.setInputPaths(job, tsfilePath);
String[] devices = {"root.car.d1"};
TSFInputFormat.setReadDeltaObjectIds(job, devices);
String[] sensors = { "s1", "s2", "s3", "s4", "s5", "s6"};
TSFInputFormat.setReadMeasurementIds(job, sensors);
TSFInputFormat.setReadDeltaObjectId(job, false);
TSFInputFormat.setReadTime(job, false);
List<InputSplit> inputSplits = inputformat.getSplits(job);
ITsRandomAccessFileReader reader = new TsRandomAccessLocalFileReader(tsfilePath);
TsFile tsFile = new TsFile(reader);
System.out.println(tsFile.getDeltaObjectRowGroupCount());
//assertEquals(tsFile.getRowGroupPosList().size(), inputSplits.size());
for (InputSplit inputSplit : inputSplits) {
System.out.println(inputSplit);
}
reader.close();
// read one split
TSFRecordReader recordReader = new TSFRecordReader();
TaskAttemptContextImpl attemptContextImpl = new TaskAttemptContextImpl(job.getConfiguration(),
new TaskAttemptID());
recordReader.initialize(inputSplits.get(0), attemptContextImpl);
while (recordReader.nextKeyValue()) {
assertEquals(recordReader.getCurrentValue().get().length, sensors.length);
for (Writable writable : recordReader.getCurrentValue().get()) {
if (writable instanceof IntWritable) {
assertEquals(writable.toString(), "1");
} else if (writable instanceof LongWritable) {
assertEquals(writable.toString(), "1");
} else if (writable instanceof FloatWritable) {
assertEquals(writable.toString(), "0.1");
} else if (writable instanceof DoubleWritable) {
assertEquals(writable.toString(), "0.1");
} else if (writable instanceof BooleanWritable) {
assertEquals(writable.toString(), "true");
} else if (writable instanceof Text) {
assertEquals(writable.toString(), "tsfile");
} else {
fail(String.format("Not support type %s", writable.getClass().getName()));
}
}
}
recordReader.close();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
} catch (InterruptedException e) {
e.printStackTrace();
fail(e.getMessage());
} catch (TSFHadoopException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test
public void RecordReaderTest() {
TsFileTestHelper.writeTsFile(tsfilePath);
try {
Job job = Job.getInstance();
// set input path to the job
TSFInputFormat.setInputPaths(job, tsfilePath);
String[] devices = {"root.car.d1"};
TSFInputFormat.setReadDeltaObjectIds(job, devices);
String[] sensors = {"s1", "s2", "s3", "s4", "s5", "s6"};
TSFInputFormat.setReadMeasurementIds(job, sensors);
TSFInputFormat.setReadDeltaObjectId(job, false);
TSFInputFormat.setReadTime(job, false);
List<InputSplit> inputSplits = inputformat.getSplits(job);
ITsRandomAccessFileReader reader = new TsRandomAccessLocalFileReader(tsfilePath);
TsFile tsFile = new TsFile(reader);
System.out.println(tsFile.getDeltaObjectRowGroupCount());
//assertEquals(tsFile.getRowGroupPosList().size(), inputSplits.size());
for (InputSplit inputSplit : inputSplits) {
System.out.println(inputSplit);
}
reader.close();
// read one split
TSFRecordReader recordReader = new TSFRecordReader();
TaskAttemptContextImpl attemptContextImpl = new TaskAttemptContextImpl(job.getConfiguration(),
new TaskAttemptID());
recordReader.initialize(inputSplits.get(0), attemptContextImpl);
while (recordReader.nextKeyValue()) {
assertEquals(recordReader.getCurrentValue().get().length, sensors.length);
for (Writable writable : recordReader.getCurrentValue().get()) {
if (writable instanceof IntWritable) {
assertEquals(writable.toString(), "1");
} else if (writable instanceof LongWritable) {
assertEquals(writable.toString(), "1");
} else if (writable instanceof FloatWritable) {
assertEquals(writable.toString(), "0.1");
} else if (writable instanceof DoubleWritable) {
assertEquals(writable.toString(), "0.1");
} else if (writable instanceof BooleanWritable) {
assertEquals(writable.toString(), "true");
} else if (writable instanceof Text) {
assertEquals(writable.toString(), "tsfile");
} else {
fail(String.format("Not support type %s", writable.getClass().getName()));
}
}
}
recordReader.close();
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
} catch (InterruptedException e) {
e.printStackTrace();
fail(e.getMessage());
} catch (TSFHadoopException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
}

113
hadoop/src/test/java/cn/edu/thu/tsfile/hadoop/TSFInputSplitTest.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -22,15 +26,13 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.junit.Before;
import org.junit.Test;
import org.apache.iotdb.tsfile.file.metadata.RowGroupMetaData;
import org.apache.iotdb.tsfile.file.metadata.TimeSeriesChunkMetaData;
import org.junit.Before;
import org.junit.Test;
/**
* Test the {@link org.apache.iotdb.tsfile.hadoop.TSFInputSplit}
@ -40,55 +42,58 @@ import org.apache.iotdb.tsfile.file.metadata.TimeSeriesChunkMetaData;
*/
public class TSFInputSplitTest {
private TSFInputSplit wInputSplit;
private TSFInputSplit rInputSplit;
private DataInputBuffer DataInputBuffer = new DataInputBuffer();
private DataOutputBuffer DataOutputBuffer = new DataOutputBuffer();
private TSFInputSplit wInputSplit;
private TSFInputSplit rInputSplit;
private DataInputBuffer DataInputBuffer = new DataInputBuffer();
private DataOutputBuffer DataOutputBuffer = new DataOutputBuffer();
@Before
public void setUp() throws Exception {
// For the test data
Path path = new Path("input");
String deviceId = "d1";
int numOfRowGroupMetaDate = 1;
List<RowGroupMetaData> rowGroupMetaDataList = new ArrayList<>();
rowGroupMetaDataList.add(new RowGroupMetaData("d1", 1, 10, new ArrayList<TimeSeriesChunkMetaData>(), "Int"));
rowGroupMetaDataList.add(new RowGroupMetaData("d1", 2, 20, new ArrayList<TimeSeriesChunkMetaData>(), "Int"));
rowGroupMetaDataList.add(new RowGroupMetaData("d2", 3, 30, new ArrayList<TimeSeriesChunkMetaData>(), "Float"));
long start = 5;
long length = 100;
String[] hosts = {"192.168.1.1", "192.168.1.0", "localhost"};
@Before
public void setUp() throws Exception {
// For the test data
Path path = new Path("input");
String deviceId = "d1";
int numOfRowGroupMetaDate = 1;
List<RowGroupMetaData> rowGroupMetaDataList = new ArrayList<>();
rowGroupMetaDataList
.add(new RowGroupMetaData("d1", 1, 10, new ArrayList<TimeSeriesChunkMetaData>(), "Int"));
rowGroupMetaDataList
.add(new RowGroupMetaData("d1", 2, 20, new ArrayList<TimeSeriesChunkMetaData>(), "Int"));
rowGroupMetaDataList
.add(new RowGroupMetaData("d2", 3, 30, new ArrayList<TimeSeriesChunkMetaData>(), "Float"));
long start = 5;
long length = 100;
String[] hosts = {"192.168.1.1", "192.168.1.0", "localhost"};
wInputSplit = new TSFInputSplit(path, rowGroupMetaDataList, start, length, hosts);
rInputSplit = new TSFInputSplit();
}
@Test
public void testInputSplitWriteAndRead() {
try {
// call the write method to serialize the object
wInputSplit.write(DataOutputBuffer);
DataOutputBuffer.flush();
DataInputBuffer.reset(DataOutputBuffer.getData(), DataOutputBuffer.getLength());
rInputSplit.readFields(DataInputBuffer);
DataInputBuffer.close();
DataOutputBuffer.close();
// assert
assertEquals(wInputSplit.getPath(), rInputSplit.getPath());
assertEquals(wInputSplit.getNumOfDeviceRowGroup(), rInputSplit.getNumOfDeviceRowGroup());
//assertEquals(wInputSplit.getDeviceRowGroupMetaDataList(), rInputSplit.getDeviceRowGroupMetaDataList());
assertEquals(wInputSplit.getStart(), rInputSplit.getStart());
try {
assertEquals(wInputSplit.getLength(), rInputSplit.getLength());
assertArrayEquals(wInputSplit.getLocations(), rInputSplit.getLocations());
} catch (InterruptedException e) {
e.printStackTrace();
fail(e.getMessage());
}
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
wInputSplit = new TSFInputSplit(path, rowGroupMetaDataList, start, length, hosts);
rInputSplit = new TSFInputSplit();
}
@Test
public void testInputSplitWriteAndRead() {
try {
// call the write method to serialize the object
wInputSplit.write(DataOutputBuffer);
DataOutputBuffer.flush();
DataInputBuffer.reset(DataOutputBuffer.getData(), DataOutputBuffer.getLength());
rInputSplit.readFields(DataInputBuffer);
DataInputBuffer.close();
DataOutputBuffer.close();
// assert
assertEquals(wInputSplit.getPath(), rInputSplit.getPath());
assertEquals(wInputSplit.getNumOfDeviceRowGroup(), rInputSplit.getNumOfDeviceRowGroup());
//assertEquals(wInputSplit.getDeviceRowGroupMetaDataList(), rInputSplit.getDeviceRowGroupMetaDataList());
assertEquals(wInputSplit.getStart(), rInputSplit.getStart());
try {
assertEquals(wInputSplit.getLength(), rInputSplit.getLength());
assertArrayEquals(wInputSplit.getLocations(), rInputSplit.getLocations());
} catch (InterruptedException e) {
e.printStackTrace();
fail(e.getMessage());
}
} catch (IOException e) {
e.printStackTrace();
fail(e.getMessage());
}
}
}

214
hadoop/src/test/java/cn/edu/thu/tsfile/hadoop/TsFileTestHelper.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -18,130 +22,126 @@ package org.apache.iotdb.tsfile.hadoop;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.json.JSONArray;
import org.json.JSONObject;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
import org.apache.iotdb.tsfile.common.utils.ITsRandomAccessFileWriter;
import org.apache.iotdb.tsfile.common.utils.TsRandomAccessFileWriter;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.read.TsRandomAccessLocalFileReader;
import org.apache.iotdb.tsfile.timeseries.basis.TsFile;
import org.apache.iotdb.tsfile.write.exception.WriteProcessException;
import org.json.JSONArray;
import org.json.JSONObject;
public class TsFileTestHelper {
public static void deleteTsFile(String filePath){
File file = new File(filePath);
file.delete();
}
public static void restoreConf(){
}
public static void writeTsFile(String filePath) {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
conf.pageSizeInByte=100;
conf.groupSizeInByte = 3000;
conf.pageCheckSizeThreshold = 1;
conf.maxStringLength = 2;
public static void deleteTsFile(String filePath) {
File file = new File(filePath);
file.delete();
}
File file = new File(filePath);
public static void restoreConf() {
if (file.exists())
file.delete();
}
JSONObject jsonSchema = getJsonSchema();
try {
ITsRandomAccessFileWriter output = new TsRandomAccessFileWriter(new File(filePath));
TsFile tsFile = new TsFile(output, jsonSchema);
String line = "";
for(int i = 1;i<10;i++){
line = "root.car.d1,"+i+",s1,1,s2,1,s3,0.1,s4,0.1,s5,true,s6,tsfile";
tsFile.writeLine(line);
}
tsFile.writeLine("root.car.d2,5, s1, 5, s2, 50, s3, 200.5, s4, 0.5");
tsFile.writeLine("root.car.d2,6, s1, 6, s2, 60, s3, 200.6, s4, 0.6");
tsFile.writeLine("root.car.d2,7, s1, 7, s2, 70, s3, 200.7, s4, 0.7");
tsFile.writeLine("root.car.d2,8, s1, 8, s2, 80, s3, 200.8, s4, 0.8");
tsFile.close();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
} catch (WriteProcessException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
public static void writeTsFile(String filePath) {
private static JSONObject getJsonSchema() {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
JSONObject s1 = new JSONObject();
s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
conf.pageSizeInByte = 100;
conf.groupSizeInByte = 3000;
conf.pageCheckSizeThreshold = 1;
conf.maxStringLength = 2;
JSONObject s2 = new JSONObject();
s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
File file = new File(filePath);
JSONObject s3 = new JSONObject();
s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
if (file.exists()) {
file.delete();
}
JSONObject s4 = new JSONObject();
s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s5 = new JSONObject();
s5.put(JsonFormatConstant.MEASUREMENT_UID, "s5");
s5.put(JsonFormatConstant.DATA_TYPE, TSDataType.BOOLEAN.toString());
s5.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s6 = new JSONObject();
s6.put(JsonFormatConstant.MEASUREMENT_UID, "s6");
s6.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
s6.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject jsonSchema = getJsonSchema();
try {
ITsRandomAccessFileWriter output = new TsRandomAccessFileWriter(new File(filePath));
TsFile tsFile = new TsFile(output, jsonSchema);
String line = "";
for (int i = 1; i < 10; i++) {
line = "root.car.d1," + i + ",s1,1,s2,1,s3,0.1,s4,0.1,s5,true,s6,tsfile";
tsFile.writeLine(line);
}
tsFile.writeLine("root.car.d2,5, s1, 5, s2, 50, s3, 200.5, s4, 0.5");
tsFile.writeLine("root.car.d2,6, s1, 6, s2, 60, s3, 200.6, s4, 0.6");
tsFile.writeLine("root.car.d2,7, s1, 7, s2, 70, s3, 200.7, s4, 0.7");
tsFile.writeLine("root.car.d2,8, s1, 8, s2, 80, s3, 200.8, s4, 0.8");
tsFile.close();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
} catch (WriteProcessException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
private static JSONObject getJsonSchema() {
TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
JSONObject s1 = new JSONObject();
s1.put(JsonFormatConstant.MEASUREMENT_UID, "s1");
s1.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT32.toString());
s1.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONArray measureGroup = new JSONArray();
measureGroup.put(s1);
measureGroup.put(s2);
measureGroup.put(s3);
measureGroup.put(s4);
measureGroup.put(s5);
measureGroup.put(s6);
JSONObject s2 = new JSONObject();
s2.put(JsonFormatConstant.MEASUREMENT_UID, "s2");
s2.put(JsonFormatConstant.DATA_TYPE, TSDataType.INT64.toString());
s2.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject jsonSchema = new JSONObject();
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup);
return jsonSchema;
}
public static void main(String[] args) throws FileNotFoundException, IOException{
String filePath = "tsfile";
File file = new File(filePath);
System.out.println(file.exists());
file.delete();
writeTsFile(filePath);
TsFile tsFile = new TsFile(new TsRandomAccessLocalFileReader(filePath));
System.out.println(tsFile.getAllColumns());
System.out.println(tsFile.getAllDeltaObject());
System.out.println(tsFile.getAllSeries());
System.out.println(tsFile.getDeltaObjectRowGroupCount());
System.out.println(tsFile.getDeltaObjectTypes());
System.out.println(tsFile.getRowGroupPosList());
tsFile.close();
}
JSONObject s3 = new JSONObject();
s3.put(JsonFormatConstant.MEASUREMENT_UID, "s3");
s3.put(JsonFormatConstant.DATA_TYPE, TSDataType.FLOAT.toString());
s3.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s4 = new JSONObject();
s4.put(JsonFormatConstant.MEASUREMENT_UID, "s4");
s4.put(JsonFormatConstant.DATA_TYPE, TSDataType.DOUBLE.toString());
s4.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s5 = new JSONObject();
s5.put(JsonFormatConstant.MEASUREMENT_UID, "s5");
s5.put(JsonFormatConstant.DATA_TYPE, TSDataType.BOOLEAN.toString());
s5.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONObject s6 = new JSONObject();
s6.put(JsonFormatConstant.MEASUREMENT_UID, "s6");
s6.put(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
s6.put(JsonFormatConstant.MEASUREMENT_ENCODING, conf.valueEncoder);
JSONArray measureGroup = new JSONArray();
measureGroup.put(s1);
measureGroup.put(s2);
measureGroup.put(s3);
measureGroup.put(s4);
measureGroup.put(s5);
measureGroup.put(s6);
JSONObject jsonSchema = new JSONObject();
jsonSchema.put(JsonFormatConstant.DELTA_TYPE, "test_type");
jsonSchema.put(JsonFormatConstant.JSON_SCHEMA, measureGroup);
return jsonSchema;
}
public static void main(String[] args) throws FileNotFoundException, IOException {
String filePath = "tsfile";
File file = new File(filePath);
System.out.println(file.exists());
file.delete();
writeTsFile(filePath);
TsFile tsFile = new TsFile(new TsRandomAccessLocalFileReader(filePath));
System.out.println(tsFile.getAllColumns());
System.out.println(tsFile.getAllDeltaObject());
System.out.println(tsFile.getAllSeries());
System.out.println(tsFile.getDeltaObjectRowGroupCount());
System.out.println(tsFile.getDeltaObjectTypes());
System.out.println(tsFile.getRowGroupPosList());
tsFile.close();
}
}

10
iotdb-cli/cli/bin/export-csv.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb-cli/cli/bin/export-csv.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb-cli/cli/bin/import-csv.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb-cli/cli/bin/import-csv.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb-cli/cli/bin/run-client.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb-cli/cli/bin/start-client.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb-cli/cli/bin/start-client.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb-cli/pom.xml
View File

@ -3,9 +3,13 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0

1129
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
View File

File diff suppressed because it is too large Load Diff

569
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/Client.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,297 +19,322 @@
*/
package org.apache.iotdb.cli.client;
import jline.console.ConsoleReader;
import jline.console.completer.ArgumentCompleter;
import jline.console.completer.Completer;
import jline.console.completer.StringsCompleter;
import org.apache.commons.cli.*;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jline.console.ConsoleReader;
import jline.console.completer.ArgumentCompleter;
import jline.console.completer.Completer;
import jline.console.completer.StringsCompleter;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
public class Client extends AbstractClient {
private static final String[] FUNCTION_NAME_LIST = new String[] { "count", "max_time", "min_time", "max_value",
"min_value", "subsequence_matching", // with parameter(s)
"now()" // without parameter
};
private static final String[] FUNCTION_NAME_LIST = new String[]{"count", "max_time", "min_time",
"max_value",
"min_value", "subsequence_matching", // with parameter(s)
"now()" // without parameter
};
private static final String[] KEY_WORD_LIST = new String[] { "SHOW", "SELECT", "DROP", "UPDATE", "DELETE", "CREATE",
"INSERT", "INDEX", "TIMESERIES", "TIME", "TIMESTAMP", "VALUES", "FROM", "WHERE", "TO", "ON", "WITH",
"USING", "AND", "OR", "USER", "ROLE", "EXIT", "QUIT", "IMPORT" };
private static final String[] KEY_WORD_LIST = new String[]{"SHOW", "SELECT", "DROP", "UPDATE",
"DELETE", "CREATE",
"INSERT", "INDEX", "TIMESERIES", "TIME", "TIMESTAMP", "VALUES", "FROM", "WHERE", "TO", "ON",
"WITH",
"USING", "AND", "OR", "USER", "ROLE", "EXIT", "QUIT", "IMPORT"};
private static final String[] CONF_NAME_LIST = new String[] { // set <key>=<value>
"time_display_type", "time_zone", // specific format
"fetch_size", "max_display_num", // integer
"storage group to" // special
};
private static final String[] CONF_NAME_LIST = new String[]{ // set <key>=<value>
"time_display_type", "time_zone", // specific format
"fetch_size", "max_display_num", // integer
"storage group to" // special
};
private static final String[] PARAM_NAME_LIST = new String[] { "datatype", "encoding", // list
"window_length", // integer
};
private static final String[] PARAM_NAME_LIST = new String[]{"datatype", "encoding", // list
"window_length", // integer
};
private static final String[] DATA_TYPE_LIST = new String[] { "BOOLEAN", "INT32", "INT64", "INT96", "FLOAT",
"DOUBLE", "TEXT", "FIXED_LEN_BYTE_ARRAY", "ENUMS", "BIGDECIMAL" };
private static final String[] DATA_TYPE_LIST = new String[]{"BOOLEAN", "INT32", "INT64", "INT96",
"FLOAT",
"DOUBLE", "TEXT", "FIXED_LEN_BYTE_ARRAY", "ENUMS", "BIGDECIMAL"};
private static final String[] ENCODING_LIST = new String[] { "PLAIN", "PLAIN_DICTIONARY", "RLE", "DIFF", "TS_2DIFF",
"BITMAP", "GORILLA" };
private static final String[] ENCODING_LIST = new String[]{"PLAIN", "PLAIN_DICTIONARY", "RLE",
"DIFF", "TS_2DIFF",
"BITMAP", "GORILLA"};
public static void main(String[] args) throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
IoTDBConnection connection = null;
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
/**
* IoTDB CLI main function.
*
* @param args launch arguments
* @throws ClassNotFoundException ClassNotFoundException
*/
public static void main(String[] args) throws ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
IoTDBConnection connection = null;
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
if (args == null || args.length == 0) {
System.out.println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) -h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
hf.printHelp(SCRIPT_HINT, options, true);
return;
}
init();
args = removePasswordArgs(args);
if (args == null || args.length == 0) {
System.out.println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) "
+ "-h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
hf.printHelp(SCRIPT_HINT, options, true);
return;
}
init();
args = removePasswordArgs(args);
try {
commandLine = parser.parse(options, args);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(SCRIPT_HINT, options, true);
return;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
commandLine = parser.parse(options, args);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(SCRIPT_HINT, options, true);
return;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
setMaxDisplayNumber(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
} catch (NumberFormatException e) {
System.out.println(IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
}
}
} catch (ParseException e) {
System.out.println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) -h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
ConsoleReader reader = null;
try {
reader = new ConsoleReader();
reader.setExpandEvents(false);
// for (Completer completer : getCommandCompleter()) {
// reader.addCompleter(completer);
// }
// ((CandidateListCompletionHandler) reader.getCompletionHandler()).setPrintSpaceAfterFullCompletion(false);
String s;
try {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, true, null);
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
try {
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
} catch (SQLException e) {
System.out.println(String.format("%s> %s. Host is %s, port is %s.", IOTDB_CLI_PREFIX,
e.getMessage(), host, port));
return;
}
} catch (ArgsErrorException e) {
// System.out.println(TSFILEDB_CLI_PREFIX + ": " + e.getMessage());
return;
}
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
s = reader.readLine(IOTDB_CLI_PREFIX + "> ", null);
if (s == null) {
continue;
} else {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !cmd.trim().equals("")) {
OPERATION_RESULT result = handleInputInputCmd(cmd, connection);
switch (result) {
case RETURN_OPER:
return;
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
}
}
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
} finally {
if (reader != null) {
reader.close();
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
}
}
setMaxDisplayNumber(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
} catch (NumberFormatException e) {
System.out.println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
}
}
} catch (ParseException e) {
System.out.println(
"Require more params input, eg. ./start-client.sh(start-client.bat if Windows) "
+ "-h xxx.xxx.xxx.xxx -p xxxx -u xxx.");
System.out.println("For more information, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
private static Completer[] getCommandCompleter() {
List<String> candidateStrings = new ArrayList<>();
for (String s : FUNCTION_NAME_LIST) {
if (!s.endsWith("()")) {
candidateStrings.add(s + "(");
} else { // for functions with no parameter, such as now().
candidateStrings.add(s + " ");
}
ConsoleReader reader = null;
try {
reader = new ConsoleReader();
reader.setExpandEvents(false);
// for (Completer completer : getCommandCompleter()) {
// reader.addCompleter(completer);
// }
// ((CandidateListCompletionHandler) reader.getCompletionHandler())
// .setPrintSpaceAfterFullCompletion(false);
String s;
try {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, true, null);
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
for (String s : KEY_WORD_LIST) {
candidateStrings.add(s + " ");
candidateStrings.add(s.toLowerCase() + " ");
try {
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
} catch (SQLException e) {
System.out.println(String.format("%s> %s. Host is %s, port is %s.", IOTDB_CLI_PREFIX,
e.getMessage(), host, port));
return;
}
StringsCompleter strCompleter = new StringsCompleter(candidateStrings);
ArgumentCompleter.ArgumentDelimiter delim = new ArgumentCompleter.AbstractArgumentDelimiter() {
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
char c = buffer.charAt(pos);
return (Character.isWhitespace(c) || c == '(' || c == ')' || c == ',');
}
};
final ArgumentCompleter argCompleter = new ArgumentCompleter(delim, strCompleter);
argCompleter.setStrict(false);
StringsCompleter confCompleter = new StringsCompleter(Arrays.asList(CONF_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter setCompleter = new StringsCompleter(Arrays.asList("set", "show")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && (buffer.equals("set") || buffer.equals("show"))
? super.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter confPropCompleter = new ArgumentCompleter(setCompleter, confCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
} catch (ArgsErrorException e) {
// System.out.println(TSFILEDB_CLI_PREFIX + ": " + e.getMessage());
return;
}
StringsCompleter insertConfCompleter = new StringsCompleter(Arrays.asList("into")) {
@Override
public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter insertCompleter = new StringsCompleter(Arrays.asList("insert")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && (buffer.equals("insert")) ? super.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter insertPropCompleter = new ArgumentCompleter(insertCompleter, insertConfCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
StringsCompleter withParamCompleter = new StringsCompleter(Arrays.asList(PARAM_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1) {
int equalsIdx = buffer.indexOf("=");
if (equalsIdx != -1) {
String confName = buffer.substring(0, equalsIdx);
String value = buffer.substring(equalsIdx + 1).toUpperCase();
if (confName.startsWith("encoding")) {
for (String str : ENCODING_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
} else if (confName.startsWith("datatype")) {
for (String str : DATA_TYPE_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
}
return cursor;
}
}
return result;
while (true) {
s = reader.readLine(IOTDB_CLI_PREFIX + "> ", null);
if (s == null) {
continue;
} else {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !cmd.trim().equals("")) {
OperationResult result = handleInputInputCmd(cmd, connection);
switch (result) {
case RETURN_OPER:
return;
case CONTINUE_OPER:
continue;
default:
break;
}
}
};
ArgumentCompleter withParamPropCompleter = new ArgumentCompleter(delim, withParamCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
withParamPropCompleter.setStrict(false);
return new Completer[] { confPropCompleter, insertPropCompleter, withParamPropCompleter, argCompleter };
}
}
}
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
} finally {
if (reader != null) {
reader.close();
}
if (connection != null) {
try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}
private static Completer[] getCommandCompleter() {
List<String> candidateStrings = new ArrayList<>();
for (String s : FUNCTION_NAME_LIST) {
if (!s.endsWith("()")) {
candidateStrings.add(s + "(");
} else { // for functions with no parameter, such as now().
candidateStrings.add(s + " ");
}
}
for (String s : KEY_WORD_LIST) {
candidateStrings.add(s + " ");
candidateStrings.add(s.toLowerCase() + " ");
}
StringsCompleter strCompleter = new StringsCompleter(candidateStrings);
ArgumentCompleter.ArgumentDelimiter delim = new ArgumentCompleter.AbstractArgumentDelimiter() {
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
char c = buffer.charAt(pos);
return (Character.isWhitespace(c) || c == '(' || c == ')' || c == ',');
}
};
final ArgumentCompleter argCompleter = new ArgumentCompleter(delim, strCompleter);
argCompleter.setStrict(false);
StringsCompleter confCompleter = new StringsCompleter(Arrays.asList(CONF_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter setCompleter = new StringsCompleter(Arrays.asList("set", "show")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && (buffer.equals("set") || buffer.equals("show"))
? super.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter confPropCompleter = new ArgumentCompleter(setCompleter, confCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
StringsCompleter insertConfCompleter = new StringsCompleter(Arrays.asList("into")) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1 && buffer.charAt(cursor - 1) == '=') {
String confName = buffer.substring(0, cursor - 1);
switch (confName) { // TODO: give config suggestion
default:
break;
}
return cursor;
}
return result;
}
};
StringsCompleter insertCompleter = new StringsCompleter(Arrays.asList("insert")) {
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
return buffer != null && (buffer.equals("insert")) ? super
.complete(buffer, cursor, candidates) : -1;
}
};
ArgumentCompleter insertPropCompleter = new ArgumentCompleter(insertCompleter,
insertConfCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
StringsCompleter withParamCompleter = new StringsCompleter(Arrays.asList(PARAM_NAME_LIST)) {
@Override
public int complete(final String buffer, final int cursor,
final List<CharSequence> candidates) {
int result = super.complete(buffer, cursor, candidates);
if (candidates.isEmpty() && cursor > 1) {
int equalsIdx = buffer.indexOf("=");
if (equalsIdx != -1) {
String confName = buffer.substring(0, equalsIdx);
String value = buffer.substring(equalsIdx + 1).toUpperCase();
if (confName.startsWith("encoding")) {
for (String str : ENCODING_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
} else if (confName.startsWith("datatype")) {
for (String str : DATA_TYPE_LIST) {
if (str.startsWith(value) && !str.equals(value)) {
candidates.add(str);
}
}
return equalsIdx + 1;
}
return cursor;
}
}
return result;
}
};
ArgumentCompleter withParamPropCompleter = new ArgumentCompleter(delim, withParamCompleter) {
@Override
public int complete(String buffer, int offset, List<CharSequence> completions) {
int ret = super.complete(buffer, offset, completions);
if (completions.size() == 1) {
completions.set(0, ((String) completions.get(0)).trim());
}
return ret;
}
};
withParamPropCompleter.setStrict(false);
return new Completer[]{confPropCompleter, insertPropCompleter, withParamPropCompleter,
argCompleter};
}
}

255
iotdb-cli/src/main/java/org/apache/iotdb/cli/client/WinClient.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,6 +19,10 @@
*/
package org.apache.iotdb.cli.client;
import java.io.Console;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Scanner;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@ -25,131 +33,132 @@ import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import java.io.Console;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Scanner;
public class WinClient extends AbstractClient {
public static void main(String[] args) throws ClassNotFoundException, SQLException {
Class.forName(Config.JDBC_DRIVER_NAME);
IoTDBConnection connection = null;
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
/**
* main function.
*
* @param args -console args
*/
public static void main(String[] args) throws ClassNotFoundException, SQLException {
Class.forName(Config.JDBC_DRIVER_NAME);
IoTDBConnection connection = null;
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
if (args == null || args.length == 0) {
System.out.println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
init();
args = removePasswordArgs(args);
try {
commandLine = parser.parse(options, args);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
maxPrintRowCount = Integer.valueOf(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
if (maxPrintRowCount < 0) {
maxPrintRowCount = Integer.MAX_VALUE;
}
} catch (NumberFormatException e) {
System.out.println(IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
}
}
} catch (ParseException e) {
System.out.println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
Scanner scanner = null;
try {
String s;
try {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, true, null);
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = readPassword();
}
try {
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
} catch (SQLException e) {
System.out.println(IOTDB_CLI_PREFIX + "> " + e.getMessage());
return;
}
} catch (ArgsErrorException e) {
// System.out.println(TSFILEDB_CLI_PREFIX + ": " + e.getMessage());
return;
}
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
scanner = new Scanner(System.in);
while (true) {
System.out.print(IOTDB_CLI_PREFIX + "> ");
s = scanner.nextLine();
if (s == null) {
continue;
} else {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !cmd.trim().equals("")) {
OPERATION_RESULT result = handleInputInputCmd(cmd, connection);
switch (result) {
case RETURN_OPER:
return;
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
}
}
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
} finally {
if (scanner != null) {
scanner.close();
}
if (connection != null) {
connection.close();
}
}
if (args == null || args.length == 0) {
System.out.println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
private static String readPassword() {
Console c = System.console();
if (c == null) { // IN ECLIPSE IDE
System.out.print(IOTDB_CLI_PREFIX + "> please input password: ");
Scanner scanner = new Scanner(System.in);
return scanner.nextLine();
} else { // Outside Eclipse IDE
return new String(c.readPassword(IOTDB_CLI_PREFIX + "> please input password: "));
init();
args = removePasswordArgs(args);
try {
commandLine = parser.parse(options, args);
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(ISO8601_ARGS)) {
setTimeFormat("long");
}
if (commandLine.hasOption(MAX_PRINT_ROW_COUNT_ARGS)) {
try {
maxPrintRowCount = Integer.valueOf(commandLine.getOptionValue(MAX_PRINT_ROW_COUNT_ARGS));
if (maxPrintRowCount < 0) {
maxPrintRowCount = Integer.MAX_VALUE;
}
} catch (NumberFormatException e) {
System.out.println(
IOTDB_CLI_PREFIX + "> error format of max print row count, it should be number");
return;
}
}
} catch (ParseException e) {
System.out.println("Require more params input, please check the following hint.");
hf.printHelp(IOTDB_CLI_PREFIX, options, true);
return;
}
Scanner scanner = null;
try {
String s;
try {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine, false, host);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine, false, port);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine, true, null);
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = readPassword();
}
try {
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
} catch (SQLException e) {
System.out.println(IOTDB_CLI_PREFIX + "> " + e.getMessage());
return;
}
} catch (ArgsErrorException e) {
// System.out.println(TSFILEDB_CLI_PREFIX + ": " + e.getMessage());
return;
}
displayLogo(properties.getVersion());
System.out.println(IOTDB_CLI_PREFIX + "> login successfully");
scanner = new Scanner(System.in);
while (true) {
System.out.print(IOTDB_CLI_PREFIX + "> ");
s = scanner.nextLine();
if (s == null) {
continue;
} else {
String[] cmds = s.trim().split(";");
for (int i = 0; i < cmds.length; i++) {
String cmd = cmds[i];
if (cmd != null && !cmd.trim().equals("")) {
OperationResult result = handleInputInputCmd(cmd, connection);
switch (result) {
case RETURN_OPER:
return;
case CONTINUE_OPER:
continue;
default:
break;
}
}
}
}
}
} catch (Exception e) {
System.out.println(IOTDB_CLI_PREFIX + "> exit client with error " + e.getMessage());
} finally {
if (scanner != null) {
scanner.close();
}
if (connection != null) {
connection.close();
}
}
}
private static String readPassword() {
Console c = System.console();
if (c == null) { // IN ECLIPSE IDE
System.out.print(IOTDB_CLI_PREFIX + "> please input password: ");
Scanner scanner = new Scanner(System.in);
return scanner.nextLine();
} else { // Outside Eclipse IDE
return new String(c.readPassword(IOTDB_CLI_PREFIX + "> please input password: "));
}
}
}

18
iotdb-cli/src/main/java/org/apache/iotdb/cli/exception/ArgsErrorException.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -17,10 +21,10 @@ package org.apache.iotdb.cli.exception;
public class ArgsErrorException extends Exception {
private static final long serialVersionUID = -3614543017182165265L;
private static final long serialVersionUID = -3614543017182165265L;
public ArgsErrorException(String msg) {
super(msg);
}
public ArgsErrorException(String msg) {
super(msg);
}
}

154
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/AbstractCsvTool.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,6 +19,8 @@
*/
package org.apache.iotdb.cli.tool;
import java.io.IOException;
import java.time.ZoneId;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
import org.apache.iotdb.cli.exception.ArgsErrorException;
@ -22,95 +28,91 @@ import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBSQLException;
import org.apache.thrift.TException;
import java.io.IOException;
import java.time.ZoneId;
public abstract class AbstractCsvTool {
protected static final String HOST_ARGS = "h";
protected static final String HOST_NAME = "host";
protected static final String HELP_ARGS = "help";
protected static final String HOST_ARGS = "h";
protected static final String HOST_NAME = "host";
protected static final String PORT_ARGS = "p";
protected static final String PORT_NAME = "port";
protected static final String HELP_ARGS = "help";
protected static final String PASSWORD_ARGS = "pw";
protected static final String PASSWORD_NAME = "password";
protected static final String PORT_ARGS = "p";
protected static final String PORT_NAME = "port";
protected static final String USERNAME_ARGS = "u";
protected static final String USERNAME_NAME = "username";
protected static final String PASSWORD_ARGS = "pw";
protected static final String PASSWORD_NAME = "password";
protected static final String TIME_FORMAT_ARGS = "tf";
protected static final String TIME_FORMAT_NAME = "timeformat";
protected static final String USERNAME_ARGS = "u";
protected static final String USERNAME_NAME = "username";
protected static final String TIME_ZONE_ARGS = "tz";
protected static final String TIME_ZONE_NAME = "timeZone";
protected static final String TIME_FORMAT_ARGS = "tf";
protected static final String TIME_FORMAT_NAME = "timeformat";
protected static String host;
protected static String port;
protected static String username;
protected static String password;
protected static final String TIME_ZONE_ARGS = "tz";
protected static final String TIME_ZONE_NAME = "timeZone";
protected static final int MAX_HELP_CONSOLE_WIDTH = 92;
protected static final String DEFAULT_TIME_FORMAT = "ISO8601";
protected static final String[] SUPPORT_TIME_FORMAT = new String[]{DEFAULT_TIME_FORMAT, "default",
"long",
"number", "timestamp", "yyyy-MM-dd HH:mm:ss", "yyyy/MM/dd HH:mm:ss", "yyyy.MM.dd HH:mm:ss",
"yyyy-MM-dd'T'HH:mm:ss", "yyyy/MM/dd'T'HH:mm:ss", "yyyy.MM.dd'T'HH:mm:ss",
"yyyy-MM-dd HH:mm:ssZZ",
"yyyy/MM/dd HH:mm:ssZZ", "yyyy.MM.dd HH:mm:ssZZ", "yyyy-MM-dd'T'HH:mm:ssZZ",
"yyyy/MM/dd'T'HH:mm:ssZZ",
"yyyy.MM.dd'T'HH:mm:ssZZ", "yyyy/MM/dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSS",
"yyyy.MM.dd HH:mm:ss.SSS",
"yyyy/MM/dd'T'HH:mm:ss.SSS", "yyyy-MM-dd'T'HH:mm:ss.SSS", "yyyy-MM-dd'T'HH:mm:ss.SSS",
"yyyy.MM.dd'T'HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSSZZ", "yyyy/MM/dd HH:mm:ss.SSSZZ",
"yyyy.MM.dd HH:mm:ss.SSSZZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyy/MM/dd'T'HH:mm:ss.SSSZZ",};
protected static String host;
protected static String port;
protected static String username;
protected static String password;
protected static ZoneId zoneId;
protected static final int MAX_HELP_CONSOLE_WIDTH = 92;
// protected static final String JDBC_DRIVER = "case hu.tsfiledb.jdbc.IoTDBDriver";
protected static String timeZoneID;
protected static String timeFormat;
protected static IoTDBConnection connection;
protected static ZoneId zoneId;
protected static String timeZoneID;
protected static String timeFormat;
// protected static final String JDBC_DRIVER = "case hu.tsfiledb.jdbc.TsfileDriver";
protected static final String DEFAULT_TIME_FORMAT = "ISO8601";
protected static final String[] SUPPORT_TIME_FORMAT = new String[] { DEFAULT_TIME_FORMAT, "default", "long",
"number", "timestamp", "yyyy-MM-dd HH:mm:ss", "yyyy/MM/dd HH:mm:ss", "yyyy.MM.dd HH:mm:ss",
"yyyy-MM-dd'T'HH:mm:ss", "yyyy/MM/dd'T'HH:mm:ss", "yyyy.MM.dd'T'HH:mm:ss", "yyyy-MM-dd HH:mm:ssZZ",
"yyyy/MM/dd HH:mm:ssZZ", "yyyy.MM.dd HH:mm:ssZZ", "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy/MM/dd'T'HH:mm:ssZZ",
"yyyy.MM.dd'T'HH:mm:ssZZ", "yyyy/MM/dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy.MM.dd HH:mm:ss.SSS",
"yyyy/MM/dd'T'HH:mm:ss.SSS", "yyyy-MM-dd'T'HH:mm:ss.SSS", "yyyy-MM-dd'T'HH:mm:ss.SSS",
"yyyy.MM.dd'T'HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSSZZ", "yyyy/MM/dd HH:mm:ss.SSSZZ",
"yyyy.MM.dd HH:mm:ss.SSSZZ", "yyyy-MM-dd'T'HH:mm:ss.SSSZZ", "yyyy/MM/dd'T'HH:mm:ss.SSSZZ", };
protected static IoTDBConnection connection;
protected static String checkRequiredArg(String arg, String name, CommandLine commandLine)
throws ArgsErrorException {
String str = commandLine.getOptionValue(arg);
if (str == null) {
String msg = String.format("Required values for option '%s' not provided", name);
System.out.println(msg);
System.out.println("Use -help for more information");
throw new ArgsErrorException(msg);
}
return str;
protected static String checkRequiredArg(String arg, String name, CommandLine commandLine)
throws ArgsErrorException {
String str = commandLine.getOptionValue(arg);
if (str == null) {
String msg = String.format("Required values for option '%s' not provided", name);
System.out.println(msg);
System.out.println("Use -help for more information");
throw new ArgsErrorException(msg);
}
return str;
}
protected static void setTimeZone() throws IoTDBSQLException, TException {
if (timeZoneID != null) {
connection.setTimeZone(timeZoneID);
}
zoneId = ZoneId.of(connection.getTimeZone());
protected static void setTimeZone() throws IoTDBSQLException, TException {
if (timeZoneID != null) {
connection.setTimeZone(timeZoneID);
}
zoneId = ZoneId.of(connection.getTimeZone());
}
protected static void parseBasicParams(CommandLine commandLine, ConsoleReader reader)
throws ArgsErrorException, IOException {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine);
protected static void parseBasicParams(CommandLine commandLine, ConsoleReader reader)
throws ArgsErrorException, IOException {
host = checkRequiredArg(HOST_ARGS, HOST_NAME, commandLine);
port = checkRequiredArg(PORT_ARGS, PORT_NAME, commandLine);
username = checkRequiredArg(USERNAME_ARGS, USERNAME_NAME, commandLine);
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
password = commandLine.getOptionValue(PASSWORD_ARGS);
if (password == null) {
password = reader.readLine("please input your password:", '\0');
}
}
protected static boolean checkTimeFormat() {
for (String format : SUPPORT_TIME_FORMAT) {
if (timeFormat.equals(format)) {
return true;
}
}
System.out.println(String.format("Input time format %s is not supported, "
+ "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS", timeFormat));
return false;
protected static boolean checkTimeFormat() {
for (String format : SUPPORT_TIME_FORMAT) {
if (timeFormat.equals(format)) {
return true;
}
}
System.out.println(String.format("Input time format %s is not supported, "
+ "please input like yyyy-MM-dd\\ HH:mm:ss.SSS or yyyy-MM-dd'T'HH:mm:ss.SSS", timeFormat));
return false;
}
}

213
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/CsvTestDataGen.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -23,104 +27,125 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
public class CsvTestDataGen {
private static String[] iso = { "Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3",
"1970-01-01T08:00:00.001+08:00,,1,pass,1,1", "1970-01-01T08:00:00.002+08:00,,2,pass,,",
"1970-01-01T08:00:00.003+08:00,,3,pass,,", "1970-01-01T08:00:00.004+08:00,4,,,4,4" };
private static String[] defaultLong = {
"Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3", "1,,1,pass,1,1",
"2,,2,pass,,", "1970-01-01T08:00:00.003+08:00,,3,pass,,", "3,4,,,4,4" };
private static String[] userSelfDefine = {
"Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3", "1971,,1,pass,1,1",
"1972,,2,pass,,", "1973-01-01T08:00:00.003+08:00,,3,pass,,", "1974,4,,,4,4" };
private static BufferedWriter bw = null;
public static String isoDataGen() {
String path = System.getProperties().getProperty("user.dir") + "/src/test/resources/iso.csv";
File file = new File(path);
private static String[] iso = {
"Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3",
"1970-01-01T08:00:00.001+08:00,,1,pass,1,1", "1970-01-01T08:00:00.002+08:00,,2,pass,,",
"1970-01-01T08:00:00.003+08:00,,3,pass,,", "1970-01-01T08:00:00.004+08:00,4,,,4,4"};
private static String[] defaultLong = {
"Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3",
"1,,1,pass,1,1",
"2,,2,pass,,", "1970-01-01T08:00:00.003+08:00,,3,pass,,", "3,4,,,4,4"};
private static String[] userSelfDefine = {
"Time,root.fit.p.s1,root.fit.d1.s1,root.fit.d1.s2,root.fit.d2.s1,root.fit.d2.s3",
"1971,,1,pass,1,1",
"1972,,2,pass,,", "1973-01-01T08:00:00.003+08:00,,3,pass,,", "1974,4,,,4,4"};
private static BufferedWriter bw = null;
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : iso) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
/**
* generate iso.csv data.
*
* @return path
*/
public static String isoDataGen() {
String path = System.getProperties().getProperty("user.dir") + "/src/test/resources/iso.csv";
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : iso) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
}
public static String defaultLongDataGen() {
String path = System.getProperties().getProperty("user.dir") + "/src/test/resources/defaultLong.csv";
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : defaultLong) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
/**
* generate default long data file: defaultLong.csv .
*
* @return path
*/
public static String defaultLongDataGen() {
String path =
System.getProperties().getProperty("user.dir") + "/src/test/resources/defaultLong.csv";
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : defaultLong) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
}
public static String userSelfDataGen() {
String path = System.getProperties().getProperty("user.dir") + "/src/test/resources/userSelfDefine.csv";
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : userSelfDefine) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
/**
* generate user defined data: userSelfDefine.csv .
*
* @return path
*/
public static String userSelfDataGen() {
String path =
System.getProperties().getProperty("user.dir") + "/src/test/resources/userSelfDefine.csv";
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)));
for (String str : userSelfDefine) {
bw.write(str + "\n");
}
bw.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
bw.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return path;
}
public static void main(String[] args) {
System.out.println(defaultLongDataGen());
}
public static void main(String[] args) {
System.out.println(defaultLongDataGen());
}
}

544
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ExportCsv.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,20 +19,6 @@
*/
package org.apache.iotdb.cli.tool;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.Config;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.thrift.TException;
import jline.console.ConsoleReader;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@ -43,275 +33,303 @@ import java.sql.Statement;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
/**
* Export CSV file.
*
* @author aru cheng
* @version 1.0.0 20170719
*/
public class ExportCsv extends AbstractCsvTool {
private static final String TARGET_FILE_ARGS = "td";
private static final String TARGET_FILE_NAME = "targetDirectory";
private static final String SQL_FILE_ARGS = "s";
private static final String SQL_FILE_NAME = "sqlfile";
private static final String TARGET_FILE_ARGS = "td";
private static final String TARGET_FILE_NAME = "targetDirectory";
private static final String TSFILEDB_CLI_PREFIX = "ExportCsv";
private static final String SQL_FILE_ARGS = "s";
private static final String SQL_FILE_NAME = "sqlfile";
private static final String DUMP_FILE_NAME = "dump";
private static final String TSFILEDB_CLI_PREFIX = "ExportCsv";
private static String targetDirectory;
private static final String DUMP_FILE_NAME = "dump";
public static void main(String[] args) throws IOException, SQLException {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setOptionComparator(null); // avoid reordering
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
private static String targetDirectory;
if (args == null || args.length == 0) {
System.out.println("[ERROR] Too few params input, please check the following hint.");
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
try {
commandLine = parser.parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
/**
* main function of export csv tool.
*/
public static void main(String[] args) throws IOException, SQLException {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setOptionComparator(null); // avoid reordering
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
ConsoleReader reader = new ConsoleReader();
reader.setExpandEvents(false);
try {
parseBasicParams(commandLine, reader);
parseSpecialParams(commandLine, reader);
if (!checkTimeFormat()) {
return;
}
Class.forName(Config.JDBC_DRIVER_NAME);
String sqlFile = commandLine.getOptionValue(SQL_FILE_ARGS);
String sql;
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
setTimeZone();
if (sqlFile == null) {
sql = reader.readLine(TSFILEDB_CLI_PREFIX + "> please input query: ");
String[] values = sql.trim().split(";");
for (int i = 0; i < values.length; i++) {
dumpResult(values[i], i);
}
return;
} else {
dumpFromSQLFile(sqlFile);
}
} catch (ClassNotFoundException e) {
System.out.println(
"[ERROR] Failed to dump data because cannot find TsFile JDBC Driver, please check whether you have imported driver or not");
} catch (SQLException e) {
System.out.println(
String.format("[ERROR] Encounter an error when dumping data, error is %s", e.getMessage()));
} catch (IOException e) {
System.out.println(String.format("[ERROR] Failed to operate on file, because %s", e.getMessage()));
} catch (TException e) {
System.out.println(
String.format("[ERROR] Encounter an error when connecting to server, because %s", e.getMessage()));
} catch (ArgsErrorException e) {
} finally {
if (reader != null) {
reader.close();
}
if (connection != null) {
connection.close();
}
}
if (args == null || args.length == 0) {
System.out.println("[ERROR] Too few params input, please check the following hint.");
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
try {
commandLine = parser.parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
private static void parseSpecialParams(CommandLine commandLine, ConsoleReader reader)
throws IOException, ArgsErrorException {
targetDirectory = checkRequiredArg(TARGET_FILE_ARGS, TARGET_FILE_NAME, commandLine);
timeFormat = commandLine.getOptionValue(TIME_FORMAT_ARGS);
if (timeFormat == null) {
timeFormat = DEFAULT_TIME_FORMAT;
}
timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS);
if (!targetDirectory.endsWith(File.separator)) {
targetDirectory += File.separator;
}
}
/**
* commandline option create
*
* @return object Options
*/
private static Options createOptions() {
Options options = new Options();
Option opHost = Option.builder(HOST_ARGS).longOpt(HOST_NAME).required().argName(HOST_NAME).hasArg()
.desc("Host Name (required)").build();
options.addOption(opHost);
Option opPort = Option.builder(PORT_ARGS).longOpt(PORT_NAME).required().argName(PORT_NAME).hasArg()
.desc("Port (required)").build();
options.addOption(opPort);
Option opUsername = Option.builder(USERNAME_ARGS).longOpt(USERNAME_NAME).required().argName(USERNAME_NAME)
.hasArg().desc("Username (required)").build();
options.addOption(opUsername);
Option opPassword = Option.builder(PASSWORD_ARGS).longOpt(PASSWORD_NAME).optionalArg(true)
.argName(PASSWORD_NAME).hasArg().desc("Password (optional)").build();
options.addOption(opPassword);
Option opTargetFile = Option.builder(TARGET_FILE_ARGS).required().argName(TARGET_FILE_NAME).hasArg()
.desc("Target File Directory (required)").build();
options.addOption(opTargetFile);
Option opSqlFile = Option.builder(SQL_FILE_ARGS).argName(SQL_FILE_NAME).hasArg()
.desc("SQL File Path (optional)").build();
options.addOption(opSqlFile);
Option opTimeFormat = Option.builder(TIME_FORMAT_ARGS).argName(TIME_FORMAT_NAME).hasArg()
.desc("Output time Format in csv file. "
+ "You can choose 1) timestamp, number, long 2) ISO8601, default 3) user-defined pattern like yyyy-MM-dd\\ HH:mm:ss, default ISO8601 (optional)")
.build();
options.addOption(opTimeFormat);
Option opTimeZone = Option.builder(TIME_ZONE_ARGS).argName(TIME_ZONE_NAME).hasArg()
.desc("Time Zone eg. +08:00 or -01:00 (optional)").build();
options.addOption(opTimeZone);
Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg(false).desc("Display help information")
.build();
options.addOption(opHelp);
return options;
}
private static void dumpFromSQLFile(String filePath) throws ClassNotFoundException, IOException {
BufferedReader reader = new BufferedReader(new FileReader(filePath));
String sql = null;
int index = 0;
while ((sql = reader.readLine()) != null) {
try {
dumpResult(sql, index);
} catch (SQLException e) {
System.out.println(
String.format("[ERROR] Cannot dump data for statment %s, because %s", sql, e.getMessage()));
}
index++;
ConsoleReader reader = new ConsoleReader();
reader.setExpandEvents(false);
try {
parseBasicParams(commandLine, reader);
parseSpecialParams(commandLine, reader);
if (!checkTimeFormat()) {
return;
}
Class.forName(Config.JDBC_DRIVER_NAME);
String sqlFile = commandLine.getOptionValue(SQL_FILE_ARGS);
String sql;
connection = (IoTDBConnection) DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password);
setTimeZone();
if (sqlFile == null) {
sql = reader.readLine(TSFILEDB_CLI_PREFIX + "> please input query: ");
String[] values = sql.trim().split(";");
for (int i = 0; i < values.length; i++) {
dumpResult(values[i], i);
}
return;
} else {
dumpFromSqlFile(sqlFile);
}
} catch (ClassNotFoundException e) {
System.out.println(
"[ERROR] Failed to dump data because cannot find TsFile JDBC Driver, "
+ "please check whether you have imported driver or not");
} catch (SQLException e) {
System.out.println(
String
.format("[ERROR] Encounter an error when dumping data, error is %s", e.getMessage()));
} catch (IOException e) {
System.out
.println(String.format("[ERROR] Failed to operate on file, because %s", e.getMessage()));
} catch (TException e) {
System.out.println(
String.format("[ERROR] Encounter an error when connecting to server, because %s",
e.getMessage()));
} catch (ArgsErrorException e) {
e.printStackTrace();
} finally {
if (reader != null) {
reader.close();
}
if (connection != null) {
connection.close();
}
}
}
private static void parseSpecialParams(CommandLine commandLine, ConsoleReader reader)
throws IOException, ArgsErrorException {
targetDirectory = checkRequiredArg(TARGET_FILE_ARGS, TARGET_FILE_NAME, commandLine);
timeFormat = commandLine.getOptionValue(TIME_FORMAT_ARGS);
if (timeFormat == null) {
timeFormat = DEFAULT_TIME_FORMAT;
}
timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS);
if (!targetDirectory.endsWith(File.separator)) {
targetDirectory += File.separator;
}
}
/**
* commandline option create.
*
* @return object Options
*/
private static Options createOptions() {
Options options = new Options();
Option opHost = Option.builder(HOST_ARGS).longOpt(HOST_NAME).required().argName(HOST_NAME)
.hasArg()
.desc("Host Name (required)").build();
options.addOption(opHost);
Option opPort = Option.builder(PORT_ARGS).longOpt(PORT_NAME).required().argName(PORT_NAME)
.hasArg()
.desc("Port (required)").build();
options.addOption(opPort);
Option opUsername = Option.builder(USERNAME_ARGS).longOpt(USERNAME_NAME).required()
.argName(USERNAME_NAME)
.hasArg().desc("Username (required)").build();
options.addOption(opUsername);
Option opPassword = Option.builder(PASSWORD_ARGS).longOpt(PASSWORD_NAME).optionalArg(true)
.argName(PASSWORD_NAME).hasArg().desc("Password (optional)").build();
options.addOption(opPassword);
Option opTargetFile = Option.builder(TARGET_FILE_ARGS).required().argName(TARGET_FILE_NAME)
.hasArg()
.desc("Target File Directory (required)").build();
options.addOption(opTargetFile);
Option opSqlFile = Option.builder(SQL_FILE_ARGS).argName(SQL_FILE_NAME).hasArg()
.desc("SQL File Path (optional)").build();
options.addOption(opSqlFile);
Option opTimeFormat = Option.builder(TIME_FORMAT_ARGS).argName(TIME_FORMAT_NAME).hasArg()
.desc("Output time Format in csv file. "
+ "You can choose 1) timestamp, number, long 2) ISO8601, default 3) "
+ "user-defined pattern like yyyy-MM-dd\\ HH:mm:ss, default ISO8601 (optional)")
.build();
options.addOption(opTimeFormat);
Option opTimeZone = Option.builder(TIME_ZONE_ARGS).argName(TIME_ZONE_NAME).hasArg()
.desc("Time Zone eg. +08:00 or -01:00 (optional)").build();
options.addOption(opTimeZone);
Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg(false)
.desc("Display help information")
.build();
options.addOption(opHelp);
return options;
}
private static void dumpFromSqlFile(String filePath) throws ClassNotFoundException, IOException {
BufferedReader reader = new BufferedReader(new FileReader(filePath));
String sql = null;
int index = 0;
while ((sql = reader.readLine()) != null) {
try {
dumpResult(sql, index);
} catch (SQLException e) {
System.out.println(
String.format("[ERROR] Cannot dump data for statment %s, because %s", sql,
e.getMessage()));
}
index++;
}
reader.close();
}
/**
* Dump files from database to CSV file.
*
* @param sql export the result of executing the sql
* @param index use to create dump file name
* @throws ClassNotFoundException if cannot find driver
* @throws SQLException if SQL is not valid
*/
private static void dumpResult(String sql, int index)
throws ClassNotFoundException, SQLException {
BufferedWriter writer = null;
final String path = targetDirectory + DUMP_FILE_NAME + index + ".csv";
try {
File tf = new File(path);
if (!tf.exists()) {
if (!tf.createNewFile()) {
System.out.println("[ERROR] Could not create target file for sql statement: " + sql);
return;
}
}
writer = new BufferedWriter(new FileWriter(tf));
} catch (IOException e) {
System.out.println(e.getMessage());
return;
}
/**
* Dump files from database to CSV file
*
* @param sql
* export the result of executing the sql
* @param index
* use to create dump file name
* @throws ClassNotFoundException
* if cannot find driver
* @throws SQLException
* if SQL is not valid
*/
private static void dumpResult(String sql, int index) throws ClassNotFoundException, SQLException {
BufferedWriter writer = null;
final String path = targetDirectory + DUMP_FILE_NAME + index + ".csv";
try {
File tf = new File(path);
if (!tf.exists()) {
if (!tf.createNewFile()) {
System.out.println("[ERROR] Could not create target file for sql statement: " + sql);
return;
}
}
writer = new BufferedWriter(new FileWriter(tf));
} catch (IOException e) {
System.out.println(e.getMessage());
return;
Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
ResultSetMetaData metadata = rs.getMetaData();
long startTime = System.currentTimeMillis();
try {
int count = metadata.getColumnCount();
// write data in csv file
for (int i = 1; i <= count; i++) {
if (i < count) {
writer.write(metadata.getColumnLabel(i) + ",");
} else {
writer.write(metadata.getColumnLabel(i) + "\n");
}
}
while (rs.next()) {
if (rs.getString(1) == null || rs.getString(1).toLowerCase().equals("null")) {
writer.write(",");
} else {
ZonedDateTime dateTime;
switch (timeFormat) {
case DEFAULT_TIME_FORMAT:
case "default":
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
writer.write(dateTime.toString() + ",");
break;
case "timestamp":
case "long":
case "nubmer":
writer.write(rs.getLong(1) + ",");
break;
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
writer.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) + ",");
break;
}
Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql);
ResultSetMetaData metadata = rs.getMetaData();
long startTime = System.currentTimeMillis();
try {
int count = metadata.getColumnCount();
// write data in csv file
for (int i = 1; i <= count; i++) {
if (i < count) {
writer.write(metadata.getColumnLabel(i) + ",");
} else {
writer.write(metadata.getColumnLabel(i) + "\n");
}
for (int j = 2; j <= count; j++) {
if (j < count) {
if (rs.getString(j).equals("null")) {
writer.write(",");
} else {
writer.write(rs.getString(j) + ",");
}
} else {
if (rs.getString(j).equals("null")) {
writer.write("\n");
} else {
writer.write(rs.getString(j) + "\n");
}
}
while (rs.next()) {
if (rs.getString(1) == null || rs.getString(1).toLowerCase().equals("null")) {
writer.write(",");
} else {
ZonedDateTime dateTime;
switch (timeFormat) {
case DEFAULT_TIME_FORMAT:
case "default":
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
writer.write(dateTime.toString() + ",");
break;
case "timestamp":
case "long":
case "nubmer":
writer.write(rs.getLong(1) + ",");
break;
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(rs.getLong(1)), zoneId);
writer.write(dateTime.format(DateTimeFormatter.ofPattern(timeFormat)) + ",");
break;
}
for (int j = 2; j <= count; j++) {
if (j < count) {
if (rs.getString(j).equals("null")) {
writer.write(",");
} else {
writer.write(rs.getString(j) + ",");
}
} else {
if (rs.getString(j).equals("null")) {
writer.write("\n");
} else {
writer.write(rs.getString(j) + "\n");
}
}
}
}
}
System.out
.println(String.format("[INFO] Statement [%s] has dumped to file %s successfully! It costs %d ms.",
sql, path, (System.currentTimeMillis() - startTime)));
} catch (IOException e) {
System.out.println(e.getMessage());
} finally {
try {
writer.flush();
writer.close();
} catch (IOException e) {
System.out.println(e.getMessage());
}
if (statement != null)
statement.close();
}
}
}
System.out
.println(String
.format("[INFO] Statement [%s] has dumped to file %s successfully! It costs %d ms.",
sql, path, (System.currentTimeMillis() - startTime)));
} catch (IOException e) {
System.out.println(e.getMessage());
} finally {
try {
writer.flush();
writer.close();
} catch (IOException e) {
System.out.println(e.getMessage());
}
if (statement != null) {
statement.close();
}
}
}
}

720
iotdb-cli/src/main/java/org/apache/iotdb/cli/tool/ImportCsv.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,20 +19,6 @@
*/
package org.apache.iotdb.cli.tool;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@ -46,371 +36,399 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import jline.console.ConsoleReader;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.thrift.TException;
/**
* CSV File To IoTDB
* read a CSV formatted data File and insert all the data into IoTDB.
*
* @author zhanggr
*/
public class ImportCsv extends AbstractCsvTool {
private static final String FILE_ARGS = "f";
private static final String FILE_NAME = "file or folder";
private static final String FILE_SUFFIX = "csv";
private static final String TSFILEDB_CLI_PREFIX = "ImportCsv";
private static final String ERROR_INFO_STR = "csvInsertError.error";
private static final String FILE_ARGS = "f";
private static final String FILE_NAME = "file or folder";
private static final String FILE_SUFFIX = "csv";
private static final String STRING_DATA_TYPE = "TEXT";
private static final int BATCH_EXECUTE_COUNT = 10;
private static final String TSFILEDB_CLI_PREFIX = "ImportCsv";
private static final String ERROR_INFO_STR = "csvInsertError.error";
private static String filename;
private static String errorInsertInfo = "";
private static final String STRING_DATA_TYPE = "TEXT";
private static final int BATCH_EXECUTE_COUNT = 10;
/**
* commandline option create
*
* @return object Options
*/
private static Options createOptions() {
Options options = new Options();
private static String filename;
private static String errorInsertInfo = "";
Option opHost = Option.builder(HOST_ARGS).longOpt(HOST_NAME).required().argName(HOST_NAME).hasArg()
.desc("Host Name (required)").build();
options.addOption(opHost);
/**
* create the commandline options.
*
* @return object Options
*/
private static Options createOptions() {
Options options = new Options();
Option opPort = Option.builder(PORT_ARGS).longOpt(PORT_NAME).required().argName(PORT_NAME).hasArg()
.desc("Port (required)").build();
options.addOption(opPort);
Option opHost = Option.builder(HOST_ARGS).longOpt(HOST_NAME).required()
.argName(HOST_NAME).hasArg().desc("Host Name (required)").build();
options.addOption(opHost);
Option opUsername = Option.builder(USERNAME_ARGS).longOpt(USERNAME_NAME).required().argName(USERNAME_NAME)
.hasArg().desc("Username (required)").build();
options.addOption(opUsername);
Option opPort = Option.builder(PORT_ARGS).longOpt(PORT_NAME).required()
.argName(PORT_NAME).hasArg().desc("Port (required)").build();
options.addOption(opPort);
Option opPassword = Option.builder(PASSWORD_ARGS).longOpt(PASSWORD_NAME).optionalArg(true)
.argName(PASSWORD_NAME).hasArg().desc("Password (optional)").build();
options.addOption(opPassword);
Option opUsername = Option.builder(USERNAME_ARGS).longOpt(USERNAME_NAME)
.required().argName(USERNAME_NAME)
.hasArg().desc("Username (required)").build();
options.addOption(opUsername);
Option opFile = Option.builder(FILE_ARGS).required().argName(FILE_NAME).hasArg().desc(
"If input a file path, load a csv file, otherwise load all csv file under this directory (required)")
.build();
options.addOption(opFile);
Option opPassword = Option.builder(PASSWORD_ARGS).longOpt(PASSWORD_NAME)
.optionalArg(true).argName(PASSWORD_NAME).hasArg().desc("Password (optional)").build();
options.addOption(opPassword);
Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS).hasArg(false).desc("Display help information")
.build();
options.addOption(opHelp);
Option opFile = Option.builder(FILE_ARGS).required().argName(FILE_NAME).hasArg().desc(
"If input a file path, load a csv file, "
+ "otherwise load all csv file under this directory (required)")
.build();
options.addOption(opFile);
Option opTimeZone = Option.builder(TIME_ZONE_ARGS).argName(TIME_ZONE_NAME).hasArg()
.desc("Time Zone eg. +08:00 or -01:00 (optional)").build();
options.addOption(opTimeZone);
Option opHelp = Option.builder(HELP_ARGS).longOpt(HELP_ARGS)
.hasArg(false).desc("Display help information")
.build();
options.addOption(opHelp);
return options;
}
Option opTimeZone = Option.builder(TIME_ZONE_ARGS).argName(TIME_ZONE_NAME).hasArg()
.desc("Time Zone eg. +08:00 or -01:00 (optional)").build();
options.addOption(opTimeZone);
/**
* Data from csv To tsfile
*/
private static void loadDataFromCSV(File file, int index) {
Statement statement = null;
BufferedReader br = null;
BufferedWriter bw = null;
File errorFile = new File(errorInsertInfo + index);
boolean errorFlag = true;
try {
br = new BufferedReader(new FileReader(file));
if (!errorFile.exists())
errorFile.createNewFile();
bw = new BufferedWriter(new FileWriter(errorFile));
return options;
}
String line = "";
String header = br.readLine();
String[] strHeadInfo = header.split(",");
/**
* Data from csv To tsfile.
*/
private static void loadDataFromCSV(File file, int index) {
Statement statement = null;
BufferedReader br = null;
BufferedWriter bw = null;
File errorFile = new File(errorInsertInfo + index);
boolean errorFlag = true;
try {
br = new BufferedReader(new FileReader(file));
if (!errorFile.exists()) {
errorFile.createNewFile();
}
bw = new BufferedWriter(new FileWriter(errorFile));
bw.write("From " + file.getAbsolutePath());
bw.newLine();
bw.newLine();
bw.write(header);
bw.newLine();
bw.newLine();
String header = br.readLine();
Map<String, ArrayList<Integer>> deviceToColumn = new HashMap<>(); // storage csv table head info
List<String> colInfo = new ArrayList<>(); // storage csv table head info
List<String> headInfo = new ArrayList<>(); // storage csv device sensor info, corresponding csv table head
bw.write("From " + file.getAbsolutePath());
bw.newLine();
bw.newLine();
bw.write(header);
bw.newLine();
bw.newLine();
if (strHeadInfo.length <= 1) {
System.out.println("[ERROR] The CSV file" + file.getName() + " illegal, please check first line");
return;
}
// storage csv table head info
Map<String, ArrayList<Integer>> deviceToColumn = new HashMap<>();
// storage csv table head info
List<String> colInfo = new ArrayList<>();
// storage csv device sensor info, corresponding csv table head
List<String> headInfo = new ArrayList<>();
long startTime = System.currentTimeMillis();
Map<String, String> timeseriesDataType = new HashMap<>();
DatabaseMetaData databaseMetaData = connection.getMetaData();
String[] strHeadInfo = header.split(",");
if (strHeadInfo.length <= 1) {
System.out.println("[ERROR] The CSV file" + file.getName()
+ " illegal, please check first line");
return;
}
for (int i = 1; i < strHeadInfo.length; i++) {
ResultSet resultSet = databaseMetaData.getColumns(null, null, strHeadInfo[i], null);
if (resultSet.next()) {
timeseriesDataType.put(resultSet.getString(1), resultSet.getString(2));
} else {
String errorInfo = String.format("[ERROR] Database cannot find %s in %s, stop import!",
strHeadInfo[i], file.getAbsolutePath());
System.out.println(errorInfo);
bw.write(errorInfo);
errorFlag = false;
return;
}
headInfo.add(strHeadInfo[i]);
String deviceInfo = strHeadInfo[i].substring(0, strHeadInfo[i].lastIndexOf("."));
long startTime = System.currentTimeMillis();
Map<String, String> timeseriesDataType = new HashMap<>();
DatabaseMetaData databaseMetaData = connection.getMetaData();
if (!deviceToColumn.containsKey(deviceInfo)) {
deviceToColumn.put(deviceInfo, new ArrayList<>());
}
// storage every device's sensor index info
deviceToColumn.get(deviceInfo).add(i - 1);
colInfo.add(strHeadInfo[i].substring(strHeadInfo[i].lastIndexOf(".") + 1));
}
statement = connection.createStatement();
int count = 0;
List<String> tmp = new ArrayList<>();
while ((line = br.readLine()) != null) {
List<String> sqls = new ArrayList<>();
try {
sqls = createInsertSQL(line, timeseriesDataType, deviceToColumn, colInfo, headInfo);
} catch (Exception e) {
bw.write(String.format("error input line, maybe it is not complete: %s", line));
bw.newLine();
errorFlag = false;
}
for (String str : sqls) {
try {
count++;
statement.addBatch(str);
tmp.add(str);
if (count == BATCH_EXECUTE_COUNT) {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
}
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
}
}
}
try {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
System.out.println(String.format("[INFO] Load data from %s successfully, it takes %dms", file.getName(),
(System.currentTimeMillis() - startTime)));
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
}
} catch (FileNotFoundException e) {
System.out.println("[ERROR] Cannot find " + file.getName());
} catch (IOException e) {
System.out.println("[ERROR] CSV file read exception!" + e.getMessage());
} catch (SQLException e) {
System.out.println("[ERROR] Database connection exception!" + e.getMessage());
} finally {
try {
if (br != null)
br.close();
if (bw != null)
bw.close();
if (statement != null)
statement.close();
if (errorFlag) {
FileUtils.forceDelete(errorFile);
} else {
System.out.println(String.format(
"[ERROR] Format of some lines in %s error, please check %s for more information",
file.getAbsolutePath(), errorFile.getAbsolutePath()));
}
} catch (SQLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
*
* @param line
* csv line data
* @param timeseriesToType
* @param deviceToColumn
* @param colInfo
* @param headInfo
* @return
* @throws IOException
*/
private static List<String> createInsertSQL(String line, Map<String, String> timeseriesToType,
Map<String, ArrayList<Integer>> deviceToColumn, List<String> colInfo, List<String> headInfo)
throws IOException {
String[] data = line.split(",", headInfo.size() + 1);
List<String> sqls = new ArrayList<>();
Iterator<Map.Entry<String, ArrayList<Integer>>> it = deviceToColumn.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, ArrayList<Integer>> entry = it.next();
StringBuilder sbd = new StringBuilder();
ArrayList<Integer> colIndex = entry.getValue();
sbd.append("insert into " + entry.getKey() + "(timestamp");
int skipcount = 0;
for (int j = 0; j < colIndex.size(); ++j) {
if (data[entry.getValue().get(j) + 1].equals("")) {
skipcount++;
continue;
}
sbd.append(", " + colInfo.get(colIndex.get(j)));
}
// define every device null value' number, if the number equal the
// sensor number, the insert operation stop
if (skipcount == entry.getValue().size())
continue;
// TODO when timestampsStr is empty,
String timestampsStr = data[0];
sbd.append(") values(").append(timestampsStr.trim().equals("") ? "NO TIMESTAMP" : timestampsStr);
// if (timestampsStr.trim().equals("")) {
// continue;
// }
// sbd.append(") values(").append(timestampsStr);
for (int j = 0; j < colIndex.size(); ++j) {
if (data[entry.getValue().get(j) + 1].equals(""))
continue;
if (timeseriesToType.get(headInfo.get(colIndex.get(j))).equals(STRING_DATA_TYPE)) {
sbd.append(", \'" + data[colIndex.get(j) + 1] + "\'");
} else {
sbd.append("," + data[colIndex.get(j) + 1]);
}
}
sbd.append(")");
sqls.add(sbd.toString());
}
return sqls;
}
public static void main(String[] args) throws IOException, SQLException {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setOptionComparator(null);
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
if (args == null || args.length == 0) {
System.out.println("[ERROR] Too few params input, please check the following hint.");
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
try {
commandLine = parser.parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
ConsoleReader reader = new ConsoleReader();
reader.setExpandEvents(false);
try {
parseBasicParams(commandLine, reader);
filename = commandLine.getOptionValue(FILE_ARGS);
if (filename == null) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
parseSpecialParams(commandLine, reader);
importCsvFromFile(host, port, username, password, filename, timeZoneID);
} catch (ArgsErrorException e) {
} catch (Exception e) {
System.out.println(String.format("[ERROR] Encounter an error, because %s", e.getMessage()));
} finally {
reader.close();
}
}
private static void parseSpecialParams(CommandLine commandLine, ConsoleReader reader)
throws IOException, ArgsErrorException {
timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS);
}
public static void importCsvFromFile(String ip, String port, String username, String password, String filename,
String timeZone) throws SQLException {
String property = System.getProperty("IOTDB_HOME");
if (property == null) {
errorInsertInfo = ERROR_INFO_STR;
for (int i = 1; i < strHeadInfo.length; i++) {
ResultSet resultSet = databaseMetaData.getColumns(null,
null, strHeadInfo[i], null);
if (resultSet.next()) {
timeseriesDataType.put(resultSet.getString(1),
resultSet.getString(2));
} else {
errorInsertInfo = property + File.separatorChar + ERROR_INFO_STR;
String errorInfo = String.format("[ERROR] Database cannot find %s in %s, stop import!",
strHeadInfo[i], file.getAbsolutePath());
System.out.println(errorInfo);
bw.write(errorInfo);
errorFlag = false;
return;
}
headInfo.add(strHeadInfo[i]);
String deviceInfo = strHeadInfo[i].substring(0, strHeadInfo[i].lastIndexOf("."));
if (!deviceToColumn.containsKey(deviceInfo)) {
deviceToColumn.put(deviceInfo, new ArrayList<>());
}
// storage every device's sensor index info
deviceToColumn.get(deviceInfo).add(i - 1);
colInfo.add(strHeadInfo[i].substring(strHeadInfo[i].lastIndexOf(".") + 1));
}
String line;
statement = connection.createStatement();
int count = 0;
List<String> tmp = new ArrayList<>();
while ((line = br.readLine()) != null) {
List<String> sqls = new ArrayList<>();
try {
Class.forName(Config.JDBC_DRIVER_NAME);
connection = (IoTDBConnection) DriverManager.getConnection(Config.IOTDB_URL_PREFIX + ip + ":" + port + "/",
username, password);
timeZoneID = timeZone;
setTimeZone();
File file = new File(filename);
if (file.isFile()) {
if (file.getName().endsWith(FILE_SUFFIX)) {
loadDataFromCSV(file, 1);
} else {
System.out.println(
"[WARN] File " + file.getName() + " should ends with '.csv' if you want to import");
}
} else if (file.isDirectory()) {
int i = 1;
for (File f : file.listFiles()) {
if (f.isFile()) {
if (f.getName().endsWith(FILE_SUFFIX)) {
loadDataFromCSV(f, i);
i++;
} else {
System.out.println(
"[WARN] File " + f.getName() + " should ends with '.csv' if you want to import");
}
}
}
}
} catch (ClassNotFoundException e) {
System.out.println(
"[ERROR] Failed to dump data because cannot find TsFile JDBC Driver, please check whether you have imported driver or not");
} catch (TException e) {
System.out.println(
String.format("[ERROR] Encounter an error when connecting to server, because %s", e.getMessage()));
sqls = createInsertSQL(line, timeseriesDataType, deviceToColumn, colInfo, headInfo);
} catch (Exception e) {
System.out.println(String.format("[ERROR] Encounter an error, because %s", e.getMessage()));
} finally {
if (connection != null) {
connection.close();
}
bw.write(String.format("error input line, maybe it is not complete: %s", line));
bw.newLine();
errorFlag = false;
}
for (String str : sqls) {
try {
count++;
statement.addBatch(str);
tmp.add(str);
if (count == BATCH_EXECUTE_COUNT) {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
}
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
}
}
}
try {
int[] result = statement.executeBatch();
for (int i = 0; i < result.length; i++) {
if (result[i] != Statement.SUCCESS_NO_INFO && i < tmp.size()) {
bw.write(tmp.get(i));
bw.newLine();
errorFlag = false;
}
}
statement.clearBatch();
count = 0;
tmp.clear();
System.out.println(String.format("[INFO] Load data from %s successfully,"
+ " it takes %dms", file.getName(),
(System.currentTimeMillis() - startTime)));
} catch (SQLException e) {
bw.write(e.getMessage());
bw.newLine();
errorFlag = false;
}
} catch (FileNotFoundException e) {
System.out.println("[ERROR] Cannot find " + file.getName());
} catch (IOException e) {
System.out.println("[ERROR] CSV file read exception!" + e.getMessage());
} catch (SQLException e) {
System.out.println("[ERROR] Database connection exception!" + e.getMessage());
} finally {
try {
if (br != null) {
br.close();
}
if (bw != null) {
bw.close();
}
if (statement != null) {
statement.close();
}
if (errorFlag) {
FileUtils.forceDelete(errorFile);
} else {
System.out.println(String.format(
"[ERROR] Format of some lines in %s error, please check %s for more information",
file.getAbsolutePath(), errorFile.getAbsolutePath()));
}
} catch (SQLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
private static List<String> createInsertSQL(String line, Map<String, String> timeseriesToType,
Map<String, ArrayList<Integer>> deviceToColumn,
List<String> colInfo, List<String> headInfo)
throws IOException {
String[] data = line.split(",", headInfo.size() + 1);
List<String> sqls = new ArrayList<>();
Iterator<Map.Entry<String, ArrayList<Integer>>> it = deviceToColumn.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<String, ArrayList<Integer>> entry = it.next();
StringBuilder sbd = new StringBuilder();
ArrayList<Integer> colIndex = entry.getValue();
sbd.append("insert into " + entry.getKey() + "(timestamp");
int skipcount = 0;
for (int j = 0; j < colIndex.size(); ++j) {
if (data[entry.getValue().get(j) + 1].equals("")) {
skipcount++;
continue;
}
sbd.append(", " + colInfo.get(colIndex.get(j)));
}
// define every device null value' number, if the number equal the
// sensor number, the insert operation stop
if (skipcount == entry.getValue().size()) {
continue;
}
// TODO when timestampsStr is empty,
String timestampsStr = data[0];
sbd.append(") values(").append(timestampsStr.trim().equals("")
? "NO TIMESTAMP" : timestampsStr);
// if (timestampsStr.trim().equals("")) {
// continue;
// }
// sbd.append(") values(").append(timestampsStr);
for (int j = 0; j < colIndex.size(); ++j) {
if (data[entry.getValue().get(j) + 1].equals("")) {
continue;
}
if (timeseriesToType.get(headInfo.get(colIndex.get(j))).equals(STRING_DATA_TYPE)) {
sbd.append(", \'" + data[colIndex.get(j) + 1] + "\'");
} else {
sbd.append("," + data[colIndex.get(j) + 1]);
}
}
sbd.append(")");
sqls.add(sbd.toString());
}
return sqls;
}
public static void main(String[] args) throws IOException, SQLException {
Options options = createOptions();
HelpFormatter hf = new HelpFormatter();
hf.setOptionComparator(null);
hf.setWidth(MAX_HELP_CONSOLE_WIDTH);
CommandLine commandLine = null;
CommandLineParser parser = new DefaultParser();
if (args == null || args.length == 0) {
System.out.println("[ERROR] Too few params input, please check the following hint.");
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
try {
commandLine = parser.parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
if (commandLine.hasOption(HELP_ARGS)) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
ConsoleReader reader = new ConsoleReader();
reader.setExpandEvents(false);
try {
parseBasicParams(commandLine, reader);
filename = commandLine.getOptionValue(FILE_ARGS);
if (filename == null) {
hf.printHelp(TSFILEDB_CLI_PREFIX, options, true);
return;
}
parseSpecialParams(commandLine, reader);
importCsvFromFile(host, port, username, password, filename, timeZoneID);
} catch (ArgsErrorException e) {
// ignored
} catch (Exception e) {
System.out.println(String.format("[ERROR] Encounter an error, because %s", e.getMessage()));
} finally {
reader.close();
}
}
private static void parseSpecialParams(CommandLine commandLine, ConsoleReader reader)
throws IOException, ArgsErrorException {
timeZoneID = commandLine.getOptionValue(TIME_ZONE_ARGS);
}
public static void importCsvFromFile(String ip, String port, String username,
String password, String filename,
String timeZone) throws SQLException {
String property = System.getProperty("IOTDB_HOME");
if (property == null) {
errorInsertInfo = ERROR_INFO_STR;
} else {
errorInsertInfo = property + File.separatorChar + ERROR_INFO_STR;
}
try {
Class.forName(Config.JDBC_DRIVER_NAME);
connection = (IoTDBConnection) DriverManager.getConnection(Config.IOTDB_URL_PREFIX
+ ip + ":" + port + "/",
username, password);
timeZoneID = timeZone;
setTimeZone();
File file = new File(filename);
if (file.isFile()) {
if (file.getName().endsWith(FILE_SUFFIX)) {
loadDataFromCSV(file, 1);
} else {
System.out.println(
"[WARN] File " + file.getName() + " should ends with '.csv' "
+ "if you want to import");
}
} else if (file.isDirectory()) {
int i = 1;
for (File f : file.listFiles()) {
if (f.isFile()) {
if (f.getName().endsWith(FILE_SUFFIX)) {
loadDataFromCSV(f, i);
i++;
} else {
System.out.println(
"[WARN] File " + f.getName() + " should ends with '.csv' "
+ "if you want to import");
}
}
}
}
} catch (ClassNotFoundException e) {
System.out.println(
"[ERROR] Failed to dump data because cannot find TsFile JDBC Driver, "
+ "please check whether you have imported driver or not");
} catch (TException e) {
System.out.println(
String.format("[ERROR] Encounter an error when connecting to server, because %s",
e.getMessage()));
} catch (Exception e) {
System.out.println(String.format("[ERROR] Encounter an error, because %s", e.getMessage()));
} finally {
if (connection != null) {
connection.close();
}
}
}
}

343
iotdb-cli/src/test/java/org/apache/iotdb/cli/client/AbstractClientTest.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,12 +19,16 @@
*/
package org.apache.iotdb.cli.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.iotdb.cli.client.AbstractClient.OPERATION_RESULT;
import org.apache.iotdb.cli.client.AbstractClient.OperationResult;
import org.apache.iotdb.cli.exception.ArgsErrorException;
import org.apache.iotdb.jdbc.IoTDBConnection;
import org.apache.iotdb.jdbc.IoTDBDatabaseMetadata;
@ -30,187 +38,200 @@ import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
public class AbstractClientTest {
@Mock
private IoTDBConnection connection;
@Mock
private IoTDBConnection connection;
@Mock
private IoTDBDatabaseMetadata databaseMetadata;
@Mock
private IoTDBDatabaseMetadata databaseMetadata;
@Before
public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
when(connection.getMetaData()).thenReturn(databaseMetadata);
when(connection.getTimeZone()).thenReturn("Asia/Shanghai");
when(databaseMetadata.getMetadataInJson()).thenReturn("test metadata");
@Before
public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
when(connection.getMetaData()).thenReturn(databaseMetadata);
when(connection.getTimeZone()).thenReturn("Asia/Shanghai");
when(databaseMetadata.getMetadataInJson()).thenReturn("test metadata");
}
@After
public void tearDown() throws Exception {
}
@Test
public void testInit() {
AbstractClient.init();
String[] keywords = {AbstractClient.HOST_ARGS, AbstractClient.HELP_ARGS,
AbstractClient.PORT_ARGS,
AbstractClient.PASSWORD_ARGS, AbstractClient.USERNAME_ARGS, AbstractClient.ISO8601_ARGS,
AbstractClient.MAX_PRINT_ROW_COUNT_ARGS,};
for (String keyword : keywords) {
if (!AbstractClient.keywordSet.contains("-" + keyword)) {
System.out.println(keyword);
fail();
}
}
}
@After
public void tearDown() throws Exception {
@Test
public void testCheckRequiredArg() throws ParseException, ArgsErrorException {
Options options = AbstractClient.createOptions();
CommandLineParser parser = new DefaultParser();
String[] args = new String[]{"-u", "user1"};
CommandLine commandLine = parser.parse(options, args);
String str = AbstractClient
.checkRequiredArg(AbstractClient.USERNAME_ARGS, AbstractClient.USERNAME_NAME,
commandLine, true, "root");
assertEquals(str, "user1");
args = new String[]{"-u", "root",};
commandLine = parser.parse(options, args);
str = AbstractClient
.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine, false,
"127.0.0.1");
assertEquals(str, "127.0.0.1");
try {
str = AbstractClient
.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine, true,
"127.0.0.1");
} catch (ArgsErrorException e) {
assertEquals(e.getMessage(), "IoTDB: Required values for option 'host' not provided");
}
@Test
public void testInit() {
AbstractClient.init();
String[] keywords = { AbstractClient.HOST_ARGS, AbstractClient.HELP_ARGS, AbstractClient.PORT_ARGS,
AbstractClient.PASSWORD_ARGS, AbstractClient.USERNAME_ARGS, AbstractClient.ISO8601_ARGS,
AbstractClient.MAX_PRINT_ROW_COUNT_ARGS, };
for (String keyword : keywords) {
if (!AbstractClient.keywordSet.contains("-" + keyword)) {
System.out.println(keyword);
fail();
}
}
try {
str = AbstractClient
.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine,
false, null);
} catch (ArgsErrorException e) {
assertEquals(e.getMessage(), "IoTDB: Required values for option 'host' is null.");
}
}
@Test
public void testCheckRequiredArg() throws ParseException, ArgsErrorException {
Options options = AbstractClient.createOptions();
CommandLineParser parser = new DefaultParser();
String[] args = new String[] { "-u", "user1" };
CommandLine commandLine = parser.parse(options, args);
String str = AbstractClient.checkRequiredArg(AbstractClient.USERNAME_ARGS, AbstractClient.USERNAME_NAME,
commandLine, true, "root");
assertEquals(str, "user1");
@Test
public void testRemovePasswordArgs() {
AbstractClient.init();
String[] input = new String[]{"-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root"};
String[] res = new String[]{"-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root"};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
args = new String[] { "-u", "root", };
commandLine = parser.parse(options, args);
str = AbstractClient.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine, false,
"127.0.0.1");
assertEquals(str, "127.0.0.1");
try {
str = AbstractClient.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine, true,
"127.0.0.1");
} catch (ArgsErrorException e) {
assertEquals(e.getMessage(), "IoTDB: Required values for option 'host' not provided");
}
try {
str = AbstractClient.checkRequiredArg(AbstractClient.HOST_ARGS, AbstractClient.HOST_NAME, commandLine,
false, null);
} catch (ArgsErrorException e) {
assertEquals(e.getMessage(), "IoTDB: Required values for option 'host' is null.");
}
input = new String[]{"-h", "127.0.0.1", "-p", "6667", "-pw", "root", "-u", "root"};
res = new String[]{"-h", "127.0.0.1", "-p", "6667", "-pw", "root", "-u", "root"};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
input = new String[]{"-h", "127.0.0.1", "-p", "6667", "root", "-u", "root", "-pw"};
res = new String[]{"-h", "127.0.0.1", "-p", "6667", "root", "-u", "root"};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
input = new String[]{"-h", "127.0.0.1", "-p", "6667", "-pw", "-u", "root"};
res = new String[]{"-h", "127.0.0.1", "-p", "6667", "-u", "root"};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
input = new String[]{"-pw", "-h", "127.0.0.1", "-p", "6667", "root", "-u", "root"};
res = new String[]{"-h", "127.0.0.1", "-p", "6667", "root", "-u", "root"};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
input = new String[]{};
res = new String[]{};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
}
private void isTwoStringArrayEqual(String[] expected, String[] actual) {
for (int i = 0; i < expected.length; i++) {
assertEquals(expected[i], actual[i]);
}
}
@Test
public void testRemovePasswordArgs() {
AbstractClient.init();
String[] input = new String[] { "-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root" };
String[] res = new String[] { "-h", "127.0.0.1", "-p", "6667", "-u", "root", "-pw", "root" };
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
@Test
public void testHandleInputInputCmd() {
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.EXIT_COMMAND, connection),
OperationResult.RETURN_OPER);
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.QUIT_COMMAND, connection),
OperationResult.RETURN_OPER);
input = new String[] { "-h", "127.0.0.1", "-p", "6667", "-pw", "root", "-u", "root" };
res = new String[] { "-h", "127.0.0.1", "-p", "6667", "-pw", "root", "-u", "root" };
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
assertEquals(
AbstractClient.handleInputInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection),
OperationResult.CONTINUE_OPER);
input = new String[] { "-h", "127.0.0.1", "-p", "6667", "root", "-u", "root", "-pw" };
res = new String[] { "-h", "127.0.0.1", "-p", "6667", "root", "-u", "root" };
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY),
connection), OperationResult.CONTINUE_OPER);
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY),
connection), OperationResult.CONTINUE_OPER);
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY),
connection),
OperationResult.CONTINUE_OPER);
testSetTimeFormat();
input = new String[] { "-h", "127.0.0.1", "-p", "6667", "-pw", "-u", "root" };
res = new String[] { "-h", "127.0.0.1", "-p", "6667", "-u", "root" };
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
assertEquals(
AbstractClient.handleInputInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OperationResult.CONTINUE_OPER);
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OperationResult.CONTINUE_OPER);
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OperationResult.CONTINUE_OPER);
testSetMaxDisplayNumber();
input = new String[] { "-pw", "-h", "127.0.0.1", "-p", "6667", "root", "-u", "root" };
res = new String[] { "-h", "127.0.0.1", "-p", "6667", "root", "-u", "root" };
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_TIMEZONE, connection),
OperationResult.CONTINUE_OPER);
assertEquals(
AbstractClient.handleInputInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection),
OperationResult.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection),
OperationResult.CONTINUE_OPER);
input = new String[] {};
res = new String[] {};
isTwoStringArrayEqual(res, AbstractClient.removePasswordArgs(input));
}
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection),
OperationResult.CONTINUE_OPER);
assertEquals(
AbstractClient.handleInputInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE),
connection), OperationResult.CONTINUE_OPER);
private void isTwoStringArrayEqual(String[] expected, String[] actual) {
for (int i = 0; i < expected.length; i++) {
assertEquals(expected[i], actual[i]);
}
}
assertEquals(
AbstractClient
.handleInputInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection),
OperationResult.CONTINUE_OPER);
assertEquals(
AbstractClient.handleInputInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE),
connection),
OperationResult.CONTINUE_OPER);
}
@Test
public void testHandleInputInputCmd() {
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.EXIT_COMMAND, connection),
OPERATION_RESULT.RETURN_OPER);
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.QUIT_COMMAND, connection),
OPERATION_RESULT.RETURN_OPER);
private void testSetTimeFormat() {
AbstractClient.setTimeFormat("long");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.maxValueLength);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_METADATA_COMMAND, connection),
OPERATION_RESULT.CONTINUE_OPER);
AbstractClient.setTimeFormat("number");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.maxValueLength);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=", AbstractClient.SET_TIMESTAMP_DISPLAY),
connection), OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=xxx", AbstractClient.SET_TIMESTAMP_DISPLAY),
connection), OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient
.handleInputInputCmd(String.format("%s=default", AbstractClient.SET_TIMESTAMP_DISPLAY), connection),
OPERATION_RESULT.CONTINUE_OPER);
testSetTimeFormat();
AbstractClient.setTimeFormat("default");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.ISO_DATETIME_LEN);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=xxx", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=1", AbstractClient.SET_MAX_DISPLAY_NUM),
connection), OPERATION_RESULT.CONTINUE_OPER);
testSetMaxDisplayNumber();
AbstractClient.setTimeFormat("iso8601");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.ISO_DATETIME_LEN);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_TIMEZONE, connection),
OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_TIMESTAMP_DISPLAY, connection),
OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(AbstractClient.SHOW_FETCH_SIZE, connection),
OPERATION_RESULT.CONTINUE_OPER);
AbstractClient.setTimeFormat("yyyy-MM-dd HH:mm:ssZZ");
assertEquals(AbstractClient.maxTimeLength, "yyyy-MM-dd HH:mm:ssZZ".length());
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=", AbstractClient.SET_TIME_ZONE), connection),
OPERATION_RESULT.CONTINUE_OPER);
assertEquals(AbstractClient.handleInputInputCmd(String.format("%s=+08:00", AbstractClient.SET_TIME_ZONE),
connection), OPERATION_RESULT.CONTINUE_OPER);
AbstractClient.setTimeFormat("dd");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.TIMESTAMP_STR.length());
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
assertEquals(
AbstractClient.handleInputInputCmd(String.format("%s=", AbstractClient.SET_FETCH_SIZE), connection),
OPERATION_RESULT.CONTINUE_OPER);
assertEquals(
AbstractClient.handleInputInputCmd(String.format("%s=111", AbstractClient.SET_FETCH_SIZE), connection),
OPERATION_RESULT.CONTINUE_OPER);
}
}
private void testSetTimeFormat() {
AbstractClient.setTimeFormat("long");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.maxValueLength);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
AbstractClient.setTimeFormat("number");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.maxValueLength);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
AbstractClient.setTimeFormat("default");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.ISO_DATETIME_LEN);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
AbstractClient.setTimeFormat("iso8601");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.ISO_DATETIME_LEN);
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
AbstractClient.setTimeFormat("yyyy-MM-dd HH:mm:ssZZ");
assertEquals(AbstractClient.maxTimeLength, "yyyy-MM-dd HH:mm:ssZZ".length());
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
AbstractClient.setTimeFormat("dd");
assertEquals(AbstractClient.maxTimeLength, AbstractClient.TIMESTAMP_STR.length());
assertEquals(AbstractClient.formatTime, "%" + AbstractClient.maxTimeLength + "s|");
}
private void testSetMaxDisplayNumber() {
AbstractClient.setMaxDisplayNumber("10");
assertEquals(AbstractClient.maxPrintRowCount, 10);
AbstractClient.setMaxDisplayNumber("111111111111111");
assertEquals(AbstractClient.maxPrintRowCount, Integer.MAX_VALUE);
AbstractClient.setMaxDisplayNumber("-10");
assertEquals(AbstractClient.maxPrintRowCount, Integer.MAX_VALUE);
}
private void testSetMaxDisplayNumber() {
AbstractClient.setMaxDisplayNumber("10");
assertEquals(AbstractClient.maxPrintRowCount, 10);
AbstractClient.setMaxDisplayNumber("111111111111111");
assertEquals(AbstractClient.maxPrintRowCount, Integer.MAX_VALUE);
AbstractClient.setMaxDisplayNumber("-10");
assertEquals(AbstractClient.maxPrintRowCount, Integer.MAX_VALUE);
}
}

145
iotdb-cli/src/test/java/org/apache/iotdb/cli/client/StartClientScriptTest.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,9 +19,7 @@
*/
package org.apache.iotdb.cli.client;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import java.io.BufferedReader;
import java.io.File;
@ -25,78 +27,83 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class StartClientScriptTest {
@Before
public void setUp() throws Exception {
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void test() throws IOException, InterruptedException {
String os = System.getProperty("os.name").toLowerCase();
if (os.startsWith("windows")) {
testStartClientOnWindows();
} else {
testStartClientOnUnix();
}
@Test
public void test() throws IOException, InterruptedException {
String os = System.getProperty("os.name").toLowerCase();
if (os.startsWith("windows")) {
testStartClientOnWindows();
} else {
testStartClientOnUnix();
}
}
private void testStartClientOnWindows() throws IOException {
final String[] output = { "````````````````````````", "Starting IoTDB Client", "````````````````````````",
"IoTDB> Connection Error, please check whether the network is avaliable or the server has started.. Host is 127.0.0.1, port is 6668." };
String dir = getCurrentPath("cmd.exe", "/c", "echo %cd%");
ProcessBuilder builder = new ProcessBuilder("cmd.exe", "/c",
dir + File.separator + "cli" + File.separator + "bin" + File.separator + "start-client.bat", "-h",
"127.0.0.1", "-p", "6668", "-u", "root", "-pw", "root");
testOutput(builder, output);
private void testStartClientOnWindows() throws IOException {
final String[] output = {"````````````````````````", "Starting IoTDB Client",
"````````````````````````",
"IoTDB> Connection Error, please check whether the network is available or the server has started.. Host is 127.0.0.1, port is 6668."};
String dir = getCurrentPath("cmd.exe", "/c", "echo %cd%");
ProcessBuilder builder = new ProcessBuilder("cmd.exe", "/c",
dir + File.separator + "cli" + File.separator + "bin" + File.separator + "start-client.bat",
"-h",
"127.0.0.1", "-p", "6668", "-u", "root", "-pw", "root");
testOutput(builder, output);
}
private void testStartClientOnUnix() throws IOException {
final String[] output = {"---------------------", "Starting IoTDB Client",
"---------------------",
"IoTDB> Connection Error, please check whether the network is available or the server has started.. Host is 127.0.0.1, port is 6668."};
String dir = getCurrentPath("pwd");
System.out.println(dir);
ProcessBuilder builder = new ProcessBuilder("sh",
dir + File.separator + "cli" + File.separator + "bin" + File.separator + "start-client.sh",
"-h",
"127.0.0.1", "-p", "6668", "-u", "root", "-pw", "root");
testOutput(builder, output);
}
private void testOutput(ProcessBuilder builder, String[] output) throws IOException {
builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
String line;
List<String> outputList = new ArrayList<>();
while (true) {
line = r.readLine();
if (line == null) {
break;
} else {
outputList.add(line);
}
}
r.close();
p.destroy();
private void testStartClientOnUnix() throws IOException {
final String[] output = { "---------------------", "Starting IoTDB Client", "---------------------",
"IoTDB> Connection Error, please check whether the network is avaliable or the server has started.. Host is 127.0.0.1, port is 6668." };
String dir = getCurrentPath("pwd");
System.out.println(dir);
ProcessBuilder builder = new ProcessBuilder("sh",
dir + File.separator + "cli" + File.separator + "bin" + File.separator + "start-client.sh", "-h",
"127.0.0.1", "-p", "6668", "-u", "root", "-pw", "root");
testOutput(builder, output);
for (int i = 0; i < output.length; i++) {
assertEquals(output[output.length - 1 - i], outputList.get(outputList.size() - 1 - i));
}
}
private void testOutput(ProcessBuilder builder, String[] output) throws IOException {
builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
String line;
List<String> outputList = new ArrayList<>();
while (true) {
line = r.readLine();
if (line == null) {
break;
} else {
outputList.add(line);
}
}
r.close();
p.destroy();
for (int i = 0; i < output.length; i++) {
assertEquals(output[output.length - 1 - i], outputList.get(outputList.size() - 1 - i));
}
}
private String getCurrentPath(String... command) throws IOException {
ProcessBuilder builder = new ProcessBuilder(command);
builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
String path = r.readLine();
return path;
}
private String getCurrentPath(String... command) throws IOException {
ProcessBuilder builder = new ProcessBuilder(command);
builder.redirectErrorStream(true);
Process p = builder.start();
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()));
String path = r.readLine();
return path;
}
}

10
iotdb-cli/src/test/resources/logback.xml
View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property name="LOG_PATH" value="logs"/>
<appender name="FILEERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEERROR">
<file>${LOG_PATH}/log_error.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/log-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -20,7 +20,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEWARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEWARN">
<file>${LOG_PATH}/log_warn.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/log-warn-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -39,7 +39,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEINFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEINFO">
<file>${LOG_PATH}/log_info.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/log-info-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -58,7 +58,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEDEBUG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEDEBUG">
<file>${LOG_PATH}/log_debug.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/log-debug-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -77,7 +77,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
<Target>System.out</Target>
<encoder>
<pattern>%-5p [%d] [%thread] %C:%L - %m %n</pattern>

10
iotdb/iotdb/bin/start-postBackClient.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb/iotdb/bin/start-postBackClient.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb/iotdb/bin/start-server.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb/iotdb/bin/start-server.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb/iotdb/bin/stop-postBackClient.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb/iotdb/bin/stop-postBackClient.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb/iotdb/bin/stop-server.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb/iotdb/bin/stop-server.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb/iotdb/conf/error_info_cn.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

10
iotdb/iotdb/conf/error_info_en.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

76
iotdb/iotdb/conf/iotdb-engine.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -14,30 +18,20 @@
# limitations under the License.
#
# JDBC Server configuration
# port which JDBC server listens to
rpc_port=6667
# Write ahead log configuration
# Is write ahead log enable
enable_wal=true
# When the total number of write ahead log in the file and memory reaches the specified size, all the logs are compressed and the unused logs are removed
# Increase this value, it will lead to short write pause. Decrease this value, it will increase IO and CPU consumption
wal_cleanup_threshold=500000
# When a certain amount ofwrite ahead log is reached, it will be flushed to disk
# It is possible to lose at most flush_wal_threshold operations
flush_wal_threshold=10000
# The cycle when write ahead log is periodically refreshed to disk(in milliseconds)
# It is possible to lose at most flush_wal_period_in_ms ms operations
flush_wal_period_in_ms=10
# database features configuration
# data dir
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/data).
# If it is absolute, system will save the data in exact location it points to.
@ -49,7 +43,6 @@ flush_wal_period_in_ms=10
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
# data_dir=/path/iotdb/data/data
# tsfile dir
# For this property, multiple directories should be set, and all directories should be separated by ",". All TsFiles will be allocated separately in all these directories. Moreover, setting absolute directories is suggested.
# If this property is unset, system will save the TsFiles in the default relative path directory under the data_dir folder(i.e., %IOTDB_HOME%/data/data/settled).
@ -62,7 +55,6 @@ flush_wal_period_in_ms=10
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
# tsfile_dir=/path/iotdb/data/data,data/data
# mult_dir_strategy
# The strategy is used to choose a directory from tsfile_dir for the system to store a new tsfile.
# System provides three strategies to choose from, or user can create his own strategy by extending org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy.
@ -74,7 +66,6 @@ flush_wal_period_in_ms=10
# If this property is unset, system will use MaxDiskUsableSpaceFirstStrategy as default strategy.
# For this property, fully-qualified class name (include package name) and simple class name are both acceptable.
# mult_dir_strategy=MaxDiskUsableSpaceFirstStrategy
# system dir
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/system).
# If it is absolute, system will save the data in exact location it points to.
@ -86,7 +77,6 @@ flush_wal_period_in_ms=10
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
# sys_dir=/path/iotdb/data/system
# wal dir
# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data).
# If it is absolute, system will save the data in the exact location it points to.
@ -98,24 +88,19 @@ flush_wal_period_in_ms=10
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
# wal_dir=/path/iotdb/data
# The maximum concurrent thread number for merging overflow
# Increase this value, it will increase IO and CPU consumption
# Decrease this value, when there is much overflow data, it will increase disk usage, which will reduce read speed
# When the value<=0 or > CPU core number, use the CPU core number.
merge_concurrent_threads=0
# Maximum number of folders open at the same time
# Increase this value, it will use more memory, random I/O becomes smaller, file fragmentation (i.e., group) is more neat.
# Decrease this value, it will use less memory, random I/O becomes greater, file fragmentation is less neat.
# group_size_in_byte * max_opened_folder = theoretical value of maximum memory occupancy
# For an application, the total amount of folder is equal to the number of storage_group settings in SQL
max_opened_folder=100
# The amount of data that is read every time when IoTDB merge data.
fetch_size=10000
# The period time of flushing data from memory to file.
# The unit is second.
period_time_for_flush_in_second=3600
@ -125,97 +110,70 @@ period_time_for_merge_in_second=7200
#When set to true, start timing flush and merge service. False, stop timing flush and merge service.
#Default is true.
enable_timing_close_and_Merge=true
# time zone of server side
# default value is +08:00
# eg. +08:00, -01:00
time_zone=+08:00
# if memory used by write reaches this threshold, auto flush will be triggered, percentile of Java heap memory
mem_threshold_warning=0.5
# if memory used by write reaches this threshold, write will be blocked, percentile of Java heap memory
mem_threshold_dangerous=0.6
# every such interval, a thread will check if memory exceeds mem_threshold_warning
# if do exceed, auto flush will be triggered, in ms, 1s by default
mem_monitor_interval=1000
# Decide how to control memory used by inserting data.
# 0 is RecordMemController, which count the size of every record (tuple).
# 1 is JVMMemController, which use JVM heap memory as threshold.
mem_controller_type=0
# When a bufferwrite's metadata size (in byte) exceed this, the bufferwrite is forced closed.
bufferwrite_meta_size_threshold=209715200
# When a bufferwrite's file size (in byte) exceeds this, the bufferwrite is forced closed.
bufferwrite_file_size_threshold=2147483648
# When a overflow's metadata size (in byte) exceeds this, the overflow is forced closed.
overflow_meta_size_threshold=20971520
# When a overflow's file size (in byte) exceeds this, the overflow is forced closed.
overflow_file_size_threshold=209715200
# How many thread can concurrently flush. When <= 0, use CPU core number.
concurrent_flush_thread=0
# Statistics Monitor configuration
# Set enable_stat_monitor true(or false) to enable(or disable) the StatMonitor that stores statistics info periodically.
# back_loop_period_sec decides the period when StatMonitor writes statistics info into IoTDB.
# stat_monitor_detect_freq_sec decides when IoTDB detects statistics info out-of-date.
# IoTDB just keeps statistics info within stat_monitor_retain_interval_sec seconds before current time.
# Note: IoTDB requires stat_monitor_detect_freq_sec >= 600s and stat_monitor_retain_interval_sec >= 600s.
# The monitor, which writes statistics info to IoTDB periodically, is disabled by default.
enable_stat_monitor = false
enable_stat_monitor=false
# The period that StatMonitor stores statistics info
back_loop_period_sec = 5
back_loop_period_sec=5
# The interval at which StatMonitor starts to check whether statistics info can be deleted due to exceeding the retention volume
stat_monitor_detect_freq_sec = 600
stat_monitor_detect_freq_sec=600
# The minimum age of statistics storage information to be eligible for deletion due to age
stat_monitor_retain_interval_sec = 600
stat_monitor_retain_interval_sec=600
# When set false, MemMonitorThread and MemStatisticThread will not be created.
enable_mem_monitor=true
# When set to true, small flush will be triggered periodically even if memory threshold is not exceeded.
enable_small_flush=true
# The interval of small flush in ms.
small_flush_interval=60000
# The threshold of lines of external sort
external_sort_threshold = 50
external_sort_threshold=50
# cache size for MManager.
# This cache is used to improve write speed where all path check and TSDataType will be cached in MManager with corresponding Path.
schema_manager_cache_size = 300000
schema_manager_cache_size=300000
# If the size in byte of a WAL log exceeds this, it won't be written.
# Generally the default value 4MB is enough.
max_log_entry_size=4194304
# IoTDB postBack server properties
# Whether to allow to post back, the default allowed
is_postback_enable = true
is_postback_enable=true
# PostBack server port address
postback_server_port = 5555
postback_server_port=5555
# White IP list of Postback client.
# Please use the form of network segment to present the range of IP, for example: 192.168.0.0/16
# If there are more than one IP segment, please separate them by commas
# The default is to allow all IP to postback
IP_white_list = 0.0.0.0/0
IP_white_list=0.0.0.0/0
# Choose a postBack strategy of merging historical data:
#1. It's more likely to update historical data, please choose "true".
#2. It's more likely not to update historical data or you don't know exactly, please choose "false".
update_historical_data_possibility = false
update_historical_data_possibility=false

10
iotdb/iotdb/conf/iotdb-env.bat
View File

@ -1,9 +1,13 @@
@REM
@REM Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
@REM
@REM Licensed under the Apache License, Version 2.0 (the "License");
@REM you may not use this file except in compliance with the License.
@REM You may obtain a copy of the License at
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM

10
iotdb/iotdb/conf/iotdb-env.sh
View File

@ -2,9 +2,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#

26
iotdb/iotdb/conf/iotdb-postbackClient.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -14,24 +18,16 @@
# limitations under the License.
#
# PostBack client configuration
# PostBack server ip address
server_ip = 127.0.0.1
server_ip=127.0.0.1
# PostBack server port address
server_port = 5555
server_port=5555
# PostBack client port
client_port = 6666
client_port=6666
# The cycle time of post data back to receiver, the unit of time is second
upload_cycle_in_seconds = 600
upload_cycle_in_seconds=600
# Set bufferWrite data absolute path of IoTDB
# It needs to be set with iotdb_schema_directory, they have to belong to the same IoTDB
# iotdb_bufferWrite_directory = D:\\iotdb\\data\\data\\settled
# Set schema file absolute path of IoTDB
# It needs to be set with iotdb_bufferWrite_directory, they have to belong to the same IoTDB
# iotdb_schema_directory = D:\\iotdb\\data\\system\\schema\\mlog.txt

24
iotdb/iotdb/conf/logback.xml
View File

@ -3,9 +3,13 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
@ -20,7 +24,7 @@
<jmxConfigurator/>
<!-- prevent logback from outputting its own status at the start of every log -->
<statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
<appender name="FILEERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEERROR">
<file>${IOTDB_HOME}/logs/log_error.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${IOTDB_HOME}/logs/log-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -39,7 +43,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEWARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEWARN">
<file>${IOTDB_HOME}/logs/log_warn.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${IOTDB_HOME}/logs/log-warn-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -58,7 +62,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEINFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEINFO">
<file>${IOTDB_HOME}/logs/log_info.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${IOTDB_HOME}/logs/log-info-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -77,7 +81,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="FILEDEBUG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILEDEBUG">
<file>${IOTDB_HOME}/logs/log_debug.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${IOTDB_HOME}/logs/log-debug-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
@ -96,7 +100,7 @@
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
<appender class="ch.qos.logback.core.ConsoleAppender" name="stdout">
<Target>System.out</Target>
<encoder>
<pattern>%d [%t] %-5p %C:%L - %m %n</pattern>
@ -106,8 +110,8 @@
<level>DEBUG</level>
</filter>
</appender>
<logger name="org.apache.iotdb.db.service" level="info"/>
<logger name="org.apache.iotdb.db.conf" level="info"/>
<logger level="info" name="org.apache.iotdb.db.service"/>
<logger level="info" name="org.apache.iotdb.db.conf"/>
<root level="info">
<appender-ref ref="FILEDEBUG"/>
<appender-ref ref="FILEINFO"/>

22
iotdb/iotdb/conf/tsfile-format.properties
View File

@ -1,9 +1,13 @@
#
# Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@ -14,38 +18,26 @@
# limitations under the License.
#
# Memory size threshold for flushing to disk or HDFS, default value is 128MB
group_size_in_byte=134217728
# The memory size for each series writer to pack page, default value is 64KB
page_size_in_byte=65536
# The maximum number of data points in a page, defalut 1024*1024
max_number_of_points_in_page=1048576
# Data type configuration
# Data type for input timestamp, TsFile supports INT32 or INT64
time_series_data_type=INT64
# Max length limitation of input string
max_string_length=128
# Floating-point precision
float_precision=2
# Encoder configuration
# Encoder of time series, TsFile supports TS_2DIFF, PLAIN and RLE(run-length encoding) and default value is TS_2DIFF
time_series_encoder=TS_2DIFF
# Encoder of value series. default value is PLAIN.
# For int, long data type, TsFile also supports TS_2DIFF and RLE(run-length encoding).
# For float, double data type, TsFile also supports TS_2DIFF, RLE(run-length encoding) and GORILLA.
# For text data type, TsFile only supports PLAIN.
value_encoder=PLAIN
# Compression configuration
# Data compression method, TsFile supports UNCOMPRESSED or SNAPPY. Default value is UNCOMPRESSED which means no compression
compressor=UNCOMPRESSED

44
iotdb/pom.xml
View File

@ -3,9 +3,13 @@
Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
@ -176,23 +180,23 @@
</plugin>
<!-- Might require this in Eclipse -->
<!--plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/target/generated-sources/antlr3</source>
</sources>
</configuration>
</execution>
</executions>
</plugin-->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/target/generated-sources/antlr3</source>
</sources>
</configuration>
</execution>
</executions>
</plugin-->
</plugins>
</build>
</project>

10
iotdb/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSLexer.g
View File

@ -1,9 +1,13 @@
//
// Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//

10
iotdb/src/main/antlr3/org/apache/iotdb/db/sql/parse/TSParser.g
View File

@ -1,9 +1,13 @@
//
// Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//

46
iotdb/src/main/java/org/apache/iotdb/db/auth/AuthException.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -16,31 +20,33 @@
package org.apache.iotdb.db.auth;
/**
* The exception for authority model
* The exception for authority model.
*
* @author liukun
*/
public class AuthException extends Exception {
private static final long serialVersionUID = 5091102941209301301L;
public AuthException(String format, String userName, String roleName) {
super();
}
private static final long serialVersionUID = 5091102941209301301L;
public AuthException(String message) {
super(message);
}
public AuthException(String format, String userName, String roleName) {
super();
}
public AuthException(String message, Throwable cause) {
super(message, cause);
}
public AuthException(String message) {
super(message);
}
public AuthException(Throwable cause) {
super(cause);
}
public AuthException(String message, Throwable cause) {
super(message, cause);
}
protected AuthException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public AuthException(Throwable cause) {
super(cause);
}
protected AuthException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

254
iotdb/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,138 +19,146 @@
*/
package org.apache.iotdb.db.auth;
import java.util.List;
import org.apache.iotdb.db.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.auth.authorizer.LocalFileAuthorizer;
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.qp.logical.Operator;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.db.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.auth.authorizer.LocalFileAuthorizer;
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.qp.logical.Operator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
public class AuthorityChecker {
private static final String SUPER_USER = IoTDBConstant.ADMIN_NAME;
private static final Logger logger = LoggerFactory.getLogger(AuthorityChecker.class);
private static final String SUPER_USER = IoTDBConstant.ADMIN_NAME;
private static final Logger logger = LoggerFactory.getLogger(AuthorityChecker.class);
public static boolean check(String username, List<Path> paths, Operator.OperatorType type, String targetUser)
throws AuthException {
if (SUPER_USER.equals(username)) {
return true;
}
int permission = translateToPermissionId(type);
if (permission == -1) {
logger.error("OperateType not found. {}", type);
return false;
} else if (permission == PrivilegeType.MODIFY_PASSWORD.ordinal() && username.equals(targetUser)) {
// a user can modify his own password
return true;
}
if (paths.size() > 0) {
for (Path path : paths) {
if (!checkOnePath(username, path, permission)) {
return false;
}
}
} else {
return checkOnePath(username, null, permission);
/**
* check permission.
*
* @param username username
* @param paths paths in List structure
* @param type Operator type
* @param targetUser target user
* @return if permission-check is passed
* @throws AuthException Authentication Exception
*/
public static boolean check(String username, List<Path> paths, Operator.OperatorType type,
String targetUser)
throws AuthException {
if (SUPER_USER.equals(username)) {
return true;
}
int permission = translateToPermissionId(type);
if (permission == -1) {
logger.error("OperateType not found. {}", type);
return false;
} else if (permission == PrivilegeType.MODIFY_PASSWORD.ordinal() && username
.equals(targetUser)) {
// a user can modify his own password
return true;
}
if (paths.size() > 0) {
for (Path path : paths) {
if (!checkOnePath(username, path, permission)) {
return false;
}
}
} else {
return checkOnePath(username, null, permission);
}
return true;
}
private static boolean checkOnePath(String username, Path path, int permission)
throws AuthException {
IAuthorizer authorizer = LocalFileAuthorizer.getInstance();
try {
String fullPath = path == null ? IoTDBConstant.PATH_ROOT : path.getFullPath();
if (authorizer.checkUserPrivileges(username, fullPath, permission)) {
return true;
}
} catch (AuthException e) {
logger.error("Error occurs when checking the seriesPath {} for user {}", path, username, e);
}
return false;
}
private static int translateToPermissionId(Operator.OperatorType type) {
switch (type) {
case GRANT_ROLE_PRIVILEGE:
return PrivilegeType.GRANT_ROLE_PRIVILEGE.ordinal();
case CREATE_ROLE:
return PrivilegeType.CREATE_ROLE.ordinal();
case CREATE_USER:
return PrivilegeType.CREATE_USER.ordinal();
case MODIFY_PASSWORD:
return PrivilegeType.MODIFY_PASSWORD.ordinal();
case GRANT_USER_PRIVILEGE:
return PrivilegeType.GRANT_USER_PRIVILEGE.ordinal();
case REVOKE_ROLE_PRIVILEGE:
return PrivilegeType.REVOKE_ROLE_PRIVILEGE.ordinal();
case REVOKE_USER_PRIVILEGE:
return PrivilegeType.REVOKE_USER_PRIVILEGE.ordinal();
case GRANT_USER_ROLE:
return PrivilegeType.GRANT_USER_ROLE.ordinal();
case DELETE_USER:
return PrivilegeType.DELETE_USER.ordinal();
case DELETE_ROLE:
return PrivilegeType.DELETE_ROLE.ordinal();
case REVOKE_USER_ROLE:
return PrivilegeType.REVOKE_USER_ROLE.ordinal();
case SET_STORAGE_GROUP:
return PrivilegeType.SET_STORAGE_GROUP.ordinal();
case DELETE_TIMESERIES:
return PrivilegeType.DELETE_TIMESERIES.ordinal();
case QUERY:
case SELECT:
case FILTER:
case GROUPBY:
case SEQTABLESCAN:
case TABLESCAN:
case INDEXQUERY:
case MERGEQUERY:
case AGGREGATION:
return PrivilegeType.READ_TIMESERIES.ordinal();
case DELETE:
return PrivilegeType.DELETE_TIMESERIES.ordinal();
case INSERT:
case LOADDATA:
case INDEX:
return PrivilegeType.INSERT_TIMESERIES.ordinal();
case UPDATE:
return PrivilegeType.UPDATE_TIMESERIES.ordinal();
case LIST_ROLE:
case LIST_ROLE_USERS:
case LIST_ROLE_PRIVILEGE:
return PrivilegeType.LIST_ROLE.ordinal();
case LIST_USER:
case LIST_USER_ROLES:
case LIST_USER_PRIVILEGE:
return PrivilegeType.LIST_USER.ordinal();
case AUTHOR:
case METADATA:
case BASIC_FUNC:
case FILEREAD:
case FROM:
case FUNC:
case HASHTABLESCAN:
case JOIN:
case LIMIT:
case MERGEJOIN:
case NULL:
case ORDERBY:
case PROPERTY:
case SFW:
case UNION:
logger.error("Illegal operator type authorization : {}", type);
return -1;
default:
return -1;
}
private static boolean checkOnePath(String username, Path path, int permission) throws AuthException {
IAuthorizer authorizer = LocalFileAuthorizer.getInstance();
try {
String fullPath = path == null ? IoTDBConstant.PATH_ROOT : path.getFullPath();
if (authorizer.checkUserPrivileges(username, fullPath, permission)) {
return true;
}
} catch (AuthException e) {
logger.error("Error occurs when checking the seriesPath {} for user {}", path, username, e);
}
return false;
}
private static int translateToPermissionId(Operator.OperatorType type) {
switch (type) {
case GRANT_ROLE_PRIVILEGE:
return PrivilegeType.GRANT_ROLE_PRIVILEGE.ordinal();
case CREATE_ROLE:
return PrivilegeType.CREATE_ROLE.ordinal();
case CREATE_USER:
return PrivilegeType.CREATE_USER.ordinal();
case MODIFY_PASSWORD:
return PrivilegeType.MODIFY_PASSWORD.ordinal();
case GRANT_USER_PRIVILEGE:
return PrivilegeType.GRANT_USER_PRIVILEGE.ordinal();
case REVOKE_ROLE_PRIVILEGE:
return PrivilegeType.REVOKE_ROLE_PRIVILEGE.ordinal();
case REVOKE_USER_PRIVILEGE:
return PrivilegeType.REVOKE_USER_PRIVILEGE.ordinal();
case GRANT_USER_ROLE:
return PrivilegeType.GRANT_USER_ROLE.ordinal();
case DELETE_USER:
return PrivilegeType.DELETE_USER.ordinal();
case DELETE_ROLE:
return PrivilegeType.DELETE_ROLE.ordinal();
case REVOKE_USER_ROLE:
return PrivilegeType.REVOKE_USER_ROLE.ordinal();
case SET_STORAGE_GROUP:
return PrivilegeType.SET_STORAGE_GROUP.ordinal();
case DELETE_TIMESERIES:
return PrivilegeType.DELETE_TIMESERIES.ordinal();
case QUERY:
case SELECT:
case FILTER:
case GROUPBY:
case SEQTABLESCAN:
case TABLESCAN:
case INDEXQUERY:
case MERGEQUERY:
case AGGREGATION:
return PrivilegeType.READ_TIMESERIES.ordinal();
case DELETE:
return PrivilegeType.DELETE_TIMESERIES.ordinal();
case INSERT:
case LOADDATA:
case INDEX:
return PrivilegeType.INSERT_TIMESERIES.ordinal();
case UPDATE:
return PrivilegeType.UPDATE_TIMESERIES.ordinal();
case LIST_ROLE:
case LIST_ROLE_USERS:
case LIST_ROLE_PRIVILEGE:
return PrivilegeType.LIST_ROLE.ordinal();
case LIST_USER:
case LIST_USER_ROLES:
case LIST_USER_PRIVILEGE:
return PrivilegeType.LIST_USER.ordinal();
case AUTHOR:
case METADATA:
case BASIC_FUNC:
case FILEREAD:
case FROM:
case FUNC:
case HASHTABLESCAN:
case JOIN:
case LIMIT:
case MERGEJOIN:
case NULL:
case ORDERBY:
case PROPERTY:
case SFW:
case UNION:
logger.error("Illegal operator type authorization : {}", type);
return -1;
default:
return -1;
}
}
}
}

159
iotdb/src/main/java/org/apache/iotdb/db/auth/Role/BasicRoleManager.java
View File

@ -1,159 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.Role;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.concurrent.HashLock;
import org.apache.iotdb.db.utils.AuthUtils;
import java.io.IOException;
import java.util.*;
/**
* This class reads roles from local files through LocalFileRoleAccessor and manages them in a hash map.
*/
public abstract class BasicRoleManager implements IRoleManager {
private Map<String, Role> roleMap;
private IRoleAccessor accessor;
private HashLock lock;
BasicRoleManager(LocalFileRoleAccessor accessor) {
this.roleMap = new HashMap<>();
this.accessor = accessor;
this.lock = new HashLock();
}
@Override
public Role getRole(String rolename) throws AuthException {
lock.readLock(rolename);
Role role = roleMap.get(rolename);
try {
if (role == null) {
role = accessor.loadRole(rolename);
if (role != null)
roleMap.put(rolename, role);
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.readUnlock(rolename);
}
return role;
}
@Override
public boolean createRole(String rolename) throws AuthException {
AuthUtils.validateRolename(rolename);
Role role = getRole(rolename);
if (role != null)
return false;
lock.writeLock(rolename);
try {
role = new Role(rolename);
accessor.saveRole(role);
roleMap.put(rolename, role);
return true;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean deleteRole(String rolename) throws AuthException {
lock.writeLock(rolename);
try {
if (accessor.deleteRole(rolename)) {
roleMap.remove(rolename);
return true;
} else
return false;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean grantPrivilegeToRole(String rolename, String path, int privilegeId) throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(rolename);
try {
Role role = getRole(rolename);
if (role == null) {
throw new AuthException(String.format("No such role %s", rolename));
}
if (role.hasPrivilege(path, privilegeId)) {
return false;
}
Set<Integer> privilegesCopy = new HashSet<>(role.getPrivileges(path));
role.addPrivilege(path, privilegeId);
try {
accessor.saveRole(role);
} catch (IOException e) {
role.setPrivileges(path, privilegesCopy);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean revokePrivilegeFromRole(String rolename, String path, int privilegeId) throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(rolename);
try {
Role role = getRole(rolename);
if (role == null) {
throw new AuthException(String.format("No such role %s", rolename));
}
if (!role.hasPrivilege(path, privilegeId)) {
return false;
}
role.removePrivilege(path, privilegeId);
try {
accessor.saveRole(role);
} catch (IOException e) {
role.addPrivilege(path, privilegeId);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public void reset() {
accessor.reset();
roleMap.clear();
lock.reset();
}
@Override
public List<String> listAllRoles() {
List<String> rtlist = accessor.listAllRoles();
rtlist.sort(null);
return rtlist;
}
}

67
iotdb/src/main/java/org/apache/iotdb/db/auth/Role/IRoleAccessor.java
View File

@ -1,67 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.Role;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.Role;
import java.io.IOException;
import java.util.List;
/**
* This interface manages the serialization/deserialization of the role objects.
*/
public interface IRoleAccessor {
/**
* Deserialize a role from lower storage.
*
* @param rolename
* The name of the role to be deserialized.
* @return The role object or null if no such role.
* @throws IOException
*/
Role loadRole(String rolename) throws IOException;
/**
* Serialize the role object to lower storage.
*
* @param role
* The role object that is to be saved.
* @throws IOException
*/
void saveRole(Role role) throws IOException;
/**
* Delete a role's in lower storage.
*
* @param rolename
* The name of the role to be deleted.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws IOException
*/
boolean deleteRole(String rolename) throws IOException;
/**
*
* @return A list contains all names of the roles.
*/
List<String> listAllRoles();
/**
* Re-initialize this object.
*/
void reset();
}

101
iotdb/src/main/java/org/apache/iotdb/db/auth/Role/IRoleManager.java
View File

@ -1,101 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.Role;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.Role;
import java.util.List;
/**
* This interface maintains roles in memory and is responsible for their modifications.
*/
public interface IRoleManager {
/**
* Get a role object.
*
* @param rolename
* The name of the role.
* @return A role object whose name is rolename or null if such role does not exist.
* @throws AuthException
*/
Role getRole(String rolename) throws AuthException;
/**
* Create a role with given rolename. New roles will only be granted no privileges.
*
* @param rolename
* is not null or empty
* @return True if the role is successfully created, false when the role already exists.
* @throws AuthException
* f the given rolename is iIllegal.
*/
boolean createRole(String rolename) throws AuthException;
/**
* Delete a role.
*
* @param rolename
* the rolename of the role.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws AuthException
*/
boolean deleteRole(String rolename) throws AuthException;
/**
* Grant a privilege on a seriesPath to a role.
*
* @param rolename
* The rolename of the role to which the privilege should be added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToRole(String rolename, String path, int privilegeId) throws AuthException;
/**
* Revoke a privilege on seriesPath from a role.
*
* @param rolename
* The rolename of the role from which the privilege should be removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully revoked, false if the permission does not exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromRole(String rolename, String path, int privilegeId) throws AuthException;
/**
* Re-initialize this object.
*/
void reset();
/**
*
* @return A list that contains names of all roles.
*/
List<String> listAllRoles();
}

147
iotdb/src/main/java/org/apache/iotdb/db/auth/Role/LocalFileRoleAccessor.java
View File

@ -1,147 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.Role;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.utils.IOUtils;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.utils.IOUtils;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.*;
/**
* This class store each role in a separate sequential file. Role file schema : Int32 role name length Utf-8 role name
* bytes Int32 seriesPath privilege number n Int32 seriesPath[1] length Utf-8 seriesPath[1] bytes Int32 privilege num k1
* Int32 privilege[1][1] Int32 privilege[1][2] ... Int32 privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2]
* bytes Int32 privilege num k2 Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32
* seriesPath[n] length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32 privilege[n][2] ...
* Int32 privilege[n][kn]
*/
public class LocalFileRoleAccessor implements IRoleAccessor {
private static final String TEMP_SUFFIX = ".temp";
private static final String STRING_ENCODING = "utf-8";
private String roleDirPath;
/**
* Reused buffer for primitive types encoding/decoding, which aim to reduce memory fragments. Use ThreadLocal for
* thread safety.
*/
private ThreadLocal<ByteBuffer> encodingBufferLocal = new ThreadLocal<>();
private ThreadLocal<byte[]> strBufferLocal = new ThreadLocal<>();
public LocalFileRoleAccessor(String roleDirPath) {
this.roleDirPath = roleDirPath;
}
@Override
public Role loadRole(String rolename) throws IOException {
File roleProfile = new File(roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
if (!roleProfile.exists() || !roleProfile.isFile()) {
// System may crush before a newer file is written, so search for back-up file.
File backProfile = new File(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (backProfile.exists() && backProfile.isFile())
roleProfile = backProfile;
else
return null;
}
FileInputStream inputStream = new FileInputStream(roleProfile);
try (DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) {
Role role = new Role();
role.name = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
int privilegeNum = dataInputStream.readInt();
List<PathPrivilege> pathPrivilegeList = new ArrayList<>();
for (int i = 0; i < privilegeNum; i++) {
pathPrivilegeList.add(IOUtils.readPathPrivilege(dataInputStream, STRING_ENCODING, strBufferLocal));
}
role.privilegeList = pathPrivilegeList;
return role;
} catch (Exception e) {
throw new IOException(e.getMessage());
}
}
@Override
public void saveRole(Role role) throws IOException {
File roleProfile = new File(
roleDirPath + File.separator + role.name + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(roleProfile));
try {
IOUtils.writeString(outputStream, role.name, STRING_ENCODING, encodingBufferLocal);
role.privilegeList.sort(PathPrivilege.referenceDescentSorter);
int privilegeNum = role.privilegeList.size();
IOUtils.writeInt(outputStream, privilegeNum, encodingBufferLocal);
for (int i = 0; i < privilegeNum; i++) {
PathPrivilege pathPrivilege = role.privilegeList.get(i);
IOUtils.writePathPrivilege(outputStream, pathPrivilege, STRING_ENCODING, encodingBufferLocal);
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
outputStream.flush();
outputStream.close();
}
File oldFile = new File(roleDirPath + File.separator + role.name + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(roleProfile, oldFile);
}
@Override
public boolean deleteRole(String rolename) throws IOException {
File roleProfile = new File(roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
File backFile = new File(roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!roleProfile.exists() && !backFile.exists())
return false;
if ((roleProfile.exists() && !roleProfile.delete()) || (backFile.exists() && !backFile.delete())) {
throw new IOException(String.format("Cannot delete role file of %s", rolename));
}
return true;
}
@Override
public List<String> listAllRoles() {
File roleDir = new File(roleDirPath);
String[] names = roleDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name.endsWith(TEMP_SUFFIX));
List<String> retList = new ArrayList<>();
if (names != null) {
// in very rare situations, normal file and backup file may exist at the same time
// so a set is used to deduplicate
Set<String> set = new HashSet<>();
for (String fileName : names) {
set.add(fileName.replace(IoTDBConstant.PROFILE_SUFFIX, "").replace(TEMP_SUFFIX, ""));
}
retList.addAll(set);
}
return retList;
}
@Override
public void reset() {
new File(roleDirPath).mkdirs();
}
}

23
iotdb/src/main/java/org/apache/iotdb/db/auth/Role/LocalFileRoleManager.java
View File

@ -1,23 +0,0 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.Role;
public class LocalFileRoleManager extends BasicRoleManager {
public LocalFileRoleManager(String roleDirPath) {
super(new LocalFileRoleAccessor(roleDirPath));
}
}

412
iotdb/src/main/java/org/apache/iotdb/db/auth/authorizer/BasicAuthorizer.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,11 +19,14 @@
*/
package org.apache.iotdb.db.auth.authorizer;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.Role.IRoleManager;
import org.apache.iotdb.db.auth.entity.PrivilegeType;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.auth.role.IRoleManager;
import org.apache.iotdb.db.auth.user.IUserManager;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.StartupException;
@ -29,225 +36,240 @@ import org.apache.iotdb.db.utils.AuthUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public abstract class BasicAuthorizer implements IAuthorizer, IService {
abstract public class BasicAuthorizer implements IAuthorizer, IService {
private static final Logger logger = LoggerFactory.getLogger(BasicAuthorizer.class);
private static final Set<Integer> ADMIN_PRIVILEGES;
private static final Logger logger = LoggerFactory.getLogger(BasicAuthorizer.class);
private static final Set<Integer> ADMIN_PRIVILEGES;
static {
ADMIN_PRIVILEGES = new HashSet<>();
for (int i = 0; i < PrivilegeType.values().length; i++)
ADMIN_PRIVILEGES.add(i);
static {
ADMIN_PRIVILEGES = new HashSet<>();
for (int i = 0; i < PrivilegeType.values().length; i++) {
ADMIN_PRIVILEGES.add(i);
}
}
private IUserManager userManager;
private IRoleManager roleManager;
private IUserManager userManager;
private IRoleManager roleManager;
BasicAuthorizer(IUserManager userManager, IRoleManager roleManager) throws AuthException {
this.userManager = userManager;
this.roleManager = roleManager;
init();
BasicAuthorizer(IUserManager userManager, IRoleManager roleManager) throws AuthException {
this.userManager = userManager;
this.roleManager = roleManager;
init();
}
protected void init() throws AuthException {
userManager.reset();
roleManager.reset();
logger.info("Initialization of Authorizer completes");
}
@Override
public boolean login(String username, String password) throws AuthException {
User user = userManager.getUser(username);
return user != null && user.password.equals(AuthUtils.encryptPassword(password));
}
@Override
public boolean createUser(String username, String password) throws AuthException {
return userManager.createUser(username, password);
}
@Override
public boolean deleteUser(String username) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username)) {
throw new AuthException("Default administrator cannot be deleted");
}
return userManager.deleteUser(username);
}
protected void init() throws AuthException {
userManager.reset();
roleManager.reset();
logger.info("Initialization of Authorizer completes");
@Override
public boolean grantPrivilegeToUser(String username, String path, int privilegeId)
throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username)) {
throw new AuthException("Invalid operation, administrator already has all privileges");
}
@Override
public boolean login(String username, String password) throws AuthException {
User user = userManager.getUser(username);
return user != null && user.password.equals(AuthUtils.encryptPassword(password));
if (!PrivilegeType.isPathRelevant(privilegeId)) {
path = IoTDBConstant.PATH_ROOT;
}
return userManager.grantPrivilegeToUser(username, path, privilegeId);
}
@Override
public boolean createUser(String username, String password) throws AuthException {
return userManager.createUser(username, password);
@Override
public boolean revokePrivilegeFromUser(String username, String path, int privilegeId)
throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username)) {
throw new AuthException("Invalid operation, administrator must have all privileges");
}
@Override
public boolean deleteUser(String username) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username))
throw new AuthException("Default administrator cannot be deleted");
return userManager.deleteUser(username);
if (!PrivilegeType.isPathRelevant(privilegeId)) {
path = IoTDBConstant.PATH_ROOT;
}
return userManager.revokePrivilegeFromUser(username, path, privilegeId);
}
@Override
public boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username))
throw new AuthException("Invalid operation, administrator already has all privileges");
if (!PrivilegeType.isPathRelevant(privilegeId))
path = IoTDBConstant.PATH_ROOT;
return userManager.grantPrivilegeToUser(username, path, privilegeId);
}
@Override
public boolean createRole(String roleName) throws AuthException {
return roleManager.createRole(roleName);
}
@Override
public boolean revokePrivilegeFromUser(String username, String path, int privilegeId) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username))
throw new AuthException("Invalid operation, administrator must have all privileges");
if (!PrivilegeType.isPathRelevant(privilegeId))
path = IoTDBConstant.PATH_ROOT;
return userManager.revokePrivilegeFromUser(username, path, privilegeId);
}
@Override
public boolean createRole(String roleName) throws AuthException {
return roleManager.createRole(roleName);
}
@Override
public boolean deleteRole(String roleName) throws AuthException {
boolean success = roleManager.deleteRole(roleName);
if (!success)
return false;
else {
// proceed to revoke the role in all users
List<String> users = userManager.listAllUsers();
for (String user : users) {
try {
userManager.revokeRoleFromUser(roleName, user);
} catch (AuthException e) {
logger.warn("Error encountered when revoking a role {} from user {} after deletion, because {}",
roleName, user, e);
}
}
}
return true;
}
@Override
public boolean grantPrivilegeToRole(String roleName, String path, int privilegeId) throws AuthException {
if (!PrivilegeType.isPathRelevant(privilegeId))
path = IoTDBConstant.PATH_ROOT;
return roleManager.grantPrivilegeToRole(roleName, path, privilegeId);
}
@Override
public boolean revokePrivilegeFromRole(String roleName, String path, int privilegeId) throws AuthException {
if (!PrivilegeType.isPathRelevant(privilegeId))
path = IoTDBConstant.PATH_ROOT;
return roleManager.revokePrivilegeFromRole(roleName, path, privilegeId);
}
@Override
public boolean grantRoleToUser(String roleName, String username) throws AuthException {
Role role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
}
// the role may be deleted before it ts granted to the user, so a double check is necessary.
boolean success = userManager.grantRoleToUser(roleName, username);
if (success) {
role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
} else
return true;
} else
return false;
}
@Override
public boolean revokeRoleFromUser(String roleName, String username) throws AuthException {
Role role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
}
return userManager.revokeRoleFromUser(roleName, username);
}
@Override
public Set<Integer> getPrivileges(String username, String path) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username))
return ADMIN_PRIVILEGES;
User user = userManager.getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user : %s", username));
}
// get privileges of the user
Set<Integer> privileges = user.getPrivileges(path);
// merge the privileges of the roles of the user
for (String roleName : user.roleList) {
Role role = roleManager.getRole(roleName);
if (role != null) {
privileges.addAll(role.getPrivileges(path));
}
}
return privileges;
}
@Override
public boolean updateUserPassword(String username, String newPassword) throws AuthException {
return userManager.updateUserPassword(username, newPassword);
}
@Override
public boolean checkUserPrivileges(String username, String path, int privilegeId) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username))
return true;
User user = userManager.getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user : %s", username));
}
// get privileges of the user
if (user.checkPrivilege(path, privilegeId))
return true;
// merge the privileges of the roles of the user
for (String roleName : user.roleList) {
Role role = roleManager.getRole(roleName);
if (role.checkPrivilege(path, privilegeId)) {
return true;
}
}
return false;
}
@Override
public void reset() throws AuthException {
init();
}
@Override
public void start() throws StartupException {
@Override
public boolean deleteRole(String roleName) throws AuthException {
boolean success = roleManager.deleteRole(roleName);
if (!success) {
return false;
} else {
// proceed to revoke the role in all users
List<String> users = userManager.listAllUsers();
for (String user : users) {
try {
init();
userManager.revokeRoleFromUser(roleName, user);
} catch (AuthException e) {
throw new StartupException(e.getMessage());
logger.warn(
"Error encountered when revoking a role {} from user {} after deletion, because {}",
roleName, user, e);
}
}
}
return true;
}
@Override
public void stop() {
@Override
public boolean grantPrivilegeToRole(String roleName, String path, int privilegeId)
throws AuthException {
if (!PrivilegeType.isPathRelevant(privilegeId)) {
path = IoTDBConstant.PATH_ROOT;
}
return roleManager.grantPrivilegeToRole(roleName, path, privilegeId);
}
@Override
public ServiceType getID() {
return ServiceType.AUTHORIZATION_SERVICE;
@Override
public boolean revokePrivilegeFromRole(String roleName, String path, int privilegeId)
throws AuthException {
if (!PrivilegeType.isPathRelevant(privilegeId)) {
path = IoTDBConstant.PATH_ROOT;
}
return roleManager.revokePrivilegeFromRole(roleName, path, privilegeId);
}
@Override
public List<String> listAllUsers() {
return userManager.listAllUsers();
@Override
public boolean grantRoleToUser(String roleName, String username) throws AuthException {
Role role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
}
// the role may be deleted before it ts granted to the user, so a double check is necessary.
boolean success = userManager.grantRoleToUser(roleName, username);
if (success) {
role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
} else {
return true;
}
} else {
return false;
}
}
@Override
public List<String> listAllRoles() {
return roleManager.listAllRoles();
@Override
public boolean revokeRoleFromUser(String roleName, String username) throws AuthException {
Role role = roleManager.getRole(roleName);
if (role == null) {
throw new AuthException(String.format("No such role : %s", roleName));
}
return userManager.revokeRoleFromUser(roleName, username);
}
@Override
public Role getRole(String roleName) throws AuthException {
return roleManager.getRole(roleName);
@Override
public Set<Integer> getPrivileges(String username, String path) throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username)) {
return ADMIN_PRIVILEGES;
}
User user = userManager.getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user : %s", username));
}
// get privileges of the user
Set<Integer> privileges = user.getPrivileges(path);
// merge the privileges of the roles of the user
for (String roleName : user.roleList) {
Role role = roleManager.getRole(roleName);
if (role != null) {
privileges.addAll(role.getPrivileges(path));
}
}
return privileges;
}
@Override
public User getUser(String username) throws AuthException {
return userManager.getUser(username);
@Override
public boolean updateUserPassword(String username, String newPassword) throws AuthException {
return userManager.updateUserPassword(username, newPassword);
}
@Override
public boolean checkUserPrivileges(String username, String path, int privilegeId)
throws AuthException {
if (IoTDBConstant.ADMIN_NAME.equals(username)) {
return true;
}
User user = userManager.getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user : %s", username));
}
// get privileges of the user
if (user.checkPrivilege(path, privilegeId)) {
return true;
}
// merge the privileges of the roles of the user
for (String roleName : user.roleList) {
Role role = roleManager.getRole(roleName);
if (role.checkPrivilege(path, privilegeId)) {
return true;
}
}
return false;
}
@Override
public void reset() throws AuthException {
init();
}
@Override
public void start() throws StartupException {
try {
init();
} catch (AuthException e) {
throw new StartupException(e.getMessage());
}
}
@Override
public void stop() {
}
@Override
public ServiceType getID() {
return ServiceType.AUTHORIZATION_SERVICE;
}
@Override
public List<String> listAllUsers() {
return userManager.listAllUsers();
}
@Override
public List<String> listAllRoles() {
return roleManager.listAllRoles();
}
@Override
public Role getRole(String roleName) throws AuthException {
return roleManager.getRole(roleName);
}
@Override
public User getUser(String username) throws AuthException {
return userManager.getUser(username);
}
}

419
iotdb/src/main/java/org/apache/iotdb/db/auth/authorizer/IAuthorizer.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,233 +19,236 @@
*/
package org.apache.iotdb.db.auth.authorizer;
import java.util.List;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.auth.entity.User;
import java.util.List;
import java.util.Set;
/**
* This interface provides all authorization-relative operations.
*/
public interface IAuthorizer {
/**
* Login for a user.
*
* @param username
* The username of the user.
* @param password
* The password of the user.
* @return True if such user exists and the given password is correct, else return false.
* @throws AuthException
*/
boolean login(String username, String password) throws AuthException;
/**
* Create a user with given username and password. New users will only be granted no privileges.
*
* @param username
* is not null or empty
* @param password
* is not null or empty
* @return True if the user is successfully created, false when the user already exists.
* @throws AuthException
* if the given username or password is illegal.
*/
boolean createUser(String username, String password) throws AuthException;
/**
* Login for a user.
*
* @param username
* The username of the user.
* @param password
* The password of the user.
* @return True if such user exists and the given password is correct, else return false.
* @throws AuthException if exception raised when searching for the user.
*/
boolean login(String username, String password) throws AuthException;
/**
* Delete a user.
*
* @param username
* the username of the user.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws AuthException
* When attempting to delete the default administrator
*/
boolean deleteUser(String username) throws AuthException;
/**
* Create a user with given username and password. New users will only be granted no privileges.
*
* @param username
* is not null or empty
* @param password
* is not null or empty
* @return True if the user is successfully created, false when the user already exists.
* @throws AuthException
* if the given username or password is illegal.
*/
boolean createUser(String username, String password) throws AuthException;
/**
* Grant a privilege on a seriesPath to a user.
*
* @param username
* The username of the user to which the privilege should be added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException;
/**
* Delete a user.
*
* @param username
* the username of the user.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws AuthException
* When attempting to delete the default administrator
*/
boolean deleteUser(String username) throws AuthException;
/**
* Revoke a privilege on seriesPath from a user.
*
* @param username
* The username of the user from which the privilege should be removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully revoked, false if the permission does not exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromUser(String username, String path, int privilegeId) throws AuthException;
/**
* Grant a privilege on a seriesPath to a user.
*
* @param username
* The username of the user to which the privilege should be added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a
* seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException;
/**
* Add a role.
*
* @param roleName
* the name of the role to be added.
* @return True if the role is successfully added, false if the role already exists
* @throws AuthException
*/
boolean createRole(String roleName) throws AuthException;
/**
* Revoke a privilege on seriesPath from a user.
*
* @param username
* The username of the user from which the privilege should be removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a
* seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully revoked,
* false if the permission does not exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromUser(String username, String path,
int privilegeId) throws AuthException;
/**
* Delete a role.
*
* @param roleName
* the name of the role tobe deleted.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws AuthException
*/
boolean deleteRole(String roleName) throws AuthException;
/**
* Add a role.
*
* @param roleName
* the name of the role to be added.
* @return True if the role is successfully added, false if the role already exists
* @throws AuthException if exception raised when adding the role.
*/
boolean createRole(String roleName) throws AuthException;
/**
* Add a privilege on a seriesPath to a role.
*
* @param roleName
* The name of the role to which the privilege is added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the privilege is successfully granted, false if the privilege already exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToRole(String roleName, String path, int privilegeId) throws AuthException;
/**
* Delete a role.
*
* @param roleName
* the name of the role tobe deleted.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws AuthException if exception raised when deleting the role.
*/
boolean deleteRole(String roleName) throws AuthException;
/**
* Remove a privilege on a seriesPath from a role.
*
* @param roleName
* The name of the role from which the privilege is removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the privilege is successfully revoked, false if the privilege does not exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromRole(String roleName, String path, int privilegeId) throws AuthException;
/**
* Add a privilege on a seriesPath to a role.
*
* @param roleName
* The name of the role to which the privilege is added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is
* a seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the privilege is successfully granted, false if the privilege already exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToRole(String roleName, String path, int privilegeId) throws AuthException;
/**
* Add a role to a user.
*
* @param roleName
* The name of the role to be added.
* @param username
* The name of the user to which the role is added.
* @return True if the role is successfully added, false if the role already exists.
* @throws AuthException
* If either the role or the user does not exist.
*/
boolean grantRoleToUser(String roleName, String username) throws AuthException;
/**
* Remove a privilege on a seriesPath from a role.
*
* @param roleName
* The name of the role from which the privilege is removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a
* seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the privilege is successfully revoked, false if the privilege does not exists.
* @throws AuthException
* If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromRole(String roleName, String path,
int privilegeId) throws AuthException;
/**
* Revoke a role from a user.
*
* @param roleName
* The name of the role to be removed.
* @param username
* The name of the user from which the role is removed.
* @return True if the role is successfully removed, false if the role already exists.
* @throws AuthException
* If either the role or the user does not exist.
*/
boolean revokeRoleFromUser(String roleName, String username) throws AuthException;
/**
* Add a role to a user.
*
* @param roleName
* The name of the role to be added.
* @param username
* The name of the user to which the role is added.
* @return True if the role is successfully added, false if the role already exists.
* @throws AuthException
* If either the role or the user does not exist.
*/
boolean grantRoleToUser(String roleName, String username) throws AuthException;
/**
* Get the all the privileges of a user on a seriesPath.
*
* @param username
* The user whose privileges are to be queried.
* @param path
* The seriesPath on which the privileges take effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @return A set of integers each present a privilege.
* @throws AuthException
*/
Set<Integer> getPrivileges(String username, String path) throws AuthException;
/**
* Revoke a role from a user.
*
* @param roleName
* The name of the role to be removed.
* @param username
* The name of the user from which the role is removed.
* @return True if the role is successfully removed, false if the role already exists.
* @throws AuthException
* If either the role or the user does not exist.
*/
boolean revokeRoleFromUser(String roleName, String username) throws AuthException;
/**
* Modify the password of a user.
*
* @param username
* The user whose password is to be modified.
* @param newPassword
* The new password.
* @return True if the password is successfully modified, false if the new password is illegal.
* @throws AuthException
* If the user does not exists.
*/
boolean updateUserPassword(String username, String newPassword) throws AuthException;
/**
* Get the all the privileges of a user on a seriesPath.
*
* @param username
* The user whose privileges are to be queried.
* @param path
* The seriesPath on which the privileges take effect. If the privilege is
* a seriesPath-free privilege, this should be "root".
* @return A set of integers each present a privilege.
* @throws AuthException if exception raised when finding the privileges.
*/
Set<Integer> getPrivileges(String username, String path) throws AuthException;
/**
* Check if the user have the privilege on the seriesPath.
*
* @param username
* The name of the user whose privileges are checked.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the user has such privilege, false if the user does not have such privilege.
* @throws AuthException
* If the seriesPath or the privilege is illegal.
*/
boolean checkUserPrivileges(String username, String path, int privilegeId) throws AuthException;
/**
* Modify the password of a user.
*
* @param username
* The user whose password is to be modified.
* @param newPassword
* The new password.
* @return True if the password is successfully modified, false if the new password is illegal.
* @throws AuthException
* If the user does not exists.
*/
boolean updateUserPassword(String username, String newPassword) throws AuthException;
/**
* Reset the Authorizer to initiative status.
*/
void reset() throws AuthException;
/**
* Check if the user have the privilege on the seriesPath.
*
* @param username
* The name of the user whose privileges are checked.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege
* is a seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the user has such privilege, false if the user does not have such privilege.
* @throws AuthException
* If the seriesPath or the privilege is illegal.
*/
boolean checkUserPrivileges(String username, String path, int privilegeId) throws AuthException;
/**
*
* @return A list contains all usernames.
*/
List<String> listAllUsers();
/**
* Reset the Authorizer to initiative status.
*/
void reset() throws AuthException;
/**
*
* @return A list contains all roleNames.
*/
List<String> listAllRoles();
/**
* List existing users in the database.
* @return A list contains all usernames.
*/
List<String> listAllUsers();
/**
*
* @param roleName
* @return A role whose name is roleName or null if such role does not exist.
*/
Role getRole(String roleName) throws AuthException;
/**
* List existing roles in the database.
* @return A list contains all roleNames.
*/
List<String> listAllRoles();
/**
*
* @param username
* @return A user whose name is username or null if such user does not exist.
*/
User getUser(String username) throws AuthException;
/**
* Find a role by its name.
* @param roleName the name of the role.
* @return A role whose name is roleName or null if such role does not exist.
*/
Role getRole(String roleName) throws AuthException;
/**
* Find a user by its name.
* @param username the name of the user.
* @return A user whose name is username or null if such user does not exist.
*/
User getUser(String username) throws AuthException;
}

63
iotdb/src/main/java/org/apache/iotdb/db/auth/authorizer/LocalFileAuthorizer.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,41 +19,46 @@
*/
package org.apache.iotdb.db.auth.authorizer;
import java.io.File;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.Role.LocalFileRoleManager;
import org.apache.iotdb.db.auth.role.LocalFileRoleManager;
import org.apache.iotdb.db.auth.user.LocalFileUserManager;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
public class LocalFileAuthorizer extends BasicAuthorizer {
private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static Logger logger = LoggerFactory.getLogger(LocalFileAuthorizer.class);
private LocalFileAuthorizer() throws AuthException {
super(new LocalFileUserManager(config.dataDir + File.separator + "users" + File.separator),
new LocalFileRoleManager(config.dataDir + File.separator + "roles" + File.separator));
private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static Logger logger = LoggerFactory.getLogger(LocalFileAuthorizer.class);
private LocalFileAuthorizer() throws AuthException {
super(new LocalFileUserManager(config.dataDir + File.separator + "users" + File.separator),
new LocalFileRoleManager(config.dataDir + File.separator + "roles" + File.separator));
}
/**
* function for getting the instance of the local file authorizer.
*/
public static LocalFileAuthorizer getInstance() throws AuthException {
if (InstanceHolder.instance == null) {
throw new AuthException("Authorizer uninitialized");
}
return InstanceHolder.instance;
}
private static class InstanceHolder {
private static LocalFileAuthorizer instance;
private static class InstanceHolder {
static {
try {
instance = new LocalFileAuthorizer();
} catch (AuthException e) {
logger.error("Authorizer initialization failed due to ", e);
instance = null;
}
}
}
public static LocalFileAuthorizer getInstance() throws AuthException {
if (InstanceHolder.instance == null)
throw new AuthException("Authorizer uninitialized");
return InstanceHolder.instance;
private static LocalFileAuthorizer instance;
static {
try {
instance = new LocalFileAuthorizer();
} catch (AuthException e) {
logger.error("Authorizer initialization failed due to ", e);
instance = null;
}
}
}
}

98
iotdb/src/main/java/org/apache/iotdb/db/auth/entity/PathPrivilege.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -22,56 +26,58 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
/**
* This class represents a privilege on a specific seriesPath. If the privilege is seriesPath-free, the seriesPath will
* be null.
* This class represents a privilege on a specific seriesPath. If the privilege is seriesPath-free,
* the seriesPath will be null.
*/
public class PathPrivilege {
public Set<Integer> privileges;
public String path;
/**
* This field records how many times this privilege is referenced during a life cycle (from being loaded to being
* discarded). When serialized to a file, this determines the order of serialization. The higher this values is, the
* sooner this privilege will be serialized. As a result, the hot privileges will be in the first place so that the
* hit time will decrease when being queried.
*/
public AtomicInteger referenceCnt = new AtomicInteger(0);
/**
* Sort PathPrivilege by referenceCnt in descent order.
*/
public static Comparator<PathPrivilege> referenceDescentSorter = (o1,
o2) -> -Integer.compare(o1.referenceCnt.get(), o2.referenceCnt.get());
public Set<Integer> privileges;
public String path;
/**
* This field records how many times this privilege is referenced during a life cycle (from being
* loaded to being discarded). When serialized to a file, this determines the order of
* serialization. The higher this values is, the sooner this privilege will be serialized. As a
* result, the hot privileges will be in the first place so that the hit time will decrease when
* being queried.
*/
public AtomicInteger referenceCnt = new AtomicInteger(0);
public PathPrivilege(String path) {
this.path = path;
this.privileges = new HashSet<>();
public PathPrivilege(String path) {
this.path = path;
this.privileges = new HashSet<>();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
PathPrivilege that = (PathPrivilege) o;
return Objects.equals(privileges, that.privileges) && Objects.equals(path, that.path);
if (o == null || getClass() != o.getClass()) {
return false;
}
PathPrivilege that = (PathPrivilege) o;
return Objects.equals(privileges, that.privileges) && Objects.equals(path, that.path);
}
@Override
public int hashCode() {
@Override
public int hashCode() {
return Objects.hash(privileges, path);
}
/**
* Sort PathPrivilege by referenceCnt in descent order.
*/
public static Comparator<PathPrivilege> referenceDescentSorter = (o1,
o2) -> -Integer.compare(o1.referenceCnt.get(), o2.referenceCnt.get());
@Override
public String toString() {
StringBuilder builder = new StringBuilder(path);
builder.append(" :");
for (Integer privilegeId : privileges) {
builder.append(" ").append(PrivilegeType.values()[privilegeId]);
}
builder.append("\n");
return builder.toString();
return Objects.hash(privileges, path);
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(path);
builder.append(" :");
for (Integer privilegeId : privileges) {
builder.append(" ").append(PrivilegeType.values()[privilegeId]);
}
builder.append("\n");
return builder.toString();
}
}

36
iotdb/src/main/java/org/apache/iotdb/db/auth/entity/PrivilegeType.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -19,17 +23,19 @@ package org.apache.iotdb.db.auth.entity;
* This enum class contains all available privileges in IoTDB.
*/
public enum PrivilegeType {
SET_STORAGE_GROUP, INSERT_TIMESERIES, UPDATE_TIMESERIES, READ_TIMESERIES, DELETE_TIMESERIES, CREATE_USER, DELETE_USER, MODIFY_PASSWORD, LIST_USER, GRANT_USER_PRIVILEGE, REVOKE_USER_PRIVILEGE, GRANT_USER_ROLE, REVOKE_USER_ROLE, CREATE_ROLE, DELETE_ROLE, LIST_ROLE, GRANT_ROLE_PRIVILEGE, REVOKE_ROLE_PRIVILEGE, ALL;
SET_STORAGE_GROUP, INSERT_TIMESERIES, UPDATE_TIMESERIES, READ_TIMESERIES, DELETE_TIMESERIES,
CREATE_USER, DELETE_USER, MODIFY_PASSWORD, LIST_USER, GRANT_USER_PRIVILEGE, REVOKE_USER_PRIVILEGE,
GRANT_USER_ROLE, REVOKE_USER_ROLE, CREATE_ROLE, DELETE_ROLE, LIST_ROLE, GRANT_ROLE_PRIVILEGE,
REVOKE_ROLE_PRIVILEGE, ALL;
/**
* Some privileges need a seriesPath as parameter, while others do not. This method returns which privileges need a
* seriesPath.
*
* @param type
* An integer that represents a privilege.
* @return Whether this privilege need a seriesPath or not.
*/
public static boolean isPathRelevant(int type) {
return type <= DELETE_TIMESERIES.ordinal();
}
/**
* Some privileges need a seriesPath as parameter, while others do not. This method returns which
* privileges need a seriesPath.
*
* @param type An integer that represents a privilege.
* @return Whether this privilege need a seriesPath or not.
*/
public static boolean isPathRelevant(int type) {
return type <= DELETE_TIMESERIES.ordinal();
}
}

116
iotdb/src/main/java/org/apache/iotdb/db/auth/entity/Role.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,66 +19,74 @@
*/
package org.apache.iotdb.db.auth.entity;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import org.apache.iotdb.db.utils.AuthUtils;
import org.apache.iotdb.db.utils.AuthUtils;
import java.util.*;
/**
* This class contains all information of a role.
*/
public class Role {
public String name;
public List<PathPrivilege> privilegeList;
public Role() {
public String name;
public List<PathPrivilege> privilegeList;
public Role() {
}
public Role(String name) {
this.name = name;
this.privilegeList = new ArrayList<>();
}
public boolean hasPrivilege(String path, int privilegeId) {
return AuthUtils.hasPrivilege(path, privilegeId, privilegeList);
}
public void addPrivilege(String path, int privilgeId) {
AuthUtils.addPrivilege(path, privilgeId, privilegeList);
}
public void removePrivilege(String path, int privilgeId) {
AuthUtils.removePrivilege(path, privilgeId, privilegeList);
}
/**
* set privileges of path.
*/
public void setPrivileges(String path, Set<Integer> privileges) {
for (PathPrivilege pathPrivilege : privilegeList) {
if (pathPrivilege.path.equals(path)) {
pathPrivilege.privileges = privileges;
}
}
}
public Role(String name) {
this.name = name;
this.privilegeList = new ArrayList<>();
public Set<Integer> getPrivileges(String path) {
return AuthUtils.getPrivileges(path, privilegeList);
}
public boolean checkPrivilege(String path, int privilegeId) {
return AuthUtils.checkPrivilege(path, privilegeId, privilegeList);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
public boolean hasPrivilege(String path, int privilegeId) {
return AuthUtils.hasPrivilege(path, privilegeId, privilegeList);
if (o == null || getClass() != o.getClass()) {
return false;
}
Role role = (Role) o;
return Objects.equals(name, role.name) && Objects.equals(privilegeList, role.privilegeList);
}
public void addPrivilege(String path, int privilgeId) {
AuthUtils.addPrivilege(path, privilgeId, privilegeList);
}
@Override
public int hashCode() {
public void removePrivilege(String path, int privilgeId) {
AuthUtils.removePrivilege(path, privilgeId, privilegeList);
}
public void setPrivileges(String path, Set<Integer> privileges) {
for (PathPrivilege pathPrivilege : privilegeList) {
if (pathPrivilege.path.equals(path))
pathPrivilege.privileges = privileges;
}
}
public Set<Integer> getPrivileges(String path) {
return AuthUtils.getPrivileges(path, privilegeList);
}
public boolean checkPrivilege(String path, int privilegeId) {
return AuthUtils.checkPrivilege(path, privilegeId, privilegeList);
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
Role role = (Role) o;
return Objects.equals(name, role.name) && Objects.equals(privilegeList, role.privilegeList);
}
@Override
public int hashCode() {
return Objects.hash(name, privilegeList);
}
return Objects.hash(name, privilegeList);
}
}

155
iotdb/src/main/java/org/apache/iotdb/db/auth/entity/User.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,80 +19,99 @@
*/
package org.apache.iotdb.db.auth.entity;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import org.apache.iotdb.db.utils.AuthUtils;
import org.apache.iotdb.db.utils.AuthUtils;
import java.util.*;
/**
* This class contains all information of a User.
*/
public class User {
public String name;
public String password;
public List<PathPrivilege> privilegeList;
public List<String> roleList;
/**
* The latest time when the user is referenced. Reserved to provide session control or LRU mechanism in the future.
*/
public long lastActiveTime;
public User() {
public String name;
public String password;
public List<PathPrivilege> privilegeList;
public List<String> roleList;
/**
* The latest time when the user is referenced. Reserved to provide session control or LRU
* mechanism in the future.
*/
public long lastActiveTime;
public User() {
}
/**
* construct function for User.
*
* @param name -user name
* @param password -user password
*/
public User(String name, String password) {
this.name = name;
this.password = password;
this.privilegeList = new ArrayList<>();
this.roleList = new ArrayList<>();
}
public boolean hasPrivilege(String path, int privilegeId) {
return AuthUtils.hasPrivilege(path, privilegeId, privilegeList);
}
public void addPrivilege(String path, int privilgeId) {
AuthUtils.addPrivilege(path, privilgeId, privilegeList);
}
public void removePrivilege(String path, int privilgeId) {
AuthUtils.removePrivilege(path, privilgeId, privilegeList);
}
/**
* set the privilege.
*
* @param path -path
* @param privileges -set of integer to determine privilege
*/
public void setPrivileges(String path, Set<Integer> privileges) {
for (PathPrivilege pathPrivilege : privilegeList) {
if (pathPrivilege.path.equals(path)) {
pathPrivilege.privileges = privileges;
}
}
}
public User(String name, String password) {
this.name = name;
this.password = password;
this.privilegeList = new ArrayList<>();
this.roleList = new ArrayList<>();
public boolean hasRole(String roleName) {
return roleList.contains(roleName);
}
public Set<Integer> getPrivileges(String path) {
return AuthUtils.getPrivileges(path, privilegeList);
}
public boolean checkPrivilege(String path, int privilegeId) {
return AuthUtils.checkPrivilege(path, privilegeId, privilegeList);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
public boolean hasPrivilege(String path, int privilegeId) {
return AuthUtils.hasPrivilege(path, privilegeId, privilegeList);
if (o == null || getClass() != o.getClass()) {
return false;
}
User user = (User) o;
return lastActiveTime == user.lastActiveTime && Objects.equals(name, user.name)
&& Objects.equals(password, user.password) && Objects
.equals(privilegeList, user.privilegeList)
&& Objects.equals(roleList, user.roleList);
}
public void addPrivilege(String path, int privilgeId) {
AuthUtils.addPrivilege(path, privilgeId, privilegeList);
}
@Override
public int hashCode() {
public void removePrivilege(String path, int privilgeId) {
AuthUtils.removePrivilege(path, privilgeId, privilegeList);
}
public void setPrivileges(String path, Set<Integer> privileges) {
for (PathPrivilege pathPrivilege : privilegeList) {
if (pathPrivilege.path.equals(path))
pathPrivilege.privileges = privileges;
}
}
public boolean hasRole(String roleName) {
return roleList.contains(roleName);
}
public Set<Integer> getPrivileges(String path) {
return AuthUtils.getPrivileges(path, privilegeList);
}
public boolean checkPrivilege(String path, int privilegeId) {
return AuthUtils.checkPrivilege(path, privilegeId, privilegeList);
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
User user = (User) o;
return lastActiveTime == user.lastActiveTime && Objects.equals(name, user.name)
&& Objects.equals(password, user.password) && Objects.equals(privilegeList, user.privilegeList)
&& Objects.equals(roleList, user.roleList);
}
@Override
public int hashCode() {
return Objects.hash(name, password, privilegeList, roleList, lastActiveTime);
}
return Objects.hash(name, password, privilegeList, roleList, lastActiveTime);
}
}

172
iotdb/src/main/java/org/apache/iotdb/db/auth/role/BasicRoleManager.java Normal file
View File

@ -0,0 +1,172 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.role;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.concurrent.HashLock;
import org.apache.iotdb.db.utils.AuthUtils;
/**
* This class reads roles from local files through LocalFileRoleAccessor and manages them in a hash
* map.
*/
public abstract class BasicRoleManager implements IRoleManager {
private Map<String, Role> roleMap;
private IRoleAccessor accessor;
private HashLock lock;
BasicRoleManager(LocalFileRoleAccessor accessor) {
this.roleMap = new HashMap<>();
this.accessor = accessor;
this.lock = new HashLock();
}
@Override
public Role getRole(String rolename) throws AuthException {
lock.readLock(rolename);
Role role = roleMap.get(rolename);
try {
if (role == null) {
role = accessor.loadRole(rolename);
if (role != null) {
roleMap.put(rolename, role);
}
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.readUnlock(rolename);
}
return role;
}
@Override
public boolean createRole(String rolename) throws AuthException {
AuthUtils.validateRolename(rolename);
Role role = getRole(rolename);
if (role != null) {
return false;
}
lock.writeLock(rolename);
try {
role = new Role(rolename);
accessor.saveRole(role);
roleMap.put(rolename, role);
return true;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean deleteRole(String rolename) throws AuthException {
lock.writeLock(rolename);
try {
if (accessor.deleteRole(rolename)) {
roleMap.remove(rolename);
return true;
} else {
return false;
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean grantPrivilegeToRole(String rolename, String path, int privilegeId)
throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(rolename);
try {
Role role = getRole(rolename);
if (role == null) {
throw new AuthException(String.format("No such role %s", rolename));
}
if (role.hasPrivilege(path, privilegeId)) {
return false;
}
Set<Integer> privilegesCopy = new HashSet<>(role.getPrivileges(path));
role.addPrivilege(path, privilegeId);
try {
accessor.saveRole(role);
} catch (IOException e) {
role.setPrivileges(path, privilegesCopy);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public boolean revokePrivilegeFromRole(String rolename, String path, int privilegeId)
throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(rolename);
try {
Role role = getRole(rolename);
if (role == null) {
throw new AuthException(String.format("No such role %s", rolename));
}
if (!role.hasPrivilege(path, privilegeId)) {
return false;
}
role.removePrivilege(path, privilegeId);
try {
accessor.saveRole(role);
} catch (IOException e) {
role.addPrivilege(path, privilegeId);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(rolename);
}
}
@Override
public void reset() {
accessor.reset();
roleMap.clear();
lock.reset();
}
@Override
public List<String> listAllRoles() {
List<String> rtlist = accessor.listAllRoles();
rtlist.sort(null);
return rtlist;
}
}

70
iotdb/src/main/java/org/apache/iotdb/db/auth/role/IRoleAccessor.java Normal file
View File

@ -0,0 +1,70 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.role;
import java.io.IOException;
import java.util.List;
import org.apache.iotdb.db.auth.entity.Role;
/**
* This interface manages the serialization/deserialization of the role objects.
*/
public interface IRoleAccessor {
/**
* Deserialize a role from lower storage.
*
* @param rolename
* The name of the role to be deserialized.
* @return The role object or null if no such role.
* @throws IOException if IOException is raised when interacting with lower storage.
*/
Role loadRole(String rolename) throws IOException;
/**
* Serialize the role object to lower storage.
*
* @param role
* The role object that is to be saved.
* @throws IOException if IOException is raised when interacting with lower storage.
*/
void saveRole(Role role) throws IOException;
/**
* Delete a role's in lower storage.
*
* @param rolename
* The name of the role to be deleted.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws IOException if IOException is raised when interacting with lower storage.
*/
boolean deleteRole(String rolename) throws IOException;
/**
* List all roles in this database.
* @return A list contains all names of the roles.
*/
List<String> listAllRoles();
/**
* Re-initialize this object.
*/
void reset();
}

94
iotdb/src/main/java/org/apache/iotdb/db/auth/role/IRoleManager.java Normal file
View File

@ -0,0 +1,94 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.role;
import java.util.List;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.Role;
/**
* This interface maintains roles in memory and is responsible for their modifications.
*/
public interface IRoleManager {
/**
* Get a role object.
*
* @param rolename The name of the role.
* @return A role object whose name is rolename or null if such role does not exist.
* @throws AuthException if exception is raised while getting the role.
*/
Role getRole(String rolename) throws AuthException;
/**
* Create a role with given rolename. New roles will only be granted no privileges.
*
* @param rolename is not null or empty
* @return True if the role is successfully created, false when the role already exists.
* @throws AuthException f the given rolename is iIllegal.
*/
boolean createRole(String rolename) throws AuthException;
/**
* Delete a role.
*
* @param rolename the rolename of the role.
* @return True if the role is successfully deleted, false if the role does not exists.
* @throws AuthException if exception is raised while finding the role.
*/
boolean deleteRole(String rolename) throws AuthException;
/**
* Grant a privilege on a seriesPath to a role.
*
* @param rolename The rolename of the role to which the privilege should be added.
* @param path The seriesPath on which the privilege takes effect. If the privilege
* is a seriesPath-free privilege, this should be "root".
* @param privilegeId An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToRole(String rolename, String path, int privilegeId) throws AuthException;
/**
* Revoke a privilege on seriesPath from a role.
*
* @param rolename The rolename of the role from which the privilege should be removed.
* @param path The seriesPath on which the privilege takes effect. If the privilege
* is a seriesPath-free privilege like 'CREATE_USER', this should be "root".
* @param privilegeId An integer that represents a privilege.
* @return True if the permission is successfully revoked,
* false if the permission does not exists.
* @throws AuthException If the role does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromRole(String rolename, String path,
int privilegeId) throws AuthException;
/**
* Re-initialize this object.
*/
void reset();
/**
* List all roles in the database.
* @return A list that contains names of all roles.
*/
List<String> listAllRoles();
}

168
iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleAccessor.java Normal file
View File

@ -0,0 +1,168 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.role;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.Role;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.utils.IOUtils;
/**
* This class store each role in a separate sequential file. Role file schema : Int32 role name
* length Utf-8 role name bytes Int32 seriesPath privilege number n Int32 seriesPath[1] length Utf-8
* seriesPath[1] bytes Int32 privilege num k1 Int32 privilege[1][1] Int32 privilege[1][2] ... Int32
* privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2] bytes Int32 privilege num yk2
* Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32 seriesPath[n]
* length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
* privilege[n][2] ... Int32 privilege[n][kn]
*/
public class LocalFileRoleAccessor implements IRoleAccessor {
private static final String TEMP_SUFFIX = ".temp";
private static final String STRING_ENCODING = "utf-8";
private String roleDirPath;
/**
* Reused buffer for primitive types encoding/decoding, which aim to reduce memory fragments. Use
* ThreadLocal for thread safety.
*/
private ThreadLocal<ByteBuffer> encodingBufferLocal = new ThreadLocal<>();
private ThreadLocal<byte[]> strBufferLocal = new ThreadLocal<>();
public LocalFileRoleAccessor(String roleDirPath) {
this.roleDirPath = roleDirPath;
}
@Override
public Role loadRole(String rolename) throws IOException {
File roleProfile = new File(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
if (!roleProfile.exists() || !roleProfile.isFile()) {
// System may crush before a newer file is written, so search for back-up file.
File backProfile = new File(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (backProfile.exists() && backProfile.isFile()) {
roleProfile = backProfile;
} else {
return null;
}
}
FileInputStream inputStream = new FileInputStream(roleProfile);
try (DataInputStream dataInputStream = new DataInputStream(
new BufferedInputStream(inputStream))) {
Role role = new Role();
role.name = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
int privilegeNum = dataInputStream.readInt();
List<PathPrivilege> pathPrivilegeList = new ArrayList<>();
for (int i = 0; i < privilegeNum; i++) {
pathPrivilegeList
.add(IOUtils.readPathPrivilege(dataInputStream, STRING_ENCODING, strBufferLocal));
}
role.privilegeList = pathPrivilegeList;
return role;
} catch (Exception e) {
throw new IOException(e.getMessage());
}
}
@Override
public void saveRole(Role role) throws IOException {
File roleProfile = new File(
roleDirPath + File.separator + role.name + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(roleProfile));
try {
IOUtils.writeString(outputStream, role.name, STRING_ENCODING, encodingBufferLocal);
role.privilegeList.sort(PathPrivilege.referenceDescentSorter);
int privilegeNum = role.privilegeList.size();
IOUtils.writeInt(outputStream, privilegeNum, encodingBufferLocal);
for (int i = 0; i < privilegeNum; i++) {
PathPrivilege pathPrivilege = role.privilegeList.get(i);
IOUtils
.writePathPrivilege(outputStream, pathPrivilege, STRING_ENCODING, encodingBufferLocal);
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
outputStream.flush();
outputStream.close();
}
File oldFile = new File(
roleDirPath + File.separator + role.name + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(roleProfile, oldFile);
}
@Override
public boolean deleteRole(String rolename) throws IOException {
File roleProfile = new File(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX);
File backFile = new File(
roleDirPath + File.separator + rolename + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!roleProfile.exists() && !backFile.exists()) {
return false;
}
if ((roleProfile.exists() && !roleProfile.delete()) || (backFile.exists() && !backFile
.delete())) {
throw new IOException(String.format("Cannot delete role file of %s", rolename));
}
return true;
}
@Override
public List<String> listAllRoles() {
File roleDir = new File(roleDirPath);
String[] names = roleDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name
.endsWith(TEMP_SUFFIX));
List<String> retList = new ArrayList<>();
if (names != null) {
// in very rare situations, normal file and backup file may exist at the same time
// so a set is used to deduplicate
Set<String> set = new HashSet<>();
for (String fileName : names) {
set.add(fileName.replace(IoTDBConstant.PROFILE_SUFFIX, "").replace(TEMP_SUFFIX, ""));
}
retList.addAll(set);
}
return retList;
}
@Override
public void reset() {
new File(roleDirPath).mkdirs();
}
}

27
iotdb/src/main/java/org/apache/iotdb/db/auth/role/LocalFileRoleManager.java Normal file
View File

@ -0,0 +1,27 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.iotdb.db.auth.role;
public class LocalFileRoleManager extends BasicRoleManager {
public LocalFileRoleManager(String roleDirPath) {
super(new LocalFileRoleAccessor(roleDirPath));
}
}

463
iotdb/src/main/java/org/apache/iotdb/db/auth/user/BasicUserManager.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,6 +19,12 @@
*/
package org.apache.iotdb.db.auth.user;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.concurrent.HashLock;
@ -23,244 +33,253 @@ import org.apache.iotdb.db.utils.AuthUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
/**
* This class stores information of each user in a separate file within a directory, and cache them in memory when a
* user is accessed.
* This class stores information of each user in a separate file within a directory, and cache them
* in memory when a user is accessed.
*/
public abstract class BasicUserManager implements IUserManager {
private static final Logger logger = LoggerFactory.getLogger(BasicUserManager.class);
private static final Logger logger = LoggerFactory.getLogger(BasicUserManager.class);
private Map<String, User> userMap;
private IUserAccessor accessor;
private HashLock lock;
private Map<String, User> userMap;
private IUserAccessor accessor;
private HashLock lock;
public BasicUserManager(IUserAccessor accessor) throws AuthException {
this.userMap = new HashMap<>();
this.accessor = accessor;
this.lock = new HashLock();
/**
* BasicUserManager Constructor.
*
* @param accessor user accessor
* @throws AuthException Authentication Exception
*/
public BasicUserManager(IUserAccessor accessor) throws AuthException {
this.userMap = new HashMap<>();
this.accessor = accessor;
this.lock = new HashLock();
reset();
reset();
}
/**
* Try to load admin. If it doesn't exist, automatically create one.
*/
private void initAdmin() throws AuthException {
User admin;
try {
admin = getUser(IoTDBConstant.ADMIN_NAME);
} catch (AuthException e) {
logger.warn("Cannot load admin because {}. Create a new one.", e.getMessage());
admin = null;
}
/**
* Try to load admin. If it doesn't exist, automatically create one.
*/
private void initAdmin() throws AuthException {
User admin;
try {
admin = getUser(IoTDBConstant.ADMIN_NAME);
} catch (AuthException e) {
logger.warn("Cannot load admin because {}. Create a new one.", e.getMessage());
admin = null;
if (admin == null) {
createUser(IoTDBConstant.ADMIN_NAME, IoTDBConstant.ADMIN_PW);
}
logger.info("Admin initialized");
}
@Override
public User getUser(String username) throws AuthException {
lock.readLock(username);
User user = userMap.get(username);
try {
if (user == null) {
user = accessor.loadUser(username);
if (user != null) {
userMap.put(username, user);
}
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.readUnlock(username);
}
if (user != null) {
user.lastActiveTime = System.currentTimeMillis();
}
return user;
}
if (admin == null) {
createUser(IoTDBConstant.ADMIN_NAME, IoTDBConstant.ADMIN_PW);
}
logger.info("Admin initialized");
@Override
public boolean createUser(String username, String password) throws AuthException {
AuthUtils.validateUsername(username);
AuthUtils.validatePassword(password);
User user = getUser(username);
if (user != null) {
return false;
}
lock.writeLock(username);
try {
user = new User(username, AuthUtils.encryptPassword(password));
accessor.saveUser(user);
userMap.put(username, user);
return true;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean deleteUser(String username) throws AuthException {
lock.writeLock(username);
try {
if (accessor.deleteUser(username)) {
userMap.remove(username);
return true;
} else {
return false;
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean grantPrivilegeToUser(String username, String path, int privilegeId)
throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (user.hasPrivilege(path, privilegeId)) {
return false;
}
Set<Integer> privilegesCopy = new HashSet<>(user.getPrivileges(path));
user.addPrivilege(path, privilegeId);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.setPrivileges(path, privilegesCopy);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean revokePrivilegeFromUser(String username, String path, int privilegeId)
throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (!user.hasPrivilege(path, privilegeId)) {
return false;
}
user.removePrivilege(path, privilegeId);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.addPrivilege(path, privilegeId);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean updateUserPassword(String username, String newPassword) throws AuthException {
try {
AuthUtils.validatePassword(newPassword);
} catch (AuthException e) {
return false;
}
@Override
public User getUser(String username) throws AuthException {
lock.readLock(username);
User user = userMap.get(username);
try {
if (user == null) {
user = accessor.loadUser(username);
if (user != null)
userMap.put(username, user);
}
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.readUnlock(username);
}
if (user != null)
user.lastActiveTime = System.currentTimeMillis();
return user;
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
String oldPassword = user.password;
user.password = AuthUtils.encryptPassword(newPassword);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.password = oldPassword;
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean createUser(String username, String password) throws AuthException {
AuthUtils.validateUsername(username);
AuthUtils.validatePassword(password);
User user = getUser(username);
if (user != null)
return false;
lock.writeLock(username);
try {
user = new User(username, AuthUtils.encryptPassword(password));
accessor.saveUser(user);
userMap.put(username, user);
return true;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(username);
}
@Override
public boolean grantRoleToUser(String roleName, String username) throws AuthException {
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (user.hasRole(roleName)) {
return false;
}
user.roleList.add(roleName);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.roleList.remove(roleName);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean deleteUser(String username) throws AuthException {
lock.writeLock(username);
try {
if (accessor.deleteUser(username)) {
userMap.remove(username);
return true;
} else
return false;
} catch (IOException e) {
throw new AuthException(e);
} finally {
lock.writeUnlock(username);
}
@Override
public boolean revokeRoleFromUser(String roleName, String username) throws AuthException {
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (!user.hasRole(roleName)) {
return false;
}
user.roleList.remove(roleName);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.roleList.add(roleName);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (user.hasPrivilege(path, privilegeId)) {
return false;
}
Set<Integer> privilegesCopy = new HashSet<>(user.getPrivileges(path));
user.addPrivilege(path, privilegeId);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.setPrivileges(path, privilegesCopy);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public void reset() throws AuthException {
accessor.reset();
userMap.clear();
lock.reset();
initAdmin();
}
@Override
public boolean revokePrivilegeFromUser(String username, String path, int privilegeId) throws AuthException {
AuthUtils.validatePrivilegeOnPath(path, privilegeId);
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (!user.hasPrivilege(path, privilegeId)) {
return false;
}
user.removePrivilege(path, privilegeId);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.addPrivilege(path, privilegeId);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean updateUserPassword(String username, String newPassword) throws AuthException {
try {
AuthUtils.validatePassword(newPassword);
} catch (AuthException e) {
return false;
}
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
String oldPassword = user.password;
user.password = AuthUtils.encryptPassword(newPassword);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.password = oldPassword;
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean grantRoleToUser(String roleName, String username) throws AuthException {
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (user.hasRole(roleName)) {
return false;
}
user.roleList.add(roleName);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.roleList.remove(roleName);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public boolean revokeRoleFromUser(String roleName, String username) throws AuthException {
lock.writeLock(username);
try {
User user = getUser(username);
if (user == null) {
throw new AuthException(String.format("No such user %s", username));
}
if (!user.hasRole(roleName)) {
return false;
}
user.roleList.remove(roleName);
try {
accessor.saveUser(user);
} catch (IOException e) {
user.roleList.add(roleName);
throw new AuthException(e);
}
return true;
} finally {
lock.writeUnlock(username);
}
}
@Override
public void reset() throws AuthException {
accessor.reset();
userMap.clear();
lock.reset();
initAdmin();
}
@Override
public List<String> listAllUsers() {
List<String> rtlist = accessor.listAllUsers();
rtlist.sort(null);
return rtlist;
}
@Override
public List<String> listAllUsers() {
List<String> rtlist = accessor.listAllUsers();
rtlist.sort(null);
return rtlist;
}
}

84
iotdb/src/main/java/org/apache/iotdb/db/auth/user/IUserAccessor.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,54 +19,52 @@
*/
package org.apache.iotdb.db.auth.user;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.auth.entity.User;
import java.io.IOException;
import java.util.List;
import org.apache.iotdb.db.auth.entity.User;
/**
* This interface manages the serialization/deserialization of the user objects.
*/
public interface IUserAccessor {
/**
* Deserialize a user from lower storage.
*
* @param username
* The name of the user to be deserialized.
* @return The user object or null if no such user.
* @throws IOException
*/
User loadUser(String username) throws IOException;
/**
* Deserialize a user from lower storage.
*
* @param username
* The name of the user to be deserialized.
* @return The user object or null if no such user.
* @throws IOException if an exception is raised when interacting with the lower storage.
*/
User loadUser(String username) throws IOException;
/**
* Serialize the user object to lower storage.
*
* @param user
* The user object that is to be saved.
* @throws IOException
*/
void saveUser(User user) throws IOException;
/**
* Serialize the user object to lower storage.
*
* @param user
* The user object that is to be saved.
* @throws IOException if an exception is raised when interacting with the lower storage.
*/
void saveUser(User user) throws IOException;
/**
* Delete a user's from lower storage.
*
* @param username
* The name of the user to be deleted.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws IOException
*/
boolean deleteUser(String username) throws IOException;
/**
* Delete a user's from lower storage.
*
* @param username
* The name of the user to be deleted.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws IOException if an exception is raised when interacting with the lower storage.
*/
boolean deleteUser(String username) throws IOException;
/**
*
* @return A list that contains names of all users.
*/
List<String> listAllUsers();
/**
* List all users existing in the database.
* @return A list that contains names of all users.
*/
List<String> listAllUsers();
/**
* Re-initialize this object.
*/
void reset();
/**
* Re-initialize this object.
*/
void reset();
}

228
iotdb/src/main/java/org/apache/iotdb/db/auth/user/IUserManager.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,130 +19,130 @@
*/
package org.apache.iotdb.db.auth.user;
import java.util.List;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.auth.entity.User;
import java.util.List;
/**
* This interface provides accesses to users.
*/
public interface IUserManager {
/**
* Get a user object.
*
* @param username
* The name of the user.
* @return A user object whose name is username or null if such user does not exist.
* @throws AuthException
*/
User getUser(String username) throws AuthException;
/**
* Get a user object.
*
* @param username
* The name of the user.
* @return A user object whose name is username or null if such user does not exist.
* @throws AuthException if an exception is raised when interacting with the lower storage.
*/
User getUser(String username) throws AuthException;
/**
* Create a user with given username and password. New users will only be granted no privileges.
*
* @param username
* is not null or empty
* @param password
* is not null or empty
* @return True if the user is successfully created, false when the user already exists.
* @throws AuthException
* if the given username or password is illegal.
*/
boolean createUser(String username, String password) throws AuthException;
/**
* Create a user with given username and password. New users will only be granted no privileges.
*
* @param username
* is not null or empty
* @param password
* is not null or empty
* @return True if the user is successfully created, false when the user already exists.
* @throws AuthException
* if the given username or password is illegal.
*/
boolean createUser(String username, String password) throws AuthException;
/**
* Delete a user.
*
* @param username
* the username of the user.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws AuthException
* .
*/
boolean deleteUser(String username) throws AuthException;
/**
* Delete a user.
*
* @param username
* the username of the user.
* @return True if the user is successfully deleted, false if the user does not exists.
* @throws AuthException
* .
*/
boolean deleteUser(String username) throws AuthException;
/**
* Grant a privilege on a seriesPath to a user.
*
* @param username
* The username of the user to which the privilege should be added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException;
/**
* Grant a privilege on a seriesPath to a user.
*
* @param username
* The username of the user to which the privilege should be added.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege
* is a seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully added, false if the permission already exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean grantPrivilegeToUser(String username, String path, int privilegeId) throws AuthException;
/**
* Revoke a privilege on seriesPath from a user.
*
* @param username
* The username of the user from which the privilege should be removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege is a seriesPath-free privilege,
* this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully revoked, false if the permission does not exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromUser(String username, String path, int privilegeId) throws AuthException;
/**
* Revoke a privilege on seriesPath from a user.
*
* @param username
* The username of the user from which the privilege should be removed.
* @param path
* The seriesPath on which the privilege takes effect. If the privilege
* is a seriesPath-free privilege, this should be "root".
* @param privilegeId
* An integer that represents a privilege.
* @return True if the permission is successfully revoked,
* false if the permission does not exists.
* @throws AuthException
* If the user does not exist or the privilege or the seriesPath is illegal.
*/
boolean revokePrivilegeFromUser(String username, String path,
int privilegeId) throws AuthException;
/**
* Modify the password of a user.
*
* @param username
* The user whose password is to be modified.
* @param newPassword
* The new password.
* @return True if the password is successfully modified, false if the new password is illegal.
* @throws AuthException
* If the user does not exists.
*/
boolean updateUserPassword(String username, String newPassword) throws AuthException;
/**
* Modify the password of a user.
*
* @param username
* The user whose password is to be modified.
* @param newPassword
* The new password.
* @return True if the password is successfully modified, false if the new password is illegal.
* @throws AuthException
* If the user does not exists.
*/
boolean updateUserPassword(String username, String newPassword) throws AuthException;
/**
* Add a role to a user.
*
* @param roleName
* The name of the role to be added.
* @param username
* The name of the user to which the role is added.
* @return True if the role is successfully added, false if the role already exists.
* @throws AuthException
* If the user does not exist.
*/
boolean grantRoleToUser(String roleName, String username) throws AuthException;
/**
* Add a role to a user.
*
* @param roleName
* The name of the role to be added.
* @param username
* The name of the user to which the role is added.
* @return True if the role is successfully added, false if the role already exists.
* @throws AuthException
* If the user does not exist.
*/
boolean grantRoleToUser(String roleName, String username) throws AuthException;
/**
* Revoke a role from a user.
*
* @param roleName
* The name of the role to be removed.
* @param username
* The name of the user from which the role is removed.
* @return True if the role is successfully removed, false if the role does not exist.
* @throws AuthException
* If the user does not exist.
*/
boolean revokeRoleFromUser(String roleName, String username) throws AuthException;
/**
* Revoke a role from a user.
*
* @param roleName
* The name of the role to be removed.
* @param username
* The name of the user from which the role is removed.
* @return True if the role is successfully removed, false if the role does not exist.
* @throws AuthException
* If the user does not exist.
*/
boolean revokeRoleFromUser(String roleName, String username) throws AuthException;
/**
* Re-initialize this object.
*/
void reset() throws AuthException;
/**
* Re-initialize this object.
*/
void reset() throws AuthException;
/**
*
* @return A list that contains all users'name.
*/
List<String> listAllUsers();
/**
* List all users in the database.
* @return A list that contains all users'name.
*/
List<String> listAllUsers();
}

322
iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserAccessor.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -15,171 +19,185 @@
*/
package org.apache.iotdb.db.auth.user;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.utils.IOUtils;
import org.apache.iotdb.db.auth.entity.PathPrivilege;
import org.apache.iotdb.db.auth.entity.User;
import org.apache.iotdb.db.utils.IOUtils;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.*;
/**
* This class loads a user's information from the corresponding file.The user file is a sequential file. User file
* schema: Int32 username bytes length Utf-8 username bytes Int32 Password bytes length Utf-8 password bytes Int32
* seriesPath privilege number n Int32 seriesPath[1] length Utf-8 seriesPath[1] bytes Int32 privilege num k1 Int32
* privilege[1][1] Int32 privilege[1][2] ... Int32 privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2] bytes
* Int32 privilege num k2 Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32 seriesPath[n]
* length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32 privilege[n][2] ... Int32
* privilege[n][kn] Int32 user name number m Int32 user name[1] length Utf-8 user name[1] bytes Int32 user name[2]
* length Utf-8 user name[2] bytes ... Int32 user name[m] length Utf-8 user name[m] bytes
* This class loads a user's information from the corresponding file.The user file is a sequential
* file. User file schema: Int32 username bytes length Utf-8 username bytes Int32 Password bytes
* length Utf-8 password bytes Int32 seriesPath privilege number n Int32 seriesPath[1] length Utf-8
* seriesPath[1] bytes Int32 privilege num k1 Int32 privilege[1][1] Int32 privilege[1][2] ... Int32
* privilege[1][k1] Int32 seriesPath[2] length Utf-8 seriesPath[2] bytes Int32 privilege num k2
* Int32 privilege[2][1] Int32 privilege[2][2] ... Int32 privilege[2][k2] ... Int32 seriesPath[n]
* length Utf-8 seriesPath[n] bytes Int32 privilege num kn Int32 privilege[n][1] Int32
* privilege[n][2] ... Int32 privilege[n][kn] Int32 user name number m Int32 user name[1] length
* Utf-8 user name[1] bytes Int32 user name[2] length Utf-8 user name[2] bytes ... Int32 user
* name[m] length Utf-8 user name[m] bytes
*/
public class LocalFileUserAccessor implements IUserAccessor {
private static final String TEMP_SUFFIX = ".temp";
private static final String STRING_ENCODING = "utf-8";
private String userDirPath;
/**
* Reused buffer for primitive types encoding/decoding, which aim to reduce memory fragments. Use ThreadLocal for
* thread safety.
*/
private ThreadLocal<ByteBuffer> encodingBufferLocal = new ThreadLocal<>();
private ThreadLocal<byte[]> strBufferLocal = new ThreadLocal<>();
private static final String TEMP_SUFFIX = ".temp";
private static final String STRING_ENCODING = "utf-8";
public LocalFileUserAccessor(String userDirPath) {
this.userDirPath = userDirPath;
private String userDirPath;
/**
* Reused buffer for primitive types encoding/decoding, which aim to reduce memory fragments. Use
* ThreadLocal for thread safety.
*/
private ThreadLocal<ByteBuffer> encodingBufferLocal = new ThreadLocal<>();
private ThreadLocal<byte[]> strBufferLocal = new ThreadLocal<>();
public LocalFileUserAccessor(String userDirPath) {
this.userDirPath = userDirPath;
}
/**
* Deserialize a user from its user file.
*
* @param username The name of the user to be deserialized.
* @return The user object or null if no such user.
*/
public User loadUser(String username) throws IOException {
File userProfile = new File(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
if (!userProfile.exists() || !userProfile.isFile()) {
// System may crush before a newer file is renamed.
File newProfile = new File(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (newProfile.exists() && newProfile.isFile()) {
newProfile.renameTo(userProfile);
userProfile = newProfile;
} else {
return null;
}
}
FileInputStream inputStream = new FileInputStream(userProfile);
try (DataInputStream dataInputStream = new DataInputStream(
new BufferedInputStream(inputStream))) {
User user = new User();
user.name = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
user.password = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
int privilegeNum = dataInputStream.readInt();
List<PathPrivilege> pathPrivilegeList = new ArrayList<>();
for (int i = 0; i < privilegeNum; i++) {
pathPrivilegeList
.add(IOUtils.readPathPrivilege(dataInputStream, STRING_ENCODING, strBufferLocal));
}
user.privilegeList = pathPrivilegeList;
int roleNum = dataInputStream.readInt();
List<String> roleList = new ArrayList<>();
for (int i = 0; i < roleNum; i++) {
String userName = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
roleList.add(userName);
}
user.roleList = roleList;
return user;
} catch (Exception e) {
throw new IOException(e);
}
}
/**
* Serialize the user object to a temp file, then replace the old user file with the new file.
*
* @param user The user object that is to be saved.
*/
public void saveUser(User user) throws IOException {
File userProfile = new File(
userDirPath + File.separator + user.name + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(userProfile));
try {
IOUtils.writeString(outputStream, user.name, STRING_ENCODING, encodingBufferLocal);
IOUtils.writeString(outputStream, user.password, STRING_ENCODING, encodingBufferLocal);
user.privilegeList.sort(PathPrivilege.referenceDescentSorter);
int privilegeNum = user.privilegeList.size();
IOUtils.writeInt(outputStream, privilegeNum, encodingBufferLocal);
for (int i = 0; i < privilegeNum; i++) {
PathPrivilege pathPrivilege = user.privilegeList.get(i);
IOUtils
.writePathPrivilege(outputStream, pathPrivilege, STRING_ENCODING, encodingBufferLocal);
}
int userNum = user.roleList.size();
IOUtils.writeInt(outputStream, userNum, encodingBufferLocal);
for (int i = 0; i < userNum; i++) {
IOUtils
.writeString(outputStream, user.roleList.get(i), STRING_ENCODING, encodingBufferLocal);
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
outputStream.flush();
outputStream.close();
}
/**
* Deserialize a user from its user file.
*
* @param username
* The name of the user to be deserialized.
* @return The user object or null if no such user.
* @throws IOException
*/
public User loadUser(String username) throws IOException {
File userProfile = new File(userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
if (!userProfile.exists() || !userProfile.isFile()) {
// System may crush before a newer file is renamed.
File newProfile = new File(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (newProfile.exists() && newProfile.isFile()) {
newProfile.renameTo(userProfile);
userProfile = newProfile;
} else
return null;
}
FileInputStream inputStream = new FileInputStream(userProfile);
try (DataInputStream dataInputStream = new DataInputStream(new BufferedInputStream(inputStream))) {
User user = new User();
user.name = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
user.password = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
File oldFile = new File(
userDirPath + File.separator + user.name + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(userProfile, oldFile);
}
int privilegeNum = dataInputStream.readInt();
List<PathPrivilege> pathPrivilegeList = new ArrayList<>();
for (int i = 0; i < privilegeNum; i++) {
pathPrivilegeList.add(IOUtils.readPathPrivilege(dataInputStream, STRING_ENCODING, strBufferLocal));
}
user.privilegeList = pathPrivilegeList;
int roleNum = dataInputStream.readInt();
List<String> roleList = new ArrayList<>();
for (int i = 0; i < roleNum; i++) {
String userName = IOUtils.readString(dataInputStream, STRING_ENCODING, strBufferLocal);
roleList.add(userName);
}
user.roleList = roleList;
return user;
} catch (Exception e) {
throw new IOException(e);
}
/**
* Delete a user's user file.
*
* @param username The name of the user to be deleted.
* @return True if the file is successfully deleted, false if the file does not exists.
* @throws IOException when the file cannot be deleted.
*/
public boolean deleteUser(String username) throws IOException {
File userProfile = new File(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
File backFile = new File(
userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!userProfile.exists() && !backFile.exists()) {
return false;
}
/**
* Serialize the user object to a temp file, then replace the old user file with the new file.
*
* @param user
* The user object that is to be saved.
* @throws IOException
*/
public void saveUser(User user) throws IOException {
File userProfile = new File(
userDirPath + File.separator + user.name + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(userProfile));
try {
IOUtils.writeString(outputStream, user.name, STRING_ENCODING, encodingBufferLocal);
IOUtils.writeString(outputStream, user.password, STRING_ENCODING, encodingBufferLocal);
user.privilegeList.sort(PathPrivilege.referenceDescentSorter);
int privilegeNum = user.privilegeList.size();
IOUtils.writeInt(outputStream, privilegeNum, encodingBufferLocal);
for (int i = 0; i < privilegeNum; i++) {
PathPrivilege pathPrivilege = user.privilegeList.get(i);
IOUtils.writePathPrivilege(outputStream, pathPrivilege, STRING_ENCODING, encodingBufferLocal);
}
int userNum = user.roleList.size();
IOUtils.writeInt(outputStream, userNum, encodingBufferLocal);
for (int i = 0; i < userNum; i++) {
IOUtils.writeString(outputStream, user.roleList.get(i), STRING_ENCODING, encodingBufferLocal);
}
} catch (Exception e) {
throw new IOException(e.getMessage());
} finally {
outputStream.flush();
outputStream.close();
}
File oldFile = new File(userDirPath + File.separator + user.name + IoTDBConstant.PROFILE_SUFFIX);
IOUtils.replaceFile(userProfile, oldFile);
if ((userProfile.exists() && !userProfile.delete()) || (backFile.exists() && !backFile
.delete())) {
throw new IOException(String.format("Cannot delete user file of %s", username));
}
return true;
}
/**
* Delete a user's user file.
*
* @param username
* The name of the user to be deleted.
* @return True if the file is successfully deleted, false if the file does not exists.
* @throws IOException
* when the file cannot be deleted.
*/
public boolean deleteUser(String username) throws IOException {
File userProfile = new File(userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX);
File backFile = new File(userDirPath + File.separator + username + IoTDBConstant.PROFILE_SUFFIX + TEMP_SUFFIX);
if (!userProfile.exists() && !backFile.exists())
return false;
if ((userProfile.exists() && !userProfile.delete()) || (backFile.exists() && !backFile.delete())) {
throw new IOException(String.format("Cannot delete user file of %s", username));
}
return true;
@Override
public List<String> listAllUsers() {
File userDir = new File(userDirPath);
String[] names = userDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name
.endsWith(TEMP_SUFFIX));
List<String> retList = new ArrayList<>();
if (names != null) {
// in very rare situations, normal file and backup file may exist at the same time
// so a set is used to deduplicate
Set<String> set = new HashSet<>();
for (String fileName : names) {
set.add(fileName.replace(IoTDBConstant.PROFILE_SUFFIX, "").replace(TEMP_SUFFIX, ""));
}
retList.addAll(set);
}
return retList;
}
@Override
public List<String> listAllUsers() {
File userDir = new File(userDirPath);
String[] names = userDir
.list((dir, name) -> name.endsWith(IoTDBConstant.PROFILE_SUFFIX) || name.endsWith(TEMP_SUFFIX));
List<String> retList = new ArrayList<>();
if (names != null) {
// in very rare situations, normal file and backup file may exist at the same time
// so a set is used to deduplicate
Set<String> set = new HashSet<>();
for (String fileName : names) {
set.add(fileName.replace(IoTDBConstant.PROFILE_SUFFIX, "").replace(TEMP_SUFFIX, ""));
}
retList.addAll(set);
}
return retList;
}
@Override
public void reset() {
new File(userDirPath).mkdirs();
}
@Override
public void reset() {
new File(userDirPath).mkdirs();
}
}

17
iotdb/src/main/java/org/apache/iotdb/db/auth/user/LocalFileUserManager.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -18,7 +22,8 @@ package org.apache.iotdb.db.auth.user;
import org.apache.iotdb.db.auth.AuthException;
public class LocalFileUserManager extends BasicUserManager {
public LocalFileUserManager(String userDirPath) throws AuthException {
super(new LocalFileUserAccessor(userDirPath));
}
public LocalFileUserManager(String userDirPath) throws AuthException {
super(new LocalFileUserAccessor(userDirPath));
}
}

108
iotdb/src/main/java/org/apache/iotdb/db/concurrent/HashLock.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -18,64 +22,68 @@ package org.apache.iotdb.db.concurrent;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* This class manages an array of locks and use the hash code of an object as an index of the array to find the
* corresponding lock, so that the operations on the same key (string) can be prevented while the number of locks remain
* controlled.
* This class manages an array of locks and use the hash code of an object as an index of
* the array to find the corresponding lock, so that the operations on the same key (string)
* can be prevented while the number of locks remain controlled.
*/
public class HashLock {
private static final int DEFAULT_LOCK_NUM = 100;
private ReentrantReadWriteLock[] locks;
private int lockSize;
private static final int DEFAULT_LOCK_NUM = 100;
public HashLock() {
this.lockSize = DEFAULT_LOCK_NUM;
init();
private ReentrantReadWriteLock[] locks;
private int lockSize;
public HashLock() {
this.lockSize = DEFAULT_LOCK_NUM;
init();
}
public HashLock(int lockSize) {
if (lockSize <= 0) {
lockSize = DEFAULT_LOCK_NUM;
}
this.lockSize = lockSize;
init();
}
public HashLock(int lockSize) {
if (lockSize <= 0)
lockSize = DEFAULT_LOCK_NUM;
this.lockSize = lockSize;
init();
private void init() {
locks = new ReentrantReadWriteLock[lockSize];
for (int i = 0; i < lockSize; i++) {
locks[i] = new ReentrantReadWriteLock();
}
}
private void init() {
locks = new ReentrantReadWriteLock[lockSize];
for (int i = 0; i < lockSize; i++) {
locks[i] = new ReentrantReadWriteLock();
}
}
public void readLock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].readLock().lock();
}
public void readLock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].readLock().lock();
}
public void readUnlock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].readLock().unlock();
}
public void readUnlock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].readLock().unlock();
}
public void writeLock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].writeLock().lock();
}
public void writeLock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].writeLock().lock();
}
public void writeUnlock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].writeLock().unlock();
}
public void writeUnlock(Object obj) {
this.locks[Math.abs(obj.hashCode()) % lockSize].writeLock().unlock();
}
/**
* This method will unlock all locks. Only for test convenience.
*/
public void reset() {
for (int i = 0; i < lockSize; i++) {
try {
locks[i].readLock().unlock();
} catch (Exception ignored) {
}
try {
locks[i].writeLock().unlock();
} catch (Exception ignored) {
}
}
/**
* This method will unlock all locks. Only for test convenience.
*/
public void reset() {
for (int i = 0; i < lockSize; i++) {
try {
locks[i].readLock().unlock();
} catch (Exception ignored) {
// ignored
}
try {
locks[i].writeLock().unlock();
} catch (Exception ignored) {
// ignored
}
}
}
}

26
iotdb/src/main/java/org/apache/iotdb/db/concurrent/IoTDBDefaultThreadExceptionHandler.java
View File

@ -1,9 +1,13 @@
/**
* Copyright © 2019 Apache IoTDB(incubating) (dev@iotdb.apache.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@ -19,14 +23,16 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class IoTDBDefaultThreadExceptionHandler implements Thread.UncaughtExceptionHandler {
private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDefaultThreadExceptionHandler.class);
public IoTDBDefaultThreadExceptionHandler() {
}
private static final Logger LOGGER = LoggerFactory
.getLogger(IoTDBDefaultThreadExceptionHandler.class);
@Override
public void uncaughtException(Thread t, Throwable e) {
LOGGER.error("Exception in thread {}-{}", t.getName(), t.getId(), e);
}
public IoTDBDefaultThreadExceptionHandler() {
}
@Override
public void uncaughtException(Thread t, Throwable e) {
LOGGER.error("Exception in thread {}-{}", t.getName(), t.getId(), e);
}
}

Some files were not shown because too many files have changed in this diff Show More