upload to trustie
This commit is contained in:
qyzh1996
parent
1c00a71758
commit
c3d8d1b23e
|
|
@ -0,0 +1,8 @@
|
|||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
# Editor-based HTTP Client requests
|
||||
/httpRequests/
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CheckStyle-IDEA">
|
||||
<option name="configuration">
|
||||
<map>
|
||||
<entry key="checkstyle-version" value="8.23" />
|
||||
<entry key="copy-libs" value="true" />
|
||||
<entry key="location-0" value="BUNDLED:(bundled):Sun Checks" />
|
||||
<entry key="location-1" value="BUNDLED:(bundled):Google Checks" />
|
||||
<entry key="scan-before-checkin" value="false" />
|
||||
<entry key="scanscope" value="JavaOnly" />
|
||||
<entry key="suppress-errors" value="false" />
|
||||
</map>
|
||||
</option>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
This folder contains libraries copied from the "stream-data-processing" project.
|
||||
It is managed by the CheckStyle-IDEA IDE plugin.
|
||||
Do not modify this folder while the IDE is running.
|
||||
When the IDE is stopped, you may delete this folder at any time. It will be recreated as needed.
|
||||
In order to prevent the CheckStyle-IDEA IDE plugin from creating this folder,
|
||||
uncheck the "Copy libraries from project directory" option in the CheckStyle-IDEA settings dialog.
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
<component name="ProjectCodeStyleConfiguration">
|
||||
<code_scheme name="Project" version="173">
|
||||
<DBN-PSQL>
|
||||
<case-options enabled="true">
|
||||
<option name="KEYWORD_CASE" value="lower" />
|
||||
<option name="FUNCTION_CASE" value="lower" />
|
||||
<option name="PARAMETER_CASE" value="lower" />
|
||||
<option name="DATATYPE_CASE" value="lower" />
|
||||
<option name="OBJECT_CASE" value="preserve" />
|
||||
</case-options>
|
||||
<formatting-settings enabled="false" />
|
||||
</DBN-PSQL>
|
||||
<DBN-SQL>
|
||||
<case-options enabled="true">
|
||||
<option name="KEYWORD_CASE" value="lower" />
|
||||
<option name="FUNCTION_CASE" value="lower" />
|
||||
<option name="PARAMETER_CASE" value="lower" />
|
||||
<option name="DATATYPE_CASE" value="lower" />
|
||||
<option name="OBJECT_CASE" value="preserve" />
|
||||
</case-options>
|
||||
<formatting-settings enabled="false">
|
||||
<option name="STATEMENT_SPACING" value="one_line" />
|
||||
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
|
||||
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
|
||||
</formatting-settings>
|
||||
</DBN-SQL>
|
||||
<DBN-PSQL>
|
||||
<case-options enabled="true">
|
||||
<option name="KEYWORD_CASE" value="lower" />
|
||||
<option name="FUNCTION_CASE" value="lower" />
|
||||
<option name="PARAMETER_CASE" value="lower" />
|
||||
<option name="DATATYPE_CASE" value="lower" />
|
||||
<option name="OBJECT_CASE" value="preserve" />
|
||||
</case-options>
|
||||
<formatting-settings enabled="false" />
|
||||
</DBN-PSQL>
|
||||
<DBN-SQL>
|
||||
<case-options enabled="true">
|
||||
<option name="KEYWORD_CASE" value="lower" />
|
||||
<option name="FUNCTION_CASE" value="lower" />
|
||||
<option name="PARAMETER_CASE" value="lower" />
|
||||
<option name="DATATYPE_CASE" value="lower" />
|
||||
<option name="OBJECT_CASE" value="preserve" />
|
||||
</case-options>
|
||||
<formatting-settings enabled="false">
|
||||
<option name="STATEMENT_SPACING" value="one_line" />
|
||||
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
|
||||
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
|
||||
</formatting-settings>
|
||||
</DBN-SQL>
|
||||
</code_scheme>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="CompilerConfiguration">
|
||||
<annotationProcessing>
|
||||
<profile name="Maven default annotation processors profile" enabled="true">
|
||||
<sourceOutputDir name="target/generated-sources/annotations" />
|
||||
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
|
||||
<outputRelativeToContentRoot value="true" />
|
||||
<module name="stream-data-processing" />
|
||||
</profile>
|
||||
</annotationProcessing>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,464 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DBNavigator.Project.DataEditorManager">
|
||||
<record-view-column-sorting-type value="BY_INDEX" />
|
||||
<value-preview-text-wrapping value="true" />
|
||||
<value-preview-pinned value="false" />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.DataExportManager">
|
||||
<export-instructions>
|
||||
<create-header value="true" />
|
||||
<quote-values-containing-separator value="true" />
|
||||
<quote-all-values value="false" />
|
||||
<value-separator value="" />
|
||||
<file-name value="" />
|
||||
<file-location value="" />
|
||||
<scope value="GLOBAL" />
|
||||
<destination value="FILE" />
|
||||
<format value="EXCEL" />
|
||||
<charset value="GBK" />
|
||||
</export-instructions>
|
||||
</component>
|
||||
<component name="DBNavigator.Project.DatabaseBrowserManager">
|
||||
<autoscroll-to-editor value="false" />
|
||||
<autoscroll-from-editor value="true" />
|
||||
<show-object-properties value="true" />
|
||||
<loaded-nodes />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.DatabaseFileManager">
|
||||
<open-files />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.EditorStateManager">
|
||||
<last-used-providers />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.MethodExecutionManager">
|
||||
<method-browser />
|
||||
<execution-history>
|
||||
<group-entries value="true" />
|
||||
<execution-inputs />
|
||||
</execution-history>
|
||||
<argument-values-cache />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.ObjectDependencyManager">
|
||||
<last-used-dependency-type value="INCOMING" />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.ObjectQuickFilterManager">
|
||||
<last-used-operator value="EQUAL" />
|
||||
<filters />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.ScriptExecutionManager" clear-outputs="true">
|
||||
<recently-used-interfaces />
|
||||
</component>
|
||||
<component name="DBNavigator.Project.Settings">
|
||||
<connections />
|
||||
<browser-settings>
|
||||
<general>
|
||||
<display-mode value="TABBED" />
|
||||
<navigation-history-size value="100" />
|
||||
<show-object-details value="false" />
|
||||
</general>
|
||||
<filters>
|
||||
<object-type-filter>
|
||||
<object-type name="SCHEMA" enabled="true" />
|
||||
<object-type name="USER" enabled="true" />
|
||||
<object-type name="ROLE" enabled="true" />
|
||||
<object-type name="PRIVILEGE" enabled="true" />
|
||||
<object-type name="CHARSET" enabled="true" />
|
||||
<object-type name="TABLE" enabled="true" />
|
||||
<object-type name="VIEW" enabled="true" />
|
||||
<object-type name="MATERIALIZED_VIEW" enabled="true" />
|
||||
<object-type name="NESTED_TABLE" enabled="true" />
|
||||
<object-type name="COLUMN" enabled="true" />
|
||||
<object-type name="INDEX" enabled="true" />
|
||||
<object-type name="CONSTRAINT" enabled="true" />
|
||||
<object-type name="DATASET_TRIGGER" enabled="true" />
|
||||
<object-type name="DATABASE_TRIGGER" enabled="true" />
|
||||
<object-type name="SYNONYM" enabled="true" />
|
||||
<object-type name="SEQUENCE" enabled="true" />
|
||||
<object-type name="PROCEDURE" enabled="true" />
|
||||
<object-type name="FUNCTION" enabled="true" />
|
||||
<object-type name="PACKAGE" enabled="true" />
|
||||
<object-type name="TYPE" enabled="true" />
|
||||
<object-type name="TYPE_ATTRIBUTE" enabled="true" />
|
||||
<object-type name="ARGUMENT" enabled="true" />
|
||||
<object-type name="DIMENSION" enabled="true" />
|
||||
<object-type name="CLUSTER" enabled="true" />
|
||||
<object-type name="DBLINK" enabled="true" />
|
||||
</object-type-filter>
|
||||
</filters>
|
||||
<sorting>
|
||||
<object-type name="COLUMN" sorting-type="NAME" />
|
||||
<object-type name="FUNCTION" sorting-type="NAME" />
|
||||
<object-type name="PROCEDURE" sorting-type="NAME" />
|
||||
<object-type name="ARGUMENT" sorting-type="POSITION" />
|
||||
</sorting>
|
||||
<default-editors>
|
||||
<object-type name="VIEW" editor-type="SELECTION" />
|
||||
<object-type name="PACKAGE" editor-type="SELECTION" />
|
||||
<object-type name="TYPE" editor-type="SELECTION" />
|
||||
</default-editors>
|
||||
</browser-settings>
|
||||
<navigation-settings>
|
||||
<lookup-filters>
|
||||
<lookup-objects>
|
||||
<object-type name="SCHEMA" enabled="true" />
|
||||
<object-type name="USER" enabled="false" />
|
||||
<object-type name="ROLE" enabled="false" />
|
||||
<object-type name="PRIVILEGE" enabled="false" />
|
||||
<object-type name="CHARSET" enabled="false" />
|
||||
<object-type name="TABLE" enabled="true" />
|
||||
<object-type name="VIEW" enabled="true" />
|
||||
<object-type name="MATERIALIZED VIEW" enabled="true" />
|
||||
<object-type name="NESTED TABLE" enabled="false" />
|
||||
<object-type name="COLUMN" enabled="false" />
|
||||
<object-type name="INDEX" enabled="true" />
|
||||
<object-type name="CONSTRAINT" enabled="true" />
|
||||
<object-type name="DATASET TRIGGER" enabled="true" />
|
||||
<object-type name="DATABASE TRIGGER" enabled="true" />
|
||||
<object-type name="SYNONYM" enabled="false" />
|
||||
<object-type name="SEQUENCE" enabled="true" />
|
||||
<object-type name="PROCEDURE" enabled="true" />
|
||||
<object-type name="FUNCTION" enabled="true" />
|
||||
<object-type name="PACKAGE" enabled="true" />
|
||||
<object-type name="TYPE" enabled="true" />
|
||||
<object-type name="TYPE ATTRIBUTE" enabled="false" />
|
||||
<object-type name="ARGUMENT" enabled="false" />
|
||||
<object-type name="DIMENSION" enabled="false" />
|
||||
<object-type name="CLUSTER" enabled="false" />
|
||||
<object-type name="DBLINK" enabled="true" />
|
||||
</lookup-objects>
|
||||
<force-database-load value="false" />
|
||||
<prompt-connection-selection value="true" />
|
||||
<prompt-schema-selection value="true" />
|
||||
</lookup-filters>
|
||||
</navigation-settings>
|
||||
<dataset-grid-settings>
|
||||
<general>
|
||||
<enable-zooming value="true" />
|
||||
<enable-column-tooltip value="true" />
|
||||
</general>
|
||||
<sorting>
|
||||
<nulls-first value="true" />
|
||||
<max-sorting-columns value="4" />
|
||||
</sorting>
|
||||
<tracking-columns>
|
||||
<columnNames value="" />
|
||||
<visible value="true" />
|
||||
<editable value="false" />
|
||||
</tracking-columns>
|
||||
</dataset-grid-settings>
|
||||
<dataset-editor-settings>
|
||||
<text-editor-popup>
|
||||
<active value="false" />
|
||||
<active-if-empty value="false" />
|
||||
<data-length-threshold value="100" />
|
||||
<popup-delay value="1000" />
|
||||
</text-editor-popup>
|
||||
<values-actions-popup>
|
||||
<show-popup-button value="true" />
|
||||
<element-count-threshold value="1000" />
|
||||
<data-length-threshold value="250" />
|
||||
</values-actions-popup>
|
||||
<general>
|
||||
<fetch-block-size value="100" />
|
||||
<fetch-timeout value="30" />
|
||||
<trim-whitespaces value="true" />
|
||||
<convert-empty-strings-to-null value="true" />
|
||||
<select-content-on-cell-edit value="true" />
|
||||
<large-value-preview-active value="true" />
|
||||
</general>
|
||||
<filters>
|
||||
<prompt-filter-dialog value="true" />
|
||||
<default-filter-type value="BASIC" />
|
||||
</filters>
|
||||
<qualified-text-editor text-length-threshold="300">
|
||||
<content-types>
|
||||
<content-type name="Text" enabled="true" />
|
||||
<content-type name="Properties" enabled="true" />
|
||||
<content-type name="XML" enabled="true" />
|
||||
<content-type name="DTD" enabled="true" />
|
||||
<content-type name="HTML" enabled="true" />
|
||||
<content-type name="XHTML" enabled="true" />
|
||||
<content-type name="CSS" enabled="true" />
|
||||
<content-type name="Java" enabled="true" />
|
||||
<content-type name="JavaScript" enabled="true" />
|
||||
<content-type name="JSP" enabled="true" />
|
||||
<content-type name="JSPx" enabled="true" />
|
||||
<content-type name="Groovy" enabled="true" />
|
||||
<content-type name="FTL" enabled="true" />
|
||||
<content-type name="TML" enabled="true" />
|
||||
<content-type name="GSP" enabled="true" />
|
||||
<content-type name="ASP" enabled="true" />
|
||||
<content-type name="VTL" enabled="true" />
|
||||
<content-type name="AIDL" enabled="true" />
|
||||
<content-type name="YAML" enabled="true" />
|
||||
<content-type name="Flex" enabled="true" />
|
||||
<content-type name="Manifest" enabled="true" />
|
||||
</content-types>
|
||||
</qualified-text-editor>
|
||||
<record-navigation>
|
||||
<navigation-target value="VIEWER" />
|
||||
</record-navigation>
|
||||
</dataset-editor-settings>
|
||||
<code-editor-settings>
|
||||
<general>
|
||||
<show-object-navigation-gutter value="false" />
|
||||
<show-spec-declaration-navigation-gutter value="true" />
|
||||
<enable-spellchecking value="true" />
|
||||
<enable-reference-spellchecking value="false" />
|
||||
</general>
|
||||
<confirmations>
|
||||
<save-changes value="false" />
|
||||
<revert-changes value="true" />
|
||||
</confirmations>
|
||||
</code-editor-settings>
|
||||
<code-completion-settings>
|
||||
<filters>
|
||||
<basic-filter>
|
||||
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="function" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
|
||||
<filter-element type="OBJECT" id="schema" selected="true" />
|
||||
<filter-element type="OBJECT" id="role" selected="true" />
|
||||
<filter-element type="OBJECT" id="user" selected="true" />
|
||||
<filter-element type="OBJECT" id="privilege" selected="true" />
|
||||
<user-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="true" />
|
||||
<filter-element type="OBJECT" id="view" selected="true" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="true" />
|
||||
<filter-element type="OBJECT" id="index" selected="true" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="true" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="true" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="false" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="true" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="true" />
|
||||
<filter-element type="OBJECT" id="function" selected="true" />
|
||||
<filter-element type="OBJECT" id="package" selected="true" />
|
||||
<filter-element type="OBJECT" id="type" selected="true" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="true" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="true" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="true" />
|
||||
</user-schema>
|
||||
<public-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="false" />
|
||||
<filter-element type="OBJECT" id="view" selected="false" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="false" />
|
||||
<filter-element type="OBJECT" id="index" selected="false" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="false" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="false" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="false" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="false" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="false" />
|
||||
<filter-element type="OBJECT" id="function" selected="false" />
|
||||
<filter-element type="OBJECT" id="package" selected="false" />
|
||||
<filter-element type="OBJECT" id="type" selected="false" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="false" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="false" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="false" />
|
||||
</public-schema>
|
||||
<any-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="true" />
|
||||
<filter-element type="OBJECT" id="view" selected="true" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="true" />
|
||||
<filter-element type="OBJECT" id="index" selected="true" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="true" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="true" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="true" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="true" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="true" />
|
||||
<filter-element type="OBJECT" id="function" selected="true" />
|
||||
<filter-element type="OBJECT" id="package" selected="true" />
|
||||
<filter-element type="OBJECT" id="type" selected="true" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="true" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="true" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="true" />
|
||||
</any-schema>
|
||||
</basic-filter>
|
||||
<extended-filter>
|
||||
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="function" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
|
||||
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
|
||||
<filter-element type="OBJECT" id="schema" selected="true" />
|
||||
<filter-element type="OBJECT" id="user" selected="true" />
|
||||
<filter-element type="OBJECT" id="role" selected="true" />
|
||||
<filter-element type="OBJECT" id="privilege" selected="true" />
|
||||
<user-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="true" />
|
||||
<filter-element type="OBJECT" id="view" selected="true" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="true" />
|
||||
<filter-element type="OBJECT" id="index" selected="true" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="true" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="true" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="true" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="true" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="true" />
|
||||
<filter-element type="OBJECT" id="function" selected="true" />
|
||||
<filter-element type="OBJECT" id="package" selected="true" />
|
||||
<filter-element type="OBJECT" id="type" selected="true" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="true" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="true" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="true" />
|
||||
</user-schema>
|
||||
<public-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="true" />
|
||||
<filter-element type="OBJECT" id="view" selected="true" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="true" />
|
||||
<filter-element type="OBJECT" id="index" selected="true" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="true" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="true" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="true" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="true" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="true" />
|
||||
<filter-element type="OBJECT" id="function" selected="true" />
|
||||
<filter-element type="OBJECT" id="package" selected="true" />
|
||||
<filter-element type="OBJECT" id="type" selected="true" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="true" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="true" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="true" />
|
||||
</public-schema>
|
||||
<any-schema>
|
||||
<filter-element type="OBJECT" id="table" selected="true" />
|
||||
<filter-element type="OBJECT" id="view" selected="true" />
|
||||
<filter-element type="OBJECT" id="materialized view" selected="true" />
|
||||
<filter-element type="OBJECT" id="index" selected="true" />
|
||||
<filter-element type="OBJECT" id="constraint" selected="true" />
|
||||
<filter-element type="OBJECT" id="trigger" selected="true" />
|
||||
<filter-element type="OBJECT" id="synonym" selected="true" />
|
||||
<filter-element type="OBJECT" id="sequence" selected="true" />
|
||||
<filter-element type="OBJECT" id="procedure" selected="true" />
|
||||
<filter-element type="OBJECT" id="function" selected="true" />
|
||||
<filter-element type="OBJECT" id="package" selected="true" />
|
||||
<filter-element type="OBJECT" id="type" selected="true" />
|
||||
<filter-element type="OBJECT" id="dimension" selected="true" />
|
||||
<filter-element type="OBJECT" id="cluster" selected="true" />
|
||||
<filter-element type="OBJECT" id="dblink" selected="true" />
|
||||
</any-schema>
|
||||
</extended-filter>
|
||||
</filters>
|
||||
<sorting enabled="true">
|
||||
<sorting-element type="RESERVED_WORD" id="keyword" />
|
||||
<sorting-element type="RESERVED_WORD" id="datatype" />
|
||||
<sorting-element type="OBJECT" id="column" />
|
||||
<sorting-element type="OBJECT" id="table" />
|
||||
<sorting-element type="OBJECT" id="view" />
|
||||
<sorting-element type="OBJECT" id="materialized view" />
|
||||
<sorting-element type="OBJECT" id="index" />
|
||||
<sorting-element type="OBJECT" id="constraint" />
|
||||
<sorting-element type="OBJECT" id="trigger" />
|
||||
<sorting-element type="OBJECT" id="synonym" />
|
||||
<sorting-element type="OBJECT" id="sequence" />
|
||||
<sorting-element type="OBJECT" id="procedure" />
|
||||
<sorting-element type="OBJECT" id="function" />
|
||||
<sorting-element type="OBJECT" id="package" />
|
||||
<sorting-element type="OBJECT" id="type" />
|
||||
<sorting-element type="OBJECT" id="dimension" />
|
||||
<sorting-element type="OBJECT" id="cluster" />
|
||||
<sorting-element type="OBJECT" id="dblink" />
|
||||
<sorting-element type="OBJECT" id="schema" />
|
||||
<sorting-element type="OBJECT" id="role" />
|
||||
<sorting-element type="OBJECT" id="user" />
|
||||
<sorting-element type="RESERVED_WORD" id="function" />
|
||||
<sorting-element type="RESERVED_WORD" id="parameter" />
|
||||
</sorting>
|
||||
<format>
|
||||
<enforce-code-style-case value="true" />
|
||||
</format>
|
||||
</code-completion-settings>
|
||||
<execution-engine-settings>
|
||||
<statement-execution>
|
||||
<fetch-block-size value="100" />
|
||||
<execution-timeout value="20" />
|
||||
<debug-execution-timeout value="600" />
|
||||
<focus-result value="false" />
|
||||
<prompt-execution value="false" />
|
||||
</statement-execution>
|
||||
<script-execution>
|
||||
<command-line-interfaces />
|
||||
<execution-timeout value="300" />
|
||||
</script-execution>
|
||||
<method-execution>
|
||||
<execution-timeout value="30" />
|
||||
<debug-execution-timeout value="600" />
|
||||
<parameter-history-size value="10" />
|
||||
</method-execution>
|
||||
</execution-engine-settings>
|
||||
<operation-settings>
|
||||
<transactions>
|
||||
<uncommitted-changes>
|
||||
<on-project-close value="ASK" />
|
||||
<on-disconnect value="ASK" />
|
||||
<on-autocommit-toggle value="ASK" />
|
||||
</uncommitted-changes>
|
||||
<multiple-uncommitted-changes>
|
||||
<on-commit value="ASK" />
|
||||
<on-rollback value="ASK" />
|
||||
</multiple-uncommitted-changes>
|
||||
</transactions>
|
||||
<session-browser>
|
||||
<disconnect-session value="ASK" />
|
||||
<kill-session value="ASK" />
|
||||
<reload-on-filter-change value="false" />
|
||||
</session-browser>
|
||||
<compiler>
|
||||
<compile-type value="KEEP" />
|
||||
<compile-dependencies value="ASK" />
|
||||
<always-show-controls value="false" />
|
||||
</compiler>
|
||||
<debugger>
|
||||
<debugger-type value="ASK" />
|
||||
<use-generic-runners value="true" />
|
||||
</debugger>
|
||||
</operation-settings>
|
||||
<ddl-file-settings>
|
||||
<extensions>
|
||||
<mapping file-type-id="VIEW" extensions="vw" />
|
||||
<mapping file-type-id="TRIGGER" extensions="trg" />
|
||||
<mapping file-type-id="PROCEDURE" extensions="prc" />
|
||||
<mapping file-type-id="FUNCTION" extensions="fnc" />
|
||||
<mapping file-type-id="PACKAGE" extensions="pkg" />
|
||||
<mapping file-type-id="PACKAGE_SPEC" extensions="pks" />
|
||||
<mapping file-type-id="PACKAGE_BODY" extensions="pkb" />
|
||||
<mapping file-type-id="TYPE" extensions="tpe" />
|
||||
<mapping file-type-id="TYPE_SPEC" extensions="tps" />
|
||||
<mapping file-type-id="TYPE_BODY" extensions="tpb" />
|
||||
</extensions>
|
||||
<general>
|
||||
<lookup-ddl-files value="true" />
|
||||
<create-ddl-files value="false" />
|
||||
<synchronize-ddl-files value="true" />
|
||||
<use-qualified-names value="false" />
|
||||
<make-scripts-rerunnable value="true" />
|
||||
</general>
|
||||
</ddl-file-settings>
|
||||
<general-settings>
|
||||
<regional-settings>
|
||||
<date-format value="MEDIUM" />
|
||||
<number-format value="UNGROUPED" />
|
||||
<locale value="SYSTEM_DEFAULT" />
|
||||
<use-custom-formats value="false" />
|
||||
</regional-settings>
|
||||
<environment>
|
||||
<environment-types>
|
||||
<environment-type id="development" name="Development" description="Development environment" color="-2430209/-12296320" readonly-code="false" readonly-data="false" />
|
||||
<environment-type id="integration" name="Integration" description="Integration environment" color="-2621494/-12163514" readonly-code="true" readonly-data="false" />
|
||||
<environment-type id="production" name="Production" description="Productive environment" color="-11574/-10271420" readonly-code="true" readonly-data="true" />
|
||||
<environment-type id="other" name="Other" description="" color="-1576/-10724543" readonly-code="false" readonly-data="false" />
|
||||
</environment-types>
|
||||
<visibility-settings>
|
||||
<connection-tabs value="true" />
|
||||
<dialog-headers value="true" />
|
||||
<object-editor-tabs value="true" />
|
||||
<script-editor-tabs value="false" />
|
||||
<execution-result-tabs value="true" />
|
||||
</visibility-settings>
|
||||
</environment>
|
||||
</general-settings>
|
||||
</component>
|
||||
<component name="DBNavigator.Project.StatementExecutionManager">
|
||||
<execution-variables />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Encoding" defaultCharsetForPropertiesFiles="UTF-8" addBOMForNewFiles="with NO BOM">
|
||||
<file url="PROJECT" charset="UTF-8" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="RemoteRepositoriesConfiguration">
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Central Repository" />
|
||||
<option name="url" value="https://repo.maven.apache.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="central" />
|
||||
<option name="name" value="Maven Central repository" />
|
||||
<option name="url" value="https://repo1.maven.org/maven2" />
|
||||
</remote-repository>
|
||||
<remote-repository>
|
||||
<option name="id" value="jboss.community" />
|
||||
<option name="name" value="JBoss Community repository" />
|
||||
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
|
||||
</remote-repository>
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<component name="libraryTable">
|
||||
<library name="GLPKSolverPack">
|
||||
<CLASSES>
|
||||
<root url="jar://$PROJECT_DIR$/lib/GLPKSolverPack.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<component name="libraryTable">
|
||||
<library name="SCPSolver">
|
||||
<CLASSES>
|
||||
<root url="jar://$PROJECT_DIR$/lib/SCPSolver.jar!/" />
|
||||
</CLASSES>
|
||||
<JAVADOC />
|
||||
<SOURCES />
|
||||
</library>
|
||||
</component>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ExternalStorageConfigurationManager" enabled="true" />
|
||||
<component name="JavaScriptSettings">
|
||||
<option name="languageLevel" value="ES6" />
|
||||
</component>
|
||||
<component name="MavenProjectsManager">
|
||||
<option name="originalFiles">
|
||||
<list>
|
||||
<option value="$PROJECT_DIR$/pom.xml" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
|
||||
<output url="file://$PROJECT_DIR$/out" />
|
||||
</component>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="Palette2">
|
||||
<group name="Swing">
|
||||
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
|
||||
</item>
|
||||
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
|
||||
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
|
||||
<initial-values>
|
||||
<property name="text" value="Button" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="RadioButton" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="CheckBox" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
|
||||
<initial-values>
|
||||
<property name="text" value="Label" />
|
||||
</initial-values>
|
||||
</item>
|
||||
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
|
||||
<preferred-size width="150" height="-1" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
|
||||
<preferred-size width="150" height="50" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
|
||||
<preferred-size width="200" height="200" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
|
||||
</item>
|
||||
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
|
||||
<preferred-size width="-1" height="20" />
|
||||
</default-constraints>
|
||||
</item>
|
||||
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
|
||||
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
|
||||
</item>
|
||||
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
|
||||
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
|
||||
</item>
|
||||
</group>
|
||||
</component>
|
||||
</project>
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,90 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>edu.nju.ics.frontier</groupId>
|
||||
<artifactId>stream-data-processing</artifactId>
|
||||
<version>1.0.0</version>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>edu.nju.ics.frontier</groupId>
|
||||
<artifactId>common-io</artifactId>
|
||||
<version>1.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
<version>2.8.5</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.opencsv</groupId>
|
||||
<artifactId>opencsv</artifactId>
|
||||
<version>4.5</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-csv</artifactId>
|
||||
<version>1.8</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<version>3.6.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>nz.ac.waikato.cms.weka</groupId>
|
||||
<artifactId>weka-stable</artifactId>
|
||||
<version>3.8.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>nz.ac.waikato.cms.weka</groupId>
|
||||
<artifactId>LibSVM</artifactId>
|
||||
<version>1.0.10</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.jfree</groupId>
|
||||
<artifactId>jfreechart</artifactId>
|
||||
<version>1.5.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<!-- use 2.9.1 for Java 7 projects -->
|
||||
<version>3.16.1</version>
|
||||
<!--<scope>test</scope>-->
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.12.4</version>
|
||||
<configuration>
|
||||
<forkMode>once</forkMode>
|
||||
<argLine>-Dfile.encoding=UTF-8</argLine>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package edu.nju.ics.frontier.bean.action;
|
||||
|
||||
public abstract class Action {
|
||||
protected String type;
|
||||
|
||||
public Action(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,450 @@
|
|||
package edu.nju.ics.frontier.bean.action;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.ActionEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.DebugEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.ExecuteEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class ExecuteAction extends IntervalAction {
|
||||
/**
|
||||
* [required] {@code id} is one of the values including {@code Run}, {@code Debug}, and {@code Coverage}
|
||||
*/
|
||||
private String id;
|
||||
|
||||
/**
|
||||
* [required] {@code executorLifeCycle} including scheduled, starting, started,
|
||||
* cancelled, terminating, terminated
|
||||
*/
|
||||
private List<ExecuteEvent> executorLifeCycle;
|
||||
|
||||
/**
|
||||
* [optional] {@code executorAction} is actions triggered by developers to start/stop
|
||||
* executing some task, such as left-single-clicking the 'Run...' button
|
||||
*/
|
||||
private List<ActionEvent> executorActions;
|
||||
|
||||
/**
|
||||
* [optional] {@code debuggerLiftCycle} consists of states and events.
|
||||
* states including STOPPED, RUNNING, WAITING_ATTACH, PAUSED, WAIT_EVALUATION, DISPOSED.
|
||||
* events including ATTACHED, DETACHED, RESUME, STEP, PAUSE, REFRESH, CONTEXT, START_WAIT_ATTACH, DISPOSE, REFRESH_WITH_STACK, THREADS_REFRESH.
|
||||
*/
|
||||
private List<DebugEvent> debuggerLiftCycle;
|
||||
|
||||
/**
|
||||
* [optional] actions to add, remove, and change break points.
|
||||
* {@code breakPointEvents} is not used because break points can be changed at any time.
|
||||
*/
|
||||
// private List<DebugEvent> breakPointEvents;
|
||||
|
||||
/**
|
||||
* [optional] actions done during debugging such as Step Over, Step In, Step Out, and so on.
|
||||
* It worth noting that there are two special debugger actions 'Attach to Process...' and
|
||||
* 'Attach Debugger to Android Process', which can convert the 'Run' task to the 'Debug' task.
|
||||
*/
|
||||
private List<ActionEvent> debuggerActions;
|
||||
|
||||
public ExecuteAction(List<ExecuteEvent> executorLifeCycle) {
|
||||
this(executorLifeCycle.get(0).getIdField(),
|
||||
executorLifeCycle.get(0).getTime(),
|
||||
executorLifeCycle.get(executorLifeCycle.size() - 1).getTime(),
|
||||
executorLifeCycle);
|
||||
}
|
||||
|
||||
public ExecuteAction(String id, long begTime, long endTime,
|
||||
List<ExecuteEvent> executorLifeCycle) {
|
||||
this(id, begTime, endTime, executorLifeCycle,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
}
|
||||
|
||||
public ExecuteAction(String id, long begTime, long endTime,
|
||||
List<ExecuteEvent> executorLifeCycle,
|
||||
List<ActionEvent> executorActions,
|
||||
List<DebugEvent> debuggerLiftCycle,
|
||||
List<ActionEvent> debuggerActions) {
|
||||
super("Execute", begTime, endTime);
|
||||
this.id = id;
|
||||
this.executorLifeCycle = executorLifeCycle;
|
||||
this.executorActions = executorActions;
|
||||
this.debuggerLiftCycle = debuggerLiftCycle;
|
||||
this.debuggerActions = debuggerActions;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public List<ExecuteEvent> getExecutorLifeCycle() {
|
||||
return executorLifeCycle;
|
||||
}
|
||||
|
||||
public List<ActionEvent> getExecutorActions() {
|
||||
return executorActions;
|
||||
}
|
||||
|
||||
public List<ActionEvent> getDebuggerActions() {
|
||||
return debuggerActions;
|
||||
}
|
||||
|
||||
public List<DebugEvent> getDebuggerLiftCycle() {
|
||||
return debuggerLiftCycle;
|
||||
}
|
||||
|
||||
public void setExecutorActions(List<ActionEvent> executorActions) {
|
||||
this.executorActions = executorActions;
|
||||
}
|
||||
|
||||
public void setDebuggerLiftCycle(List<DebugEvent> debuggerLiftCycle) {
|
||||
this.debuggerLiftCycle = debuggerLiftCycle;
|
||||
}
|
||||
|
||||
public void setDebuggerActions(List<ActionEvent> debuggerActions) {
|
||||
this.debuggerActions = debuggerActions;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{id:%s,executorLifeCycle:%s,executorActions:%s,debuggerLifeCycle:%s,debuggerActions:%s}",
|
||||
id,
|
||||
(executorLifeCycle == null ? "null" : executorLifeCycle.toString()),
|
||||
(executorActions == null ? "null" : executorActions.toString()),
|
||||
(debuggerLiftCycle == null ? "null" : debuggerLiftCycle.toString()),
|
||||
(debuggerActions == null ? "null" : debuggerActions.toString()));
|
||||
}
|
||||
|
||||
/**
|
||||
* construct execute actions from the given IntelliJ event sequence. An execute action
|
||||
* consists an unique id and a life cycle of the corresponding executor, if the execute
|
||||
* action represents a debug task, it also includes a life cycle of the corresponding
|
||||
* debugger and several debug-related actions. In addition, an execute action is triggered
|
||||
* by an executor action, which also be involved in an execute action.
|
||||
* @param events the given IntelliJ event sequence
|
||||
* @return the execute actions
|
||||
*/
|
||||
public static List<ExecuteAction> createExecuteActions(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<ExecuteEvent> executorLifeCycleEvents = new ArrayList<ExecuteEvent>();
|
||||
List<DebugEvent> debuggerLifeCycleEvents = new ArrayList<DebugEvent>();
|
||||
List<ActionEvent> executorActions = new ArrayList<ActionEvent>();
|
||||
List<ActionEvent> debuggerActions = new ArrayList<ActionEvent>();
|
||||
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isExecuteEvent(event)) {
|
||||
ExecuteEvent executeEvent = IntelliJEvent.toExecuteEvent(event);
|
||||
if (executeEvent.getIdField() != null) {
|
||||
executorLifeCycleEvents.add(executeEvent);
|
||||
}
|
||||
} else if (IntelliJEvent.isDebugEvent(event)) {
|
||||
DebugEvent debugEvent = IntelliJEvent.toDebugEvent(event);
|
||||
if (debugEvent.isWhenContextChanged() &&
|
||||
debugEvent.getProjectField() != null &&
|
||||
debugEvent.getSessionField() != null) {
|
||||
debuggerLifeCycleEvents.add(debugEvent);
|
||||
}
|
||||
} else if (IntelliJEvent.isActionEvent(event)) {
|
||||
ActionEvent actionEvent = IntelliJEvent.toActionEvent(event);
|
||||
String actionClass = actionEvent.getClassField();
|
||||
// if (ActionCategory.EXECUTE_CATEGORY.match(actionClass)) {
|
||||
// executorActions.add(actionEvent);
|
||||
// } else if (ActionCategory.DEBUG_CATEGORY.match(actionClass)) {
|
||||
// debuggerActions.add(actionEvent);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
List<List<ExecuteEvent>> executorLifeCycles = createExecutorLifeCycles(executorLifeCycleEvents);
|
||||
List<List<DebugEvent>> debuggerLifeCycles = createDebuggerLifeCycles(debuggerLifeCycleEvents);
|
||||
|
||||
if (executorLifeCycles == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<ExecuteAction> executeActions = new ArrayList<ExecuteAction>();
|
||||
for (List<ExecuteEvent> executorLifeCycle : executorLifeCycles) {
|
||||
ExecuteAction executeAction = new ExecuteAction(executorLifeCycle);
|
||||
executeActions.add(executeAction);
|
||||
}
|
||||
|
||||
attachDebuggerLifeCycles(executeActions, debuggerLifeCycles);
|
||||
attachExecutorActions(executeActions, executorActions);
|
||||
attachDebuggerActions(executeActions, debuggerActions);
|
||||
|
||||
return executeActions;
|
||||
}
|
||||
|
||||
private static List<List<ExecuteEvent>> createExecutorLifeCycles(List<ExecuteEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Map<String, List<ExecuteEvent>> map = new HashMap<String, List<ExecuteEvent>>();
|
||||
for (ExecuteEvent event : events) {
|
||||
String key = event.getIdField();
|
||||
List<ExecuteEvent> value = map.get(key);
|
||||
if (value == null) {
|
||||
value = new ArrayList<ExecuteEvent>();
|
||||
}
|
||||
value.add(event);
|
||||
map.put(key, value);
|
||||
}
|
||||
|
||||
List<List<ExecuteEvent>> result = new ArrayList<List<ExecuteEvent>>();
|
||||
for (Map.Entry<String, List<ExecuteEvent>> entry : map.entrySet()) {
|
||||
List<List<ExecuteEvent>> tmp = createSpecificExecutorLifeCycles(entry.getValue());
|
||||
if (tmp != null) {
|
||||
result.addAll(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Collections.sort(result, new Comparator<List<ExecuteEvent>>() {
|
||||
public int compare(List<ExecuteEvent> o1, List<ExecuteEvent> o2) {
|
||||
long t1 = o1.get(0).getTime();
|
||||
long t2 = o2.get(0).getTime();
|
||||
return (int) (t1 - t2);
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<List<ExecuteEvent>> createSpecificExecutorLifeCycles(List<ExecuteEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<List<ExecuteEvent>> result = new ArrayList<List<ExecuteEvent>>();
|
||||
|
||||
List<ExecuteEvent> tmp = null;
|
||||
for (ExecuteEvent event : events) {
|
||||
if (event.isWhenScheduled()) {
|
||||
if (tmp == null) {
|
||||
tmp = new ArrayList<ExecuteEvent>();
|
||||
tmp.add(event);
|
||||
}
|
||||
} else if (event.isWhenTerminated()) {
|
||||
if (tmp != null) {
|
||||
tmp.add(event);
|
||||
result.add(tmp);
|
||||
tmp = null;
|
||||
}
|
||||
} else {
|
||||
if (tmp != null) {
|
||||
tmp.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tmp != null) {
|
||||
throw new IllegalArgumentException("[incomplete executor life cycle] " + tmp);
|
||||
}
|
||||
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static List<List<DebugEvent>> createDebuggerLifeCycles(List<DebugEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Map<String, List<DebugEvent>> map = new HashMap<String, List<DebugEvent>>();
|
||||
for (DebugEvent event : events) {
|
||||
String project = event.getProjectField();
|
||||
String session = event.getSessionField();
|
||||
String key = project + "#" + session;
|
||||
List<DebugEvent> value = map.get(key);
|
||||
if (value == null) {
|
||||
value = new ArrayList<DebugEvent>();
|
||||
}
|
||||
value.add(event);
|
||||
map.put(key, value);
|
||||
}
|
||||
|
||||
List<List<DebugEvent>> result = new ArrayList<List<DebugEvent>>();
|
||||
for (Map.Entry<String, List<DebugEvent>> entry : map.entrySet()) {
|
||||
List<List<DebugEvent>> tmp = createSpecificDebuggerLifeCycles(entry.getValue());
|
||||
if (tmp != null) {
|
||||
result.addAll(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Collections.sort(result, new Comparator<List<DebugEvent>>() {
|
||||
public int compare(List<DebugEvent> o1, List<DebugEvent> o2) {
|
||||
long t1 = o1.get(0).getTime();
|
||||
long t2 = o2.get(0).getTime();
|
||||
return (int) (t1 - t2);
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<List<DebugEvent>> createSpecificDebuggerLifeCycles(List<DebugEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<List<DebugEvent>> result = new ArrayList<List<DebugEvent>>();
|
||||
|
||||
List<DebugEvent> tmp = null;
|
||||
for (DebugEvent event : events) {
|
||||
if (event.isWaitingAttachState() && event.isContextEvent()) {
|
||||
if (tmp == null) {
|
||||
tmp = new ArrayList<DebugEvent>();
|
||||
tmp.add(event);
|
||||
}
|
||||
} else if (event.isDisposedState() && event.isDisposeEvent()) {
|
||||
if (tmp != null) {
|
||||
tmp.add(event);
|
||||
result.add(tmp);
|
||||
tmp = null;
|
||||
}
|
||||
} else {
|
||||
if (tmp != null) {
|
||||
tmp.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tmp != null) {
|
||||
throw new IllegalArgumentException("[incomplete debugger life cycle] " + tmp);
|
||||
}
|
||||
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static void attachDebuggerLifeCycles(List<ExecuteAction> executeActions, List<List<DebugEvent>> debuggerLifeCycles) {
|
||||
if (executeActions == null || executeActions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (debuggerLifeCycles == null || debuggerLifeCycles.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (List<DebugEvent> debuggerLifeCycle : debuggerLifeCycles) {
|
||||
long debuggerBegTime = debuggerLifeCycle.get(0).getTime();
|
||||
|
||||
ExecuteAction nearest = null;
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
boolean isInvolved = false;
|
||||
for (ExecuteAction executeAction : executeActions) {
|
||||
long executorBegTime = executeAction.getBegTime();
|
||||
long executorEndTime = executeAction.getEndTime();
|
||||
if (debuggerBegTime >= executorBegTime && debuggerBegTime < executorEndTime) {
|
||||
nearest = executeAction;
|
||||
isInvolved = true;
|
||||
break;
|
||||
}
|
||||
long delta = Math.abs(debuggerBegTime - executorBegTime);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
nearest = executeAction;
|
||||
}
|
||||
}
|
||||
|
||||
if (!"Debug".equals(nearest.getId())) {
|
||||
throw new IllegalArgumentException("[attach debugger life cycle: class] executor:" + nearest +
|
||||
", debugger:" + debuggerLifeCycle);
|
||||
}
|
||||
if (nearest.getDebuggerLiftCycle() != null) {
|
||||
throw new IllegalArgumentException("[attach debugger life cycle: value] executor:" + nearest +
|
||||
", oldDebugger:" + nearest.getDebuggerLiftCycle() +
|
||||
", newDebugger:" + debuggerLifeCycle);
|
||||
}
|
||||
if (!isInvolved) {
|
||||
if (minDelta > 1000) {
|
||||
throw new IllegalArgumentException("[attach debugger life cycle: delta] executor:" + nearest +
|
||||
", debugger:" + debuggerLifeCycle);
|
||||
}
|
||||
}
|
||||
nearest.setDebuggerLiftCycle(debuggerLifeCycle);
|
||||
}
|
||||
}
|
||||
|
||||
private static void attachExecutorActions(List<ExecuteAction> executeActions, List<ActionEvent> executorActions) {
|
||||
if (executeActions == null || executeActions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (executorActions == null || executorActions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (ActionEvent executorAction : executorActions) {
|
||||
long actionTime = executorAction.getTime();
|
||||
|
||||
ExecuteAction nearest = null;
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
for (ExecuteAction executeAction : executeActions) {
|
||||
long executorBegTime = executeAction.getBegTime();
|
||||
long executorEndTime = executeAction.getEndTime();
|
||||
if (actionTime >= executorBegTime && actionTime < executorEndTime) {
|
||||
nearest = executeAction;
|
||||
break;
|
||||
}
|
||||
long delta = Math.abs(executorBegTime - actionTime);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
nearest = executeAction;
|
||||
}
|
||||
}
|
||||
|
||||
List<ActionEvent> container = nearest.getExecutorActions();
|
||||
if (container == null) {
|
||||
container = new ArrayList<ActionEvent>();
|
||||
nearest.setExecutorActions(container);
|
||||
}
|
||||
container.add(executorAction);
|
||||
}
|
||||
}
|
||||
|
||||
private static void attachDebuggerActions(List<ExecuteAction> executeActions, List<ActionEvent> debuggerActions) {
|
||||
if (executeActions == null || executeActions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (debuggerActions == null || debuggerActions.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (ActionEvent debuggerAction : debuggerActions) {
|
||||
long actionTime = debuggerAction.getTime();
|
||||
|
||||
ExecuteAction nearest = null;
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
for (ExecuteAction executeAction : executeActions) {
|
||||
long executorBegTime = executeAction.getBegTime();
|
||||
long executorEndTime = executeAction.getEndTime();
|
||||
if (actionTime >= executorBegTime && actionTime < executorEndTime) {
|
||||
nearest = executeAction;
|
||||
break;
|
||||
}
|
||||
long delta = Math.abs(executorBegTime - actionTime);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
nearest = executeAction;
|
||||
}
|
||||
}
|
||||
|
||||
List<ActionEvent> container = nearest.getDebuggerActions();
|
||||
if (container == null) {
|
||||
container = new ArrayList<ActionEvent>();
|
||||
nearest.setDebuggerActions(container);
|
||||
}
|
||||
container.add(debuggerAction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
package edu.nju.ics.frontier.bean.action;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.CursorEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.FileEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class FileAction extends IntervalAction {
|
||||
private String path;
|
||||
private List<FileEvent> fileChangedTrajectory;
|
||||
private List<CursorEvent> cursorChangedTrajectory;
|
||||
|
||||
public FileAction(String path, long begTime, long endTime,
|
||||
List<FileEvent> fileChangedTrajectory,
|
||||
List<CursorEvent> cursorChangedTrajectory) {
|
||||
super("File", begTime, endTime);
|
||||
this.path = path;
|
||||
this.fileChangedTrajectory = fileChangedTrajectory;
|
||||
this.cursorChangedTrajectory = cursorChangedTrajectory;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public List<FileEvent> getFileChangedTrajectory() {
|
||||
return fileChangedTrajectory;
|
||||
}
|
||||
|
||||
public List<CursorEvent> getCursorChangedTrajectory() {
|
||||
return cursorChangedTrajectory;
|
||||
}
|
||||
|
||||
public boolean isCursorChangedInSuccession(boolean isPrintCheckInfo) {
|
||||
if (cursorChangedTrajectory == null || cursorChangedTrajectory.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
String lastPoint = cursorChangedTrajectory.get(0).getNewPosField();
|
||||
for (int i = 1, s = cursorChangedTrajectory.size(); i < s; i++) {
|
||||
CursorEvent cursorEvent = cursorChangedTrajectory.get(i);
|
||||
if (!lastPoint.equals(cursorEvent.getOldPosField())) {
|
||||
if (isPrintCheckInfo) {
|
||||
int begIndex = Math.max(0, i - 5);
|
||||
int endIndex = Math.min(i + 4, cursorChangedTrajectory.size() - 1);
|
||||
System.out.printf("[%d] %s [%d] {\n", begTime, path, cursorChangedTrajectory.size());
|
||||
for (int j = begIndex; j < i; j++) {
|
||||
CursorEvent ce = cursorChangedTrajectory.get(j);
|
||||
System.out.printf(" [%d] %s -> %s\n", ce.getTime(), ce.getOldPosField(), ce.getNewPosField());
|
||||
}
|
||||
System.out.printf("* [%d] %s -> %s\n", cursorEvent.getTime(), cursorEvent.getOldPosField(), cursorEvent.getNewPosField());
|
||||
for (int j = i + 1; j <= endIndex; j++) {
|
||||
CursorEvent ce = cursorChangedTrajectory.get(j);
|
||||
System.out.printf(" [%d] %s -> %s\n", ce.getTime(), ce.getOldPosField(), ce.getNewPosField());
|
||||
}
|
||||
System.out.println("}");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
lastPoint = cursorEvent.getNewPosField();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public void printCursorChangedTrajectory() {
|
||||
int size = cursorChangedTrajectory == null ? 0 : cursorChangedTrajectory.size();
|
||||
System.out.printf("[%d] %s [%d] {\n", begTime, path, size);
|
||||
if (!(cursorChangedTrajectory == null || cursorChangedTrajectory.isEmpty())) {
|
||||
for (int i = 0, s = cursorChangedTrajectory.size(); i < s; i++) {
|
||||
CursorEvent ce = cursorChangedTrajectory.get(i);
|
||||
System.out.printf(" [%d] %s -> %s\n", ce.getTime(), ce.getOldPosField(), ce.getNewPosField());
|
||||
}
|
||||
}
|
||||
System.out.println("}");
|
||||
}
|
||||
|
||||
/**
|
||||
* For the given event sequence, segmenting it into several file actions by
|
||||
* splitting the sequence at each file selected point.
|
||||
* @param events IntelliJ event sequence
|
||||
* @return list of file actions
|
||||
* @see FileAction#removeOutliers(List, int, int)
|
||||
*/
|
||||
public static List<FileAction> createFileActions(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<FileAction> fileActions = new ArrayList<FileAction>();
|
||||
|
||||
FileEvent splitPoint = null;
|
||||
List<FileEvent> fileChangedEvents = null;
|
||||
List<CursorEvent> cursorChangedEvents = null;
|
||||
List<IntelliJEvent> terminalForFileAction = null;
|
||||
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isFileEvent(event)) {
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
// when file was selected. The new path must be not null and different from the old one.
|
||||
String oldPath = fileEvent.getOldPathField();
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
if (newPath == null || newPath.equals(oldPath)) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint != null) {
|
||||
String path = splitPoint.getNewPathField();
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = fileEvent.getTime();
|
||||
if (!terminalForFileAction.isEmpty()) {
|
||||
long t = terminalForFileAction.get(0).getTime();
|
||||
if (t < endTime) {
|
||||
endTime = t;
|
||||
}
|
||||
}
|
||||
removeOutliers(cursorChangedEvents);
|
||||
FileAction fileAction = new FileAction(
|
||||
path, begTime, endTime,
|
||||
fileChangedEvents,
|
||||
cursorChangedEvents);
|
||||
fileActions.add(fileAction);
|
||||
}
|
||||
splitPoint = fileEvent;
|
||||
fileChangedEvents = new ArrayList<FileEvent>();
|
||||
cursorChangedEvents = new ArrayList<CursorEvent>();
|
||||
terminalForFileAction = new ArrayList<IntelliJEvent>();
|
||||
} else if (fileEvent.isWhenChanged()) {
|
||||
// when file changed. The path must be same as the last split point.
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint.getNewPathField().equals(fileEvent.getPathField())) {
|
||||
fileChangedEvents.add(fileEvent);
|
||||
}
|
||||
} else if (fileEvent.isWhenClosed()) {
|
||||
// when file closed. The last file closed event is used to determine the end time of the last file filter.
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint.getNewPathField().equals(fileEvent.getPathField())) {
|
||||
terminalForFileAction.add(fileEvent);
|
||||
}
|
||||
}
|
||||
} else if (IntelliJEvent.isCursorEvent(event)) {
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
CursorEvent cursorEvent = IntelliJEvent.toCursorEvent(event);
|
||||
if (cursorEvent.isWhenCursorChanged()) {
|
||||
// when cursor changed. The editor must be 'UNTYPED' and the old position does not equals (0,0).
|
||||
String oldPos = cursorEvent.getOldPosField();
|
||||
String newPos = cursorEvent.getNewPosField();
|
||||
if (cursorEvent.isUntypedEditor() && oldPos != null &&
|
||||
newPos != null && (!"(0,0)".equals(oldPos))) {
|
||||
cursorChangedEvents.add(cursorEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (splitPoint != null) {
|
||||
String path = splitPoint.getNewPathField();
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = events.get(events.size() - 1).getTime();
|
||||
if (!terminalForFileAction.isEmpty()) {
|
||||
long t = terminalForFileAction.get(0).getTime();
|
||||
if (t < endTime) {
|
||||
endTime = t;
|
||||
}
|
||||
}
|
||||
removeOutliers(cursorChangedEvents);
|
||||
FileAction fileAction = new FileAction(
|
||||
path, begTime, endTime,
|
||||
fileChangedEvents,
|
||||
cursorChangedEvents);
|
||||
fileActions.add(fileAction);
|
||||
}
|
||||
|
||||
return fileActions.isEmpty() ? null : fileActions;
|
||||
}
|
||||
|
||||
/**
|
||||
* @see FileAction#removeOutliers(List, int, int)
|
||||
* @param events
|
||||
*/
|
||||
private static void removeOutliers(List<CursorEvent> events) {
|
||||
int maxDis = 5;
|
||||
int minHop = Math.min(events.size(), Math.max(5, events.size() / 10));
|
||||
removeOutliers(events, maxDis, minHop);
|
||||
}
|
||||
|
||||
/**
|
||||
* remove outliers in the cursor changed sequence.
|
||||
* @param events IntelliJ event sequence
|
||||
* @param maxDis for any two adjacent cursor changed events, the difference between
|
||||
* the former's {@code newPos} and the latter's {@code oldPos} must be
|
||||
* less than the given {@code maxDis}.
|
||||
* @param minHop the continuous sub cursor changed sequence is valid
|
||||
* if and only if its hops exceed {@code minHop}.
|
||||
*/
|
||||
private static void removeOutliers(List<CursorEvent> events, int maxDis, int minHop) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<List<CursorEvent>> traces = new ArrayList<List<CursorEvent>>();
|
||||
List<CursorEvent> tmp = null;
|
||||
|
||||
for (CursorEvent event : events) {
|
||||
if (tmp == null) {
|
||||
tmp = new ArrayList<CursorEvent>();
|
||||
tmp.add(event);
|
||||
traces.add(tmp);
|
||||
} else {
|
||||
long[] lastNewPos = CursorEvent.getLineAndColumn(tmp.get(tmp.size() - 1).getNewPosField());
|
||||
long lastNewLine = lastNewPos[0];
|
||||
long[] currOldPos = CursorEvent.getLineAndColumn(event.getOldPosField());
|
||||
long currOldLine = currOldPos[0];
|
||||
|
||||
if (Math.abs(currOldLine - lastNewLine) < maxDis) {
|
||||
tmp.add(event);
|
||||
} else {
|
||||
tmp = new ArrayList<CursorEvent>();
|
||||
tmp.add(event);
|
||||
traces.add(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
// if (traces.isEmpty()) {
|
||||
// List<CursorEvent> trace = new ArrayList<CursorEvent>();
|
||||
// trace.add(event);
|
||||
// traces.add(trace);
|
||||
// } else {
|
||||
// long[] currOldPos = CursorEvent.getLineAndColumn(event.getOldPosField());
|
||||
// long currOldLine = currOldPos[0];
|
||||
// long currOldColumn = currOldPos[1];
|
||||
//
|
||||
// long minDelta = Long.MAX_VALUE;
|
||||
// long minLineDelta = Long.MAX_VALUE;
|
||||
// List<CursorEvent> pinnedTrace = null;
|
||||
//
|
||||
// for (List<CursorEvent> trace : traces) {
|
||||
// long[] lastNewPos = CursorEvent.getLineAndColumn(trace.get(trace.size() - 1).getNewPosField());
|
||||
// long lastNewLine = lastNewPos[0];
|
||||
// long lastNewColumn = lastNewPos[1];
|
||||
//
|
||||
// long lineDelta = Math.abs(currOldLine - lastNewLine);
|
||||
// long columnDelta = Math.abs(currOldColumn - lastNewColumn);
|
||||
// long delta = lineDelta + columnDelta;
|
||||
// if (delta < minDelta) {
|
||||
// minDelta = delta;
|
||||
// minLineDelta = lineDelta;
|
||||
// pinnedTrace = trace;
|
||||
// if (delta == 0) {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (minLineDelta < maxDis) {
|
||||
// pinnedTrace.add(event);
|
||||
// } else {
|
||||
// List<CursorEvent> trace = new ArrayList<CursorEvent>();
|
||||
// trace.add(event);
|
||||
// traces.add(trace);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
for (List<CursorEvent> trace : traces) {
|
||||
if (trace.size() < minHop) {
|
||||
events.removeAll(trace);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package edu.nju.ics.frontier.bean.action;
|
||||
|
||||
public abstract class InstantAction extends Action {
|
||||
protected long time;
|
||||
|
||||
public InstantAction(String type, long time) {
|
||||
super(type);
|
||||
this.time = time;
|
||||
}
|
||||
|
||||
public long getTime() {
|
||||
return time;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
package edu.nju.ics.frontier.bean.action;
|
||||
|
||||
public abstract class IntervalAction extends Action {
|
||||
protected long begTime;
|
||||
protected long endTime;
|
||||
|
||||
public IntervalAction(String type, long begTime, long endTime) {
|
||||
super(type);
|
||||
this.begTime = begTime;
|
||||
this.endTime = endTime;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
return begTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public abstract class Block {
|
||||
protected String type;
|
||||
protected long begTime;
|
||||
protected long endTime;
|
||||
|
||||
public Block(String type, long begTime, long endTime) {
|
||||
this.type = type;
|
||||
this.begTime = begTime;
|
||||
this.endTime = endTime;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
return begTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public long getDeltaTime() {
|
||||
return endTime - begTime;
|
||||
}
|
||||
|
||||
public abstract Block slice(long t1, long t2);
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
return String.format("{type:%s,begTime:%s,endTime:%s}",
|
||||
type,
|
||||
sdf.format(new Date(begTime)),
|
||||
sdf.format(new Date(endTime)));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
public class BlockChain {
|
||||
public static BlockChain createBlockChain(List<IntelliJEvent> events, long shortThreshold, long longThreshold) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<ProjectBlock> projectBlocks = ProjectBlock.createProjectBlocks(events);
|
||||
List<FileBlock> fileBlocks = FileBlock.createFileBlocks(events);
|
||||
List<EditorBlock> editorBlocks = EditorBlock.createEditorBlocks(events);
|
||||
List<IdleBlock> idleBlocks = IdleBlock.createIdleBlocks(events, shortThreshold);
|
||||
List<OutsideBlock> outsideBlocks = OutsideBlock.createOutsideBlocks(events);
|
||||
|
||||
List<? extends Block> fileEditorBlocks = linkBlocks(editorBlocks, fileBlocks);
|
||||
List<? extends Block> fileEditorIdleBlocks = linkBlocks(idleBlocks, fileEditorBlocks);
|
||||
List<? extends Block> fileEditorIdleOutsideBlocks = linkBlocks(outsideBlocks, fileEditorIdleBlocks);
|
||||
|
||||
List<IdleBlock> longIdleBlocks = null;
|
||||
if (!(idleBlocks == null || idleBlocks.isEmpty())) {
|
||||
longIdleBlocks = new ArrayList<IdleBlock>();
|
||||
for (IdleBlock idleBlock : idleBlocks) {
|
||||
if ((idleBlock.getEndTime() - idleBlock.getBegTime()) >= longThreshold) {
|
||||
longIdleBlocks.add(idleBlock);
|
||||
}
|
||||
}
|
||||
if (longIdleBlocks.isEmpty()) {
|
||||
longIdleBlocks = null;
|
||||
}
|
||||
}
|
||||
|
||||
List<? extends Block> projectLongIdleBlocks = linkBlocks(longIdleBlocks, projectBlocks);
|
||||
if (projectLongIdleBlocks != null) {
|
||||
projectBlocks = new ArrayList<ProjectBlock>();
|
||||
for (Block projectLongIdleBlock : projectLongIdleBlocks) {
|
||||
if (projectLongIdleBlock instanceof ProjectBlock) {
|
||||
projectBlocks.add((ProjectBlock) projectLongIdleBlock);
|
||||
}
|
||||
}
|
||||
if (projectBlocks.isEmpty()) {
|
||||
projectBlocks = null;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO
|
||||
return null;
|
||||
}
|
||||
|
||||
public static <A extends Block, B extends Block> List<? extends Block> linkBlocks(
|
||||
List<A> aboveBlocks, List<B> belowBlocks) {
|
||||
if (aboveBlocks == null && belowBlocks == null) {
|
||||
return null;
|
||||
} else if (aboveBlocks == null) {
|
||||
return belowBlocks;
|
||||
} else if (belowBlocks == null) {
|
||||
return aboveBlocks;
|
||||
}
|
||||
|
||||
List<Block> result = new ArrayList<Block>();
|
||||
|
||||
LinkedList<Block> copiedAboveBlocks = new LinkedList<Block>(aboveBlocks);
|
||||
LinkedList<Block> copiedBelowBlocks = new LinkedList<Block>(belowBlocks);
|
||||
while (!(copiedAboveBlocks.isEmpty() || copiedBelowBlocks.isEmpty())) {
|
||||
Block aboveBlock = copiedAboveBlocks.removeFirst();
|
||||
Block belowBlock = copiedBelowBlocks.removeFirst();
|
||||
|
||||
if (aboveBlock.getBegTime() <= belowBlock.getBegTime()) {
|
||||
result.add(aboveBlock);
|
||||
if (aboveBlock.getEndTime() < belowBlock.getEndTime()) {
|
||||
copiedBelowBlocks.addFirst(belowBlock.slice(aboveBlock.getEndTime(), belowBlock.getEndTime()));
|
||||
} else {
|
||||
while (!copiedBelowBlocks.isEmpty()) {
|
||||
belowBlock = copiedBelowBlocks.removeFirst();
|
||||
if (belowBlock.getEndTime() <= aboveBlock.getEndTime()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (belowBlock.getBegTime() >= aboveBlock.getEndTime()) {
|
||||
copiedBelowBlocks.addFirst(belowBlock);
|
||||
break;
|
||||
}
|
||||
|
||||
copiedBelowBlocks.addFirst(belowBlock.slice(aboveBlock.getEndTime(), belowBlock.getEndTime()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (belowBlock.getEndTime() <= aboveBlock.getBegTime()) {
|
||||
result.add(belowBlock);
|
||||
} else {
|
||||
result.add(belowBlock.slice(belowBlock.getBegTime(), aboveBlock.getBegTime()));
|
||||
if (belowBlock.getEndTime() > aboveBlock.getEndTime()) {
|
||||
copiedBelowBlocks.addFirst(belowBlock.slice(aboveBlock.getEndTime(), belowBlock.getEndTime()));
|
||||
}
|
||||
}
|
||||
copiedAboveBlocks.addFirst(aboveBlock);
|
||||
}
|
||||
}
|
||||
if (!copiedAboveBlocks.isEmpty()) {
|
||||
result.addAll(copiedAboveBlocks);
|
||||
}
|
||||
if (!copiedBelowBlocks.isEmpty()) {
|
||||
result.addAll(copiedBelowBlocks);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.CursorEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class EditorBlock extends Block {
|
||||
protected List<CursorEvent> cursorChangedTrajectory;
|
||||
|
||||
public EditorBlock(String type, long begTime, long endTime, List<CursorEvent> cursorChangedTrajectory) {
|
||||
super(type, begTime, endTime);
|
||||
this.cursorChangedTrajectory = cursorChangedTrajectory;
|
||||
}
|
||||
|
||||
public List<CursorEvent> getCursorChangedTrajectory() {
|
||||
return cursorChangedTrajectory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public EditorBlock slice(long t1, long t2) {
|
||||
if (begTime <= t1 && t1 < t2 && t2 <= endTime) {
|
||||
List<CursorEvent> slicedCursorEvents = EditorBlock.copyEvents(cursorChangedTrajectory, t1, t2);
|
||||
return new EditorBlock(type, t1, t2, slicedCursorEvents);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static <E extends IntelliJEvent> List<E> copyEvents(List<E> src, long t1, long t2) {
|
||||
if (src == null || src.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<E> tmp = new ArrayList<E>();
|
||||
for (E e : src) {
|
||||
long t = e.getTime();
|
||||
if (t >= t1 && t < t2) {
|
||||
tmp.add(e);
|
||||
} else if (t >= t2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return tmp.isEmpty() ? null : tmp;
|
||||
}
|
||||
|
||||
public static List<EditorBlock> createEditorBlocks(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<EditorBlock> editorBlocks = new ArrayList<EditorBlock>();
|
||||
|
||||
String lastEditor = null;
|
||||
List<CursorEvent> cursorChangedEvents = null;
|
||||
|
||||
for (IntelliJEvent event : events) {
|
||||
if (!IntelliJEvent.isCursorEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
CursorEvent cursorEvent = IntelliJEvent.toCursorEvent(event);
|
||||
if (!cursorEvent.isWhenCursorChanged()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String editor = cursorEvent.getEditorField();
|
||||
if (lastEditor == null) {
|
||||
if (!"UNTYPED".equals(editor)) {
|
||||
lastEditor = editor;
|
||||
cursorChangedEvents = new ArrayList<CursorEvent>();
|
||||
cursorChangedEvents.add(cursorEvent);
|
||||
}
|
||||
} else {
|
||||
if (lastEditor.equals(editor)) {
|
||||
cursorChangedEvents.add(cursorEvent);
|
||||
} else {
|
||||
removeOutliers(cursorChangedEvents);
|
||||
if (!cursorChangedEvents.isEmpty()) {
|
||||
editorBlocks.add(new EditorBlock(
|
||||
lastEditor,
|
||||
cursorChangedEvents.get(0).getTime(),
|
||||
cursorChangedEvents.get(cursorChangedEvents.size() - 1).getTime(),
|
||||
cursorChangedEvents));
|
||||
}
|
||||
|
||||
if (!"UNTYPED".equals(editor)) {
|
||||
lastEditor = editor;
|
||||
cursorChangedEvents = new ArrayList<CursorEvent>();
|
||||
cursorChangedEvents.add(cursorEvent);
|
||||
} else {
|
||||
lastEditor = null;
|
||||
cursorChangedEvents = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lastEditor != null) {
|
||||
removeOutliers(cursorChangedEvents);
|
||||
if (!cursorChangedEvents.isEmpty()) {
|
||||
editorBlocks.add(new EditorBlock(
|
||||
lastEditor,
|
||||
cursorChangedEvents.get(0).getTime(),
|
||||
cursorChangedEvents.get(cursorChangedEvents.size() - 1).getTime(),
|
||||
cursorChangedEvents));
|
||||
}
|
||||
}
|
||||
|
||||
return editorBlocks.isEmpty() ? null : editorBlocks;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @see EditorBlock#removeOutliers(List, int, int)
|
||||
* @param events
|
||||
*/
|
||||
public static void removeOutliers(List<CursorEvent> events) {
|
||||
int maxDis = 5;
|
||||
int minHop = Math.min(events.size(), Math.max(5, events.size() / 10));
|
||||
removeOutliers(events, maxDis, minHop);
|
||||
}
|
||||
|
||||
/**
|
||||
* remove outliers in the cursor changed sequence.
|
||||
* @param events IntelliJ event sequence
|
||||
* @param maxDis for any two adjacent cursor changed events, the difference between
|
||||
* the former's {@code newPos} and the latter's {@code oldPos} must be
|
||||
* less than the given {@code maxDis}.
|
||||
* @param minHop the continuous sub cursor changed sequence is valid
|
||||
* if and only if its hops exceed {@code minHop}.
|
||||
*/
|
||||
public static void removeOutliers(List<CursorEvent> events, int maxDis, int minHop) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<List<CursorEvent>> traces = new ArrayList<List<CursorEvent>>();
|
||||
List<CursorEvent> tmp = null;
|
||||
|
||||
for (CursorEvent event : events) {
|
||||
if (tmp == null) {
|
||||
tmp = new ArrayList<CursorEvent>();
|
||||
tmp.add(event);
|
||||
traces.add(tmp);
|
||||
} else {
|
||||
long[] lastNewPos = CursorEvent.getLineAndColumn(tmp.get(tmp.size() - 1).getNewPosField());
|
||||
long lastNewLine = lastNewPos[0];
|
||||
long[] currOldPos = CursorEvent.getLineAndColumn(event.getOldPosField());
|
||||
long currOldLine = currOldPos[0];
|
||||
|
||||
if (Math.abs(currOldLine - lastNewLine) < maxDis) {
|
||||
tmp.add(event);
|
||||
} else {
|
||||
tmp = new ArrayList<CursorEvent>();
|
||||
tmp.add(event);
|
||||
traces.add(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (List<CursorEvent> trace : traces) {
|
||||
if (trace.size() < minHop) {
|
||||
events.removeAll(trace);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* sampling {@code events} in a fixed {@code frequency}. The parameter {@code lineParser}
|
||||
* is an interface whose function is parsing the line in the event.
|
||||
* @param events file or cursor events
|
||||
* @param frequency sampling frequency
|
||||
* @param lineParser instance that implements the {@code LineParser} interface
|
||||
* @param <E> FileEvent or CursorEvent
|
||||
* @return samples
|
||||
*/
|
||||
public static <E extends IntelliJEvent> long[][] samplingLines(
|
||||
List<E> events, long frequency, LineParser<E> lineParser) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
long begTime = aligning(events.get(0).getTime(), frequency);
|
||||
long endTime = aligning(events.get(events.size() - 1).getTime(), frequency);
|
||||
int sampleNum = (int) ((endTime - begTime) / frequency) + 1;
|
||||
|
||||
long[][] result = new long[sampleNum][2];
|
||||
|
||||
// initialize
|
||||
result[0][0] = begTime;
|
||||
result[0][1] = -1;
|
||||
for (int i = 1; i < sampleNum; i++) {
|
||||
result[i][0] = result[i - 1][0] + frequency;
|
||||
result[i][1] = -1;
|
||||
}
|
||||
|
||||
// assignment
|
||||
for (int i = events.size() - 1; i >= 0; i--) {
|
||||
E event = events.get(i);
|
||||
long time = aligning(event.getTime(), frequency);
|
||||
int index = (int) ((time - begTime) / frequency);
|
||||
if (result[index][1] == -1) {
|
||||
result[index][1] = lineParser.parse(event);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* aligning the time to the sampling point. Following below steps:
|
||||
* 1) calculating the quotient and remainder of {@code time} divided by {@code frequency};
|
||||
* 2) return {@code (quotient + 1) * frequency} if {@code remainder >= frequency / 2},
|
||||
* else return {@code quotient * frequency}.
|
||||
* @param time time (millisecond)
|
||||
* @param frequency sampling frequency (millisecond)
|
||||
* @return aligned time
|
||||
*/
|
||||
private static long aligning(long time, long frequency) {
|
||||
long q = time / frequency;
|
||||
long r = time % frequency;
|
||||
long h = frequency / 2;
|
||||
if (r >= h) {
|
||||
return (q + 1) * frequency;
|
||||
} else {
|
||||
return q * frequency;
|
||||
}
|
||||
}
|
||||
|
||||
public interface LineParser<E> {
|
||||
long parse(E e);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.CursorEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.FileEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class FileBlock extends EditorBlock {
|
||||
private String path;
|
||||
// private long[][] samples;
|
||||
private List<FileEvent> fileChangedTrajectory;
|
||||
// private List<CursorEvent> cursorChangedTrajectory;
|
||||
|
||||
public FileBlock(long begTime, long endTime, String path,
|
||||
// long[][] samples,
|
||||
List<FileEvent> fileChangedTrajectory,
|
||||
List<CursorEvent> cursorChangedTrajectory) {
|
||||
super("File", begTime, endTime, cursorChangedTrajectory);
|
||||
this.path = path;
|
||||
// this.samples = samples;
|
||||
this.fileChangedTrajectory = fileChangedTrajectory;
|
||||
// this.cursorChangedTrajectory = cursorChangedTrajectory;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public List<FileEvent> getFileChangedTrajectory() {
|
||||
return fileChangedTrajectory;
|
||||
}
|
||||
|
||||
public long[][] getSamples() {
|
||||
return samplingLines(fileChangedTrajectory, cursorChangedTrajectory, 10000);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileBlock slice(long t1, long t2) {
|
||||
if (begTime <= t1 && t1 < t2 && t2 <= endTime) {
|
||||
List<FileEvent> slicedFileEvents = copyEvents(fileChangedTrajectory, t1, t2);
|
||||
List<CursorEvent> slicedCursorEvents = copyEvents(cursorChangedTrajectory, t1, t2);
|
||||
return new FileBlock(t1, t2, path, slicedFileEvents, slicedCursorEvents);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* For the given event sequence, segmenting it into several file blocks by
|
||||
* splitting the sequence at each file selected point.
|
||||
* @param events IntelliJ event sequence
|
||||
* @return list of file blocks
|
||||
*/
|
||||
public static List<FileBlock> createFileBlocks(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<FileBlock> fileBlocks = new ArrayList<FileBlock>();
|
||||
|
||||
FileEvent splitPoint = null;
|
||||
List<FileEvent> fileChangedEvents = null;
|
||||
List<CursorEvent> cursorChangedEvents = null;
|
||||
List<IntelliJEvent> terminalForFileBlock = null;
|
||||
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isFileEvent(event)) {
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
// when file was selected. The new path must be not null and different from the old one.
|
||||
String oldPath = fileEvent.getOldPathField();
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
if (newPath == null || newPath.equals(oldPath)) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint != null) {
|
||||
String path = splitPoint.getNewPathField();
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = fileEvent.getTime();
|
||||
if (!terminalForFileBlock.isEmpty()) {
|
||||
long t = terminalForFileBlock.get(0).getTime();
|
||||
if (t < endTime) {
|
||||
endTime = t;
|
||||
}
|
||||
}
|
||||
removeOutliers(cursorChangedEvents);
|
||||
FileBlock fileBlock = new FileBlock(
|
||||
begTime, endTime, path,
|
||||
// samplingLines(fileChangedEvents, cursorChangedEvents, 10000),
|
||||
fileChangedEvents,
|
||||
cursorChangedEvents);
|
||||
fileBlocks.add(fileBlock);
|
||||
}
|
||||
splitPoint = fileEvent;
|
||||
fileChangedEvents = new ArrayList<FileEvent>();
|
||||
cursorChangedEvents = new ArrayList<CursorEvent>();
|
||||
terminalForFileBlock = new ArrayList<IntelliJEvent>();
|
||||
} else if (fileEvent.isWhenChanged()) {
|
||||
// when file changed. The path must be same as the last split point.
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint.getNewPathField().equals(fileEvent.getPathField())) {
|
||||
fileChangedEvents.add(fileEvent);
|
||||
}
|
||||
} else if (fileEvent.isWhenClosed()) {
|
||||
// when file closed. The last file closed event is used to determine the end time of the last file filter.
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (splitPoint.getNewPathField().equals(fileEvent.getPathField())) {
|
||||
terminalForFileBlock.add(fileEvent);
|
||||
}
|
||||
}
|
||||
} else if (IntelliJEvent.isCursorEvent(event)) {
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
CursorEvent cursorEvent = IntelliJEvent.toCursorEvent(event);
|
||||
if (cursorEvent.isWhenCursorChanged()) {
|
||||
// when cursor changed. The editor must be 'UNTYPED' and the old position does not equals (0,0).
|
||||
String oldPos = cursorEvent.getOldPosField();
|
||||
String newPos = cursorEvent.getNewPosField();
|
||||
if (cursorEvent.isUntypedEditor() && oldPos != null &&
|
||||
newPos != null && (!"(0,0)".equals(oldPos))) {
|
||||
cursorChangedEvents.add(cursorEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (splitPoint != null) {
|
||||
String path = splitPoint.getNewPathField();
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = events.get(events.size() - 1).getTime();
|
||||
if (!terminalForFileBlock.isEmpty()) {
|
||||
long t = terminalForFileBlock.get(0).getTime();
|
||||
if (t < endTime) {
|
||||
endTime = t;
|
||||
}
|
||||
}
|
||||
removeOutliers(cursorChangedEvents);
|
||||
FileBlock fileBlock = new FileBlock(
|
||||
begTime, endTime, path,
|
||||
// samplingLines(fileChangedEvents, cursorChangedEvents, 10000),
|
||||
fileChangedEvents,
|
||||
cursorChangedEvents);
|
||||
fileBlocks.add(fileBlock);
|
||||
}
|
||||
|
||||
return fileBlocks.isEmpty() ? null : fileBlocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* sampling the Lines of Code and Line of Cursor in a fixed frequency.
|
||||
* @param fileEvents file events
|
||||
* @param cursorEvents cursor events
|
||||
* @param frequency sampling frequency
|
||||
* @return samples
|
||||
*/
|
||||
private static long[][] samplingLines(List<FileEvent> fileEvents, List<CursorEvent> cursorEvents, long frequency) {
|
||||
long[][] linesOfCode = EditorBlock.samplingLines(fileEvents, frequency, new EditorBlock.LineParser<FileEvent>() {
|
||||
public long parse(FileEvent event) {
|
||||
return Long.parseLong(event.getLineField());
|
||||
}
|
||||
});
|
||||
|
||||
long[][] lineOfCursor = EditorBlock.samplingLines(cursorEvents, frequency, new EditorBlock.LineParser<CursorEvent>() {
|
||||
public long parse(CursorEvent event) {
|
||||
String pos = event.getNewPosField();
|
||||
String line = pos.substring(1, pos.indexOf(","));
|
||||
return Long.parseLong(line);
|
||||
}
|
||||
});
|
||||
|
||||
long begTime = -1;
|
||||
long endTime = -1;
|
||||
if (linesOfCode != null) {
|
||||
begTime = linesOfCode[0][0];
|
||||
endTime = linesOfCode[linesOfCode.length - 1][0];
|
||||
}
|
||||
if (lineOfCursor != null) {
|
||||
begTime = (begTime == -1) ? lineOfCursor[0][0] : Math.min(begTime, lineOfCursor[0][0]);
|
||||
endTime = (endTime == -1) ? lineOfCursor[lineOfCursor.length - 1][0] : Math.max(endTime, lineOfCursor[lineOfCursor.length - 1][0]);
|
||||
}
|
||||
if (begTime == -1 || endTime == -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int sampleNum = (int) ((endTime - begTime) / frequency) + 1;
|
||||
long[][] buffer = new long[sampleNum][3];
|
||||
|
||||
// initialize
|
||||
buffer[0][0] = begTime;
|
||||
buffer[0][1] = -1;
|
||||
buffer[0][2] = -1;
|
||||
for (int i = 1; i < sampleNum; i++) {
|
||||
buffer[i][0] = buffer[i - 1][0] + frequency;
|
||||
buffer[i][1] = -1;
|
||||
buffer[i][2] = -1;
|
||||
}
|
||||
|
||||
// placing
|
||||
placing(buffer, linesOfCode, 1);
|
||||
placing(buffer, lineOfCursor, 2);
|
||||
|
||||
List<long[]> tmp = new ArrayList<long[]>();
|
||||
for (int i = 0; i < sampleNum; i++) {
|
||||
if ((buffer[i][1] != -1) || (buffer[i][2] != -1)) {
|
||||
tmp.add(buffer[i]);
|
||||
}
|
||||
}
|
||||
|
||||
long[][] result = new long[tmp.size()][];
|
||||
for (int i = 0, s = tmp.size(); i < s; i++) {
|
||||
result[i] = tmp.get(i);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* placing the Lines of Code/Cursor into a common array.
|
||||
* @param dest array to store the value of Lines of Code/Cursor.
|
||||
* @param src array of Lines of Code/Cursor
|
||||
* @param column column of {@code dest} to store {@code src}.
|
||||
*/
|
||||
private static void placing(long[][] dest, long[][] src, int column) {
|
||||
if (src == null || src.length <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int offset = -1;
|
||||
for (int i = 0, l = dest.length; i < l; i++) {
|
||||
if (dest[i][0] == src[0][0]) {
|
||||
offset = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (offset == -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0, l = src.length; i < l; i++) {
|
||||
dest[i + offset][column] = src[i][1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class IdleBlock extends Block {
|
||||
public IdleBlock(long begTime, long endTime) {
|
||||
super("Idle", begTime, endTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public IdleBlock slice(long t1, long t2) {
|
||||
if (begTime <= t1 && t1 < t2 && t2 <= endTime) {
|
||||
return new IdleBlock(t1, t2);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static List<IdleBlock> createIdleBlocks(List<IntelliJEvent> events, long threshold) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<IdleBlock> idleBlocks = new ArrayList<IdleBlock>();
|
||||
|
||||
IntelliJEvent lastEvent = null;
|
||||
for (IntelliJEvent currEvent : events) {
|
||||
if (lastEvent != null) {
|
||||
long lastTime = lastEvent.getTime();
|
||||
long currTime = currEvent.getTime();
|
||||
if ((currTime - lastTime) >= threshold) {
|
||||
idleBlocks.add(new IdleBlock(lastTime, currTime));
|
||||
}
|
||||
}
|
||||
lastEvent = currEvent;
|
||||
}
|
||||
|
||||
return idleBlocks.isEmpty() ? null : idleBlocks;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.AppEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class OutsideBlock extends Block {
|
||||
public OutsideBlock(long begTime, long endTime) {
|
||||
super("Outside", begTime, endTime);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OutsideBlock slice(long t1, long t2) {
|
||||
if (begTime <= t1 && t1 < t2 && t2 <= endTime) {
|
||||
return new OutsideBlock(t1, t2);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static List<OutsideBlock> createOutsideBlocks(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<Long> tmp = new ArrayList<Long>();
|
||||
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isAppEvent(event)) {
|
||||
AppEvent appEvent = IntelliJEvent.toAppEvent(event);
|
||||
if (appEvent.isWhenDeactivated()) {
|
||||
if (tmp.size() % 2 == 0) {
|
||||
tmp.add(appEvent.getTime());
|
||||
}
|
||||
} else if (appEvent.isWhenActivated()) {
|
||||
if (tmp.size() % 2 == 1) {
|
||||
tmp.add(appEvent.getTime());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (tmp.size() % 2 == 1) {
|
||||
tmp.remove(tmp.size() - 1);
|
||||
}
|
||||
|
||||
if (tmp.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<OutsideBlock> outsideBlocks = new ArrayList<OutsideBlock>();
|
||||
for (int i = 0, s = tmp.size() - 1; i < s; i++) {
|
||||
long begTime = tmp.get(i);
|
||||
long endTime = tmp.get(i + 1);
|
||||
outsideBlocks.add(new OutsideBlock(begTime, endTime));
|
||||
}
|
||||
return outsideBlocks;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
package edu.nju.ics.frontier.bean.block;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.ProjectEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.WindowEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class ProjectBlock extends Block {
|
||||
private String projectName;
|
||||
private String frameName;
|
||||
|
||||
public ProjectBlock(long begTime, long endTime, String projectName, String frameName) {
|
||||
super("Project", begTime, endTime);
|
||||
this.projectName = projectName;
|
||||
this.frameName = frameName;
|
||||
}
|
||||
|
||||
public String getProjectName() {
|
||||
return projectName;
|
||||
}
|
||||
|
||||
public String getFrameName() {
|
||||
return frameName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ProjectBlock slice(long t1, long t2) {
|
||||
if (begTime <= t1 && t1 < t2 && t2 <= endTime) {
|
||||
return new ProjectBlock(t1, t2, projectName, frameName);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{begTime:%d,endTime:%d,projectName:%s,frameName:%s}",
|
||||
begTime, endTime, projectName, frameName);
|
||||
}
|
||||
|
||||
public static List<ProjectBlock> createProjectBlocks(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// find out all project events. For such opened event with unknown 'frame' field,
|
||||
// just try to temperately scan the subsequent window events and select out the
|
||||
// opened event with a clear 'frame' field, further assigning the window's 'frame'
|
||||
// field to the project event.
|
||||
// Noting that an IllegalArgumentException will be thrown out if there is no suitable
|
||||
// opened window event found in limited step.
|
||||
List<ProjectEvent> projectEvents = new ArrayList<ProjectEvent>();
|
||||
List<WindowEvent> windowEvents = new ArrayList<WindowEvent>();
|
||||
for (int i = 0, s = events.size(); i < s; i++) {
|
||||
IntelliJEvent event = events.get(i);
|
||||
if (IntelliJEvent.isProjectEvent(event)) {
|
||||
ProjectEvent projectEvent = IntelliJEvent.toProjectEvent(event);
|
||||
if (projectEvent.isWhenOpened() && projectEvent.getFrameField() == null) {
|
||||
boolean isFrameAttached = false;
|
||||
for (int j = i + 1; j < s; j++) {
|
||||
IntelliJEvent e = events.get(j);
|
||||
if (!IntelliJEvent.isWindowEvent(e)) {
|
||||
continue;
|
||||
}
|
||||
WindowEvent windowEvent = IntelliJEvent.toWindowEvent(e);
|
||||
if (!windowEvent.isWhenWindowOpened()) {
|
||||
continue;
|
||||
}
|
||||
String frame = windowEvent.getThisWindowName();
|
||||
if (frame != null && frame.startsWith("frame")) {
|
||||
projectEvent.addExtraData("frame", frame);
|
||||
isFrameAttached = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!isFrameAttached) {
|
||||
throw new IllegalArgumentException("project " + projectEvent + " does not attach any frame!");
|
||||
}
|
||||
}
|
||||
projectEvents.add(projectEvent);
|
||||
} else if (IntelliJEvent.isWindowEvent(event)) {
|
||||
WindowEvent windowEvent = IntelliJEvent.toWindowEvent(event);
|
||||
if (windowEvent.isWhenWindowActivated()) {
|
||||
String selfName = windowEvent.getThisWindowName();
|
||||
if (selfName != null && selfName.startsWith("frame")) {
|
||||
windowEvents.add(windowEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (projectEvents.isEmpty() || windowEvents.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// try to transform window events to project events. More specifically,
|
||||
// for such window event whose 'this' field matches the 'frame[0-9]+,[A-Z]+' regex,
|
||||
// and occurred when window activated:
|
||||
// 1) trying to find the nearest opened project event with the same 'frame' field.
|
||||
// If found, jump to 3);
|
||||
// 2) trying to find the nearest closed project event with the same 'frame' field.
|
||||
// If not found, jump to 5);
|
||||
// 3) transforming the window event to the project event in following rule:
|
||||
// a) the project's 'when' field is same as the window's;
|
||||
// b) the project's 'name' field is same as the one that found in 1) or 2);
|
||||
// c) the project's 'frame' field is same as the one that found in 1) or 2).
|
||||
// 4) Jumping to 1) if there are remain window events, else go to 6);
|
||||
// 5) throwing out an IllegalArgumentException to warn there is a lonely window event;
|
||||
// 6) terminating.
|
||||
List<ProjectEvent> transforms = new ArrayList<ProjectEvent>();
|
||||
for (WindowEvent windowEvent : windowEvents) {
|
||||
long winTime = windowEvent.getTime();
|
||||
String winFrame = windowEvent.getThisWindowName();
|
||||
|
||||
ProjectEvent nearestOpened = null;
|
||||
ProjectEvent nearestClosed = null;
|
||||
for (ProjectEvent projectEvent : projectEvents) {
|
||||
String proFrame = projectEvent.getFrameField();
|
||||
if (!proFrame.equals(winFrame)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long proTime = projectEvent.getTime();
|
||||
if (proTime <= winTime) {
|
||||
if (projectEvent.isWhenOpened()) {
|
||||
nearestOpened = projectEvent;
|
||||
} else {
|
||||
nearestClosed = projectEvent;
|
||||
}
|
||||
} else {
|
||||
if (projectEvent.isWhenOpened()) {
|
||||
if (nearestOpened == null) {
|
||||
nearestOpened = projectEvent;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (nearestClosed == null) {
|
||||
nearestClosed = projectEvent;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ProjectEvent match = (nearestOpened != null) ? nearestOpened : nearestClosed;
|
||||
if (match != null) {
|
||||
ProjectEvent transform = new ProjectEvent(
|
||||
winTime,
|
||||
match.getType(),
|
||||
windowEvent.isWhenWindowActivated() ? "activated" : "deactivated",
|
||||
match.getData());
|
||||
transforms.add(transform);
|
||||
} else {
|
||||
Map<String, String> data = new HashMap<String, String>();
|
||||
data.put("name", ("unknown_" + winFrame));
|
||||
data.put("frame", winFrame);
|
||||
ProjectEvent transform = new ProjectEvent(
|
||||
winTime,
|
||||
"Project",
|
||||
windowEvent.isWhenWindowActivated() ? "activated" : "deactivated",
|
||||
data);
|
||||
transforms.add(transform);
|
||||
// throw new IllegalArgumentException("the lonely window event " + windowEvent + ".");
|
||||
}
|
||||
}
|
||||
|
||||
if (transforms.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// removing the successive similar activated project events.
|
||||
List<ProjectEvent> prepared = new ArrayList<ProjectEvent>();
|
||||
ProjectEvent lastEvent = null;
|
||||
for (int i = 0, s = transforms.size(); i < s; i++) {
|
||||
ProjectEvent currEvent = transforms.get(i);
|
||||
if (lastEvent == null) {
|
||||
lastEvent = currEvent;
|
||||
prepared.add(currEvent);
|
||||
} else {
|
||||
String lastName = lastEvent.getNameField();
|
||||
String currName = currEvent.getNameField();
|
||||
if (!lastName.equals(currName)) {
|
||||
lastEvent = currEvent;
|
||||
prepared.add(currEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (prepared.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (ProjectEvent projectEvent : projectEvents) {
|
||||
if (projectEvent.isWhenClosed()) {
|
||||
prepared.add(projectEvent);
|
||||
}
|
||||
}
|
||||
Collections.sort(prepared, new Comparator<ProjectEvent>() {
|
||||
public int compare(ProjectEvent o1, ProjectEvent o2) {
|
||||
long t1 = o1.getTime();
|
||||
long t2 = o2.getTime();
|
||||
return (int) (t1 - t2);
|
||||
}
|
||||
});
|
||||
|
||||
// constructing project blocks based on the transformed activated project events.
|
||||
List<ProjectBlock> projectBlocks = new ArrayList<ProjectBlock>();
|
||||
|
||||
ProjectEvent splitPoint = null;
|
||||
List<IntelliJEvent> terminalForProjectBlock = null;
|
||||
|
||||
for (ProjectEvent projectEvent : prepared) {
|
||||
if (projectEvent.isWhenActivated()) {
|
||||
if (splitPoint != null) {
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = projectEvent.getTime();
|
||||
if (!terminalForProjectBlock.isEmpty()) {
|
||||
long time = terminalForProjectBlock.get(0).getTime();
|
||||
if (time < endTime) {
|
||||
endTime = time;
|
||||
}
|
||||
}
|
||||
ProjectBlock projectBlock = new ProjectBlock(begTime, endTime,
|
||||
splitPoint.getNameField(), splitPoint.getFrameField());
|
||||
projectBlocks.add(projectBlock);
|
||||
}
|
||||
splitPoint = projectEvent;
|
||||
terminalForProjectBlock = new ArrayList<IntelliJEvent>();
|
||||
} else if (projectEvent.isWhenClosed()) {
|
||||
if (splitPoint == null) {
|
||||
continue;
|
||||
}
|
||||
if (!splitPoint.getNameField().equals(projectEvent.getNameField())) {
|
||||
continue;
|
||||
}
|
||||
if (!splitPoint.getFrameField().equals(projectEvent.getFrameField())) {
|
||||
continue;
|
||||
}
|
||||
terminalForProjectBlock.add(projectEvent);
|
||||
}
|
||||
}
|
||||
|
||||
if (splitPoint != null) {
|
||||
long begTime = splitPoint.getTime();
|
||||
long endTime = events.get(events.size() - 1).getTime();
|
||||
if (!terminalForProjectBlock.isEmpty()) {
|
||||
long time = terminalForProjectBlock.get(0).getTime();
|
||||
if (time < endTime) {
|
||||
endTime = time;
|
||||
}
|
||||
}
|
||||
ProjectBlock projectBlock = new ProjectBlock(begTime, endTime,
|
||||
splitPoint.getNameField(), splitPoint.getFrameField());
|
||||
projectBlocks.add(projectBlock);
|
||||
}
|
||||
|
||||
return projectBlocks.isEmpty() ? null : projectBlocks;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,49 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class ActionEvent extends IntelliJEvent {
|
||||
public ActionEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public ActionEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenPerformed() {
|
||||
return "performed".equals(when);
|
||||
}
|
||||
|
||||
public String getActionField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("action");
|
||||
}
|
||||
}
|
||||
|
||||
public String getIdField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("id");
|
||||
}
|
||||
}
|
||||
|
||||
public String getClassField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("class");
|
||||
}
|
||||
}
|
||||
|
||||
public String getCategoryField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("category");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class AppEvent extends IntelliJEvent {
|
||||
public AppEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public AppEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenOpened() {
|
||||
return "opened".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenClosed() {
|
||||
return "closed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenActivated() {
|
||||
return "activated".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenDeactivated() {
|
||||
return "deactivated".equals(when);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class CompileEvent extends IntelliJEvent {
|
||||
public CompileEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public CompileEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenFinished() {
|
||||
return "finished".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenAutoFinished() {
|
||||
return "auto_finished".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenFileGenerated() {
|
||||
return "file_generated".equals(when);
|
||||
}
|
||||
|
||||
public String getAbortedField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("aborted");
|
||||
}
|
||||
}
|
||||
|
||||
public String getErrorsField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("errors");
|
||||
}
|
||||
}
|
||||
|
||||
public String getWarningsField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("warnings");
|
||||
}
|
||||
}
|
||||
|
||||
public String getRootField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("root");
|
||||
}
|
||||
}
|
||||
|
||||
public String getPathField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("path");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class CursorEvent extends IntelliJEvent {
|
||||
public CursorEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public CursorEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenCursorChanged() {
|
||||
return "cursor_changed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenCursorAdded() {
|
||||
return "cursor_added".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenCursorRemoved() {
|
||||
return "cursor_removed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenSelectChanged() {
|
||||
return "select_changed".equals(when);
|
||||
}
|
||||
|
||||
public String getEditorField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("editor");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldPosField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("oldPos");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewPosField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("newPos");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("old");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("new");
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isUntypedEditor() {
|
||||
return "UNTYPED".equals(getEditorField());
|
||||
}
|
||||
|
||||
public boolean isConsoleEditor() {
|
||||
return "CONSOLE".equals(getEditorField());
|
||||
}
|
||||
|
||||
public boolean isPreviewEditor() {
|
||||
String editor = getEditorField();
|
||||
return (editor != null) && editor.startsWith("PREVIEW");
|
||||
}
|
||||
|
||||
public boolean isDiffEditor() {
|
||||
return "DIFF".equals(getEditorField());
|
||||
}
|
||||
|
||||
public static boolean isInSameEditor(CursorEvent event1, CursorEvent event2) {
|
||||
if (event1 == null || event2 == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (event1.isUntypedEditor()) {
|
||||
return event2.isUntypedEditor();
|
||||
} else if (event1.isConsoleEditor()) {
|
||||
return event2.isConsoleEditor();
|
||||
} else if (event1.isPreviewEditor()) {
|
||||
return event2.isPreviewEditor();
|
||||
} else if (event1.isDiffEditor()) {
|
||||
return event2.isDiffEditor();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static long[] getLineAndColumn(String pos) {
|
||||
int commaIndex = pos.indexOf(",");
|
||||
String lineStr = pos.substring(1, commaIndex);
|
||||
String columnStr = pos.substring(commaIndex + 1, pos.length() - 1);
|
||||
long line = -1;
|
||||
long column = -1;
|
||||
try {
|
||||
line = Long.parseLong(lineStr);
|
||||
column = Long.parseLong(columnStr);
|
||||
} catch (NumberFormatException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return new long[]{line, column};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class DebugEvent extends IntelliJEvent {
|
||||
public DebugEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public DebugEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenContextChanged() {
|
||||
return "context_changed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenBreakpointAdded() {
|
||||
return "breakpoint_added".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenBreakpointRemoved() {
|
||||
return "breakpoint_removed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenBreakpointChanged() {
|
||||
return "breakpoint_changed".equals(when);
|
||||
}
|
||||
|
||||
public String getProjectField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("project");
|
||||
}
|
||||
}
|
||||
|
||||
public String getSessionField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("segment");
|
||||
}
|
||||
}
|
||||
|
||||
public String getEventField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("event");
|
||||
}
|
||||
}
|
||||
|
||||
public String getStateField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("state");
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isWaitingAttachState() {
|
||||
return "WAITING_ATTACH".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isRunningState() {
|
||||
return "RUNNING".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isPausedState() {
|
||||
return "PAUSED".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isWaitEvaluationState() {
|
||||
return "WAIT_EVALUATION".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isStoppedState() {
|
||||
return "STOPPED".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isDisposedState() {
|
||||
return "DISPOSED".equals(getStateField());
|
||||
}
|
||||
|
||||
public boolean isContextEvent() {
|
||||
return "CONTEXT".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isStartWaitAttachEvent() {
|
||||
return "START_WAIT_ATTACH".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isAttachedEvent() {
|
||||
return "ATTACHED".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isThreadsRefreshEvent() {
|
||||
return "THREADS_REFRESH".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isResumeEvent() {
|
||||
return "RESUME".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isStepEvent() {
|
||||
return "STEP".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isPauseEvent() {
|
||||
return "PAUSE".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isRefreshEvent() {
|
||||
return "REFRESH".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isRefreshWithStackEvent() {
|
||||
return "REFRESH_WITH_STACK".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isDetachedEvent() {
|
||||
return "DETACHED".equals(getEventField());
|
||||
}
|
||||
|
||||
public boolean isDisposeEvent() {
|
||||
return "DISPOSE".equals(getEventField());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class ExecuteEvent extends IntelliJEvent {
|
||||
public ExecuteEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public ExecuteEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenScheduled() {
|
||||
return "scheduled".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenStarting() {
|
||||
return "starting".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenCancelled() {
|
||||
return "cancelled".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenStarted() {
|
||||
return "started".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenTerminating() {
|
||||
return "terminating".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenTerminated() {
|
||||
return "terminated".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenActiveTargetChanged() {
|
||||
return "active_target_changed".equals(when);
|
||||
}
|
||||
|
||||
public String getIdField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("id");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNameField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("name");
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isRun() {
|
||||
return "Run".equals(getIdField());
|
||||
}
|
||||
|
||||
public boolean isDebug() {
|
||||
return "Debug".equals(getIdField());
|
||||
}
|
||||
|
||||
public boolean isCoverage() {
|
||||
return "Coverage".equals(getIdField());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class FileEvent extends IntelliJEvent {
|
||||
public FileEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public FileEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenPropertyChanged() {
|
||||
return "property_changed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenContentChanged() {
|
||||
return "content_changed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenCreated() {
|
||||
return "created".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenDeleted() {
|
||||
return "deleted".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMoved() {
|
||||
return "moved".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenCopied() {
|
||||
return "copied".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenOpened() {
|
||||
return "opened".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenClosed() {
|
||||
return "closed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenSelected() {
|
||||
return "selected".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenChanged() {
|
||||
return "changed".equals(when);
|
||||
}
|
||||
|
||||
public String getPathField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("path");
|
||||
}
|
||||
}
|
||||
|
||||
public String getFileLenField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("fileLen");
|
||||
}
|
||||
}
|
||||
|
||||
public String getDocLenField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("docLen");
|
||||
}
|
||||
}
|
||||
|
||||
public String getLineField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("line");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldPathField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("oldPath");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewPathField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("newPath");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldDocLenField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("oldDocLen");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewDocLenField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("newDocLen");
|
||||
}
|
||||
}
|
||||
|
||||
public String getPropertyField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("property");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldValueField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("oldValue");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewValueField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("newValue");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOldParentField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("oldParent");
|
||||
}
|
||||
}
|
||||
|
||||
public String getNewParentField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("newParent");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOriginalField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("original");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import edu.nju.ics.frontier.util.StringUtil;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class IntelliJEvent implements Comparable<IntelliJEvent> {
|
||||
public enum Type {
|
||||
App, Project, File, Action, Key, Mouse, Window, Cursor, Compile, Execute, Debug,
|
||||
}
|
||||
|
||||
protected long time;
|
||||
protected String type;
|
||||
protected String when;
|
||||
protected Map<String, String> data;
|
||||
|
||||
public IntelliJEvent(long time, String type, String when, Map<String, String> data) {
|
||||
this.time = time;
|
||||
this.type = type;
|
||||
this.when = when;
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public long getTime() {
|
||||
return time;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public String getWhen() {
|
||||
return when;
|
||||
}
|
||||
|
||||
public Map<String, String> getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(Map<String, String> data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public boolean hasData() {
|
||||
return (!(data == null || data.isEmpty()));
|
||||
}
|
||||
|
||||
public void addExtraData(String k, String v) {
|
||||
if (k != null && v != null) {
|
||||
if (data == null) {
|
||||
data = new HashMap<String, String>();
|
||||
}
|
||||
data.put(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
public String getExtraData(String k) {
|
||||
if (data != null && k != null) {
|
||||
return data.get(k);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{time:%d(%s),type:%s,when:%s,data:%s}",
|
||||
time, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(time)),
|
||||
type, when, StringUtil.toString(data));
|
||||
}
|
||||
|
||||
public static boolean isAppEvent(IntelliJEvent event) {
|
||||
return Type.App.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isProjectEvent(IntelliJEvent event) {
|
||||
return Type.Project.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isFileEvent(IntelliJEvent event) {
|
||||
return Type.File.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isActionEvent(IntelliJEvent event) {
|
||||
return Type.Action.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isKeyEvent(IntelliJEvent event) {
|
||||
return Type.Key.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isMouseEvent(IntelliJEvent event) {
|
||||
return Type.Mouse.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isWindowEvent(IntelliJEvent event) {
|
||||
return Type.Window.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isCursorEvent(IntelliJEvent event) {
|
||||
return Type.Cursor.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isCompileEvent(IntelliJEvent event) {
|
||||
return Type.Compile.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isExecuteEvent(IntelliJEvent event) {
|
||||
return Type.Execute.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static boolean isDebugEvent(IntelliJEvent event) {
|
||||
return Type.Debug.name().equals(event.type);
|
||||
}
|
||||
|
||||
public static AppEvent toAppEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isAppEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new AppEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static ProjectEvent toProjectEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isProjectEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new ProjectEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static FileEvent toFileEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isFileEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new FileEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static ActionEvent toActionEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isActionEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new ActionEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static KeyEvent toKeyEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isKeyEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new KeyEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static MouseEvent toMouseEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isMouseEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new MouseEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static WindowEvent toWindowEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isWindowEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new WindowEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static CursorEvent toCursorEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isCursorEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new CursorEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static CompileEvent toCompileEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isCompileEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new CompileEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static ExecuteEvent toExecuteEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isExecuteEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new ExecuteEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public static DebugEvent toDebugEvent(IntelliJEvent event) {
|
||||
if (event == null || (!isDebugEvent(event))) {
|
||||
return null;
|
||||
} else {
|
||||
return new DebugEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
public int compareTo(IntelliJEvent o) {
|
||||
long t1 = this.time;
|
||||
long t2 = o.time;
|
||||
return (int) (t1 - t2);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,237 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class KeyEvent extends IntelliJEvent {
|
||||
public KeyEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public KeyEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenKeyPressed() {
|
||||
return "KEY_PRESSED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenKeyReleased() {
|
||||
return "KEY_RELEASED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenKeyTyped() {
|
||||
return "KEY_TYPED".equals(when);
|
||||
}
|
||||
|
||||
public String getTextField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("text");
|
||||
}
|
||||
}
|
||||
|
||||
public String getModifiersField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("modifiers");
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isLetter() {
|
||||
return "Letter".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isNumber() {
|
||||
return "Number".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isPunctuation() {
|
||||
return "Punctuation".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isEnter() {
|
||||
return "Enter".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isEscape() {
|
||||
return "Escape".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isTab() {
|
||||
return "Tab".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isSpace() {
|
||||
return "Space".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isBackspace() {
|
||||
return "Backspace".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isInsert() {
|
||||
return "Insert".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isDelete() {
|
||||
return "Delete".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isDirection() {
|
||||
return isLeft() || isRight() || isUp() || isDown() ||
|
||||
isHome() || isEnd() || isPageUp() || isPageDown();
|
||||
}
|
||||
|
||||
public boolean isLeft() {
|
||||
return "Left".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isRight() {
|
||||
return "Right".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isUp() {
|
||||
return "Up".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isDown() {
|
||||
return "Down".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isHome() {
|
||||
return "Home".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isEnd() {
|
||||
return "End".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isPageUp() {
|
||||
return "Page Up".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isPageDown() {
|
||||
return "Page Down".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isModifier() {
|
||||
return isControl() || isAlt() || isShift();
|
||||
}
|
||||
|
||||
public boolean isControl() {
|
||||
return "Control".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isAlt() {
|
||||
return "Alt".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isShift() {
|
||||
return "Shift".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isFunction() {
|
||||
return isF1() || isF2() || isF3() || isF4() ||
|
||||
isF5() || isF6() || isF7() || isF8() ||
|
||||
isF9() || isF10() || isF11() || isF12();
|
||||
}
|
||||
|
||||
public boolean isF1() {
|
||||
return "F1".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF2() {
|
||||
return "F2".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF3() {
|
||||
return "F3".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF4() {
|
||||
return "F4".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF5() {
|
||||
return "F5".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF6() {
|
||||
return "F6".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF7() {
|
||||
return "F7".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF8() {
|
||||
return "F8".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF9() {
|
||||
return "F9".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF10() {
|
||||
return "F10".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF11() {
|
||||
return "F11".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isF12() {
|
||||
return "F12".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isCapsLock() {
|
||||
return "Caps Lock".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isNumLock() {
|
||||
return "Num Lock".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isScrollLock() {
|
||||
return "Scroll Lock".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isPause() {
|
||||
return "Pause".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isPrintScreen() {
|
||||
return "Print Screen".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isWindows() {
|
||||
return "Windows".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isContextMenu() {
|
||||
return "Context Menu".equals(getTextField());
|
||||
}
|
||||
|
||||
public boolean isNone() {
|
||||
return getTextField() == null;
|
||||
}
|
||||
|
||||
public boolean isPureOrShiftLetter() {
|
||||
return isLetter() && isNoneOrShiftModifier();
|
||||
}
|
||||
|
||||
public boolean isPureOrShiftNumber() {
|
||||
return isNumber() && isNoneOrShiftModifier();
|
||||
}
|
||||
|
||||
public boolean isPureOrShiftPunctuation() {
|
||||
return isPunctuation() && isNoneOrShiftModifier();
|
||||
}
|
||||
|
||||
private boolean isNoneOrShiftModifier() {
|
||||
String modifiers = getModifiersField();
|
||||
return (modifiers == null || "Shift".equals(modifiers));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class MouseEvent extends IntelliJEvent {
|
||||
public MouseEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public MouseEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenMousePressed() {
|
||||
return "MOUSE_PRESSED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseReleased() {
|
||||
return "MOUSE_RELEASED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseClicked() {
|
||||
return "MOUSE_CLICKED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseEntered() {
|
||||
return "MOUSE_ENTERED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseExited() {
|
||||
return "MOUSE_EXITED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseMoved() {
|
||||
return "MOUSE_MOVED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseDragged() {
|
||||
return "MOUSE_DRAGGED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenMouseWheel() {
|
||||
return "MOUSE_WHEEL".equals(when);
|
||||
}
|
||||
|
||||
public String getButtonField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("button");
|
||||
}
|
||||
}
|
||||
|
||||
public String getClickCountField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("clickCount");
|
||||
}
|
||||
}
|
||||
|
||||
public String getCoordinatesField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("coordinates");
|
||||
}
|
||||
}
|
||||
|
||||
public String getModifiersField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("modifiers");
|
||||
}
|
||||
}
|
||||
|
||||
public int[] getAbsoluteCoordinate() {
|
||||
String coordinates = getCoordinatesField();
|
||||
if (coordinates == null) {
|
||||
return null;
|
||||
}
|
||||
String point = coordinates.substring(0, coordinates.indexOf("/"));
|
||||
point = point.substring(1, point.length() - 1);
|
||||
String[] xy = point.split(",");
|
||||
int x = Integer.parseInt(xy[0]);
|
||||
int y = Integer.parseInt(xy[1]);
|
||||
return new int[]{x, y};
|
||||
}
|
||||
|
||||
public int[] getRelativeCoordinate() {
|
||||
String coordinates = getCoordinatesField();
|
||||
if (coordinates == null) {
|
||||
return null;
|
||||
}
|
||||
String point = coordinates.substring(coordinates.indexOf("/") + 1);
|
||||
point = point.substring(1, point.length() - 1);
|
||||
String[] xy = point.split(",");
|
||||
int x = Integer.parseInt(xy[0]);
|
||||
int y = Integer.parseInt(xy[1]);
|
||||
return new int[]{x, y};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class ProjectEvent extends IntelliJEvent {
|
||||
public ProjectEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public ProjectEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenOpened() {
|
||||
return "opened".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenClosed() {
|
||||
return "closed".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenActivated() {
|
||||
return "activated".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenDeactivated() {
|
||||
return "deactivated".equals(when);
|
||||
}
|
||||
|
||||
public String getNameField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("name");
|
||||
}
|
||||
}
|
||||
|
||||
public String getFrameField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("frame");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package edu.nju.ics.frontier.bean.intellij;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class WindowEvent extends IntelliJEvent {
|
||||
public WindowEvent(long time, String type, String when, Map<String, String> data) {
|
||||
super(time, type, when, data);
|
||||
}
|
||||
|
||||
public WindowEvent(IntelliJEvent event) {
|
||||
super(event.time, event.type, event.when, event.data);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowOpened() {
|
||||
return "WINDOW_OPENED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowClosing() {
|
||||
return "WINDOW_CLOSING".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowClosed() {
|
||||
return "WINDOW_CLOSED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowIconified() {
|
||||
return "WINDOW_ICONIFIED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowDeiconified() {
|
||||
return "WINDOW_DEICONIFIED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowActivated() {
|
||||
return "WINDOW_ACTIVATED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowDeactivated() {
|
||||
return "WINDOW_DEACTIVATED".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowGainedFocus() {
|
||||
return "WINDOW_GAINED_FOCUS".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowLostFocus() {
|
||||
return "WINDOW_LOST_FOCUS".equals(when);
|
||||
}
|
||||
|
||||
public boolean isWhenWindowStateChanged() {
|
||||
return "WINDOW_STATE_CHANGED".equals(when);
|
||||
}
|
||||
|
||||
public String getThisField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("this");
|
||||
}
|
||||
}
|
||||
|
||||
public String getOppositeField() {
|
||||
if (data == null) {
|
||||
return null;
|
||||
} else {
|
||||
return data.get("opposite");
|
||||
}
|
||||
}
|
||||
|
||||
public String getThisWindowName() {
|
||||
String s = getThisField();
|
||||
return (s == null) ? null : s.split(",")[0];
|
||||
}
|
||||
|
||||
public String getOppositeWindowName() {
|
||||
String s = getOppositeField();
|
||||
return (s == null) ? null : s.split(",")[0];
|
||||
}
|
||||
|
||||
public String getThisWindowType() {
|
||||
return name2type(getThisWindowName());
|
||||
}
|
||||
|
||||
public String getOppositeWindowType() {
|
||||
return name2type(getOppositeWindowName());
|
||||
}
|
||||
|
||||
public static String name2type(String winName) {
|
||||
if (winName == null) {
|
||||
return null;
|
||||
}
|
||||
int index;
|
||||
for (index = 0; index < winName.length(); index++) {
|
||||
char c = winName.charAt(index);
|
||||
if ('0' <= c && c <= '9') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return winName.substring(0, index);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
public abstract class Action {
|
||||
public enum Type {
|
||||
SELECT, SUBMIT, REFUSE
|
||||
}
|
||||
|
||||
protected long time;
|
||||
protected Type type;
|
||||
|
||||
public Action(long time, Type type) {
|
||||
this.time = time;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public long getTime() {
|
||||
return time;
|
||||
}
|
||||
|
||||
public Type getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public boolean isSelectAction() {
|
||||
return type == Type.SELECT;
|
||||
}
|
||||
|
||||
public boolean isSubmitAction() {
|
||||
return type == Type.SUBMIT;
|
||||
}
|
||||
|
||||
public boolean isRefuseAction() {
|
||||
return type == Type.REFUSE;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
import com.google.gson.*;
|
||||
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.Set;
|
||||
|
||||
public class ActionTypeConverter implements JsonSerializer<Action>, JsonDeserializer<Action> {
|
||||
public JsonElement serialize(Action action, Type type, JsonSerializationContext jsonSerializationContext) {
|
||||
Class clazz = null;
|
||||
|
||||
Action.Type actionType = action.getType();
|
||||
switch (actionType) {
|
||||
case SELECT:
|
||||
clazz = SelectAction.class;
|
||||
break;
|
||||
case REFUSE:
|
||||
clazz = RefuseAction.class;
|
||||
break;
|
||||
case SUBMIT:
|
||||
clazz = SubmitAction.class;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (clazz != null) {
|
||||
return jsonSerializationContext.serialize(action, clazz);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public Action deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
|
||||
Class clazz = null;
|
||||
|
||||
Set<String> keySet = jsonElement.getAsJsonObject().keySet();
|
||||
if (keySet != null && keySet.contains("type")) {
|
||||
String actionTypeName = jsonElement.getAsJsonObject().get("type").getAsString();
|
||||
if (Action.Type.SELECT.name().equals(actionTypeName)) {
|
||||
clazz = SelectAction.class;
|
||||
} else if (Action.Type.REFUSE.name().equals(actionTypeName)) {
|
||||
clazz = RefuseAction.class;
|
||||
} else if (Action.Type.SUBMIT.name().equals(actionTypeName)) {
|
||||
clazz = SubmitAction.class;
|
||||
}
|
||||
}
|
||||
|
||||
if (clazz != null) {
|
||||
return jsonDeserializationContext.deserialize(jsonElement, clazz);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
import edu.nju.ics.frontier.util.StringUtil;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public class Label {
|
||||
private long begTime;
|
||||
private long endTime;
|
||||
private List<Action> actions;
|
||||
|
||||
public enum Type {
|
||||
CLOSE,
|
||||
CLOSE_WITHOUT_SELECT,
|
||||
CLOSE_WITH_SELECT,
|
||||
REFUSE,
|
||||
REFUSE_WITHOUT_SELECT,
|
||||
REFUSE_WITH_SELECT,
|
||||
SUBMIT,
|
||||
}
|
||||
|
||||
public Label(long begTime, long endTime, List<Action> actions) {
|
||||
this.begTime = begTime;
|
||||
this.endTime = endTime;
|
||||
this.actions = actions;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
return begTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public List<Action> getActions() {
|
||||
return actions;
|
||||
}
|
||||
|
||||
public boolean hasAction() {
|
||||
return (!actions.isEmpty());
|
||||
}
|
||||
|
||||
public boolean containSelectAction(SelectOrSubmitAction.Dimension dimension) {
|
||||
List<SelectAction> actions = getSelectActions(dimension);
|
||||
return actions != null;
|
||||
}
|
||||
|
||||
public boolean containSelectAction() {
|
||||
if (actions == null || actions.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (Action action : actions) {
|
||||
if (action.isSelectAction()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean containSubmitAction() {
|
||||
if (actions == null || actions.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (Action action : actions) {
|
||||
if (action.isSubmitAction()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean containRefuseAction() {
|
||||
if (actions == null || actions.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (Action action : actions) {
|
||||
if (action.isRefuseAction()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public List<SelectAction> getSelectActions(SelectOrSubmitAction.Dimension dimension) {
|
||||
if (!containSelectAction()) {
|
||||
return null;
|
||||
}
|
||||
List<SelectAction> selectActions = new ArrayList<SelectAction>();
|
||||
for (Action action : actions) {
|
||||
if (action.isSelectAction()) {
|
||||
SelectAction selectAction = (SelectAction) action;
|
||||
if (selectAction.getDimension() == dimension) {
|
||||
selectActions.add(selectAction);
|
||||
}
|
||||
}
|
||||
}
|
||||
return selectActions.isEmpty() ? null : selectActions;
|
||||
}
|
||||
|
||||
public SubmitAction getSubmitAction(SelectOrSubmitAction.Dimension dimension) {
|
||||
if (!containSubmitAction()) {
|
||||
return null;
|
||||
}
|
||||
for (Action action : actions) {
|
||||
if (action.isSubmitAction()) {
|
||||
SubmitAction submitAction = (SubmitAction) action;
|
||||
if (submitAction.getDimension() == dimension) {
|
||||
return submitAction;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public RefuseAction getRefuseAction() {
|
||||
if (!containRefuseAction()) {
|
||||
return null;
|
||||
}
|
||||
for (Action action : actions) {
|
||||
if (action.isRefuseAction()) {
|
||||
return ((RefuseAction) action);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public int getLabelScore(SelectOrSubmitAction.Dimension dimension) {
|
||||
List<SelectAction> actions = getSelectActions(dimension);
|
||||
return (actions == null) ? -1 : actions.get(actions.size() - 1).getScore();
|
||||
}
|
||||
|
||||
public boolean isMultiTimesSelected(SelectOrSubmitAction.Dimension dimension) {
|
||||
List<SelectAction> selectActions = getSelectActions(dimension);
|
||||
return (selectActions != null && selectActions.size() > 1);
|
||||
}
|
||||
|
||||
public Type getSimpleType() {
|
||||
boolean isRefused = containRefuseAction();
|
||||
boolean isSubmitted = containSubmitAction();
|
||||
|
||||
if (isRefused) {
|
||||
return Type.REFUSE;
|
||||
} else if (isSubmitted) {
|
||||
return Type.SUBMIT;
|
||||
} else {
|
||||
return Type.CLOSE;
|
||||
}
|
||||
}
|
||||
|
||||
public Type getCanonicalType() {
|
||||
boolean isEngagementSelected = containSelectAction(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
boolean isProductivitySelected = containSelectAction(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
boolean isSelected = (isEngagementSelected && isProductivitySelected);
|
||||
boolean isRefused = containRefuseAction();
|
||||
boolean isSubmitted = containSubmitAction();
|
||||
|
||||
if (isRefused) {
|
||||
return isSelected ? Type.REFUSE_WITH_SELECT : Type.REFUSE_WITHOUT_SELECT;
|
||||
} else if (isSubmitted) {
|
||||
return Type.SUBMIT;
|
||||
} else {
|
||||
return isSelected ? Type.CLOSE_WITH_SELECT : Type.CLOSE_WITHOUT_SELECT;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
return String.format("{begTime:%d(%s),endTime:%d(%s),actions:%s}",
|
||||
begTime, sdf.format(new Date(begTime)),
|
||||
endTime, sdf.format(new Date(endTime)),
|
||||
StringUtil.toString(actions));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public class RefuseAction extends Action {
|
||||
public enum OutOfIDEActivity {
|
||||
MEETING, REST, TALK, OTHER
|
||||
}
|
||||
|
||||
private OutOfIDEActivity activity;
|
||||
|
||||
public RefuseAction(long time, OutOfIDEActivity activity) {
|
||||
super(time, Type.REFUSE);
|
||||
this.activity = activity;
|
||||
}
|
||||
|
||||
public OutOfIDEActivity getActivity() {
|
||||
return activity;
|
||||
}
|
||||
|
||||
public boolean isMeetingActivity() {
|
||||
return activity == OutOfIDEActivity.MEETING;
|
||||
}
|
||||
|
||||
public boolean isRestActivity() {
|
||||
return activity == OutOfIDEActivity.REST;
|
||||
}
|
||||
|
||||
public boolean isTalkActivity() {
|
||||
return activity == OutOfIDEActivity.TALK;
|
||||
}
|
||||
|
||||
public boolean isOtherActivity() {
|
||||
return activity == OutOfIDEActivity.OTHER;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{time:%d(%s),type:%s,activity:%s}",
|
||||
time, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(time)), type.name(), activity.name());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
public class SelectAction extends SelectOrSubmitAction {
|
||||
public SelectAction(long time, Dimension dimension, int index, int score) {
|
||||
super(time, Type.SELECT, dimension, index, score);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public abstract class SelectOrSubmitAction extends Action {
|
||||
public enum Dimension {
|
||||
ENGAGEMENT, PRODUCTIVITY
|
||||
}
|
||||
|
||||
protected Dimension dimension;
|
||||
protected int index;
|
||||
protected int score;
|
||||
|
||||
public SelectOrSubmitAction(long time, Type type, Dimension dimension, int index, int score) {
|
||||
super(time, type);
|
||||
this.dimension = dimension;
|
||||
this.index = index;
|
||||
this.score = score;
|
||||
}
|
||||
|
||||
public Dimension getDimension() {
|
||||
return dimension;
|
||||
}
|
||||
|
||||
public int getIndex() {
|
||||
return index;
|
||||
}
|
||||
|
||||
public int getScore() {
|
||||
return score;
|
||||
}
|
||||
|
||||
public boolean isEngagementDimension() {
|
||||
return dimension == Dimension.ENGAGEMENT;
|
||||
}
|
||||
|
||||
public boolean isProductivityDimension() {
|
||||
return dimension == Dimension.PRODUCTIVITY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{time:%d(%s),type:%s,dimension:%s,index:%d,score:%d}",
|
||||
time, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date(time)), type.name(), dimension.name(), index, score);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package edu.nju.ics.frontier.bean.label;
|
||||
|
||||
public class SubmitAction extends SelectOrSubmitAction {
|
||||
public SubmitAction(long time, Dimension dimension, int index, int score) {
|
||||
super(time, Type.SUBMIT, dimension, index, score);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package edu.nju.ics.frontier.bean.session;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Session {
|
||||
private final List<IntelliJEvent> events;
|
||||
private final List<Label> labels;
|
||||
|
||||
public Session(List<IntelliJEvent> events, List<Label> labels) {
|
||||
this.events = events;
|
||||
this.labels = labels;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return -1;
|
||||
} else {
|
||||
return events.get(0).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
if (events == null ||events.isEmpty()) {
|
||||
return -1;
|
||||
} else {
|
||||
return events.get(events.size() - 1).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
public List<IntelliJEvent> getEvents() {
|
||||
return events;
|
||||
}
|
||||
|
||||
public List<Label> getLabels() {
|
||||
return labels;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package edu.nju.ics.frontier.bean.slice;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.StringUtil;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Slice {
|
||||
private List<IntelliJEvent> events;
|
||||
private Label label;
|
||||
|
||||
public Slice(List<IntelliJEvent> events, Label label) {
|
||||
this.events = events;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public List<IntelliJEvent> getEvents() {
|
||||
return events;
|
||||
}
|
||||
|
||||
public Label getLabel() {
|
||||
return label;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return -1;
|
||||
} else {
|
||||
return events.get(0).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return -1;
|
||||
} else {
|
||||
return events.get(events.size() - 1).getTime();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{events:%s,label:%s}",
|
||||
StringUtil.toString(events), StringUtil.toString(label));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
package edu.nju.ics.frontier.cleaning;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.learning.TimeDatabase;
|
||||
import edu.nju.ics.frontier.learning.TimePoint;
|
||||
import edu.nju.ics.frontier.learning.TimeSeries;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import org.apache.commons.math3.stat.StatUtils;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Max;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Median;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Min;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class CleaningMain {
|
||||
public static void main(String[] args) {
|
||||
String csvPath = "C:\\Users\\zzw\\Desktop\\fujitsu\\time_aligned_instances.csv";
|
||||
TimeDatabase db = new TimeDatabase();
|
||||
db.load(csvPath);
|
||||
db.positiveNeutralNegative();
|
||||
|
||||
List<double[]> badFeatures = new ArrayList<double[]>();
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open("C:\\Users\\zzw\\Desktop\\fujitsu\\bad_data.csv");
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] info = line.split(",");
|
||||
Assertion.assertEqual(info.length, 2);
|
||||
String user = info[0];
|
||||
long timestamp = Long.parseLong(info[1]);
|
||||
TimePoint tp = db.findTimeSeriesByUser(user).findClosestTimePoint(timestamp, 0, true, false, false);
|
||||
Assertion.assertNotNull(tp);
|
||||
badFeatures.add(tp.getFeatures());
|
||||
tp.setLabel(-1);
|
||||
Assertion.assertTrue(tp.isUnlabeled());
|
||||
}
|
||||
reader.close();
|
||||
|
||||
List<double[]> goodFeatures = new ArrayList<double[]>();
|
||||
for (TimeSeries ts : db.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
goodFeatures.add(tp.getFeatures());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String[] featureNames = db.getFeatureNames();
|
||||
double[][] goodData = transpose(goodFeatures);
|
||||
double[][] badData = transpose(badFeatures);
|
||||
Assertion.assertEqual(featureNames.length, goodData.length);
|
||||
Assertion.assertEqual(goodData.length, badData.length);
|
||||
for (int i = 0; i < goodData.length; i++) {
|
||||
System.out.printf("%30s: %110s\n%30s %110s\n",
|
||||
featureNames[i],
|
||||
boxPlot(goodData[i], "good"),
|
||||
"",
|
||||
boxPlot(badData[i], "bad"));
|
||||
}
|
||||
}
|
||||
|
||||
private static double[][] transpose(List<double[]> list) {
|
||||
double[][] result = new double[list.get(0).length][list.size()];
|
||||
for (int j = 0; j < result[0].length; j++) {
|
||||
double[] item = list.get(j);
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
result[i][j] = item[i];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String boxPlot(double[] vector, String tag) {
|
||||
double min = new Min().evaluate(vector);
|
||||
double max = new Max().evaluate(vector);
|
||||
double q1 = StatUtils.percentile(vector, 25);
|
||||
double median = new Median().evaluate(vector);
|
||||
double q3 = StatUtils.percentile(vector, 75);
|
||||
double mean = new Mean().evaluate(vector);
|
||||
return String.format("%s:{avg:%f,min:%f,max:%f,q1:%f,q2:%f,q3:%f}",
|
||||
tag, mean, min, max, q1, median, q3);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package edu.nju.ics.frontier.configuration;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
|
||||
public class Configuration {
|
||||
public static final long WINDOW_SIZE_IN_MINUTE = 30;
|
||||
public static final double OVERLAP = 0.5;
|
||||
public static final boolean STANDARDIZE = false;
|
||||
public static final boolean NORMALIZE = true;
|
||||
public static final long WINDOW_SIZE_IN_MILLISECOND = WINDOW_SIZE_IN_MINUTE * 60 * 1000;
|
||||
public static final int N = 8;
|
||||
public static final double BASE = 0.99;
|
||||
public static final double SAME_TEAM_WEIGHT = 1.0;
|
||||
public static final double DIFF_TEAM_WEIGHT = 0.0;
|
||||
public static final boolean IS_INTRA_ENABLED = true;
|
||||
public static final boolean IS_INTER_ENABLED = true;
|
||||
public static final int EPOCH = 10;
|
||||
public static final int BATCH = 1;
|
||||
public static final double ERASE_RATE = 0.1;
|
||||
public static final boolean IS_DEBUG_MODE = false;
|
||||
public static final String DATA_ROOT = "C:\\Users\\zzw\\Desktop\\fujitsu\\data\\data";
|
||||
// public static final PrintStream out = System.out;
|
||||
public static final OkTextWriter out = new OkTextWriter();
|
||||
|
||||
static {
|
||||
String datetime = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
|
||||
out.open("C:\\Users\\zzw\\Desktop\\fujitsu\\log\\" + datetime + ".txt");
|
||||
}
|
||||
|
||||
public static void update(String name) {
|
||||
out.close();
|
||||
String datetime = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date());
|
||||
out.open("C:\\Users\\zzw\\Desktop\\fujitsu\\log\\" + datetime + "_" + name + ".txt");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
package edu.nju.ics.frontier.deep_learning;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DataPreparation {
|
||||
public static void main(String[] args) {
|
||||
String[] head;
|
||||
List<String[]> tmp = new ArrayList<String[]>();
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open("C:\\Users\\zzw\\Desktop\\fujitsu\\time_aligned_instances.csv");
|
||||
head = reader.readLine().split(",");
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] info = line.split(",");
|
||||
tmp.add(info);
|
||||
}
|
||||
reader.close();
|
||||
|
||||
String[][] body = new String[tmp.size()][];
|
||||
for (int i = 0; i < body.length; i++) {
|
||||
body[i] = tmp.get(i);
|
||||
Assertion.assertEqual(head.length, body[i].length);
|
||||
}
|
||||
|
||||
// create a zero-filled array for padding
|
||||
String[] padding = new String[head.length];
|
||||
for (int i = 0; i < padding.length; i++) {
|
||||
padding[i] = "0.0";
|
||||
}
|
||||
|
||||
List<String[][]> sampleTimeStepFeatureList = new ArrayList<String[][]>();
|
||||
boolean isOverSampling = true;
|
||||
int timeSteps = 3;
|
||||
int forePoint = getIndexOfNextLabeledExample(body, timeSteps - 1);
|
||||
while (forePoint != -1) {
|
||||
int backPoint = forePoint - timeSteps + 1;
|
||||
int foreSessionId = Integer.parseInt(body[forePoint][7]);
|
||||
String[][] timeStepFeature = new String[timeSteps][];
|
||||
for (int i = 0; i < timeSteps; i++) {
|
||||
int sessionId = Integer.parseInt(body[backPoint + i][7]);
|
||||
if (sessionId == foreSessionId) {
|
||||
timeStepFeature[i] = body[backPoint + i];
|
||||
} else {
|
||||
timeStepFeature[i] = padding;
|
||||
}
|
||||
}
|
||||
sampleTimeStepFeatureList.add(timeStepFeature);
|
||||
|
||||
// update forePoint
|
||||
if (isOverSampling) {
|
||||
forePoint = getIndexOfNextLabeledExample(body, forePoint + 1);
|
||||
} else {
|
||||
forePoint = getIndexOfNextLabeledExample(body, forePoint + timeSteps);
|
||||
}
|
||||
}
|
||||
|
||||
// generate time sequence dataset
|
||||
System.out.println("generate features...");
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
String root = "C:\\Users\\zzw\\Desktop\\fujitsu\\deep_learning";
|
||||
for (int featureIndex = 8; featureIndex < head.length - 2; featureIndex++) {
|
||||
System.out.printf(" %2d: %s\n", featureIndex - 7, head[featureIndex]);
|
||||
writer.open(FileUtil.combine(root, "features", String.format("feature%02d.csv", featureIndex - 8)));
|
||||
for (int sampleIndex = 0; sampleIndex < sampleTimeStepFeatureList.size(); sampleIndex++) {
|
||||
String[][] sample = sampleTimeStepFeatureList.get(sampleIndex);
|
||||
for (int timeStepIndex = 0; timeStepIndex < timeSteps; timeStepIndex++) {
|
||||
writer.print(sample[timeStepIndex][featureIndex]);
|
||||
writer.print(timeStepIndex < timeSteps - 1 ? "," : "\n");
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
}
|
||||
|
||||
System.out.println("generate labels...");
|
||||
writer.open(FileUtil.combine(root, "labels.csv"));
|
||||
for (int sampleIndex = 0; sampleIndex < sampleTimeStepFeatureList.size(); sampleIndex++) {
|
||||
String s = sampleTimeStepFeatureList.get(sampleIndex)[timeSteps - 1][head.length - 2];
|
||||
int label = Integer.parseInt(s);
|
||||
if (label < 3) {
|
||||
writer.println("1");
|
||||
} else if (label == 3) {
|
||||
writer.println("2");
|
||||
} else {
|
||||
writer.println("3");
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
|
||||
System.out.println("done");
|
||||
}
|
||||
|
||||
private static int getIndexOfNextLabeledExample(String[][] body, int offset) {
|
||||
if (offset < 0 || offset >= body.length) {
|
||||
return -1;
|
||||
}
|
||||
int colIndex = body[0].length - 2; // internalScore
|
||||
for (int rowIndex = offset; rowIndex < body.length; rowIndex++) {
|
||||
int label = Integer.parseInt(body[rowIndex][colIndex]);
|
||||
if (label != -1) {
|
||||
return rowIndex;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
package edu.nju.ics.frontier.deep_learning;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.recognition.feature.MeasureUtil;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
public class FoldSplitting {
|
||||
public static void main(String[] args) {
|
||||
String root = "C:\\Users\\zzw\\Desktop\\fujitsu\\deep_learning";
|
||||
|
||||
List<List<String>> features = new ArrayList<List<String>>();
|
||||
File[] featureFiles = FileUtil.listSubFiles(new File(FileUtil.combine(root, "features")), "feature", ".csv");
|
||||
for (File featureFile : featureFiles) {
|
||||
features.add(loadData(featureFile.getAbsolutePath()));
|
||||
}
|
||||
|
||||
List<String> labels = loadData(FileUtil.combine(root, "labels.csv"));
|
||||
int[][][] trainAndTestFolds = splitTrainAndTestByStratifiedKFold(labels, 10);
|
||||
|
||||
for (int foldIndex = 0; foldIndex < 10; foldIndex++) {
|
||||
String foldRoot = FileUtil.combine(root, "fold" + foldIndex);
|
||||
System.out.printf(" > fold %d\n", foldIndex);
|
||||
int[] trainIndices = trainAndTestFolds[foldIndex][0];
|
||||
int[] testIndices = trainAndTestFolds[foldIndex][1];
|
||||
|
||||
for (int featureIndex = 0; featureIndex < features.size(); featureIndex++) {
|
||||
List<String> feature = features.get(featureIndex);
|
||||
// X_train
|
||||
saveData(FileUtil.combine(foldRoot, "train", "features", String.format("feature%02d.csv", featureIndex)), feature, trainIndices);
|
||||
// X_test
|
||||
saveData(FileUtil.combine(foldRoot, "test", "features", String.format("feature%02d.csv", featureIndex)), feature, testIndices);
|
||||
}
|
||||
// y_train
|
||||
saveData(FileUtil.combine(foldRoot, "train", "labels.csv"), labels, trainIndices);
|
||||
// y_test
|
||||
saveData(FileUtil.combine(foldRoot, "test", "labels.csv"), labels, testIndices);
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> loadData(String filePath) {
|
||||
List<String> result = new ArrayList<String>();
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(filePath);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
result.add(line);
|
||||
}
|
||||
reader.close();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void saveData(String filePath, List<String> data, int[] indices) {
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
writer.open(filePath);
|
||||
for (int index : indices) {
|
||||
writer.println(data.get(index));
|
||||
}
|
||||
writer.close();
|
||||
}
|
||||
|
||||
private static int[][][] splitTrainAndTestByStratifiedKFold(List<String> labels, int fold) {
|
||||
// count number of labels with specific nominal class
|
||||
Map<String, List<Integer>> labelToIndex = new HashMap<String, List<Integer>>();
|
||||
for (int i = 0; i < labels.size(); i++) {
|
||||
MeasureUtil.updateListMap(labelToIndex, labels.get(i), i);
|
||||
}
|
||||
List<String> nominalClasses = new ArrayList<String>(labelToIndex.keySet());
|
||||
Collections.sort(nominalClasses);
|
||||
System.out.print("#classes:");
|
||||
for (String nominalClass : nominalClasses) {
|
||||
System.out.printf(" #(%s)=%3d,", nominalClass, labelToIndex.get(nominalClass).size());
|
||||
}
|
||||
System.out.println();
|
||||
|
||||
// split training and test sets
|
||||
int[][][] result = new int[fold][][];
|
||||
for (String nominalClass : nominalClasses) {
|
||||
List<Integer> indices = labelToIndex.get(nominalClass);
|
||||
int length = indices.size();
|
||||
int size = length / fold;
|
||||
for (int i = 0; i < fold; i++) {
|
||||
int testBegIndex = size * i;
|
||||
int testEndIndex = (i < fold - 1) ? (testBegIndex + size) : length;
|
||||
|
||||
// generate training set
|
||||
int[] trainIndices = new int[length - (testEndIndex - testBegIndex)];
|
||||
int trainIndex = 0;
|
||||
for (int j = 0; j < testBegIndex; j++) {
|
||||
trainIndices[trainIndex++] = indices.get(j);
|
||||
}
|
||||
for (int j = testEndIndex; j < length; j++) {
|
||||
trainIndices[trainIndex++] = indices.get(j);
|
||||
}
|
||||
Assertion.assertTrue(trainIndex == trainIndices.length);
|
||||
|
||||
// generate test set
|
||||
int[] testIndices = new int[testEndIndex - testBegIndex];
|
||||
int testIndex = 0;
|
||||
for (int j = testBegIndex; j < testEndIndex; j++) {
|
||||
testIndices[testIndex++] = indices.get(j);
|
||||
}
|
||||
Assertion.assertTrue(testIndex == testIndices.length);
|
||||
|
||||
// joint
|
||||
Assertion.assertEqual(length, trainIndex + testIndex);
|
||||
int[][] oldTrainAndTestIndices = result[i];
|
||||
if (oldTrainAndTestIndices == null) {
|
||||
result[i] = new int[][]{trainIndices, testIndices};
|
||||
} else {
|
||||
int[] oldTrainIndices = oldTrainAndTestIndices[0];
|
||||
int[] newTrainIndices = arrayJoin(oldTrainIndices, trainIndices);
|
||||
int[] oldTestIndices = oldTrainAndTestIndices[1];
|
||||
int[] newTestIndices = arrayJoin(oldTestIndices, testIndices);
|
||||
result[i] = new int[][]{newTrainIndices, newTestIndices};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sort
|
||||
for (int[][] trainAndTestIndices : result) {
|
||||
int[] trainIndices = trainAndTestIndices[0];
|
||||
int[] testIndices = trainAndTestIndices[1];
|
||||
Arrays.sort(trainIndices);
|
||||
Arrays.sort(testIndices);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int[] arrayJoin(int[] array1, int[] array2) {
|
||||
int[] result = new int[array1.length + array2.length];
|
||||
for (int i = 0; i < array1.length; i++) {
|
||||
result[i] = array1[i];
|
||||
}
|
||||
for (int i = 0; i < array2.length; i++) {
|
||||
result[array1.length + i] = array2[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
public abstract class BatchSemiSupervisedLearningStrategy extends SemiSupervisedLearningStrategy {
|
||||
protected int epoch;
|
||||
protected BatchRate batchRate;
|
||||
|
||||
public BatchSemiSupervisedLearningStrategy(String name, int epoch, BatchRate batchRate) {
|
||||
super(name);
|
||||
this.epoch = epoch;
|
||||
this.batchRate = batchRate;
|
||||
}
|
||||
|
||||
public int getEpoch() {
|
||||
return epoch;
|
||||
}
|
||||
|
||||
public BatchRate getBatchRate() {
|
||||
return batchRate;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isStopBeforeUpdate(TimeDatabase trainDb, int epoch) {
|
||||
return epoch >= this.epoch;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isStopAfterUpdate(TimeDatabase trainDb, int epoch) {
|
||||
return false;
|
||||
}
|
||||
|
||||
public interface BatchRate {
|
||||
public abstract int getBatch(int epoch);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
public class DataSource {
|
||||
public static TimeDatabase initDatabase() {
|
||||
String csvPath = "C:\\Users\\zzw\\Desktop\\fujitsu\\time_aligned_instances.csv";
|
||||
TimeDatabase db = new TimeDatabase();
|
||||
// load dataset
|
||||
db.load(csvPath);
|
||||
// transform labels
|
||||
db.positiveNeutralNegative();
|
||||
return db.align();
|
||||
}
|
||||
|
||||
public static int[][][] splitTrainAndTestByKFold(int length, int fold) {
|
||||
int[][][] result = new int[fold][][];
|
||||
int size = length / fold;
|
||||
for (int i = 0; i < fold; i++) {
|
||||
int testBegIndex = size * i;
|
||||
int testEndIndex = (i < fold - 1) ? (testBegIndex + size) : length;
|
||||
|
||||
// generate training set
|
||||
int[] trainIndices = new int[length - (testEndIndex - testBegIndex)];
|
||||
int trainIndex = 0;
|
||||
for (int j = 0; j < testBegIndex; j++) {
|
||||
trainIndices[trainIndex++] = j;
|
||||
}
|
||||
for (int j = testEndIndex; j < length; j++) {
|
||||
trainIndices[trainIndex++] = j;
|
||||
}
|
||||
Assertion.assertTrue(trainIndex == trainIndices.length);
|
||||
|
||||
// generate test set
|
||||
int[] testIndices = new int[testEndIndex - testBegIndex];
|
||||
int testIndex = 0;
|
||||
for (int j = testBegIndex; j < testEndIndex; j++) {
|
||||
testIndices[testIndex++] = j;
|
||||
}
|
||||
Assertion.assertTrue(testIndex == testIndices.length);
|
||||
|
||||
// reserve
|
||||
Assertion.assertEqual(length, trainIndex + testIndex);
|
||||
result[i] = new int[][]{trainIndices, testIndices};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static int[][][] splitTrainValidAndTestByKFold(int length, int fold, double validRate) {
|
||||
Assertion.assertTrue(validRate >= 0.0 && validRate < 1.0);
|
||||
int[][][] result = new int[fold][][];
|
||||
int size = length / fold;
|
||||
for (int i = 0; i < fold; i++) {
|
||||
int testBegIndex = size * i;
|
||||
int testEndIndex = (i < fold - 1) ? (testBegIndex + size) : length;
|
||||
|
||||
// generate training and validation sets
|
||||
int[] trainAndValidIndices = new int[length - (testEndIndex - testBegIndex)];
|
||||
int trainAndValidIndex = 0;
|
||||
for (int j = 0; j < testBegIndex; j++) {
|
||||
trainAndValidIndices[trainAndValidIndex++] = j;
|
||||
}
|
||||
for (int j = testEndIndex; j < length; j++) {
|
||||
trainAndValidIndices[trainAndValidIndex++] = j;
|
||||
}
|
||||
Assertion.assertTrue(trainAndValidIndex == trainAndValidIndices.length);
|
||||
|
||||
// generate test set
|
||||
int[] testIndices = new int[testEndIndex - testBegIndex];
|
||||
int testIndex = 0;
|
||||
for (int j = testBegIndex; j < testEndIndex; j++) {
|
||||
testIndices[testIndex++] = j;
|
||||
}
|
||||
Assertion.assertTrue(testIndex == testIndices.length);
|
||||
Assertion.assertEqual(length, trainAndValidIndex + testIndex);
|
||||
|
||||
if (validRate > 0.0) {
|
||||
// divide training and validation sets
|
||||
int validLength = (int) (trainAndValidIndices.length * validRate);
|
||||
Assertion.assertPositive(validLength);
|
||||
int trainLength = trainAndValidIndices.length - validLength;
|
||||
Assertion.assertPositive(trainLength);
|
||||
int[] trainIndices = new int[trainLength];
|
||||
int[] validIndices = new int[validLength];
|
||||
System.arraycopy(trainAndValidIndices, 0, trainIndices, 0, trainLength);
|
||||
System.arraycopy(trainAndValidIndices, trainLength, validIndices, 0, validLength);
|
||||
result[i] = new int[][]{trainIndices, validIndices, testIndices};
|
||||
} else {
|
||||
result[i] = new int[][]{trainAndValidIndices, null, testIndices};
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static TimeDatabase[] splitTrainingAndValidationSets(TimeDatabase trainAndValidDb, double validRate) {
|
||||
if (validRate == 0.0) {
|
||||
return new TimeDatabase[]{trainAndValidDb, null};
|
||||
}
|
||||
// get the number of labeled data per time point,
|
||||
// and the number of labeled data of the total database
|
||||
int numTps = trainAndValidDb.getNumberOfTimePoints();
|
||||
int[] labeledCountPerTp = new int[numTps];
|
||||
int labeledCountAllTp = 0;
|
||||
for (int i = 0; i < numTps; i++) {
|
||||
int labeledCount = 0;
|
||||
for (TimeSeries ts : trainAndValidDb.getTimeSeries()) {
|
||||
TimePoint tp = ts.getTimePoints().get(i);
|
||||
if (tp.isLabeled()) {
|
||||
labeledCount++;
|
||||
}
|
||||
}
|
||||
labeledCountPerTp[i] = labeledCount;
|
||||
labeledCountAllTp += labeledCount;
|
||||
}
|
||||
// compute the number of labeled data of the validation set
|
||||
int labeledCountOfValid = (int) (validRate * labeledCountAllTp);
|
||||
// get the range of accumulation closest to the number of labeled data of validation set
|
||||
int[] validRange = rangeOfAccumulationClosestToThreshold(labeledCountPerTp, labeledCountOfValid);
|
||||
// get the time point indices of training set and that of validation set
|
||||
int[] validIndices = new int[validRange[1] - validRange[0] + 1];
|
||||
int[] trainIndices = new int[numTps - validIndices.length];
|
||||
int validIndex = 0;
|
||||
for (int i = validRange[0]; i <= validRange[1]; i++) {
|
||||
validIndices[validIndex++] = i;
|
||||
}
|
||||
Assertion.assertEqual(validIndices.length, validIndex);
|
||||
int trainIndex = 0;
|
||||
for (int i = 0; i < validRange[0]; i++) {
|
||||
trainIndices[trainIndex++] = i;
|
||||
}
|
||||
for (int i = validRange[1] + 1; i < numTps; i++) {
|
||||
trainIndices[trainIndex++] = i;
|
||||
}
|
||||
Assertion.assertEqual(trainIndices.length, trainIndex);
|
||||
// split training set and validation set
|
||||
TimeDatabase trainDb = trainAndValidDb.slice(trainIndices);
|
||||
TimeDatabase validDb = trainAndValidDb.slice(validIndices);
|
||||
return new TimeDatabase[]{trainDb, validDb};
|
||||
}
|
||||
|
||||
private static int[] rangeOfAccumulationClosestToThreshold(int[] values, int threshold) {
|
||||
List<int[]> ranges = new ArrayList<int[]>();
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
int[] range = rangeOfAccumulationClosestToThreshold(values, i, threshold);
|
||||
if (range != null) {
|
||||
ranges.add(range);
|
||||
if (range[2] == 0) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Assertion.assertFalse(ranges.isEmpty());
|
||||
Collections.sort(ranges, new Comparator<int[]>() {
|
||||
public int compare(int[] o1, int[] o2) {
|
||||
int absDelta1 = o1[2];
|
||||
int absDelta2 = o2[2];
|
||||
return absDelta1 - absDelta2;
|
||||
}
|
||||
});
|
||||
int[] closest = ranges.get(0);
|
||||
return new int[]{closest[0], closest[1]};
|
||||
}
|
||||
|
||||
private static int[] rangeOfAccumulationClosestToThreshold(int[] values, int offset, int threshold) {
|
||||
Assertion.assertTrue(offset >= 0 && offset < values.length);
|
||||
int accumulation = 0;
|
||||
for (int i = offset; i < values.length; i++) {
|
||||
int oldAccumulation = accumulation;
|
||||
int newAccumulation = accumulation + values[i];
|
||||
if (newAccumulation > threshold) {
|
||||
int oldAbsDelta = Math.abs(oldAccumulation - threshold);
|
||||
int newAbsDelta = Math.abs(newAccumulation - threshold);
|
||||
if (oldAbsDelta <= newAbsDelta) {
|
||||
return new int[]{offset, i - 1, oldAbsDelta};
|
||||
} else {
|
||||
return new int[]{offset, i, newAbsDelta};
|
||||
}
|
||||
} else if (newAccumulation < threshold) {
|
||||
accumulation = newAccumulation;
|
||||
} else {
|
||||
return new int[]{offset, i, 0};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,907 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.core.Instance;
|
||||
import weka.core.Instances;
|
||||
import weka.core.Utils;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class Evaluation {
|
||||
private int m_NumClasses;
|
||||
private String[] m_ClassNames;
|
||||
private List<int[]> m_TruePredLabelPairs;
|
||||
private double[][] m_ConfusionMatrix;
|
||||
private double m_Correct;
|
||||
private double m_Incorrect;
|
||||
private double m_WithClass;
|
||||
|
||||
public Evaluation(int[] classes, List<int[]> truePredLabelPairs) {
|
||||
m_NumClasses = classes.length;
|
||||
m_ClassNames = initClassNames(classes);
|
||||
m_TruePredLabelPairs = truePredLabelPairs;
|
||||
m_ConfusionMatrix = getConfusionMatrix(classes, truePredLabelPairs);
|
||||
for (int i = 0; i < m_ConfusionMatrix.length; i++) {
|
||||
for (int j = 0; j < m_ConfusionMatrix[i].length; j++) {
|
||||
if (i == j) {
|
||||
m_Correct += m_ConfusionMatrix[i][j];
|
||||
} else {
|
||||
m_Incorrect += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
m_WithClass = m_Correct + m_Incorrect;
|
||||
}
|
||||
|
||||
public final List<int[]> getTruePredLabelPairs() {
|
||||
return m_TruePredLabelPairs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of instances incorrectly classified (that is, for which an
|
||||
* incorrect prediction was made). (Actually the sum of the weights of these
|
||||
* instances)
|
||||
*
|
||||
* @return the number of incorrectly classified instances
|
||||
*/
|
||||
public final double incorrect() {
|
||||
return m_Incorrect;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the percentage of instances incorrectly classified (that is, for which
|
||||
* an incorrect prediction was made).
|
||||
*
|
||||
* @return the percent of incorrectly classified instances (between 0 and 100)
|
||||
*/
|
||||
public final double pctIncorrect() {
|
||||
return 100 * m_Incorrect / m_WithClass;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of instances correctly classified (that is, for which a
|
||||
* correct prediction was made). (Actually the sum of the weights of these
|
||||
* instances)
|
||||
*
|
||||
* @return the number of correctly classified instances
|
||||
*/
|
||||
public final double correct() {
|
||||
return m_Correct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the percentage of instances correctly classified (that is, for which a
|
||||
* correct prediction was made).
|
||||
*
|
||||
* @return the percent of correctly classified instances (between 0 and 100)
|
||||
*/
|
||||
public final double pctCorrect() {
|
||||
return 100 * m_Correct / m_WithClass;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the weighted (by class size) true positive rate.
|
||||
*
|
||||
* @return the weighted true positive rate.
|
||||
*/
|
||||
public double weightedTruePositiveRate() {
|
||||
double[] classCounts = new double[m_NumClasses];
|
||||
double classCountSum = 0;
|
||||
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
classCounts[i] += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
classCountSum += classCounts[i];
|
||||
}
|
||||
|
||||
double truePosTotal = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
double temp = truePositiveRate(i);
|
||||
if (classCounts[i] > 0) { // If temp is NaN, we want the sum to also be NaN if count > 0
|
||||
truePosTotal += (temp * classCounts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return truePosTotal / classCountSum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the true positive rate with respect to a particular class. This
|
||||
* is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* correctly classified positives
|
||||
* ------------------------------
|
||||
* total positives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the true positive rate
|
||||
*/
|
||||
public double truePositiveRate(int classIndex) {
|
||||
|
||||
double correct = 0, total = 0;
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j == classIndex) {
|
||||
correct += m_ConfusionMatrix[classIndex][j];
|
||||
}
|
||||
total += m_ConfusionMatrix[classIndex][j];
|
||||
}
|
||||
return correct / total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the weighted (by class size) false positive rate.
|
||||
*
|
||||
* @return the weighted false positive rate.
|
||||
*/
|
||||
public double weightedFalsePositiveRate() {
|
||||
double[] classCounts = new double[m_NumClasses];
|
||||
double classCountSum = 0;
|
||||
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
classCounts[i] += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
classCountSum += classCounts[i];
|
||||
}
|
||||
|
||||
double falsePosTotal = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
double temp = falsePositiveRate(i);
|
||||
if (classCounts[i] > 0) { // If temp is NaN, we want the sum to also be NaN if count > 0
|
||||
falsePosTotal += (temp * classCounts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return falsePosTotal / classCountSum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the false positive rate with respect to a particular class. This
|
||||
* is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* incorrectly classified negatives
|
||||
* --------------------------------
|
||||
* total negatives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the false positive rate
|
||||
*/
|
||||
public double falsePositiveRate(int classIndex) {
|
||||
|
||||
double incorrect = 0, total = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (i != classIndex) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j == classIndex) {
|
||||
incorrect += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
total += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
return incorrect / total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the weighted (by class size) precision.
|
||||
*
|
||||
* @return the weighted precision.
|
||||
*/
|
||||
public double weightedPrecision() {
|
||||
double[] classCounts = new double[m_NumClasses];
|
||||
double classCountSum = 0;
|
||||
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
classCounts[i] += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
classCountSum += classCounts[i];
|
||||
}
|
||||
|
||||
double precisionTotal = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
double temp = precision(i);
|
||||
if (classCounts[i] > 0) { // If temp is NaN, we want the sum to also be NaN if count > 0
|
||||
precisionTotal += (temp * classCounts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return precisionTotal / classCountSum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the precision with respect to a particular class. This is defined
|
||||
* as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* correctly classified positives
|
||||
* ------------------------------
|
||||
* total predicted as positive
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the precision
|
||||
*/
|
||||
public double precision(int classIndex) {
|
||||
|
||||
double correct = 0, total = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (i == classIndex) {
|
||||
correct += m_ConfusionMatrix[i][classIndex];
|
||||
}
|
||||
total += m_ConfusionMatrix[i][classIndex];
|
||||
}
|
||||
return correct / total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the weighted (by class size) recall.
|
||||
*
|
||||
* @return the weighted recall.
|
||||
*/
|
||||
public double weightedRecall() {
|
||||
return weightedTruePositiveRate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the macro weighted (by class size) average F-Measure.
|
||||
*
|
||||
* @return the weighted F-Measure.
|
||||
*/
|
||||
public double weightedFMeasure() {
|
||||
double[] classCounts = new double[m_NumClasses];
|
||||
double classCountSum = 0;
|
||||
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
classCounts[i] += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
classCountSum += classCounts[i];
|
||||
}
|
||||
|
||||
double fMeasureTotal = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
double temp = fMeasure(i);
|
||||
if (classCounts[i] > 0) { // If temp is NaN, we want the sum to also be NaN if count > 0
|
||||
fMeasureTotal += (temp * classCounts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return fMeasureTotal / classCountSum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the F-Measure with respect to a particular class. This is defined
|
||||
* as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* 2 * recall * precision
|
||||
* ----------------------
|
||||
* recall + precision
|
||||
* </pre>
|
||||
*
|
||||
* Returns zero when both precision and recall are zero
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the F-Measure
|
||||
*/
|
||||
public double fMeasure(int classIndex) {
|
||||
|
||||
double precision = precision(classIndex);
|
||||
double recall = recall(classIndex);
|
||||
if ((precision == 0) && (recall == 0)) {
|
||||
return 0;
|
||||
}
|
||||
return 2 * precision * recall / (precision + recall);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the recall with respect to a particular class. This is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* correctly classified positives
|
||||
* ------------------------------
|
||||
* total positives
|
||||
* </pre>
|
||||
* <p/>
|
||||
* (Which is also the same as the truePositiveRate.)
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the recall
|
||||
*/
|
||||
public double recall(int classIndex) {
|
||||
return truePositiveRate(classIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the weighted (by class size) matthews correlation coefficient.
|
||||
*
|
||||
* @return the weighted matthews correlation coefficient.
|
||||
*/
|
||||
public double weightedMatthewsCorrelation() {
|
||||
double[] classCounts = new double[m_NumClasses];
|
||||
double classCountSum = 0;
|
||||
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
classCounts[i] += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
classCountSum += classCounts[i];
|
||||
}
|
||||
|
||||
double mccTotal = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
double temp = matthewsCorrelationCoefficient(i);
|
||||
if (classCounts[i] > 0) { // If temp is NaN, we want the sum to also be NaN if count > 0
|
||||
mccTotal += (temp * classCounts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return mccTotal / classCountSum;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the matthews correlation coefficient (sometimes called phi
|
||||
* coefficient) for the supplied class
|
||||
*
|
||||
* @param classIndex the index of the class to compute the matthews
|
||||
* correlation coefficient for
|
||||
*
|
||||
* @return the mathews correlation coefficient
|
||||
*/
|
||||
public double matthewsCorrelationCoefficient(int classIndex) {
|
||||
double numTP = numTruePositives(classIndex);
|
||||
double numTN = numTrueNegatives(classIndex);
|
||||
double numFP = numFalsePositives(classIndex);
|
||||
double numFN = numFalseNegatives(classIndex);
|
||||
double n = (numTP * numTN) - (numFP * numFN);
|
||||
double d =
|
||||
(numTP + numFP) * (numTP + numFN) * (numTN + numFP) * (numTN + numFN);
|
||||
d = Math.sqrt(d);
|
||||
|
||||
return n / d;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of true positives with respect to a particular class.
|
||||
* This is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* correctly classified positives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the true positive rate
|
||||
*/
|
||||
public double numTruePositives(int classIndex) {
|
||||
|
||||
double correct = 0;
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j == classIndex) {
|
||||
correct += m_ConfusionMatrix[classIndex][j];
|
||||
}
|
||||
}
|
||||
return correct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the number of true negatives with respect to a particular class.
|
||||
* This is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* correctly classified negatives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the true positive rate
|
||||
*/
|
||||
public double numTrueNegatives(int classIndex) {
|
||||
|
||||
double correct = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (i != classIndex) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j != classIndex) {
|
||||
correct += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return correct;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate number of false positives with respect to a particular class.
|
||||
* This is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* incorrectly classified negatives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the false positive rate
|
||||
*/
|
||||
public double numFalsePositives(int classIndex) {
|
||||
|
||||
double incorrect = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (i != classIndex) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j == classIndex) {
|
||||
incorrect += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return incorrect;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate number of false negatives with respect to a particular class.
|
||||
* This is defined as
|
||||
* <p/>
|
||||
*
|
||||
* <pre>
|
||||
* incorrectly classified positives
|
||||
* </pre>
|
||||
*
|
||||
* @param classIndex the index of the class to consider as "positive"
|
||||
* @return the false positive rate
|
||||
*/
|
||||
public double numFalseNegatives(int classIndex) {
|
||||
|
||||
double incorrect = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (i == classIndex) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
if (j != classIndex) {
|
||||
incorrect += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return incorrect;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns value of kappa statistic if class is nominal.
|
||||
*
|
||||
* @return the value of the kappa statistic
|
||||
*/
|
||||
public final double kappa() {
|
||||
double[] sumRows = new double[m_ConfusionMatrix.length];
|
||||
double[] sumColumns = new double[m_ConfusionMatrix.length];
|
||||
double sumOfWeights = 0;
|
||||
for (int i = 0; i < m_ConfusionMatrix.length; i++) {
|
||||
for (int j = 0; j < m_ConfusionMatrix.length; j++) {
|
||||
sumRows[i] += m_ConfusionMatrix[i][j];
|
||||
sumColumns[j] += m_ConfusionMatrix[i][j];
|
||||
sumOfWeights += m_ConfusionMatrix[i][j];
|
||||
}
|
||||
}
|
||||
double correct = 0, chanceAgreement = 0;
|
||||
for (int i = 0; i < m_ConfusionMatrix.length; i++) {
|
||||
chanceAgreement += (sumRows[i] * sumColumns[i]);
|
||||
correct += m_ConfusionMatrix[i][i];
|
||||
}
|
||||
chanceAgreement /= (sumOfWeights * sumOfWeights);
|
||||
correct /= sumOfWeights;
|
||||
|
||||
if (chanceAgreement < 1) {
|
||||
return (correct - chanceAgreement) / (1 - chanceAgreement);
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Outputs the performance statistics in summary form. Lists number (and
|
||||
* percentage) of instances classified correctly, incorrectly and
|
||||
* unclassified. Outputs the total number of instances classified, and the
|
||||
* number of instances (if any) that had no class value provided.
|
||||
*
|
||||
* @param title the title for the statistics
|
||||
* @return the summary as a String
|
||||
*/
|
||||
public String toSummaryString(String title) {
|
||||
StringBuffer text = new StringBuffer();
|
||||
|
||||
text.append(title + "\n");
|
||||
|
||||
text.append("Correctly Classified Instances ");
|
||||
text.append(Utils.doubleToString(correct(), 12, 4) + " "
|
||||
+ Utils.doubleToString(pctCorrect(), 12, 4) + " %\n");
|
||||
|
||||
text.append("Incorrectly Classified Instances ");
|
||||
text.append(Utils.doubleToString(incorrect(), 12, 4) + " "
|
||||
+ Utils.doubleToString(pctIncorrect(), 12, 4) + " %\n");
|
||||
|
||||
text.append("Kappa statistic ");
|
||||
text.append(Utils.doubleToString(kappa(), 12, 4) + "\n");
|
||||
|
||||
text.append("Total Number of Instances ");
|
||||
text.append(Utils.doubleToString(m_WithClass, 12, 4) + "\n");
|
||||
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Outputs the performance statistics as a classification confusion matrix.
|
||||
* For each class value, shows the distribution of predicted class values.
|
||||
*
|
||||
* @param title the title for the confusion matrix
|
||||
* @return the confusion matrix as a String
|
||||
* @throws Exception if the class is numeric
|
||||
*/
|
||||
public String toMatrixString(String title) {
|
||||
StringBuffer text = new StringBuffer();
|
||||
char[] IDChars =
|
||||
{ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
|
||||
'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };
|
||||
int IDWidth;
|
||||
boolean fractional = false;
|
||||
|
||||
// Find the maximum value in the matrix
|
||||
// and check for fractional display requirement
|
||||
double maxval = 0;
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
double current = m_ConfusionMatrix[i][j];
|
||||
if (current < 0) {
|
||||
current *= -10;
|
||||
}
|
||||
if (current > maxval) {
|
||||
maxval = current;
|
||||
}
|
||||
double fract = current - Math.rint(current);
|
||||
if (!fractional && ((Math.log(fract) / Math.log(10)) >= -2)) {
|
||||
fractional = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
IDWidth =
|
||||
1 + Math.max(
|
||||
(int) (Math.log(maxval) / Math.log(10) + (fractional ? 3 : 0)),
|
||||
(int) (Math.log(m_NumClasses) / Math.log(IDChars.length)));
|
||||
text.append(title).append("\n");
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
if (fractional) {
|
||||
text.append(" ").append(num2ShortID(i, IDChars, IDWidth - 3))
|
||||
.append(" ");
|
||||
} else {
|
||||
text.append(" ").append(num2ShortID(i, IDChars, IDWidth));
|
||||
}
|
||||
}
|
||||
text.append(" <-- classified as\n");
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
for (int j = 0; j < m_NumClasses; j++) {
|
||||
text.append(" ").append(
|
||||
Utils.doubleToString(m_ConfusionMatrix[i][j], IDWidth,
|
||||
(fractional ? 2 : 0)));
|
||||
}
|
||||
text.append(" | ").append(num2ShortID(i, IDChars, IDWidth)).append(" = ")
|
||||
.append(m_ClassNames[i]).append("\n");
|
||||
}
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a breakdown of the accuracy for each class, incorporating various
|
||||
* information-retrieval statistics, such as true/false positive rate,
|
||||
* precision/recall/F-Measure. Should be useful for ROC curves,
|
||||
* recall/precision curves.
|
||||
*
|
||||
* @param title the title to prepend the stats string with
|
||||
* @return the statistics presented as a string
|
||||
* @throws Exception if class is not nominal
|
||||
*/
|
||||
public String toClassDetailsString(String title) {
|
||||
boolean displayTP = true;
|
||||
boolean displayFP = true;
|
||||
boolean displayP = true;
|
||||
boolean displayR = true;
|
||||
boolean displayFM = true;
|
||||
boolean displayMCC = true;
|
||||
|
||||
StringBuffer text =
|
||||
new StringBuffer(title + "\n "
|
||||
+ "TP Rate " + "FP Rate "
|
||||
+ "Precision " + "Recall "
|
||||
+ "F-Measure " + "MCC ");
|
||||
|
||||
text.append("Class\n");
|
||||
for (int i = 0; i < m_NumClasses; i++) {
|
||||
text.append(" ");
|
||||
if (displayTP) {
|
||||
double tpr = truePositiveRate(i);
|
||||
if (Utils.isMissingValue(tpr)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", tpr));
|
||||
}
|
||||
}
|
||||
if (displayFP) {
|
||||
double fpr = falsePositiveRate(i);
|
||||
if (Utils.isMissingValue(fpr)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", fpr));
|
||||
}
|
||||
}
|
||||
if (displayP) {
|
||||
double p = precision(i);
|
||||
if (Utils.isMissingValue(p)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-11.3f", precision(i)));
|
||||
}
|
||||
}
|
||||
if (displayR) {
|
||||
double r = recall(i);
|
||||
if (Utils.isMissingValue(r)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", recall(i)));
|
||||
}
|
||||
}
|
||||
if (displayFM) {
|
||||
double fm = fMeasure(i);
|
||||
if (Utils.isMissingValue(fm)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-11.3f", fMeasure(i)));
|
||||
}
|
||||
}
|
||||
if (displayMCC) {
|
||||
double mat = matthewsCorrelationCoefficient(i);
|
||||
if (Utils.isMissingValue(mat)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String
|
||||
.format("%-9.3f", matthewsCorrelationCoefficient(i)));
|
||||
}
|
||||
}
|
||||
text.append(m_ClassNames[i]).append('\n');
|
||||
}
|
||||
|
||||
text.append("Weighted Avg. ");
|
||||
if (displayTP) {
|
||||
double wtpr = weightedTruePositiveRate();
|
||||
if (Utils.isMissingValue(wtpr)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", wtpr));
|
||||
}
|
||||
}
|
||||
if (displayFP) {
|
||||
double wfpr = weightedFalsePositiveRate();
|
||||
if (Utils.isMissingValue(wfpr)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", wfpr));
|
||||
}
|
||||
}
|
||||
if (displayP) {
|
||||
double wp = weightedPrecision();
|
||||
if (Utils.isMissingValue(wp)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-11.3f", wp));
|
||||
}
|
||||
}
|
||||
if (displayR) {
|
||||
double wr = weightedRecall();
|
||||
if (Utils.isMissingValue(wr)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", wr));
|
||||
}
|
||||
}
|
||||
if (displayFM) {
|
||||
double wf = weightedFMeasure();
|
||||
if (Utils.isMissingValue(wf)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-11.3f", wf));
|
||||
}
|
||||
}
|
||||
if (displayMCC) {
|
||||
double wmc = weightedMatthewsCorrelation();
|
||||
if (Utils.isMissingValue(wmc)) {
|
||||
text.append("? ");
|
||||
} else {
|
||||
text.append(String.format("%-9.3f", wmc));
|
||||
}
|
||||
}
|
||||
text.append("\n");
|
||||
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("[");
|
||||
for (int rowIndex = 0; rowIndex < m_ConfusionMatrix.length; rowIndex++) {
|
||||
builder.append("[");
|
||||
for (int colIndex = 0; colIndex < m_ConfusionMatrix[rowIndex].length; colIndex++) {
|
||||
int value = (int) (m_ConfusionMatrix[rowIndex][colIndex]);
|
||||
builder.append(value);
|
||||
if (colIndex < m_ConfusionMatrix[rowIndex].length - 1) {
|
||||
builder.append(",");
|
||||
}
|
||||
}
|
||||
builder.append("]");
|
||||
}
|
||||
builder.append("]");
|
||||
return String.format("{Accuracy:%.3f,Precision:%.3f,Recall:%.3f,F1 Score:%.3f,Matrix:%s}",
|
||||
pctCorrect() / 100.0, weightedPrecision(), weightedRecall(), weightedFMeasure(), builder.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Method for generating indices for the confusion matrix.
|
||||
*
|
||||
* @param num integer to format
|
||||
* @param IDChars the characters to use
|
||||
* @param IDWidth the width of the entry
|
||||
* @return the formatted integer as a string
|
||||
*/
|
||||
private String num2ShortID(int num, char[] IDChars, int IDWidth) {
|
||||
|
||||
char ID[] = new char[IDWidth];
|
||||
int i;
|
||||
|
||||
for (i = IDWidth - 1; i >= 0; i--) {
|
||||
ID[i] = IDChars[num % IDChars.length];
|
||||
num = num / IDChars.length - 1;
|
||||
if (num < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (i--; i >= 0; i--) {
|
||||
ID[i] = ' ';
|
||||
}
|
||||
|
||||
return new String(ID);
|
||||
}
|
||||
|
||||
public static double[][] initConfusionMatrix(int classNum) {
|
||||
double[][] confusionMatrix = new double[classNum][classNum];
|
||||
for (double[] row : confusionMatrix) {
|
||||
Arrays.fill(row, 0);
|
||||
}
|
||||
return confusionMatrix;
|
||||
}
|
||||
|
||||
public static void matrixAddMatrix(double[][] dest, double[][] src) {
|
||||
Assertion.assertEqual(dest.length, src.length);
|
||||
for (int i = 0; i < dest.length; i++) {
|
||||
Assertion.assertEqual(dest[i].length, src[i].length);
|
||||
for (int j = 0; j < dest[i].length; j++) {
|
||||
dest[i][j] += src[i][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static String[] initClassNames(int[] classes) {
|
||||
String[] classNames = new String[classes.length];
|
||||
for (int i = 0; i < classes.length; i++) {
|
||||
classNames[i] = String.valueOf(classes[i]);
|
||||
}
|
||||
return classNames;
|
||||
}
|
||||
|
||||
public static double[][] getConfusionMatrix(int[] classes, List<Integer> trueLabels, List<Integer> predLabels) {
|
||||
double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
for (int i = 0; i < trueLabels.size(); i++) {
|
||||
int trueLabel = trueLabels.get(i);
|
||||
int predLabel = predLabels.get(i);
|
||||
confusionMatrix[trueLabel][predLabel]++;
|
||||
}
|
||||
return confusionMatrix;
|
||||
}
|
||||
|
||||
public static double[][] getConfusionMatrix(int[] classes, List<int[]> truePredLabelPairs) {
|
||||
double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
for (int[] truePredLabelPair : truePredLabelPairs) {
|
||||
int trueLabel = truePredLabelPair[0];
|
||||
int predLabel = truePredLabelPair[1];
|
||||
int trueLabelIndex = TimeDatabase.argWhere(classes, trueLabel);
|
||||
int predLabelIndex = TimeDatabase.argWhere(classes, predLabel);
|
||||
confusionMatrix[trueLabelIndex][predLabelIndex]++;
|
||||
}
|
||||
return confusionMatrix;
|
||||
}
|
||||
|
||||
public static Evaluation evaluate(String title, int[] classes, List<int[]> truePredLabelPairs, boolean isPrint) {
|
||||
Evaluation evaluation = new Evaluation(classes, truePredLabelPairs);
|
||||
if (isPrint) {
|
||||
Configuration.out.println("================================================================================");
|
||||
Configuration.out.print(evaluation.toSummaryString("[" + title + "]"));
|
||||
Configuration.out.print(evaluation.toMatrixString(""));
|
||||
Configuration.out.print(evaluation.toClassDetailsString(""));
|
||||
Configuration.out.println("================================================================================");
|
||||
}
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
public static double[] softmax(double[] probDist) {
|
||||
double[] tmp = new double[probDist.length];
|
||||
for (int i = 0; i < probDist.length; i++) {
|
||||
tmp[i] = Math.exp(probDist[i]);
|
||||
}
|
||||
return KStepTransitionMatrix.normalize(tmp);
|
||||
}
|
||||
|
||||
public static void describe(Instances instances, String tag) {
|
||||
int[] classCount = TimeDatabase.zeros(instances.numClasses());
|
||||
for (int i = 0; i < instances.numInstances(); i++) {
|
||||
Instance instance = instances.instance(i);
|
||||
classCount[(int) instance.classValue()]++;
|
||||
}
|
||||
for (int i = 0; i < classCount.length; i++) {
|
||||
Configuration.out.printf("#(%d)=%4d ", i, classCount[i]);
|
||||
}
|
||||
Configuration.out.printf("<- %s\n", tag);
|
||||
}
|
||||
|
||||
public static double entropy(double[] probDist) {
|
||||
double result = 0.0;
|
||||
for (double prob : probDist) {
|
||||
result -= (prob * Math.log(prob));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static int min(int[] array) {
|
||||
int min = array[0];
|
||||
for (int i = 1; i < array.length; i++) {
|
||||
if (array[i] < min) {
|
||||
min = array[i];
|
||||
}
|
||||
}
|
||||
return min;
|
||||
}
|
||||
|
||||
public static int argMin(int[] array) {
|
||||
int min = array[0];
|
||||
int minIndex = 0;
|
||||
for (int i = 1; i < array.length; i++) {
|
||||
if (array[i] < min) {
|
||||
min = array[i];
|
||||
minIndex = i;
|
||||
}
|
||||
}
|
||||
return minIndex;
|
||||
}
|
||||
|
||||
public static double distance(double[][] m1, double[][] m2) {
|
||||
double result = 0.0;
|
||||
Assertion.assertEqual(m1.length, m2.length);
|
||||
for (int row = 0; row < m1.length; row++) {
|
||||
Assertion.assertEqual(m1[row].length, m2[row].length);
|
||||
for (int col = 0; col < m1[row].length; col++) {
|
||||
double v1 = m1[row][col];
|
||||
double v2 = m2[row][col];
|
||||
result += Math.abs(v2 - v1);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,444 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.classifiers.Classifier;
|
||||
import weka.classifiers.bayes.NaiveBayes;
|
||||
import weka.classifiers.trees.J48;
|
||||
import weka.classifiers.trees.RandomForest;
|
||||
import weka.core.Instance;
|
||||
import weka.core.Instances;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.*;
|
||||
|
||||
public class HOCMEMM extends Model {
|
||||
private int n; // order of Markov chain model
|
||||
private long offset; // time interval of the time point (i.e., instance)
|
||||
private double[] lambdas; // weights of each order, used in high-order Markov chain model
|
||||
private double sameTeamWeight; // weight for users from the same team
|
||||
private double diffTeamWeight; // weight for users from different teams
|
||||
private boolean isIntraEnabled; // whether the intra-personal correlation is enabling
|
||||
private boolean isInterEnabled; // whether the inter-personal correlation is enabling
|
||||
private int seed = 1; // initial seed for the meta classifier
|
||||
|
||||
public static class MyLearningEpochListener implements SemiSupervisedLearningStrategy.LearningEpochListener {
|
||||
public void learnerInitialized(
|
||||
Classifier classifier, Model model, ModelPerformanceSampler sampler,
|
||||
LearningStrategy strategy, TimeDatabase trainDb, TimeDatabase validDb) throws Exception {
|
||||
String identifier = String.format("%s+%s+init", model.getName(), strategy.getName());
|
||||
Configuration.out.println(identifier + "{");
|
||||
// information of both the training and the validation sets
|
||||
Configuration.out.println(trainDb.simpleDescribe("train"));
|
||||
Configuration.out.println(validDb == null ? "valid{null}" : validDb.simpleDescribe("valid"));
|
||||
// k-step transition matrix
|
||||
Map<String, KStepTransitionMatrix> ktms = trainDb.getKStepTransitionMatrix(Configuration.N, Configuration.WINDOW_SIZE_IN_MILLISECOND);
|
||||
List<String> keys1 = new ArrayList<String>(ktms.keySet());
|
||||
Collections.sort(keys1);
|
||||
for (String key : keys1) {
|
||||
Configuration.out.println(ktms.get(key).describeIn1D());
|
||||
}
|
||||
// valid
|
||||
if (validDb != null) {
|
||||
Map<String, List<int[]>> truePredLabelPairsMap = model.predict(classifier, trainDb, validDb, sampler);
|
||||
List<String> keys2 = new ArrayList<String>(truePredLabelPairsMap.keySet());
|
||||
Collections.sort(keys2);
|
||||
for (String key : keys2) {
|
||||
Evaluation evaluation = new Evaluation(validDb.getClasses(), truePredLabelPairsMap.get(key));
|
||||
Configuration.out.println(key + evaluation.toString());
|
||||
}
|
||||
validDb.cleanCache();
|
||||
}
|
||||
Configuration.out.println("}");
|
||||
}
|
||||
|
||||
public void learnerRetrained(int epoch, Classifier classifier, Model model, ModelPerformanceSampler sampler, LearningStrategy strategy, TimeDatabase oldTrainDb, TimeDatabase newTrainDb, TimeDatabase validDb) throws Exception {
|
||||
String identifier = String.format("%s+%s+%d", model.getName(), strategy.getName(), epoch);
|
||||
Configuration.out.println(identifier + "{");
|
||||
// information of both the training and the validation sets
|
||||
Configuration.out.println(newTrainDb.simpleDescribe("train"));
|
||||
Configuration.out.println(validDb == null ? "valid{null}" : validDb.simpleDescribe("valid"));
|
||||
// distance
|
||||
double distance = SemiSupervisedLearningStrategy.distance(
|
||||
oldTrainDb.getKStepTransitionMatrix(Configuration.N, Configuration.WINDOW_SIZE_IN_MILLISECOND),
|
||||
newTrainDb.getKStepTransitionMatrix(Configuration.N, Configuration.WINDOW_SIZE_IN_MILLISECOND), false, "");
|
||||
if (distance == 0.0) {
|
||||
Configuration.out.println("retain");
|
||||
} else {
|
||||
Configuration.out.println("change:" + new BigDecimal(distance).toString());
|
||||
}
|
||||
// k-step transition matrix
|
||||
Map<String, KStepTransitionMatrix> ktms = newTrainDb.getKStepTransitionMatrix(Configuration.N, Configuration.WINDOW_SIZE_IN_MILLISECOND);
|
||||
List<String> keys1 = new ArrayList<String>(ktms.keySet());
|
||||
Collections.sort(keys1);
|
||||
for (String key : keys1) {
|
||||
Configuration.out.println(ktms.get(key).describeIn1D());
|
||||
}
|
||||
// valid
|
||||
if (validDb != null) {
|
||||
Map<String, List<int[]>> truePredLabelPairsMap = model.predict(classifier, newTrainDb, validDb, sampler);
|
||||
List<String> keys2 = new ArrayList<String>(truePredLabelPairsMap.keySet());
|
||||
Collections.sort(keys2);
|
||||
for (String key : keys2) {
|
||||
Evaluation evaluation = new Evaluation(validDb.getClasses(), truePredLabelPairsMap.get(key));
|
||||
Configuration.out.println(key + evaluation.toString());
|
||||
}
|
||||
validDb.cleanCache();
|
||||
}
|
||||
Configuration.out.println("}");
|
||||
}
|
||||
}
|
||||
|
||||
public static class MyModelPerformanceSampler extends Model.DefaultModelPerformanceSampler {
|
||||
public static final String OBSER_TAG = "obser";
|
||||
public static final String INTRA_TAG = "intra";
|
||||
public static final String INTER_TAG = "inter";
|
||||
|
||||
public MyModelPerformanceSampler(Model model, boolean isPrintInfo) {
|
||||
super(model, isPrintInfo);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public List<int[]> collectTrueAndPredLabels(String tag, TimeDatabase testDb, Object... args) {
|
||||
if (OBSER_TAG.equals(tag)) {
|
||||
return afterObserPredictionDone(tag, testDb);
|
||||
}
|
||||
if (INTRA_TAG.equals(tag)) {
|
||||
Map<String, KStepTransitionMatrix> tcm = (Map<String, KStepTransitionMatrix>) args[0];
|
||||
return afterIntraPredictionDone(tag, testDb, tcm);
|
||||
}
|
||||
if (INTER_TAG.equals(tag)) {
|
||||
Map<String, KStepTransitionMatrix> scm = (Map<String, KStepTransitionMatrix>) args[0];
|
||||
return afterInterPredictionDone(tag, testDb, scm);
|
||||
}
|
||||
if (FINAL_TAG.equals(tag)) {
|
||||
return afterFinalPredictionDone(tag, testDb);
|
||||
}
|
||||
throw new IllegalArgumentException(String.format("Tag '%s' is not defined for sampling %s's performance.", tag, this.model.getName()));
|
||||
}
|
||||
|
||||
public List<int[]> afterObserPredictionDone(String tag, TimeDatabase testDb) {
|
||||
int[] classes = testDb.getClasses();
|
||||
// double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
List<int[]> truePredLabelPairs = new ArrayList<int[]>();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int trueLabel = tp.getLabel();
|
||||
Assertion.assertNotNegative(trueLabel);
|
||||
double[] obserProbDist = tp.getObserProbDist();
|
||||
Assertion.assertNotNull(obserProbDist);
|
||||
int predLabel = classes[KStepTransitionMatrix.argMax(obserProbDist)];
|
||||
// confusionMatrix[trueLabel][predLabel]++;
|
||||
truePredLabelPairs.add(new int[]{trueLabel, predLabel});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.isPrintInfo) {
|
||||
Evaluation.evaluate(String.format("%s:%s", this.model.getName(), tag), classes, truePredLabelPairs, true);
|
||||
}
|
||||
// return confusionMatrix;
|
||||
return truePredLabelPairs;
|
||||
}
|
||||
|
||||
public List<int[]> afterIntraPredictionDone(String tag, TimeDatabase testDb, Map<String, KStepTransitionMatrix> tcm) {
|
||||
int[] classes = testDb.getClasses();
|
||||
// double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
List<int[]> truePredLabelPairs = new ArrayList<int[]>();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int trueLabel = tp.getLabel();
|
||||
Assertion.assertNotNegative(trueLabel);
|
||||
double[] obserProbDist = tp.getObserProbDist();
|
||||
Assertion.assertNotNull(obserProbDist);
|
||||
double[] intraProbDist = tp.getIntraProbDist();
|
||||
Assertion.assertNotNull(intraProbDist);
|
||||
double[] compoProbDist = KStepTransitionMatrix.zeros(classes.length);
|
||||
KStepTransitionMatrix.vectorAddVector(compoProbDist, obserProbDist);
|
||||
KStepTransitionMatrix.vectorAddVector(compoProbDist, intraProbDist);
|
||||
compoProbDist = Evaluation.softmax(compoProbDist);
|
||||
int predLabel = classes[KStepTransitionMatrix.argMax(compoProbDist)];
|
||||
// confusionMatrix[trueLabel][predLabel]++;
|
||||
truePredLabelPairs.add(new int[]{trueLabel, predLabel});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.isPrintInfo) {
|
||||
Evaluation.evaluate(String.format("%s:%s", this.model.getName(), tag), classes, truePredLabelPairs, true);
|
||||
for (KStepTransitionMatrix matrix : tcm.values()) {
|
||||
Configuration.out.print(matrix.toString());
|
||||
}
|
||||
}
|
||||
// return confusionMatrix;
|
||||
return truePredLabelPairs;
|
||||
}
|
||||
|
||||
public List<int[]> afterInterPredictionDone(String tag, TimeDatabase testDb, Map<String, KStepTransitionMatrix> scm) {
|
||||
int[] classes = testDb.getClasses();
|
||||
// double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
List<int[]> truePredLabelPairs = new ArrayList<int[]>();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int trueLabel = tp.getLabel();
|
||||
Assertion.assertNotNegative(trueLabel);
|
||||
double[] obserProbDist = tp.getObserProbDist();
|
||||
Assertion.assertNotNull(obserProbDist);
|
||||
double[] interProbDist = tp.getInterProbDist();
|
||||
Assertion.assertNotNull(interProbDist);
|
||||
double[] compoProbDist = KStepTransitionMatrix.zeros(classes.length);
|
||||
KStepTransitionMatrix.vectorAddVector(compoProbDist, obserProbDist);
|
||||
KStepTransitionMatrix.vectorAddVector(compoProbDist, interProbDist);
|
||||
compoProbDist = Evaluation.softmax(compoProbDist);
|
||||
int predLabel = classes[KStepTransitionMatrix.argMax(compoProbDist)];
|
||||
// confusionMatrix[trueLabel][predLabel]++;
|
||||
truePredLabelPairs.add(new int[]{trueLabel, predLabel});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.isPrintInfo) {
|
||||
Evaluation.evaluate(String.format("%s:%s", this.model.getName(), tag), classes, truePredLabelPairs, true);
|
||||
for (KStepTransitionMatrix matrix : scm.values()) {
|
||||
Configuration.out.print(matrix.toString());
|
||||
}
|
||||
}
|
||||
// return confusionMatrix;
|
||||
return truePredLabelPairs;
|
||||
}
|
||||
|
||||
public List<int[]> afterFinalPredictionDone(String tag, TimeDatabase testDb) {
|
||||
int[] classes = testDb.getClasses();
|
||||
// double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
List<int[]> truePredLabelPairs = new ArrayList<int[]>();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int trueLabel = tp.getLabel();
|
||||
Assertion.assertNotNegative(trueLabel);
|
||||
double[] finalProbDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(finalProbDist);
|
||||
int predLabel = classes[KStepTransitionMatrix.argMax(finalProbDist)];
|
||||
// confusionMatrix[trueLabel][predLabel]++;
|
||||
truePredLabelPairs.add(new int[]{trueLabel, predLabel});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.isPrintInfo) {
|
||||
Evaluation.evaluate(String.format("%s:%s", this.model.getName(), tag), classes, truePredLabelPairs, true);
|
||||
}
|
||||
// return confusionMatrix;
|
||||
return truePredLabelPairs;
|
||||
}
|
||||
}
|
||||
|
||||
public HOCMEMM(String name, int n, long offset, double base, double sameTeamWeight, double diffTeamWeight, boolean isIntraEnabled, boolean isInterEnabled, int seed) {
|
||||
super(name);
|
||||
this.n = n;
|
||||
this.offset = offset;
|
||||
this.lambdas = new double[this.n];
|
||||
this.lambdas[0] = base;
|
||||
for (int i = 1; i < n; i++) {
|
||||
lambdas[i] = lambdas[i - 1] * base;
|
||||
}
|
||||
this.sameTeamWeight = sameTeamWeight;
|
||||
this.diffTeamWeight = diffTeamWeight;
|
||||
this.isIntraEnabled = isIntraEnabled;
|
||||
this.isInterEnabled = isInterEnabled;
|
||||
this.seed = seed;
|
||||
}
|
||||
|
||||
public HOCMEMM(String name, int n, long offset, double base, double sameTeamWeight, double diffTeamWeight, boolean isIntraEnabled, boolean isInterEnabled) {
|
||||
this(name, n, offset, base, sameTeamWeight, diffTeamWeight, isIntraEnabled, isInterEnabled, 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Classifier fit(TimeDatabase trainDb) throws Exception {
|
||||
Instances instances = trainDb.getLabeledInstances();
|
||||
if (instances == null) {
|
||||
throw new NullPointerException("No labeled data in the training set!");
|
||||
}
|
||||
RandomForest metaClassifier = new RandomForest();
|
||||
metaClassifier.setSeed(this.seed);
|
||||
metaClassifier.buildClassifier(instances);
|
||||
return metaClassifier;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<int[]>> predict(Classifier classifier, TimeDatabase trainDb, TimeDatabase testDb, ModelPerformanceSampler sampler) throws Exception {
|
||||
Map<String, List<int[]>> confusionMatrices = new HashMap<String, List<int[]>>();
|
||||
|
||||
predictObserProbDist(classifier, testDb);
|
||||
confusionMatrices.put(MyModelPerformanceSampler.OBSER_TAG,
|
||||
sampler.collectTrueAndPredLabels(MyModelPerformanceSampler.OBSER_TAG, testDb));
|
||||
|
||||
if (isIntraEnabled) {
|
||||
Map<String, KStepTransitionMatrix> tcm = predictIntraProbDist(trainDb, testDb);
|
||||
confusionMatrices.put(MyModelPerformanceSampler.INTRA_TAG,
|
||||
sampler.collectTrueAndPredLabels(MyModelPerformanceSampler.INTRA_TAG, testDb, tcm));
|
||||
}
|
||||
|
||||
if (isInterEnabled) {
|
||||
Map<String, KStepTransitionMatrix> scm = predictInterProbDist(trainDb, testDb);
|
||||
confusionMatrices.put(MyModelPerformanceSampler.INTER_TAG,
|
||||
sampler.collectTrueAndPredLabels(MyModelPerformanceSampler.INTER_TAG, testDb, scm));
|
||||
}
|
||||
|
||||
predictFinalProbDist(testDb);
|
||||
confusionMatrices.put(MyModelPerformanceSampler.FINAL_TAG,
|
||||
sampler.collectTrueAndPredLabels(MyModelPerformanceSampler.FINAL_TAG, testDb));
|
||||
|
||||
return confusionMatrices;
|
||||
}
|
||||
|
||||
private void predictObserProbDist(Classifier classifier, TimeDatabase testDb) throws Exception {
|
||||
// to tackle the issue: weka.core.UnassignedDatasetException: Instance doesn't have access to a dataset!
|
||||
testDb.getLabeledAndUnlabeledInstances();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isEmpty()) {
|
||||
Instance instance = tp.getInstance();
|
||||
Assertion.assertNotNull(instance);
|
||||
double[] probDist = classifier.distributionForInstance(instance);
|
||||
tp.setObserProbDist(probDist);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, KStepTransitionMatrix> predictIntraProbDist(TimeDatabase trainDb, TimeDatabase testDb) {
|
||||
// estimate intra-personal k-step transition matrix from the training set
|
||||
Map<String, KStepTransitionMatrix> tcm = trainDb.getTemporalCorrelationMatrix(n, offset);
|
||||
String[] users = testDb.getUsers();
|
||||
int classNum = testDb.getClasses().length;
|
||||
|
||||
for (int toTpIndex = 0; toTpIndex < testDb.getNumberOfTimePoints(); toTpIndex++) {
|
||||
for (String user : users) {
|
||||
List<TimePoint> tps = testDb.findTimeSeriesByUser(user).getTimePoints();
|
||||
TimePoint toTp = tps.get(toTpIndex);
|
||||
if (toTp.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
double[][][] Q_hat = tcm.get(user + "->" + user).getQ_hat();
|
||||
double[] intraProbDist = KStepTransitionMatrix.zeros(classNum);
|
||||
adjustPredictionInOneKStepLoop(toTp, toTpIndex, tps, Q_hat, intraProbDist);
|
||||
|
||||
toTp.setIntraProbDist(intraProbDist);
|
||||
}
|
||||
}
|
||||
return tcm;
|
||||
}
|
||||
|
||||
private Map<String, KStepTransitionMatrix> predictInterProbDist(TimeDatabase trainDb, TimeDatabase testDb) {
|
||||
// estimate inter-personal k-step transition matrix from the training set
|
||||
Map<String, KStepTransitionMatrix> scm = trainDb.getSpatialCorrelationMatrix(n, offset);
|
||||
String[] users = testDb.getUsers();
|
||||
int classNum = testDb.getClasses().length;
|
||||
|
||||
for (int toTpIndex = 0; toTpIndex < testDb.getNumberOfTimePoints(); toTpIndex++) {
|
||||
for (String toUser : users) {
|
||||
TimePoint toTp = testDb.findTimeSeriesByUser(toUser).getTimePoints().get(toTpIndex);
|
||||
if (toTp.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
String toTeam = testDb.findTimeSeriesByUser(toUser).getTeam();
|
||||
|
||||
double[] interProbDist = KStepTransitionMatrix.zeros(classNum);
|
||||
for (String fromUser : users) {
|
||||
if (fromUser.equals(toUser)) {
|
||||
continue;
|
||||
}
|
||||
String fromTeam = testDb.findTimeSeriesByUser(fromUser).getTeam();
|
||||
|
||||
List<TimePoint> fromTps = testDb.findTimeSeriesByUser(fromUser).getTimePoints();
|
||||
double[][][] Q_hat = scm.get(fromUser + "->" + toUser).getQ_hat();
|
||||
double[] probDist = KStepTransitionMatrix.zeros(classNum);
|
||||
adjustPredictionInOneKStepLoop(toTp, toTpIndex, fromTps, Q_hat, probDist);
|
||||
|
||||
if (toTeam.equals(fromTeam)) {
|
||||
// in same team
|
||||
probDist = KStepTransitionMatrix.constantTimesVector(sameTeamWeight, probDist);
|
||||
} else {
|
||||
// in different team
|
||||
probDist = KStepTransitionMatrix.constantTimesVector(diffTeamWeight, probDist);
|
||||
}
|
||||
|
||||
KStepTransitionMatrix.vectorAddVector(interProbDist, probDist);
|
||||
}
|
||||
toTp.setInterProbDist(interProbDist);
|
||||
}
|
||||
}
|
||||
return scm;
|
||||
}
|
||||
|
||||
/**
|
||||
* one k-step loop (k == n here):
|
||||
* toTpIndex - n, ..., toTpIndex - 1, toTpIndex
|
||||
* fromUser: * ... *
|
||||
* | ... |
|
||||
* ----------------------------------
|
||||
* |
|
||||
* toUser: *
|
||||
* @param toTp which toTp to update
|
||||
* @param toTpIndex index of toTp
|
||||
* @param fromTps fromTps used to update toTp
|
||||
* @param Q_hat correlation matrix from fromUser to toUser
|
||||
* @param probDist placeholder of the k-step loop's result
|
||||
*/
|
||||
private void adjustPredictionInOneKStepLoop(TimePoint toTp, int toTpIndex, List<TimePoint> fromTps, double[][][] Q_hat, double[] probDist) {
|
||||
int fromTpIndexBeg = toTpIndex - 1;
|
||||
int fromTpIndexEnd = toTpIndex - n;
|
||||
for (int fromTpIndex = fromTpIndexBeg, step = 0; fromTpIndex >= 0 && fromTpIndex >= fromTpIndexEnd; fromTpIndex--, step++) {
|
||||
TimePoint fromTp = fromTps.get(fromTpIndex);
|
||||
if (fromTp.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
long deltaTime = toTp.getTimestamp() - fromTp.getTimestamp();
|
||||
Assertion.assertPositive(deltaTime);
|
||||
if (deltaTime > offset * (step + 1)) {
|
||||
continue;
|
||||
}
|
||||
// lambda * Q_i * X_i
|
||||
double lambda_i = lambdas[step];
|
||||
double[][] Q_i = Q_hat[step];
|
||||
double[] fromProbDist = fromTp.getObserProbDist();
|
||||
Assertion.assertNotNull(fromProbDist);
|
||||
double[] QX_i = KStepTransitionMatrix.matrixTimesVector(Q_i, fromProbDist);
|
||||
double[] lambdaQX_i = KStepTransitionMatrix.constantTimesVector(lambda_i, QX_i);
|
||||
Assertion.assertAllNotNegative(lambdaQX_i);
|
||||
KStepTransitionMatrix.vectorAddVector(probDist, lambdaQX_i);
|
||||
}
|
||||
}
|
||||
|
||||
private void predictFinalProbDist(TimeDatabase db) {
|
||||
for (TimeSeries ts : db.getTimeSeries()) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (TimePoint tp : tps) {
|
||||
if (!tp.isEmpty()) {
|
||||
double[] obserProbDist = tp.getObserProbDist();
|
||||
double[] intraProbDist = tp.getIntraProbDist();
|
||||
double[] interProbDist = tp.getInterProbDist();
|
||||
|
||||
double[] finalProbDist = KStepTransitionMatrix.zeros(db.getClasses().length);
|
||||
// add observation probability distribution
|
||||
Assertion.assertNotNull(obserProbDist);
|
||||
KStepTransitionMatrix.vectorAddVector(finalProbDist, obserProbDist);
|
||||
// add intra-personal probability distribution
|
||||
if (isIntraEnabled) {
|
||||
Assertion.assertNotNull(intraProbDist);
|
||||
KStepTransitionMatrix.vectorAddVector(finalProbDist, intraProbDist);
|
||||
}
|
||||
// add inter-personal probability distribution
|
||||
if (isInterEnabled) {
|
||||
Assertion.assertNotNull(interProbDist);
|
||||
KStepTransitionMatrix.vectorAddVector(finalProbDist, interProbDist);
|
||||
}
|
||||
// restore the final probability distribution
|
||||
tp.setFinalProbDist(Evaluation.softmax(finalProbDist));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import org.apache.commons.math3.util.MathArrays;
|
||||
|
||||
public class KStepTransitionMatrix {
|
||||
private String fromUser;
|
||||
private String toUser;
|
||||
private double[][][] Q_hat;
|
||||
|
||||
public KStepTransitionMatrix(String fromUser, String toUser, double[][][] Q_hat) {
|
||||
this.fromUser = fromUser;
|
||||
this.toUser = toUser;
|
||||
this.Q_hat = Q_hat;
|
||||
}
|
||||
|
||||
public String getFromUser() {
|
||||
return fromUser;
|
||||
}
|
||||
|
||||
public String getToUser() {
|
||||
return toUser;
|
||||
}
|
||||
|
||||
public double[][][] getQ_hat() {
|
||||
return Q_hat;
|
||||
}
|
||||
|
||||
public static double[] zeros(int length) {
|
||||
double[] result = new double[length];
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
result[i] = 0;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static double[] matrixTimesVector(double[][] matrix, double[] vector) {
|
||||
// linear combination is indeed the combination of column vectors,
|
||||
// and the row of the matrix is actually the column vector.
|
||||
Assertion.assertEqual(matrix.length, vector.length);
|
||||
double[] result = zeros(matrix[0].length);
|
||||
for (int i = 0; i < vector.length; i++) {
|
||||
double[] tmp = constantTimesVector(vector[i], matrix[i]);
|
||||
vectorAddVector(result, tmp);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void vectorAddVector(double[] dest, double[] src) {
|
||||
Assertion.assertTrue(dest.length == src.length);
|
||||
for (int i = 0; i < dest.length; i++) {
|
||||
dest[i] += src[i];
|
||||
}
|
||||
}
|
||||
|
||||
public static double[] constantTimesVector(double constant, double[] vector) {
|
||||
double[] result = new double[vector.length];
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
result[i] = constant * vector[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static double sumVector(double[] vector) {
|
||||
double sum = 0;
|
||||
for (double value : vector) {
|
||||
sum += value;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
public static double[] normalize(double[] vector) {
|
||||
double sum = sumVector(vector);
|
||||
Assertion.assertNotEqual(sum, 0.0);
|
||||
return MathArrays.normalizeArray(vector, 1.0);
|
||||
}
|
||||
|
||||
public static int argMax(double[] vector) {
|
||||
double maxValue = vector[0];
|
||||
int maxIndex = 0;
|
||||
for (int i = 1; i < vector.length; i++) {
|
||||
if (vector[i] > maxValue) {
|
||||
maxValue = vector[i];
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
return maxIndex;
|
||||
}
|
||||
|
||||
public static boolean isAllZero(double[] vector) {
|
||||
for (double value : vector) {
|
||||
if (value != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public static boolean isUniform(double[] vector) {
|
||||
for (int i = 1; i < vector.length; i++) {
|
||||
if (vector[i] != vector[0]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public static double shannonEntropy(double[] vector) {
|
||||
double[] vector_hat = normalize(vector);
|
||||
// -\Sigma (p * ln(p))
|
||||
double result = 0;
|
||||
for (int i = 0; i < vector_hat.length; i++) {
|
||||
if (vector_hat[i] == 0.0) {
|
||||
continue;
|
||||
}
|
||||
result -= (vector_hat[i] * Math.log(vector_hat[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static double klDivergence(double[] P, double[] Q) {
|
||||
Assertion.assertEqual(P.length, Q.length);
|
||||
double[] P_hat = normalize(P);
|
||||
double[] Q_hat = normalize(Q);
|
||||
|
||||
double result = 0;
|
||||
for (int i = 0; i < P_hat.length; i++) {
|
||||
double p = P_hat[i];
|
||||
double q = Q_hat[i];
|
||||
if (p == 0.0) {
|
||||
continue;
|
||||
}
|
||||
if (q == 0.0) {
|
||||
return Double.POSITIVE_INFINITY;
|
||||
}
|
||||
result += (p * Math.log(p / q));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static double jsDivergence(double[] P, double[] Q) {
|
||||
Assertion.assertEqual(P.length, Q.length);
|
||||
double[] M = new double[P.length];
|
||||
for (int i = 0; i < M.length; i++) {
|
||||
M[i] = (P[i] + Q[i]) / 2.0;
|
||||
}
|
||||
return 0.5 * klDivergence(P, M) + 0.5 * klDivergence(Q, M);
|
||||
}
|
||||
|
||||
public static double cosineDistance(double[] P, double[] Q) {
|
||||
P = normalize(P);
|
||||
Q = normalize(Q);
|
||||
Assertion.assertEqual(P.length, Q.length);
|
||||
double innerProduct = 0;
|
||||
for (int i = 0; i < P.length; i++) {
|
||||
innerProduct += (P[i] * Q[i]);
|
||||
}
|
||||
return innerProduct / (norm(P) * norm(Q));
|
||||
}
|
||||
|
||||
public static double norm(double[] vector) {
|
||||
double result = 0;
|
||||
for (double value : vector) {
|
||||
result += (value * value);
|
||||
}
|
||||
return Math.sqrt(result);
|
||||
}
|
||||
|
||||
private static double[] createAndAssign(int length, double[] src) {
|
||||
double[] result = zeros(length);
|
||||
for (int i = 0; i < Math.min(length, src.length); i++) {
|
||||
result[i] = src[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public String describeIn1D() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(this.fromUser).append("->").append(this.toUser).append("{");
|
||||
for (int orderIndex = 0; orderIndex < this.Q_hat.length; orderIndex++) {
|
||||
builder.append(orderIndex).append("[");
|
||||
for (int rowIndex = 0; rowIndex < this.Q_hat[orderIndex].length; rowIndex++) {
|
||||
builder.append("[");
|
||||
for (int colIndex = 0; colIndex < this.Q_hat[orderIndex][rowIndex].length; colIndex++) {
|
||||
double value = this.Q_hat[orderIndex][rowIndex][colIndex];
|
||||
Assertion.assertTrue(value >= 0.0 && value <= 1.0);
|
||||
builder.append(String.format("%.3f", this.Q_hat[orderIndex][rowIndex][colIndex]));
|
||||
if (colIndex < this.Q_hat[orderIndex][rowIndex].length - 1) {
|
||||
builder.append(",");
|
||||
}
|
||||
}
|
||||
builder.append("]");
|
||||
}
|
||||
builder.append("]");
|
||||
if (orderIndex < this.Q_hat.length - 1) {
|
||||
builder.append(",");
|
||||
}
|
||||
}
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String describeIn2D() {
|
||||
String head = this.fromUser + "->" + this.toUser + "\n";
|
||||
StringBuilder[] rows = new StringBuilder[this.Q_hat[0].length];
|
||||
for (int i = 0; i < rows.length; i++) {
|
||||
rows[i] = new StringBuilder();
|
||||
}
|
||||
|
||||
for (int nIndex = 0; nIndex < this.Q_hat.length; nIndex++) {
|
||||
for (int rowIndex = 0; rowIndex < this.Q_hat[nIndex].length; rowIndex++) {
|
||||
StringBuilder builder = rows[rowIndex];
|
||||
if (this.Q_hat[nIndex][rowIndex][0] > 0) {
|
||||
builder.append(String.format("[%3.1f", this.Q_hat[nIndex][rowIndex][0]));
|
||||
} else {
|
||||
builder.append("[ ");
|
||||
}
|
||||
for (int colIndex = 1; colIndex < this.Q_hat[nIndex][rowIndex].length; colIndex++) {
|
||||
if (this.Q_hat[nIndex][rowIndex][colIndex] > 0) {
|
||||
builder.append(String.format(",%3.1f", this.Q_hat[nIndex][rowIndex][colIndex]));
|
||||
} else {
|
||||
builder.append(", ");
|
||||
}
|
||||
}
|
||||
builder.append("]");
|
||||
if (nIndex < this.Q_hat.length - 1) {
|
||||
builder.append(" ");
|
||||
} else {
|
||||
builder.append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
StringBuilder matrices = new StringBuilder();
|
||||
matrices.append(head);
|
||||
for (StringBuilder row : rows) {
|
||||
matrices.append(row.toString());
|
||||
}
|
||||
return matrices.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return describeIn2D();
|
||||
}
|
||||
|
||||
public static double distance(KStepTransitionMatrix m1, KStepTransitionMatrix m2) {
|
||||
double result = 0.0;
|
||||
double[][][] q1 = m1.Q_hat;
|
||||
double[][][] q2 = m2.Q_hat;
|
||||
Assertion.assertEqual(q1.length, q2.length);
|
||||
for (int i = 0; i < q1.length; i++) {
|
||||
double[][] matrix1 = q1[i];
|
||||
double[][] matrix2 = q2[i];
|
||||
result += Evaluation.distance(matrix1, matrix2);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Set;
|
||||
|
||||
public class LabelAllAtOnceLearningStrategy extends SemiSupervisedLearningStrategy {
|
||||
private int epoch;
|
||||
|
||||
public LabelAllAtOnceLearningStrategy(String name, int epoch) {
|
||||
super(name);
|
||||
this.epoch = epoch;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int assignPseudoLabels(TimeDatabase trainDb, int epoch) {
|
||||
// int[] classes = trainDb.getClasses();
|
||||
// // get indices of all the unlabeled data
|
||||
// Set<TimePointWithProbDist> unlabeledIndices = SemiSupervisedLearningStrategy.indexOfUnlabeledData(trainDb, classes);
|
||||
// // assign pseudo labels
|
||||
// SemiSupervisedLearningStrategy.assignPseudoLabelsByIndices(trainDb, new ArrayList<TimePointWithProbDist>(unlabeledIndices));
|
||||
// return unlabeledIndices.size();
|
||||
|
||||
int[] classes = trainDb.getClasses();
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isEmpty()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
int label = classes[KStepTransitionMatrix.argMax(probDist)];
|
||||
tp.setLabel(label);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isStopBeforeUpdate(TimeDatabase trainDb, int epoch) {
|
||||
return epoch >= this.epoch;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isStopAfterUpdate(TimeDatabase trainDb, int epoch) {
|
||||
if ((epoch + 1) % 10 == 0) {
|
||||
System.out.print(epoch + 1);
|
||||
} else {
|
||||
System.out.print(".");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public class LabelKMostCertainLearningStrategy extends BatchSemiSupervisedLearningStrategy {
|
||||
public LabelKMostCertainLearningStrategy(String name, int epoch, BatchRate batchRate) {
|
||||
super(name, epoch, batchRate);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int assignPseudoLabels(TimeDatabase trainDb, int epoch) {
|
||||
int[] classes = trainDb.getClasses();
|
||||
// get indices of unlabeled data
|
||||
Set<TimePointWithProbDist> unlabeledIndices = SemiSupervisedLearningStrategy.indexOfUnlabeledData(trainDb, classes);
|
||||
if (unlabeledIndices.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
// divide data by class
|
||||
List<List<TimePointWithProbDist>> classIndices = SemiSupervisedLearningStrategy.divideDataByClass(unlabeledIndices, classes);
|
||||
// compute unlabeled batch under each class
|
||||
int[] classCounts = SemiSupervisedLearningStrategy.scaleDataUnderEachClass(trainDb, classes, batchRate.getBatch(epoch));
|
||||
// select out indices of unlabeled data to be added into database
|
||||
List<TimePointWithProbDist> candidateIndices = new ArrayList<TimePointWithProbDist>();
|
||||
for (int i = 0; i < classCounts.length; i++) {
|
||||
candidateIndices.addAll(SemiSupervisedLearningStrategy.selectKMostCertainData(
|
||||
classIndices.get(i), classCounts[i]));
|
||||
}
|
||||
// assign pseudo labels
|
||||
SemiSupervisedLearningStrategy.assignPseudoLabelsByIndices(trainDb, candidateIndices);
|
||||
return candidateIndices.size();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class LabelNeighborFirstLearningStrategy extends BatchSemiSupervisedLearningStrategy {
|
||||
public static final int INTRA_DIMENSION = 1;
|
||||
public static final int INTER_DIMENSION = INTRA_DIMENSION << 1;
|
||||
public static final int BOTH_DIMENSION = INTRA_DIMENSION + INTER_DIMENSION;
|
||||
|
||||
private static final int INTRA_DIMENSION_MASK = INTRA_DIMENSION;
|
||||
private static final int INTER_DIMENSION_MASK = INTER_DIMENSION;
|
||||
|
||||
public static final int BACKWARD = 1;
|
||||
public static final int FORWARD = BACKWARD << 1;
|
||||
public static final int BIDIRECTIONAL = BACKWARD + FORWARD;
|
||||
|
||||
private static final int BACKWARD_MASK = BACKWARD;
|
||||
private static final int FORWARD_MASK = FORWARD;
|
||||
|
||||
private int dimension;
|
||||
private int direction;
|
||||
private int n;
|
||||
private boolean isCrossTeam;
|
||||
|
||||
public LabelNeighborFirstLearningStrategy(String name, int epoch, BatchRate batchRate,
|
||||
int dimension, int direction, int n, boolean isCrossTeam) {
|
||||
super(name, epoch, batchRate);
|
||||
this.dimension = dimension;
|
||||
this.direction = direction;
|
||||
this.n = n;
|
||||
this.isCrossTeam = isCrossTeam;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int assignPseudoLabels(TimeDatabase trainDb, int epoch) {
|
||||
int[] classes = trainDb.getClasses();
|
||||
// get indices of all unlabeled data
|
||||
Set<TimePointWithProbDist> unlabeledIndices = SemiSupervisedLearningStrategy.indexOfUnlabeledData(trainDb, classes);
|
||||
if (unlabeledIndices.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
// get number of unlabeled data which will be assigned label soon
|
||||
int[] classCount = SemiSupervisedLearningStrategy.scaleDataUnderEachClass(trainDb, classes, this.batchRate.getBatch(epoch));
|
||||
// divide data by class
|
||||
List<List<TimePointWithProbDist>> classLists = SemiSupervisedLearningStrategy.divideDataByClass(unlabeledIndices, classes);
|
||||
List<TimePointWithProbDist> candidateIndices = new ArrayList<TimePointWithProbDist>();
|
||||
for (int i = 0; i < classLists.size(); i++) {
|
||||
List<TimePointWithProbDist> classList = classLists.get(i);
|
||||
if (classList.size() > classCount[i]) {
|
||||
List<TimePointWithNeighbor> unlabeledIndexList = new ArrayList<TimePointWithNeighbor>();
|
||||
// count number of neighbors of each unlabeled data
|
||||
for (TimePointWithProbDist unlabeledIndex : classList) {
|
||||
int[] neighborCount = countNeighbors(trainDb, unlabeledIndex, this.dimension, this.direction, this.n, this.isCrossTeam);
|
||||
unlabeledIndexList.add(new TimePointWithNeighbor(unlabeledIndex, neighborCount[0], neighborCount[1]));
|
||||
}
|
||||
// sort by number of labeled neighbor: more neighbor first
|
||||
Collections.sort(unlabeledIndexList, new Comparator<TimePointWithNeighbor>() {
|
||||
public int compare(TimePointWithNeighbor o1, TimePointWithNeighbor o2) {
|
||||
if (o2.labeledNeighbor == o1.labeledNeighbor) {
|
||||
return o2.unlabeledNeighbor - o1.unlabeledNeighbor;
|
||||
} else {
|
||||
return o2.labeledNeighbor - o1.labeledNeighbor;
|
||||
}
|
||||
}
|
||||
});
|
||||
// select the first k unlabeled data
|
||||
for (int j = 0; j < classCount[i]; j++) {
|
||||
candidateIndices.add(unlabeledIndexList.get(j).tp);
|
||||
}
|
||||
} else {
|
||||
candidateIndices.addAll(classList);
|
||||
}
|
||||
}
|
||||
// assign pseudo labels
|
||||
SemiSupervisedLearningStrategy.assignPseudoLabelsByIndices(trainDb, candidateIndices);
|
||||
return candidateIndices.size();
|
||||
|
||||
// int[] classes = trainDb.getClasses();
|
||||
// // find indices of neighbors
|
||||
// Set<TimePointWithProbDist> unlabeledIndices = indexOfNeighbors(trainDb, classes, dimension, direction, n, isCrossTeam);
|
||||
// // no unlabeled data
|
||||
// if (unlabeledIndices.isEmpty()) {
|
||||
// return 0;
|
||||
// }
|
||||
// // assign pseudo labels to all unlabeled neighbors
|
||||
// SemiSupervisedLearningStrategy.assignPseudoLabelsByIndices(trainDb, new ArrayList<TimePointWithProbDist>(unlabeledIndices));
|
||||
// return unlabeledIndices.size();
|
||||
|
||||
// // determine how many unlabeled data will be assigned pseudo labels soon
|
||||
// int[] classCount = SemiSupervisedLearningStrategy.scaleDataUnderEachClass(trainDb, classes, this.batchRate.getBatch(epoch));
|
||||
// // divide data by class
|
||||
// List<List<TimePointWithProbDist>> classLists = SemiSupervisedLearningStrategy.divideDataByClass(unlabeledIndices, classes);
|
||||
// // select k most certain data
|
||||
// List<TimePointWithProbDist> candidateList = new ArrayList<TimePointWithProbDist>();
|
||||
// for (int i = 0; i < classLists.size(); i++) {
|
||||
// List<TimePointWithProbDist> classList = SemiSupervisedLearningStrategy.selectKMostCertainData(classLists.get(i), classCount[i]);
|
||||
// candidateList.addAll(classList);
|
||||
// }
|
||||
// // assign pseudo labels
|
||||
// SemiSupervisedLearningStrategy.assignPseudoLabelsByIndices(trainDb, candidateList);
|
||||
// return candidateList.size();
|
||||
}
|
||||
|
||||
public static Set<TimePointWithProbDist> indexOfNeighbors(TimeDatabase trainDb, int[] classes, int dimension, int direction, int n, boolean isCrossTeam) {
|
||||
Set<TimePointWithProbDist> result = new HashSet<TimePointWithProbDist>();
|
||||
Set<TimePointWithProbDist> labeledIndices = SemiSupervisedLearningStrategy.indexOfLabeledData(trainDb, classes);
|
||||
if ((dimension & INTRA_DIMENSION_MASK) == INTRA_DIMENSION) {
|
||||
if ((direction & BACKWARD_MASK) == BACKWARD) {
|
||||
result.addAll(backwardIndexOfIntraNeighbors(trainDb, labeledIndices, classes, n));
|
||||
}
|
||||
if ((direction & FORWARD_MASK) == FORWARD) {
|
||||
result.addAll(forwardIndexOfIntraNeighbors(trainDb, labeledIndices, classes, n));
|
||||
}
|
||||
}
|
||||
if ((dimension & INTER_DIMENSION_MASK) == INTER_DIMENSION) {
|
||||
if ((direction & BACKWARD_MASK) == BACKWARD) {
|
||||
result.addAll(backwardIndexOfInterNeighbors(trainDb, labeledIndices, classes, n, isCrossTeam));
|
||||
}
|
||||
if ((direction & FORWARD_MASK) == FORWARD) {
|
||||
result.addAll(forwardIndexOfInterNeighbors(trainDb, labeledIndices, classes, n, isCrossTeam));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Set<TimePointWithProbDist> backwardIndexOfIntraNeighbors(TimeDatabase trainDb, Set<TimePointWithProbDist> labeledIndices, int[] classes, int n) {
|
||||
Set<TimePointWithProbDist> result = new HashSet<TimePointWithProbDist>();
|
||||
for (TimePointWithProbDist labeledIndex : labeledIndices) {
|
||||
List<TimePoint> tps = trainDb.findTimeSeriesByUser(labeledIndex.user).getTimePoints();
|
||||
for (int i = Math.max(0, labeledIndex.index - n); i < labeledIndex.index; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isUnlabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
result.add(new TimePointWithProbDist(labeledIndex.team, labeledIndex.user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Set<TimePointWithProbDist> forwardIndexOfIntraNeighbors(TimeDatabase trainDb, Set<TimePointWithProbDist> labeledIndices, int[] classes, int n) {
|
||||
Set<TimePointWithProbDist> result = new HashSet<TimePointWithProbDist>();
|
||||
for (TimePointWithProbDist labeledIndex : labeledIndices) {
|
||||
List<TimePoint> tps = trainDb.findTimeSeriesByUser(labeledIndex.user).getTimePoints();
|
||||
for (int i = labeledIndex.index + 1, t = Math.min(tps.size(), labeledIndex.index + 1 + n); i < t; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isUnlabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
result.add(new TimePointWithProbDist(labeledIndex.team, labeledIndex.user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Set<TimePointWithProbDist> backwardIndexOfInterNeighbors(TimeDatabase trainDb, Set<TimePointWithProbDist> labeledIndices, int[] classes, int n, boolean isCrossTeam) {
|
||||
Set<TimePointWithProbDist> result = new HashSet<TimePointWithProbDist>();
|
||||
for (TimePointWithProbDist labeledIndex : labeledIndices) {
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
if (ts.getUser().equals(labeledIndex.user)) {
|
||||
continue;
|
||||
}
|
||||
if (isCrossTeam && (!ts.getTeam().equals(labeledIndex.team))) {
|
||||
continue;
|
||||
}
|
||||
String team = ts.getTeam();
|
||||
String user = ts.getUser();
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (int i = Math.max(0, labeledIndex.index - n); i < labeledIndex.index; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isUnlabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
result.add(new TimePointWithProbDist(team, user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Set<TimePointWithProbDist> forwardIndexOfInterNeighbors(TimeDatabase trainDb, Set<TimePointWithProbDist> labeledIndices, int[] classes, int n, boolean isCrossTeam) {
|
||||
Set<TimePointWithProbDist> result = new HashSet<TimePointWithProbDist>();
|
||||
for (TimePointWithProbDist labeledIndex : labeledIndices) {
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
if (ts.getUser().equals(labeledIndex.user)) {
|
||||
continue;
|
||||
}
|
||||
if (isCrossTeam && (!ts.getTeam().equals(labeledIndex.team))) {
|
||||
continue;
|
||||
}
|
||||
String team = ts.getTeam();
|
||||
String user = ts.getUser();
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (int i = labeledIndex.index + 1, t = Math.min(tps.size(), labeledIndex.index + 1 + n); i < t; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isUnlabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
result.add(new TimePointWithProbDist(team, user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static class TimePointWithNeighbor {
|
||||
public final TimePointWithProbDist tp;
|
||||
public final int labeledNeighbor;
|
||||
public final int unlabeledNeighbor;
|
||||
public int entropyRank;
|
||||
public int labeledNeighborRank;
|
||||
public int unlabeledNeighborRank;
|
||||
|
||||
public TimePointWithNeighbor(TimePointWithProbDist tp, int labeledNeighbor, int unlabeledNeighbor) {
|
||||
this.tp = tp;
|
||||
this.labeledNeighbor = labeledNeighbor;
|
||||
this.unlabeledNeighbor = unlabeledNeighbor;
|
||||
}
|
||||
}
|
||||
|
||||
public static int[] countNeighbors(TimeDatabase trainDb, TimePointWithProbDist unlabeledIndex,
|
||||
int dimension, int direction, int n, boolean isCrossTeam) {
|
||||
int labeledNeighborCount = 0;
|
||||
int unlabeledNeighborCount = 0;
|
||||
if ((dimension & INTRA_DIMENSION_MASK) == INTRA_DIMENSION) {
|
||||
List<TimePoint> tps = trainDb.findTimeSeriesByUser(unlabeledIndex.user).getTimePoints();
|
||||
if ((direction & BACKWARD_MASK) == BACKWARD) {
|
||||
for (int i = Math.max(0, unlabeledIndex.index - n); i < unlabeledIndex.index; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isLabeled()) {
|
||||
labeledNeighborCount++;
|
||||
} else if (tp.isUnlabeled()) {
|
||||
unlabeledNeighborCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((direction & FORWARD_MASK) == FORWARD) {
|
||||
for (int i = unlabeledIndex.index + 1, t = Math.min(trainDb.getNumberOfTimePoints(), unlabeledIndex.index + n + 1); i < t; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isLabeled()) {
|
||||
labeledNeighborCount++;
|
||||
} else if (tp.isUnlabeled()) {
|
||||
unlabeledNeighborCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((dimension & INTER_DIMENSION_MASK) == INTER_DIMENSION) {
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
if (ts.getUser().equals(unlabeledIndex.user)) {
|
||||
continue;
|
||||
}
|
||||
if (!(isCrossTeam || ts.getTeam().equals(unlabeledIndex.team))) {
|
||||
continue;
|
||||
}
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
if ((direction & BACKWARD_MASK) == BACKWARD) {
|
||||
for (int i = Math.max(0, unlabeledIndex.index - n); i < unlabeledIndex.index; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isLabeled()) {
|
||||
labeledNeighborCount++;
|
||||
} else if (tp.isUnlabeled()) {
|
||||
unlabeledNeighborCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((direction & FORWARD_MASK) == FORWARD) {
|
||||
for (int i = unlabeledIndex.index + 1, t = Math.min(trainDb.getNumberOfTimePoints(), unlabeledIndex.index + n + 1); i < t; i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isLabeled()) {
|
||||
labeledNeighborCount++;
|
||||
} else if (tp.isUnlabeled()) {
|
||||
unlabeledNeighborCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new int[]{labeledNeighborCount, unlabeledNeighborCount};
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,18 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public abstract class LearningStrategy {
|
||||
protected String name;
|
||||
|
||||
public LearningStrategy(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public abstract Map<String, List<int[]>> fitAndPredict(Model model, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb, Model.ModelPerformanceSampler sampler) throws Exception;
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class Main {
|
||||
public static void main(String[] args) throws Exception {
|
||||
// define models and samplers
|
||||
Model hocmemm = new HOCMEMM("HOCMEMM",
|
||||
Configuration.N, Configuration.WINDOW_SIZE_IN_MILLISECOND, Configuration.BASE,
|
||||
Configuration.SAME_TEAM_WEIGHT, Configuration.DIFF_TEAM_WEIGHT,
|
||||
Configuration.IS_INTRA_ENABLED, Configuration.IS_INTER_ENABLED);
|
||||
Model.ModelPerformanceSampler hocmemmSampler = new HOCMEMM.MyModelPerformanceSampler(hocmemm, false);
|
||||
Model rf = new RFModel("RF");
|
||||
Model.ModelPerformanceSampler rfSampler = new Model.DefaultModelPerformanceSampler(rf, false);
|
||||
|
||||
// define learning strategies
|
||||
LearningStrategy supervised = new SupervisedLearningStrategy("supervised");
|
||||
LearningStrategy labelAllAtOnce = new LabelAllAtOnceLearningStrategy("labelAllAtOnce", 100);
|
||||
LearningStrategy kmc = new LabelKMostCertainLearningStrategy("kmc", 10000, new BatchSemiSupervisedLearningStrategy.BatchRate() {
|
||||
public int getBatch(int epoch) {
|
||||
return 300;
|
||||
}
|
||||
});
|
||||
|
||||
// double[] eraseRates = new double[]{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9};
|
||||
double[] eraseRates = new double[]{/*0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, */0.99};
|
||||
int start = 0;
|
||||
int round = start + 1;
|
||||
for (double eraseRate : eraseRates) {
|
||||
for (int i = start; i < round; i++) {
|
||||
Configuration.update(String.format("erase_rate_%.2f", eraseRate));
|
||||
System.out.printf("[%d] removing %.1f%% labeled data from the training set: ", i, eraseRate * 100.0);
|
||||
long begTime = System.currentTimeMillis();
|
||||
learningWithRemovingPartialLabels(
|
||||
hocmemm,
|
||||
hocmemmSampler,
|
||||
new LearningStrategy[]{
|
||||
// supervised,
|
||||
labelAllAtOnce,
|
||||
// kmc,
|
||||
},
|
||||
eraseRate,
|
||||
String.format("%s_erase_%.2f_round_%02d", Configuration.DATA_ROOT, eraseRate, i));
|
||||
long endTime = System.currentTimeMillis();
|
||||
long deltaTime = (endTime - begTime) / 1000;
|
||||
long minute = deltaTime / 60;
|
||||
long second = deltaTime % 60;
|
||||
System.out.printf("%2d min %2d sec.\n", minute, second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void learningWithRemovingPartialLabels(
|
||||
Model model, Model.ModelPerformanceSampler sampler, LearningStrategy[] learningStrategies,
|
||||
final double eraseRate, String root) throws Exception {
|
||||
MultiLearningStrategyTrainAndTestSession mss = new MultiLearningStrategyTrainAndTestSession(
|
||||
"mss", 10, 0.0, false) {
|
||||
@Override
|
||||
protected TimeDatabase[] prepareTrainValidAndTestSets(TimeDatabase trainDb, TimeDatabase testDb) {
|
||||
trainDb.erasePartialLabels(eraseRate);
|
||||
return new TimeDatabase[]{trainDb, testDb.copy(), testDb};
|
||||
}
|
||||
};
|
||||
mss.setModel(model, sampler);
|
||||
SemiSupervisedLearningStrategy.LearningEpochListener learningEpochListener = new HOCMEMM.MyLearningEpochListener();
|
||||
for (LearningStrategy learningStrategy : learningStrategies) {
|
||||
if (learningStrategy instanceof SemiSupervisedLearningStrategy) {
|
||||
((SemiSupervisedLearningStrategy) learningStrategy).setLearningEpochListener(learningEpochListener);
|
||||
}
|
||||
mss.addLearningStrategies(learningStrategy);
|
||||
}
|
||||
Map<String, Evaluation> evaluationMap = mss.crossValidation(root);
|
||||
|
||||
// get combinations of model and strategy
|
||||
Set<String> modelPlusStrategySet = new HashSet<String>();
|
||||
Set<String> phaseSet = new HashSet<String>();
|
||||
for (String key : evaluationMap.keySet()) {
|
||||
modelPlusStrategySet.add(key.substring(0, key.lastIndexOf(":")));
|
||||
phaseSet.add(key.substring(key.lastIndexOf(":") + 1));
|
||||
}
|
||||
List<String> modelPlusStrategyList = new ArrayList<String>(modelPlusStrategySet);
|
||||
Collections.sort(modelPlusStrategyList);
|
||||
List<String> phaseList = new ArrayList<String>(phaseSet);
|
||||
Collections.sort(phaseList);
|
||||
|
||||
// compare difference of prediction between every pair of phase
|
||||
for (String modelPlusStrategy : modelPlusStrategyList) {
|
||||
for (int fromPhaseIndex = 0, s = phaseList.size(); fromPhaseIndex < s - 1; fromPhaseIndex++) {
|
||||
String fromPhase = phaseList.get(fromPhaseIndex);
|
||||
List<int[]> fromTruePredLabelPairs = evaluationMap.get(modelPlusStrategy + ":" + fromPhase).getTruePredLabelPairs();
|
||||
for (int toPhaseIndex = fromPhaseIndex + 1; toPhaseIndex < s; toPhaseIndex++) {
|
||||
String toPhase = phaseList.get(toPhaseIndex);
|
||||
List<int[]> toTruePredLabelPairs = evaluationMap.get(modelPlusStrategy + ":" + toPhase).getTruePredLabelPairs();
|
||||
Assertion.assertEqual(fromTruePredLabelPairs.size(), toTruePredLabelPairs.size());
|
||||
int[][] comparisionMatrix = new int[][]{{0, 0}, {0, 0}};
|
||||
for (int pairIndex = 0; pairIndex < fromTruePredLabelPairs.size(); pairIndex++) {
|
||||
int[] fromTruePredLabelPair = fromTruePredLabelPairs.get(pairIndex);
|
||||
int[] toTruePredLabelPair = toTruePredLabelPairs.get(pairIndex);
|
||||
// check whether two true labels are identical
|
||||
Assertion.assertEqual(fromTruePredLabelPair[0], toTruePredLabelPair[0]);
|
||||
// check whether the predicted label is same as the true label
|
||||
boolean isFromPredLabelTrue = fromTruePredLabelPair[0] == fromTruePredLabelPair[1];
|
||||
boolean isToPredLabelTrue = toTruePredLabelPair[0] == toTruePredLabelPair[1];
|
||||
// get index
|
||||
int fromIndex = isFromPredLabelTrue ? 0 : 1;
|
||||
int toIndex = isToPredLabelTrue ? 0 : 1;
|
||||
comparisionMatrix[fromIndex][toIndex]++;
|
||||
}
|
||||
Configuration.out.printf("%s: %s vs %s -> [[%4d, %4d], [%4d, %4d]]\n",
|
||||
modelPlusStrategy, fromPhase, toPhase,
|
||||
comparisionMatrix[0][0], comparisionMatrix[0][1],
|
||||
comparisionMatrix[1][0], comparisionMatrix[1][1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.classifiers.Classifier;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public abstract class Model {
|
||||
protected String name;
|
||||
|
||||
public Model(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public abstract Classifier fit(TimeDatabase trainDb) throws Exception;
|
||||
|
||||
public abstract Map<String, List<int[]>> predict(Classifier classifier, TimeDatabase trainDb, TimeDatabase testDb, ModelPerformanceSampler sampler) throws Exception;
|
||||
|
||||
public static abstract class ModelPerformanceSampler {
|
||||
protected Model model;
|
||||
protected boolean isPrintInfo;
|
||||
|
||||
public ModelPerformanceSampler(Model model, boolean isPrintInfo) {
|
||||
this.model = model;
|
||||
this.isPrintInfo = isPrintInfo;
|
||||
}
|
||||
|
||||
public Model getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public boolean isPrintInfo() {
|
||||
return isPrintInfo;
|
||||
}
|
||||
|
||||
public abstract List<int[]> collectTrueAndPredLabels(String tag, TimeDatabase testDb, Object... args);
|
||||
}
|
||||
|
||||
public static class DefaultModelPerformanceSampler extends ModelPerformanceSampler {
|
||||
public static final String FINAL_TAG = "final";
|
||||
|
||||
public DefaultModelPerformanceSampler(Model model, boolean isPrintInfo) {
|
||||
super(model, isPrintInfo);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<int[]> collectTrueAndPredLabels(String tag, TimeDatabase testDb, Object... args) {
|
||||
Assertion.assertEqual("final", tag);
|
||||
int[] classes = testDb.getClasses();
|
||||
// double[][] confusionMatrix = Evaluation.initConfusionMatrix(classes.length);
|
||||
List<int[]> truePredLabelPairs = new ArrayList<int[]>();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int trueLabel = tp.getLabel();
|
||||
Assertion.assertNotNegative(trueLabel);
|
||||
double[] finalProbDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(finalProbDist);
|
||||
int predLabel = classes[KStepTransitionMatrix.argMax(finalProbDist)];
|
||||
// confusionMatrix[trueLabel][predLabel]++;
|
||||
truePredLabelPairs.add(new int[]{trueLabel, predLabel});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.isPrintInfo) {
|
||||
Evaluation.evaluate(String.format("%s:%s", this.model.getName(), tag), classes, truePredLabelPairs, true);
|
||||
}
|
||||
// return confusionMatrix;
|
||||
return truePredLabelPairs;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public abstract class MultiLearningStrategyTrainAndTestSession extends TrainAndTestSession {
|
||||
protected Model model;
|
||||
protected Model.ModelPerformanceSampler sampler;
|
||||
protected List<LearningStrategy> learningStrategies;
|
||||
|
||||
public MultiLearningStrategyTrainAndTestSession(String name, int fold, double validRate, boolean isPrintInfo) {
|
||||
super(name, fold, validRate, isPrintInfo);
|
||||
}
|
||||
|
||||
public void setModel(Model model, Model.ModelPerformanceSampler sampler) {
|
||||
this.model = model;
|
||||
this.sampler = sampler;
|
||||
}
|
||||
|
||||
public void addLearningStrategies(LearningStrategy learningStrategy) {
|
||||
if (learningStrategy == null) {
|
||||
return;
|
||||
}
|
||||
if (this.learningStrategies == null) {
|
||||
this.learningStrategies = new ArrayList<LearningStrategy>();
|
||||
}
|
||||
this.learningStrategies.add(learningStrategy);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, List<int[]>> validation(int foldIndex, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb) throws Exception {
|
||||
Assertion.assertNotNull(this.model);
|
||||
Assertion.assertNotNull(this.sampler);
|
||||
Assertion.assertNotNull(this.learningStrategies);
|
||||
|
||||
Map<String, List<int[]>> results = new HashMap<String, List<int[]>>();
|
||||
|
||||
for (int i = 0; i < this.learningStrategies.size(); i++) {
|
||||
LearningStrategy learningStrategy = this.learningStrategies.get(i);
|
||||
Map<String, List<int[]>> confusionMatrices = learningStrategy.fitAndPredict(this.model, trainDb.copy(), (validDb == null) ? null : validDb.copy(), testDb.copy(), this.sampler);
|
||||
for (Map.Entry<String, List<int[]>> entry : confusionMatrices.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
List<int[]> value = entry.getValue();
|
||||
results.put(String.format("%s:%s:%s", this.model.getName(), learningStrategy.getName(), key), value);
|
||||
}
|
||||
|
||||
if (this.isPrintInfo) {
|
||||
List<String> keys = new ArrayList<String>(confusionMatrices.keySet());
|
||||
Collections.sort(keys);
|
||||
for (String key : keys) {
|
||||
List<int[]> confusionMatrix = confusionMatrices.get(key);
|
||||
Evaluation.evaluate(String.format("%s:%s:%s:fold %d", this.model.getName(), learningStrategy.getName(), key, foldIndex), trainDb.getClasses(), confusionMatrix, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public static MultiLearningStrategyTrainAndTestSession getDefaultInstance(String name, int fold, double validRate, boolean isPrintInfo) {
|
||||
return new MultiLearningStrategyTrainAndTestSession(name, fold, validRate, isPrintInfo) {
|
||||
@Override
|
||||
protected TimeDatabase[] prepareTrainValidAndTestSets(TimeDatabase trainDb, TimeDatabase testDb) {
|
||||
TimeDatabase[] tmp = DataSource.splitTrainingAndValidationSets(trainDb, this.validRate);
|
||||
return new TimeDatabase[]{tmp[0], tmp[1], testDb};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(this.name).append("{model:");
|
||||
if (this.model == null) {
|
||||
builder.append("null");
|
||||
} else {
|
||||
builder.append(this.model.getName());
|
||||
}
|
||||
builder.append(",strategies:");
|
||||
if (this.learningStrategies == null || this.learningStrategies.isEmpty()) {
|
||||
builder.append("null");
|
||||
} else {
|
||||
builder.append("[").append(this.learningStrategies.get(0).getName());
|
||||
for (int i = 1; i < this.learningStrategies.size(); i++) {
|
||||
builder.append(",").append(this.learningStrategies.get(i).getName());
|
||||
}
|
||||
builder.append("]");
|
||||
}
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public abstract class MultiModelTrainAndTestSession extends TrainAndTestSession {
|
||||
protected List<Model> models;
|
||||
protected List<Model.ModelPerformanceSampler> samplers;
|
||||
protected LearningStrategy learningStrategy;
|
||||
|
||||
public MultiModelTrainAndTestSession(String name, int fold, double validRate, boolean isPrintInfo) {
|
||||
super(name, fold, validRate, isPrintInfo);
|
||||
}
|
||||
|
||||
public void addModel(Model model, Model.ModelPerformanceSampler sampler) {
|
||||
if (model == null || sampler == null) {
|
||||
return;
|
||||
}
|
||||
if (this.models == null) {
|
||||
this.models = new ArrayList<Model>();
|
||||
Assertion.assertNull(this.samplers);
|
||||
this.samplers = new ArrayList<Model.ModelPerformanceSampler>();
|
||||
}
|
||||
this.models.add(model);
|
||||
this.samplers.add(sampler);
|
||||
}
|
||||
|
||||
public void setLearningStrategy(LearningStrategy learningStrategy) {
|
||||
this.learningStrategy = learningStrategy;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Map<String, List<int[]>> validation(int foldIndex, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb) throws Exception {
|
||||
Assertion.assertNotNull(this.models);
|
||||
Assertion.assertNotNull(this.samplers);
|
||||
Assertion.assertNotNull(this.learningStrategy);
|
||||
|
||||
Map<String, List<int[]>> results = new HashMap<String, List<int[]>>();
|
||||
|
||||
for (int i = 0; i < this.models.size(); i++) {
|
||||
Model model = this.models.get(i);
|
||||
Model.ModelPerformanceSampler sampler = this.samplers.get(i);
|
||||
Map<String, List<int[]>> confusionMatrices = this.learningStrategy.fitAndPredict(
|
||||
model, trainDb.copy(), (validDb == null) ? null : validDb.copy(), testDb.copy(), sampler);
|
||||
for (Map.Entry<String, List<int[]>> entry : confusionMatrices.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
List<int[]> value = entry.getValue();
|
||||
results.put(String.format("%s:%s:%s", model.getName(), this.learningStrategy.getName(), key), value);
|
||||
}
|
||||
|
||||
if (this.isPrintInfo) {
|
||||
List<String> keys = new ArrayList<String>(confusionMatrices.keySet());
|
||||
Collections.sort(keys);
|
||||
for (String key : keys) {
|
||||
List<int[]> confusionMatrix = confusionMatrices.get(key);
|
||||
Evaluation.evaluate(String.format("%s:%s:%s:fold %d", model.getName(), this.learningStrategy.getName(), key, foldIndex), trainDb.getClasses(), confusionMatrix, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
public static MultiModelTrainAndTestSession getDefaultInstance(String name, int fold, double validRate, boolean isPrintInfo) {
|
||||
return new MultiModelTrainAndTestSession(name, fold, validRate, isPrintInfo) {
|
||||
@Override
|
||||
protected TimeDatabase[] prepareTrainValidAndTestSets(TimeDatabase trainDb, TimeDatabase testDb) {
|
||||
TimeDatabase[] tmp = DataSource.splitTrainingAndValidationSets(trainDb, this.validRate);
|
||||
return new TimeDatabase[]{tmp[0], tmp[1], testDb};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(this.name).append("{models:");
|
||||
if (this.models == null || this.models.isEmpty()) {
|
||||
builder.append("null");
|
||||
} else {
|
||||
builder.append("[").append(this.models.get(0).getName());
|
||||
for (int i = 1; i < this.models.size(); i++) {
|
||||
builder.append(",").append(this.models.get(i).getName());
|
||||
}
|
||||
builder.append("]");
|
||||
}
|
||||
builder.append(",strategy:");
|
||||
if (this.learningStrategy == null) {
|
||||
builder.append("null");
|
||||
} else {
|
||||
builder.append(this.learningStrategy.getName());
|
||||
}
|
||||
builder.append("}");
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.classifiers.Classifier;
|
||||
import weka.classifiers.trees.RandomForest;
|
||||
import weka.core.Instance;
|
||||
import weka.core.Instances;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class RFModel extends Model {
|
||||
public RFModel(String name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Classifier fit(TimeDatabase trainDb) throws Exception {
|
||||
Instances instances = trainDb.getLabeledInstances();
|
||||
if (instances == null) {
|
||||
throw new NullPointerException("No labeled data in the training set!");
|
||||
}
|
||||
RandomForest metaClassifier = new RandomForest();
|
||||
metaClassifier.buildClassifier(instances);
|
||||
return metaClassifier;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<int[]>> predict(Classifier classifier, TimeDatabase trainDb, TimeDatabase testDb, ModelPerformanceSampler sampler) throws Exception {
|
||||
Map<String, List<int[]>> confusionMatrices = new HashMap<String, List<int[]>>();
|
||||
testDb.getLabeledAndUnlabeledInstances();
|
||||
for (TimeSeries ts : testDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isEmpty()) {
|
||||
Instance instance = tp.getInstance();
|
||||
Assertion.assertNotNull(instance);
|
||||
double[] probDist = classifier.distributionForInstance(instance);
|
||||
tp.setFinalProbDist(probDist);
|
||||
}
|
||||
}
|
||||
}
|
||||
confusionMatrices.put("final", sampler.collectTrueAndPredLabels("final", testDb));
|
||||
return confusionMatrices;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.classifiers.Classifier;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.*;
|
||||
|
||||
public abstract class SemiSupervisedLearningStrategy extends LearningStrategy {
|
||||
protected LearningEpochListener learningEpochListener;
|
||||
|
||||
public SemiSupervisedLearningStrategy(String name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
public interface LearningEpochListener {
|
||||
void learnerInitialized(Classifier classifier, Model model, Model.ModelPerformanceSampler sampler, LearningStrategy strategy, TimeDatabase trainDb, TimeDatabase validDb) throws Exception;
|
||||
void learnerRetrained(int epoch, Classifier classifier, Model model, Model.ModelPerformanceSampler sampler, LearningStrategy strategy, TimeDatabase oldTrainDb, TimeDatabase newTrainDb, TimeDatabase validDb) throws Exception;
|
||||
}
|
||||
|
||||
public LearningEpochListener getLearningEpochListener() {
|
||||
return learningEpochListener;
|
||||
}
|
||||
|
||||
public void setLearningEpochListener(LearningEpochListener learningEpochListener) {
|
||||
this.learningEpochListener = learningEpochListener;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<int[]>> fitAndPredict(Model model, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb, Model.ModelPerformanceSampler sampler) throws Exception {
|
||||
// initial meta classifier
|
||||
Classifier classifier = model.fit(trainDb);
|
||||
if (this.learningEpochListener != null) {
|
||||
this.learningEpochListener.learnerInitialized(classifier, model, sampler, this, trainDb, validDb);
|
||||
}
|
||||
// iteratively learning
|
||||
int i = 0;
|
||||
while (true) {
|
||||
// check whether exit is required
|
||||
if (isStopBeforeUpdate(trainDb, i)) {
|
||||
break;
|
||||
}
|
||||
// predict the probability distributions of both labeled and unlabeled data in training set
|
||||
model.predict(classifier, trainDb, trainDb, sampler);
|
||||
// add pseudo labeled data into training set
|
||||
TimeDatabase oldTrainDb = null;
|
||||
if (this.learningEpochListener != null) {
|
||||
oldTrainDb = trainDb.copy();
|
||||
}
|
||||
if (assignPseudoLabels(trainDb, i) <= 0) {
|
||||
break;
|
||||
}
|
||||
// clean predicted probability distribution
|
||||
trainDb.cleanCache();
|
||||
// retrain meta classifier
|
||||
classifier = model.fit(trainDb);
|
||||
if (this.learningEpochListener != null) {
|
||||
this.learningEpochListener.learnerRetrained(i, classifier, model, sampler, this, oldTrainDb, trainDb, validDb);
|
||||
}
|
||||
// check whether exit is required
|
||||
if (isStopAfterUpdate(trainDb, i)) {
|
||||
break;
|
||||
}
|
||||
// update epoch
|
||||
i++;
|
||||
}
|
||||
return model.predict(classifier, trainDb, testDb, sampler);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param trainDb
|
||||
* @param epoch
|
||||
* @return number of pseudo labels assigned
|
||||
*/
|
||||
protected abstract int assignPseudoLabels(TimeDatabase trainDb, int epoch);
|
||||
|
||||
protected abstract boolean isStopBeforeUpdate(TimeDatabase trainDb, int epoch);
|
||||
|
||||
protected abstract boolean isStopAfterUpdate(TimeDatabase trainDb, int epoch);
|
||||
|
||||
public static class TimePointWithProbDist implements Comparable<TimePointWithProbDist> {
|
||||
public final String team;
|
||||
public final String user;
|
||||
public final int index;
|
||||
public final double[] probDist;
|
||||
public final int label;
|
||||
public final double entropy;
|
||||
|
||||
public TimePointWithProbDist(String team, String user, int index, double[] probDist, int[] classes) {
|
||||
this.team = team;
|
||||
this.user = user;
|
||||
this.index = index;
|
||||
this.probDist = probDist;
|
||||
this.label = classes[KStepTransitionMatrix.argMax(probDist)];
|
||||
this.entropy = Evaluation.entropy(probDist);
|
||||
}
|
||||
|
||||
/**
|
||||
* the less entropy, the more certainty.
|
||||
* @param o the instance to compare
|
||||
* @return negative number if this.entropy is less than o.entropy,
|
||||
* positive number if this.entropy is greater than o.entropy,
|
||||
* zero if this.entropy equals to o.entropy.
|
||||
*/
|
||||
public int compareTo(TimePointWithProbDist o) {
|
||||
return Double.compare(this.entropy, o.entropy);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (!(obj instanceof TimePointWithProbDist)) {
|
||||
return false;
|
||||
}
|
||||
TimePointWithProbDist o = (TimePointWithProbDist) obj;
|
||||
return this.user.equals(o.user) && this.index == o.index;
|
||||
}
|
||||
|
||||
public TimePoint findTimePoint(TimeDatabase db) {
|
||||
return db.findTimeSeriesByUser(this.user).getTimePoints().get(this.index);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds indices of all the labeled data in the database.
|
||||
* @param trainDb
|
||||
* @param classes
|
||||
* @return
|
||||
*/
|
||||
public static Set<TimePointWithProbDist> indexOfLabeledData(TimeDatabase trainDb, int[] classes) {
|
||||
Set<TimePointWithProbDist> labeledIndices = new HashSet<TimePointWithProbDist>();
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
String team = ts.getTeam();
|
||||
String user = ts.getUser();
|
||||
for (int i = 0; i < tps.size(); i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isLabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
labeledIndices.add(new TimePointWithProbDist(team, user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
return labeledIndices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds indices of all the unlabeled data in the database.
|
||||
* @param trainDb
|
||||
* @param classes
|
||||
* @return
|
||||
*/
|
||||
public static Set<TimePointWithProbDist> indexOfUnlabeledData(TimeDatabase trainDb, int[] classes) {
|
||||
Set<TimePointWithProbDist> unlabeledIndices = new HashSet<TimePointWithProbDist>();
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
String team = ts.getTeam();
|
||||
String user = ts.getUser();
|
||||
for (int i = 0; i < tps.size(); i++) {
|
||||
TimePoint tp = tps.get(i);
|
||||
if (tp.isUnlabeled()) {
|
||||
double[] probDist = tp.getFinalProbDist();
|
||||
Assertion.assertNotNull(probDist);
|
||||
unlabeledIndices.add(new TimePointWithProbDist(team, user, i, probDist, classes));
|
||||
}
|
||||
}
|
||||
}
|
||||
return unlabeledIndices;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of data under each class.
|
||||
* @param trainDb
|
||||
* @param classes
|
||||
* @return
|
||||
*/
|
||||
public static int[] countDataUnderEachClass(TimeDatabase trainDb, int[] classes) {
|
||||
// computing proportion of each class
|
||||
int[] classCount = TimeDatabase.zeros(classes.length);
|
||||
for (TimeSeries ts : trainDb.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int index = TimeDatabase.argWhere(classes, tp.getLabel());
|
||||
classCount[index]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return classCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets scaled number of data whose label is as the specific class.
|
||||
* @param trainDb
|
||||
* @param classes
|
||||
* @param batch number of unlabeled added in this epoch
|
||||
* @return
|
||||
*/
|
||||
public static int[] scaleDataUnderEachClass(TimeDatabase trainDb, int[] classes, int batch) {
|
||||
// compute proportion of each class
|
||||
int[] classCounts = countDataUnderEachClass(trainDb, classes);
|
||||
int sum = 0;
|
||||
for (int classCount : classCounts) {
|
||||
Assertion.assertPositive(classCount);
|
||||
sum += classCount;
|
||||
}
|
||||
int accumulation = 0;
|
||||
for (int i = 0; i < classCounts.length - 1; i++) {
|
||||
classCounts[i] = (int) (((double) (classCounts[i] * batch)) / ((double) sum));
|
||||
Assertion.assertPositive(classCounts[i]);
|
||||
accumulation += classCounts[i];
|
||||
}
|
||||
classCounts[classCounts.length - 1] = batch - accumulation;
|
||||
Assertion.assertPositive(classCounts[classCounts.length - 1]);
|
||||
return classCounts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classifies data into different group according to their labels.
|
||||
* @param indices
|
||||
* @param classes
|
||||
* @return
|
||||
*/
|
||||
public static List<List<TimePointWithProbDist>> divideDataByClass(Set<TimePointWithProbDist> indices, int[] classes) {
|
||||
// initial containers
|
||||
List<List<TimePointWithProbDist>> classLists = new ArrayList<List<TimePointWithProbDist>>();
|
||||
for (int i = 0; i < classes.length; i++) {
|
||||
classLists.add(new ArrayList<TimePointWithProbDist>());
|
||||
}
|
||||
for (TimePointWithProbDist tp : indices) {
|
||||
int index = TimeDatabase.argWhere(classes, tp.label);
|
||||
classLists.get(index).add(tp);
|
||||
}
|
||||
return classLists;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selects the k most certain data. The certainty is measured by
|
||||
* the entropy of the data's estimated probability distribution.
|
||||
* @param indices
|
||||
* @param k
|
||||
* @return
|
||||
*/
|
||||
public static List<TimePointWithProbDist> selectKMostCertainData(List<TimePointWithProbDist> indices, int k) {
|
||||
if (indices.size() <= k) {
|
||||
return indices;
|
||||
} else {
|
||||
// sort by entropy
|
||||
Collections.sort(indices);
|
||||
return indices.subList(0, k);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns pseudo labels to unlabeled data.
|
||||
* @param trainDb
|
||||
* @param indices
|
||||
*/
|
||||
public static void assignPseudoLabelsByIndices(TimeDatabase trainDb, List<TimePointWithProbDist> indices) {
|
||||
for (TimePointWithProbDist candidate : indices) {
|
||||
TimePoint tp = candidate.findTimePoint(trainDb);
|
||||
tp.setLabel(candidate.label);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computing distance between the old set of k-step transition matrices and the new one.
|
||||
* @param oldKtms
|
||||
* @param newKtms
|
||||
* @param isPrintInfo
|
||||
* @return
|
||||
*/
|
||||
public static double distance(Map<String, KStepTransitionMatrix> oldKtms, Map<String, KStepTransitionMatrix> newKtms, boolean isPrintInfo, String name) {
|
||||
double distance = 0.0;
|
||||
for (String key : newKtms.keySet()) {
|
||||
KStepTransitionMatrix oldKtm = oldKtms.get(key);
|
||||
KStepTransitionMatrix newKtm = newKtms.get(key);
|
||||
Assertion.assertNotNull(oldKtm);
|
||||
Assertion.assertNotNull(newKtm);
|
||||
distance += KStepTransitionMatrix.distance(oldKtm, newKtm);
|
||||
}
|
||||
if (isPrintInfo) {
|
||||
if (distance > 0.0) {
|
||||
Configuration.out.printf("%s: transition matrices are changed (%s).\n", name, new BigDecimal(distance).toString());
|
||||
} else {
|
||||
Configuration.out.println(name + ": transition matrices are unchanged.");
|
||||
}
|
||||
}
|
||||
return distance;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import weka.classifiers.Classifier;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class SupervisedLearningStrategy extends LearningStrategy {
|
||||
public SupervisedLearningStrategy(String name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<int[]>> fitAndPredict(Model model, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb, Model.ModelPerformanceSampler sampler) throws Exception {
|
||||
Classifier classifier = model.fit(trainDb);
|
||||
return model.predict(classifier, trainDb, testDb, sampler);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextTransfer;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
|
||||
public class Temp {
|
||||
public static void main(String[] args) {
|
||||
transformJsonToCsv();
|
||||
}
|
||||
|
||||
private static void copyTestSetToValidSet() {
|
||||
OkTextTransfer transfer = new OkTextTransfer();
|
||||
File rootDir = new File("C:\\Users\\zzw\\Desktop\\fujitsu\\data");
|
||||
File[] eraseAndRoundDirs = rootDir.listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isDirectory() &&
|
||||
pathname.getName().matches("data_erase_0[.][0-9]{2}_round_[0-9]{2}");
|
||||
}
|
||||
});
|
||||
Assertion.assertTrue(eraseAndRoundDirs != null && eraseAndRoundDirs.length > 0);
|
||||
for (File eraseAndRoundDir : eraseAndRoundDirs) {
|
||||
File[] foldDirs = eraseAndRoundDir.listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isDirectory() && pathname.getName().matches("fold_[0-9]");
|
||||
}
|
||||
});
|
||||
Assertion.assertTrue(foldDirs != null && foldDirs.length > 0);
|
||||
for (File foldDir : foldDirs) {
|
||||
File testFile = new File(foldDir.getAbsolutePath() + File.separator + "test.json");
|
||||
Assertion.assertTrue(testFile.exists());
|
||||
File validFile = new File(foldDir.getAbsolutePath() + File.separator + "valid.json");
|
||||
transfer.transfer(testFile, validFile);
|
||||
System.out.println(validFile.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void transformJsonToCsv() {
|
||||
File rootDir = new File("C:\\Users\\zzw\\Desktop\\fujitsu\\data");
|
||||
File[] eraseAndRoundDirs = rootDir.listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isDirectory() &&
|
||||
pathname.getName().matches("data_erase_0[.][0-9]{2}_round_[0-9]{2}");
|
||||
}
|
||||
});
|
||||
Assertion.assertTrue(eraseAndRoundDirs != null && eraseAndRoundDirs.length > 0);
|
||||
for (File eraseAndRoundDir : eraseAndRoundDirs) {
|
||||
File[] foldDirs = eraseAndRoundDir.listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isDirectory() && pathname.getName().matches("fold_[0-9]");
|
||||
}
|
||||
});
|
||||
Assertion.assertTrue(foldDirs != null && foldDirs.length > 0);
|
||||
for (File foldDir : foldDirs) {
|
||||
File trainFile = new File(foldDir.getAbsolutePath() + File.separator + "train.json");
|
||||
Assertion.assertTrue(trainFile.exists());
|
||||
transformJsonToCsv(trainFile);
|
||||
File testFile = new File(foldDir.getAbsolutePath() + File.separator + "test.json");
|
||||
Assertion.assertTrue(testFile.exists());
|
||||
transformJsonToCsv(testFile);
|
||||
// File validFile = new File(foldDir.getAbsolutePath() + File.separator + "valid.json");
|
||||
// Assertion.assertTrue(validFile.exists());
|
||||
// transformJsonToCsv(validFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void transformJsonToCsv(File jsonFile) {
|
||||
String jsonPath = jsonFile.getAbsolutePath();
|
||||
String csvPath = jsonPath.substring(0, jsonPath.lastIndexOf(".")) + ".csv";
|
||||
TimeDatabase db = TimeDatabase.deserialize(jsonPath);
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
writer.open(csvPath);
|
||||
// csv header
|
||||
for (String featureName : db.getFeatureNames()) {
|
||||
writer.print(featureName + ",");
|
||||
}
|
||||
writer.println(db.getLabelName());
|
||||
// csv body
|
||||
for (TimeSeries ts : db.getTimeSeries()) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isEmpty()) {
|
||||
for (double feature : tp.getFeatures()) {
|
||||
writer.print(feature + ",");
|
||||
}
|
||||
writer.println(tp.getLabel());
|
||||
}
|
||||
}
|
||||
}
|
||||
writer.close();
|
||||
System.out.println(csvPath);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,786 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import weka.core.*;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
public class TimeDatabase {
|
||||
private String[] users;
|
||||
private int[] classes;
|
||||
private String[] featureNames;
|
||||
private String labelName;
|
||||
private List<TimeSeries> timeSeries;
|
||||
|
||||
public TimeDatabase() {
|
||||
this.timeSeries = new ArrayList<TimeSeries>();
|
||||
}
|
||||
|
||||
public void load(String csvPath) {
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(csvPath);
|
||||
|
||||
// head
|
||||
String[] head = reader.readLine().split(",");
|
||||
this.featureNames = new String[head.length - 10];
|
||||
for (int i = 0, l = this.featureNames.length; i < l; i++) {
|
||||
this.featureNames[i] = head[i + 8];
|
||||
}
|
||||
this.labelName = head[head.length - 2];
|
||||
|
||||
// body
|
||||
Set<Integer> classSet = new HashSet<Integer>();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
// parse data
|
||||
String[] info = line.split(",");
|
||||
String team = info[0];
|
||||
String user = info[1];
|
||||
// 徐敏敏组, 江超组, 缪忍忍组
|
||||
if (team.equals("徐敏敏组")) {
|
||||
continue;
|
||||
}
|
||||
long timestamp = Long.parseLong(info[3]);
|
||||
int sessionId = Integer.parseInt(info[7]);
|
||||
double[] features = new double[info.length - 10];
|
||||
for (int i = 0, l = features.length; i < l; i++) {
|
||||
features[i] = Double.parseDouble(info[i + 8]);
|
||||
}
|
||||
// zero-based classes
|
||||
int label = Integer.parseInt(info[info.length - 2]);
|
||||
if (label != -1) {
|
||||
label = label - 1;
|
||||
}
|
||||
|
||||
// add data into database
|
||||
TimeSeries ts = findTimeSeriesByUser(user);
|
||||
if (ts == null) {
|
||||
ts = new TimeSeries(team, user);
|
||||
timeSeries.add(ts);
|
||||
}
|
||||
ts.addTimePoint(new TimePoint(timestamp, sessionId, features, label));
|
||||
|
||||
// update classes
|
||||
if (label != -1) {
|
||||
classSet.add(label);
|
||||
}
|
||||
}
|
||||
|
||||
reader.close();
|
||||
|
||||
// assign value to users
|
||||
this.users = new String[this.timeSeries.size()];
|
||||
for (int i = 0, l = this.users.length; i < l; i++) {
|
||||
this.users[i] = this.timeSeries.get(i).getUser();
|
||||
}
|
||||
|
||||
// assign value to classes
|
||||
List<Integer> classList = new ArrayList<Integer>(classSet);
|
||||
Collections.sort(classList, new Comparator<Integer>() {
|
||||
public int compare(Integer o1, Integer o2) {
|
||||
return o1 - o2;
|
||||
}
|
||||
});
|
||||
this.classes = new int[classList.size()];
|
||||
for (int i = 0, l = this.classes.length; i < l; i++) {
|
||||
this.classes[i] = classList.get(i);
|
||||
}
|
||||
}
|
||||
|
||||
public Instances defineDataFormat() {
|
||||
ArrayList<Attribute> attrs = new ArrayList<Attribute>();
|
||||
// add features
|
||||
for (String attrName : this.featureNames) {
|
||||
attrs.add(new Attribute(attrName));
|
||||
}
|
||||
// add label
|
||||
ArrayList<String> classes = new ArrayList<String>();
|
||||
for (int classValue : this.classes) {
|
||||
classes.add(String.valueOf(classValue));
|
||||
}
|
||||
attrs.add(new Attribute(this.labelName, classes));
|
||||
return new Instances("work_engagement", attrs, 0);
|
||||
}
|
||||
|
||||
public Instances getLabeledInstances() {
|
||||
Instances instances = defineDataFormat();
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (TimePoint tp : tps) {
|
||||
if (!tp.isLabeled()) {
|
||||
continue;
|
||||
}
|
||||
Instance instance = tp.transferToInstance();
|
||||
instances.add(instance);
|
||||
instance.setDataset(instances);
|
||||
}
|
||||
}
|
||||
instances.setClassIndex(instances.numAttributes() - 1);
|
||||
return instances.isEmpty() ? null : instances;
|
||||
}
|
||||
|
||||
public Instances getLabeledAndUnlabeledInstances() {
|
||||
Instances instances = defineDataFormat();
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (TimePoint tp : tps) {
|
||||
if (tp.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
Instance instance = tp.transferToInstance();
|
||||
instances.add(instance);
|
||||
instance.setDataset(instances);
|
||||
}
|
||||
}
|
||||
instances.setClassIndex(instances.numAttributes() - 1);
|
||||
return instances.isEmpty() ? null : instances;
|
||||
}
|
||||
|
||||
public void positiveNeutralNegative() {
|
||||
this.classes = new int[]{0, 1, 2};
|
||||
|
||||
if (this.timeSeries.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int label = tp.getLabel();
|
||||
if (label <= 1) {
|
||||
tp.setLabel(0);
|
||||
} else if (label == 2) {
|
||||
tp.setLabel(1);
|
||||
} else {
|
||||
tp.setLabel(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void positiveNegative() {
|
||||
this.classes = new int[]{0, 1};
|
||||
|
||||
if (this.timeSeries.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
int label = tp.getLabel();
|
||||
if (label <= 1) {
|
||||
tp.setLabel(0);
|
||||
} else if (label == 2) {
|
||||
tp.setLabel(-1);
|
||||
} else {
|
||||
tp.setLabel(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public TimeDatabase copy() {
|
||||
TimeDatabase newDb = new TimeDatabase();
|
||||
newDb.users = this.users;
|
||||
newDb.classes = this.classes;
|
||||
newDb.featureNames = this.featureNames;
|
||||
newDb.labelName = this.labelName;
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
newTs.addTimePoint(new TimePoint(tp));
|
||||
}
|
||||
newDb.timeSeries.add(newTs);
|
||||
}
|
||||
return newDb;
|
||||
}
|
||||
|
||||
public TimeDatabase align() {
|
||||
TimeDatabase newDb = new TimeDatabase();
|
||||
newDb.users = this.users;
|
||||
newDb.classes = this.classes;
|
||||
newDb.featureNames = this.featureNames;
|
||||
newDb.labelName = this.labelName;
|
||||
|
||||
// retrieve all users' timestamps, and sort in ascending order
|
||||
Set<Long> timestampSet = new HashSet<Long>();
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (TimePoint tp : tps) {
|
||||
timestampSet.add(tp.getTimestamp());
|
||||
}
|
||||
}
|
||||
List<Long> timestampList = new ArrayList<Long>(timestampSet);
|
||||
Collections.sort(timestampList, new Comparator<Long>() {
|
||||
public int compare(Long o1, Long o2) {
|
||||
if (o1 < o2) {
|
||||
return -1;
|
||||
} else if (o1 > o2) {
|
||||
return 1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// align all users' time points by adding empty time points as placeholders
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||||
for (long timestamp : timestampList) {
|
||||
TimePoint newTp = ts.findClosestTimePoint(timestamp, 0, true, true, false);
|
||||
if (newTp == null) {
|
||||
newTp = new TimePoint(timestamp, -1, null, -1);
|
||||
}
|
||||
newTs.addTimePoint(newTp);
|
||||
}
|
||||
newDb.timeSeries.add(newTs);
|
||||
}
|
||||
|
||||
return newDb;
|
||||
}
|
||||
|
||||
public boolean fillingEmpty(boolean[] classFlags, long threshold) {
|
||||
if (this.timeSeries.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
boolean isDbChanged = false;
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
TimePoint closestTp = ts.findClosestTimePoint(tp.getTimestamp(), threshold, true, true, false);
|
||||
if (closestTp == null) {
|
||||
continue;
|
||||
}
|
||||
if (closestTp.isLabeled()) {
|
||||
int label = closestTp.getLabel();
|
||||
if (classFlags[argWhere(this.classes, label)]) {
|
||||
tp.setFeatures(closestTp.getFeatures());
|
||||
tp.setLabel(label);
|
||||
isDbChanged = true;
|
||||
}
|
||||
} else {
|
||||
tp.setFeatures(closestTp.getFeatures());
|
||||
isDbChanged = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return isDbChanged;
|
||||
}
|
||||
|
||||
public boolean spreadLabels(boolean[] classFlags, long threshold) {
|
||||
if (this.timeSeries.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
boolean isDbChanged = false;
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (!tp.isUnlabeled()) {
|
||||
continue;
|
||||
}
|
||||
int label = ts.findClosestLabel(tp.getTimestamp(), threshold);
|
||||
if (label != -1 && classFlags[argWhere(this.classes, label)]) {
|
||||
tp.setLabel(label);
|
||||
isDbChanged = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return isDbChanged;
|
||||
}
|
||||
|
||||
public TimeDatabase slice(int[] indices) {
|
||||
TimeDatabase newDb = new TimeDatabase();
|
||||
newDb.users = this.users;
|
||||
newDb.classes = this.classes;
|
||||
newDb.featureNames = this.featureNames;
|
||||
newDb.labelName = this.labelName;
|
||||
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
TimeSeries newTs = new TimeSeries(ts.getTeam(), ts.getUser());
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (int index : indices) {
|
||||
newTs.addTimePoint(new TimePoint(tps.get(index)));
|
||||
}
|
||||
newDb.timeSeries.add(newTs);
|
||||
}
|
||||
|
||||
return newDb;
|
||||
}
|
||||
|
||||
public String simpleDescribe(String tag) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
int[] classCount = zeros(this.classes.length);
|
||||
int labeledCount = 0;
|
||||
int unlabeledCount = 0;
|
||||
int emptyCount = 0;
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
if (tp.isLabeled()) {
|
||||
labeledCount++;
|
||||
int index = argWhere(this.classes, tp.getLabel());
|
||||
classCount[index]++;
|
||||
} else if (tp.isUnlabeled()) {
|
||||
unlabeledCount++;
|
||||
} else {
|
||||
emptyCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
builder.append(tag).
|
||||
append("{all:").append(labeledCount + unlabeledCount + emptyCount).
|
||||
append(",lbl:").append(labeledCount).
|
||||
append(",unl:").append(unlabeledCount).
|
||||
append(",emp:").append(emptyCount).
|
||||
append(",cls[").append(classCount[0]);
|
||||
for (int i = 1; i < classCount.length; i++) {
|
||||
builder.append(",").append(classCount[i]);
|
||||
}
|
||||
builder.append("]}");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String describe(String tag) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("id | #all #lbl #unl #emp |");
|
||||
for (int cls : this.classes) {
|
||||
builder.append(" #(").append(cls).append(")");
|
||||
}
|
||||
builder.append(" | #workday | team#user <- ").append(tag).append("\n");
|
||||
|
||||
int allLblNum = 0;
|
||||
int allUnlNum = 0;
|
||||
int[] allClsNums = zeros(this.classes.length);
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
|
||||
|
||||
for (int tsIndex = 0, tsLength = this.timeSeries.size(); tsIndex < tsLength; tsIndex++) {
|
||||
int lblNum = 0;
|
||||
int unlNum = 0;
|
||||
int empNum = 0;
|
||||
int[] clsNums = zeros(this.classes.length);
|
||||
Set<String> workdays = new HashSet<String>();
|
||||
|
||||
TimeSeries ts = this.timeSeries.get(tsIndex);
|
||||
String team = ts.getTeam();
|
||||
String user = ts.getUser();
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (TimePoint tp : tps) {
|
||||
switch (tp.getType()) {
|
||||
case TimePoint.LABELED:
|
||||
lblNum++;
|
||||
clsNums[argWhere(this.classes, tp.getLabel())]++;
|
||||
workdays.add(sdf.format(new Date(tp.getTimestamp())));
|
||||
break;
|
||||
case TimePoint.UNLABELED:
|
||||
unlNum++;
|
||||
workdays.add(sdf.format(new Date(tp.getTimestamp())));
|
||||
break;
|
||||
case TimePoint.EMPTY:
|
||||
empNum++;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
int allNum = tps.size();
|
||||
allLblNum += lblNum;
|
||||
allUnlNum += unlNum;
|
||||
addVector(allClsNums, clsNums);
|
||||
int workdayNum = workdays.size();
|
||||
|
||||
builder.append(String.format("%2d | %4d %4d %4d %4d |",
|
||||
(tsIndex + 1), allNum, lblNum, unlNum, empNum));
|
||||
for (int clsNum : clsNums) {
|
||||
if (clsNum > 0) {
|
||||
builder.append(String.format(" %4d", clsNum));
|
||||
} else {
|
||||
builder.append(" ");
|
||||
}
|
||||
}
|
||||
builder.append(String.format(" | %8d | %s#%s\n", workdayNum, team, user));
|
||||
}
|
||||
|
||||
builder.append(String.format(" | %4d %4d |", allLblNum, allUnlNum));
|
||||
for (int allClsNum : allClsNums) {
|
||||
if (allClsNum > 0) {
|
||||
builder.append(String.format(" %4d", allClsNum));
|
||||
} else {
|
||||
builder.append(" ");
|
||||
}
|
||||
}
|
||||
builder.append(" |");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public String describe() {
|
||||
return describe("");
|
||||
}
|
||||
|
||||
public Map<String, KStepTransitionMatrix> getTemporalCorrelationMatrix(int n, long offset) {
|
||||
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||||
for (String user : this.users) {
|
||||
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(user, user, n, offset);
|
||||
Assertion.assertNotNull(transitionMatrix);
|
||||
matricesMap.put(String.format("%s->%s", user, user), transitionMatrix);
|
||||
}
|
||||
return matricesMap;
|
||||
}
|
||||
|
||||
public Map<String, KStepTransitionMatrix> getSpatialCorrelationMatrix(int n, long offset) {
|
||||
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||||
for (String fromUser : this.users) {
|
||||
for (String toUser : this.users) {
|
||||
if (toUser.equals(fromUser)) {
|
||||
continue;
|
||||
}
|
||||
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(fromUser, toUser, n, offset);
|
||||
Assertion.assertNotNull(transitionMatrix);
|
||||
matricesMap.put(String.format("%s->%s", fromUser, toUser), transitionMatrix);
|
||||
}
|
||||
}
|
||||
return matricesMap;
|
||||
}
|
||||
|
||||
public Map<String, KStepTransitionMatrix> getKStepTransitionMatrix(int n, long offset) {
|
||||
Map<String, KStepTransitionMatrix> matricesMap = new HashMap<String, KStepTransitionMatrix>();
|
||||
for (String fromUser : this.users) {
|
||||
for (String toUser : this.users) {
|
||||
KStepTransitionMatrix transitionMatrix = getKStepTransitionMatrix(fromUser, toUser, n, offset);
|
||||
Assertion.assertNotNull(transitionMatrix);
|
||||
matricesMap.put(String.format("%s->%s", fromUser, toUser), transitionMatrix);
|
||||
}
|
||||
}
|
||||
return matricesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the 1- to n-order state transition matrices from {@code fromUser} to {@code toUser}.
|
||||
* @param fromUser requester
|
||||
* @param toUser recipient
|
||||
* @param n n-order Markov Chain Model.
|
||||
* @param offset upper limitation of the time difference between 1-step pair of time point.
|
||||
* @return 1- to n-order state transition matrices from {@code fromUser} to {@code toUser},
|
||||
* or {@code null} if {@code n} less than 1.
|
||||
*/
|
||||
public KStepTransitionMatrix getKStepTransitionMatrix(String fromUser, String toUser, int n, long offset) {
|
||||
if (fromUser == null || toUser == null || n < 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
TimeSeries fromTs = findTimeSeriesByUser(fromUser);
|
||||
TimeSeries toTs = findTimeSeriesByUser(toUser);
|
||||
if (fromTs == null || toTs == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<TimePoint> fromTps = fromTs.getTimePoints();
|
||||
List<TimePoint> toTps = toTs.getTimePoints();
|
||||
if (fromTps.size() != toTps.size()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// initialize F
|
||||
int[][][] F = new int[n][this.classes.length][this.classes.length];
|
||||
for (int i = 0; i < F.length; i++) {
|
||||
for (int j = 0; j < F[i].length; j++) {
|
||||
for (int k = 0; k < F[i][j].length; k++) {
|
||||
F[i][j][k] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// statistic F
|
||||
for (int step = 1; step <= n; step++) {
|
||||
long threshold = offset * step;
|
||||
for (int i = 0, j = step, l = toTps.size(); j < l; i++, j++) {
|
||||
TimePoint fromTp = fromTps.get(i);
|
||||
TimePoint toTp = toTps.get(j);
|
||||
if (!(fromTp.isLabeled() && toTp.isLabeled())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long deltaTime = toTp.getTimestamp() - fromTp.getTimestamp();
|
||||
Assertion.assertPositive(deltaTime);
|
||||
if (deltaTime > threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int fromIndex = argWhere(this.classes, fromTp.getLabel());
|
||||
int toIndex = argWhere(this.classes, toTp.getLabel());
|
||||
F[step - 1][fromIndex][toIndex]++;
|
||||
}
|
||||
}
|
||||
|
||||
// transform F to Q_hat
|
||||
double[][][] Q_hat = new double[n][this.classes.length][this.classes.length];
|
||||
for (int i = 0; i < F.length; i++) {
|
||||
for (int j = 0; j < F[i].length; j++) {
|
||||
int sum = sumVector(F[i][j]);
|
||||
if (sum == 0) {
|
||||
for (int k = 0; k < F[i][j].length; k++) {
|
||||
Q_hat[i][j][k] = 0;
|
||||
}
|
||||
} else {
|
||||
for (int k = 0; k < F[i][j].length; k++) {
|
||||
Q_hat[i][j][k] = ((double) F[i][j][k]) / ((double) sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new KStepTransitionMatrix(fromUser, toUser, Q_hat);
|
||||
}
|
||||
|
||||
public String[] getUsers() {
|
||||
return users;
|
||||
}
|
||||
|
||||
public int[] getClasses() {
|
||||
return classes;
|
||||
}
|
||||
|
||||
public String[] getFeatureNames() {
|
||||
return featureNames;
|
||||
}
|
||||
|
||||
public String getLabelName() {
|
||||
return labelName;
|
||||
}
|
||||
|
||||
public List<TimeSeries> getTimeSeries() {
|
||||
return timeSeries;
|
||||
}
|
||||
|
||||
public int getNumberOfTimeSeries() {
|
||||
return this.timeSeries.size();
|
||||
}
|
||||
|
||||
public int[] getNumberOfTimePointsOfEachUser() {
|
||||
if (this.timeSeries.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int[] result = new int[this.timeSeries.size()];
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
result[i] = this.timeSeries.get(i).getTimePoints().size();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public int getNumberOfTimePoints() {
|
||||
int[] tmp = getNumberOfTimePointsOfEachUser();
|
||||
if (tmp == null) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 1; i < tmp.length; i++) {
|
||||
if (tmp[i] != tmp[0]) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return tmp[0];
|
||||
}
|
||||
|
||||
public TimeSeries findTimeSeriesByUser(String user) {
|
||||
if (timeSeries.isEmpty() || user == null) {
|
||||
return null;
|
||||
}
|
||||
for (TimeSeries ts : timeSeries) {
|
||||
if (ts.getUser().equals(user)) {
|
||||
return ts;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static int[] zeros(int length) {
|
||||
int[] result = new int[length];
|
||||
for (int i = 0; i < length; i++) {
|
||||
result[i] = 0;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static int argWhere(int[] array, int value) {
|
||||
for (int i = 0, l = array.length; i < l; i++) {
|
||||
if (array[i] == value) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static void addVector(int[] dest, int[] src) {
|
||||
Assertion.assertTrue(dest.length == src.length);
|
||||
for (int i = 0, l = dest.length; i < l; i++) {
|
||||
dest[i] += src[i];
|
||||
}
|
||||
}
|
||||
|
||||
public static int sumVector(int[] vector) {
|
||||
int sum = 0;
|
||||
for (int i = 0, l = vector.length; i < l; i++) {
|
||||
sum += vector[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
public static boolean isAllZero(int[] vector) {
|
||||
for (int value : vector) {
|
||||
if (value != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void cleanCache() {
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
for (TimePoint tp : ts.getTimePoints()) {
|
||||
tp.cleanCache();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void erasePartialLabels(double rate) {
|
||||
Assertion.assertTrue(rate > 0.0 && rate < 1.0);
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (int cls : this.classes) {
|
||||
List<Integer> indices = ts.getIndicesWithClass(cls);
|
||||
if (indices == null) {
|
||||
continue;
|
||||
}
|
||||
int eraseNum = (int) (indices.size() * rate);
|
||||
if (eraseNum <= 0) {
|
||||
continue;
|
||||
}
|
||||
int[] eraseIndices = randInt(indices, eraseNum);
|
||||
for (int eraseIndex : eraseIndices) {
|
||||
TimePoint tp = tps.get(eraseIndex);
|
||||
Assertion.assertTrue(tp.isLabeled() && tp.getLabel() == cls);
|
||||
tp.setLabel(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets n random integers fall in the interval of [min, max).
|
||||
* @param min inclusive
|
||||
* @param max exclusive
|
||||
* @param length number of random to generate
|
||||
* @return
|
||||
*/
|
||||
public static int[] randInt(int min, int max, int length) {
|
||||
Assertion.assertTrue(min < max && max - min >= length);
|
||||
List<Integer> tmp = new ArrayList<Integer>();
|
||||
for (int i = min; i < max; i++) {
|
||||
tmp.add(i);
|
||||
}
|
||||
int[] result = new int[length];
|
||||
for (int i = 0; i < length; i++) {
|
||||
int index = (int) (Math.random() * tmp.size());
|
||||
result[i] = tmp.remove(index);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets n random integers from a specific integer list.
|
||||
* @param list
|
||||
* @param length number of random to generate
|
||||
* @return
|
||||
*/
|
||||
public static int[] randInt(List<Integer> list, int length) {
|
||||
Assertion.assertNotNull(list);
|
||||
Assertion.assertTrue(list.size() >= length);
|
||||
int[] indices = randInt(0, list.size(), length);
|
||||
int[] result = new int[length];
|
||||
for (int i = 0; i < length; i++) {
|
||||
result[i] = list.get(indices[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public String drawSparseDatabase() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
int tpNum = this.getNumberOfTimePoints();
|
||||
int notLabeledTpNum = 0;
|
||||
List<Integer> tmp = new ArrayList<Integer>();
|
||||
for (int i = 0; i < tpNum; i++) {
|
||||
boolean isAllNotLabeledInAColumn = true;
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
if (ts.getTimePoints().get(i).isLabeled()) {
|
||||
isAllNotLabeledInAColumn = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (isAllNotLabeledInAColumn) {
|
||||
notLabeledTpNum++;
|
||||
} else {
|
||||
if (notLabeledTpNum > 0) {
|
||||
tmp.add(-notLabeledTpNum);
|
||||
}
|
||||
tmp.add(i);
|
||||
notLabeledTpNum = 0;
|
||||
}
|
||||
}
|
||||
if (notLabeledTpNum > 0) {
|
||||
tmp.add(-notLabeledTpNum);
|
||||
}
|
||||
|
||||
// print head
|
||||
for (int value : tmp) {
|
||||
if (value < 0) {
|
||||
builder.append(String.format("[%4d]", -value));
|
||||
} else {
|
||||
builder.append("*");
|
||||
}
|
||||
}
|
||||
builder.append("\n");
|
||||
|
||||
// print body
|
||||
for (TimeSeries ts : this.timeSeries) {
|
||||
List<TimePoint> tps = ts.getTimePoints();
|
||||
for (int value : tmp) {
|
||||
if (value < 0) {
|
||||
builder.append(" ");
|
||||
} else {
|
||||
TimePoint tp = tps.get(value);
|
||||
if (tp.isLabeled()) {
|
||||
builder.append(tp.getLabel());
|
||||
} else {
|
||||
builder.append("|");
|
||||
}
|
||||
}
|
||||
}
|
||||
builder.append("<-").append(ts.getTeam()).append("#").append(ts.getUser()).append("\n");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
public static void serialize(TimeDatabase db, String path) {
|
||||
Gson gson = new Gson();
|
||||
String json = gson.toJson(db);
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
writer.open(path);
|
||||
writer.println(json);
|
||||
writer.close();
|
||||
}
|
||||
|
||||
public static TimeDatabase deserialize(String path) {
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(path);
|
||||
String json = reader.readLine();
|
||||
reader.close();
|
||||
Gson gson = new Gson();
|
||||
return gson.fromJson(json, TimeDatabase.class);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import weka.core.DenseInstance;
|
||||
import weka.core.Instance;
|
||||
import weka.core.Utils;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
|
||||
public class TimePoint {
|
||||
public static final int LABELED = 0;
|
||||
public static final int UNLABELED = 1;
|
||||
public static final int EMPTY = 2;
|
||||
|
||||
private static final SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
private long timestamp;
|
||||
private int sessionId;
|
||||
private double[] features;
|
||||
private int label;
|
||||
|
||||
private Instance instance;
|
||||
|
||||
/**
|
||||
* $f_{obser}(o^{v}_{t},s^{v}_{t})$
|
||||
*/
|
||||
private double[] obserProbDist;
|
||||
|
||||
/**
|
||||
* $f_{intra}(s^{v}_{t-i},s^{v}_{t})$
|
||||
*/
|
||||
private double[] intraProbDist;
|
||||
|
||||
/**
|
||||
* $f_{inter}(s^{u}_{t-i},s^{v}_{t})$
|
||||
*/
|
||||
private double[] interProbDist;
|
||||
|
||||
private double[] finalProbDist;
|
||||
|
||||
public TimePoint(long timestamp, int sessionId, double[] features, int label) {
|
||||
this.timestamp = timestamp;
|
||||
this.sessionId = sessionId;
|
||||
this.features = features;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public TimePoint(TimePoint tp) {
|
||||
this.timestamp = tp.timestamp;
|
||||
this.sessionId = tp.sessionId;
|
||||
this.features = tp.features;
|
||||
this.label = tp.label;
|
||||
}
|
||||
|
||||
public long getTimestamp() {
|
||||
return timestamp;
|
||||
}
|
||||
|
||||
public void setTimestamp(long timestamp) {
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
public int getSessionId() {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public void setSessionId(int sessionId) {
|
||||
this.sessionId = sessionId;
|
||||
}
|
||||
|
||||
public double[] getFeatures() {
|
||||
return features;
|
||||
}
|
||||
|
||||
public void setFeatures(double[] features) {
|
||||
this.features = features;
|
||||
}
|
||||
|
||||
public int getLabel() {
|
||||
return label;
|
||||
}
|
||||
|
||||
public void setLabel(int label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public boolean isLabeled() {
|
||||
return features != null && label != -1;
|
||||
}
|
||||
|
||||
public boolean isUnlabeled() {
|
||||
return features != null && label == -1;
|
||||
}
|
||||
|
||||
public boolean isEmpty() {
|
||||
return features == null;
|
||||
}
|
||||
|
||||
public int getType() {
|
||||
if (features != null) {
|
||||
return (label != -1) ? LABELED : UNLABELED;
|
||||
} else {
|
||||
return EMPTY;
|
||||
}
|
||||
}
|
||||
|
||||
public Instance getInstance() {
|
||||
return instance;
|
||||
}
|
||||
|
||||
public Instance transferToInstance() {
|
||||
if (this.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
double[] features = this.getFeatures();
|
||||
double[] values = new double[features.length + 1];
|
||||
System.arraycopy(features, 0, values, 0, features.length);
|
||||
if (this.isLabeled()) {
|
||||
values[values.length - 1] = this.getLabel();
|
||||
} else {
|
||||
values[values.length - 1] = Utils.missingValue();
|
||||
}
|
||||
this.instance = new DenseInstance(1.0, values);
|
||||
return this.instance;
|
||||
}
|
||||
|
||||
public double[] getObserProbDist() {
|
||||
return obserProbDist;
|
||||
}
|
||||
|
||||
public void setObserProbDist(double[] obserProbDist) {
|
||||
this.obserProbDist = obserProbDist;
|
||||
}
|
||||
|
||||
public double[] getIntraProbDist() {
|
||||
return intraProbDist;
|
||||
}
|
||||
|
||||
public void setIntraProbDist(double[] intraProbDist) {
|
||||
this.intraProbDist = intraProbDist;
|
||||
}
|
||||
|
||||
public double[] getInterProbDist() {
|
||||
return interProbDist;
|
||||
}
|
||||
|
||||
public void setInterProbDist(double[] interProbDist) {
|
||||
this.interProbDist = interProbDist;
|
||||
}
|
||||
|
||||
public double[] getFinalProbDist() {
|
||||
return finalProbDist;
|
||||
}
|
||||
|
||||
public void setFinalProbDist(double[] finalProbDist) {
|
||||
this.finalProbDist = finalProbDist;
|
||||
}
|
||||
|
||||
public void cleanCache() {
|
||||
this.instance = null;
|
||||
this.obserProbDist = null;
|
||||
this.intraProbDist = null;
|
||||
this.interProbDist = null;
|
||||
this.finalProbDist = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("timestamp:%s,sessionId:%d,features:%s,internalScore:%d",
|
||||
SDF.format(new Date(timestamp)), sessionId, Arrays.toString(features), label);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class TimeSeries {
|
||||
private String team;
|
||||
private String user;
|
||||
private List<TimePoint> timePoints;
|
||||
|
||||
public TimeSeries(String team, String user) {
|
||||
this.team = team;
|
||||
this.user = user;
|
||||
this.timePoints = new ArrayList<TimePoint>();
|
||||
}
|
||||
|
||||
public void addTimePoint(TimePoint tp) {
|
||||
if (tp != null) {
|
||||
timePoints.add(tp);
|
||||
}
|
||||
}
|
||||
|
||||
public String getTeam() {
|
||||
return team;
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public List<TimePoint> getTimePoints() {
|
||||
return timePoints;
|
||||
}
|
||||
|
||||
public TimePoint findClosestTimePoint(long timestamp, long threshold, boolean labeled, boolean unlabeled, boolean empty) {
|
||||
if (timePoints.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
int minIndex = -1;
|
||||
for (int index = 0, l = timePoints.size(); index < l; index++) {
|
||||
TimePoint tp = timePoints.get(index);
|
||||
|
||||
if (tp.isLabeled() && (!labeled)) {
|
||||
continue;
|
||||
}
|
||||
if (tp.isUnlabeled() && (!unlabeled)) {
|
||||
continue;
|
||||
}
|
||||
if (tp.isEmpty() && (!empty)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long delta = Math.abs(tp.getTimestamp() - timestamp);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
minIndex = index;
|
||||
}
|
||||
}
|
||||
|
||||
return (minDelta <= threshold) ? timePoints.get(minIndex) : null;
|
||||
}
|
||||
|
||||
public int findClosestIndex(long timestamp, long threshold, boolean labeled, boolean unlabeled, boolean empty) {
|
||||
if (timePoints.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
int minIndex = -1;
|
||||
for (int index = 0, l = timePoints.size(); index < l; index++) {
|
||||
TimePoint tp = timePoints.get(index);
|
||||
|
||||
if (tp.isLabeled() && (!labeled)) {
|
||||
continue;
|
||||
}
|
||||
if (tp.isUnlabeled() && (!unlabeled)) {
|
||||
continue;
|
||||
}
|
||||
if (tp.isEmpty() && (!empty)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long delta = Math.abs(tp.getTimestamp() - timestamp);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
minIndex = index;
|
||||
}
|
||||
}
|
||||
|
||||
return (minDelta <= threshold) ? minIndex : -1;
|
||||
}
|
||||
|
||||
public int findClosestLabel(long timestamp, long threshold) {
|
||||
if (timePoints.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
long minDelta = Long.MAX_VALUE;
|
||||
int minLabel = -1;
|
||||
for (int index = 0, l = timePoints.size(); index < l; index++) {
|
||||
TimePoint tp = timePoints.get(index);
|
||||
if (tp.isLabeled()) {
|
||||
long delta = Math.abs(tp.getTimestamp() - timestamp);
|
||||
if (delta < minDelta) {
|
||||
minDelta = delta;
|
||||
minLabel = tp.getLabel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (minDelta <= threshold) ? minLabel : -1;
|
||||
}
|
||||
|
||||
public int[] countLabeledDataWithClass(int[] classes) {
|
||||
int[] result = TimeDatabase.zeros(classes.length);
|
||||
for (TimePoint tp : this.timePoints) {
|
||||
if (tp.isLabeled()) {
|
||||
int label = tp.getLabel();
|
||||
Assertion.assertNotNegative(label);
|
||||
result[TimeDatabase.argWhere(classes, label)]++;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<Integer> getIndicesWithClass(int cls) {
|
||||
List<Integer> result = new ArrayList<Integer>();
|
||||
for (int i = 0; i < this.timePoints.size(); i++) {
|
||||
TimePoint tp = this.timePoints.get(i);
|
||||
if (tp.isLabeled() && tp.getLabel() == cls) {
|
||||
result.add(i);
|
||||
}
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
public abstract class TrainAndTestSession {
|
||||
protected String name;
|
||||
protected int fold;
|
||||
protected double validRate;
|
||||
protected boolean isPrintInfo;
|
||||
|
||||
public TrainAndTestSession(String name, int fold, double validRate, boolean isPrintInfo) {
|
||||
this.name = name;
|
||||
this.fold = fold;
|
||||
this.validRate = validRate;
|
||||
this.isPrintInfo = isPrintInfo;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public int getFold() {
|
||||
return fold;
|
||||
}
|
||||
|
||||
public void setFold(int fold) {
|
||||
this.fold = fold;
|
||||
}
|
||||
|
||||
public double getValidRate() {
|
||||
return validRate;
|
||||
}
|
||||
|
||||
public void setValidRate(double validRate) {
|
||||
this.validRate = validRate;
|
||||
}
|
||||
|
||||
public boolean isPrintInfo() {
|
||||
return isPrintInfo;
|
||||
}
|
||||
|
||||
public void setPrintInfo(boolean printInfo) {
|
||||
isPrintInfo = printInfo;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public Map<String, Evaluation> crossValidation(String root) throws Exception {
|
||||
Configuration.out.println(toString());
|
||||
|
||||
// attempt to load training set, validation set, and test set from local file system
|
||||
boolean isLoadDataFromLocalFileSystem;
|
||||
if (root != null) {
|
||||
isLoadDataFromLocalFileSystem = true;
|
||||
for (int foldIndex = 0; foldIndex < this.fold; foldIndex++) {
|
||||
File trainFile = new File(root + File.separator + "fold_" + foldIndex + File.separator + "train.json");
|
||||
if (!trainFile.exists()) {
|
||||
isLoadDataFromLocalFileSystem = false;
|
||||
break;
|
||||
}
|
||||
File testFile = new File(root + File.separator + "fold_" + foldIndex + File.separator + "test.json");
|
||||
if (!testFile.exists()) {
|
||||
isLoadDataFromLocalFileSystem = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
isLoadDataFromLocalFileSystem = false;
|
||||
}
|
||||
|
||||
// container of confusion matrices generated in each fold
|
||||
Map<String, List<int[]>> tmp = new HashMap<String, List<int[]>>();
|
||||
int[] classes = null;
|
||||
if (isLoadDataFromLocalFileSystem) {
|
||||
System.out.println("load data from local file system: " + root);
|
||||
// cross validation
|
||||
for (int foldIndex = 0; foldIndex < this.fold; foldIndex++) {
|
||||
Configuration.out.printf(" > %s:fold %d\n", this.name, foldIndex);
|
||||
// get training and test sets
|
||||
File trainFile = new File(root + File.separator + "fold_" + foldIndex + File.separator + "train.json");
|
||||
TimeDatabase trainDb = TimeDatabase.deserialize(trainFile.getAbsolutePath());
|
||||
File testFile = new File(root + File.separator + "fold_" + foldIndex + File.separator + "test.json");
|
||||
TimeDatabase testDb = TimeDatabase.deserialize(testFile.getAbsolutePath());
|
||||
File validFile = new File(root + File.separator + "fold_" + foldIndex + File.separator + "valid.json");
|
||||
TimeDatabase validDb = validFile.exists() ? TimeDatabase.deserialize(validFile.getAbsolutePath()) : null;
|
||||
// initial classes
|
||||
if (classes == null) {
|
||||
classes = trainDb.getClasses();
|
||||
}
|
||||
// log
|
||||
if (this.isPrintInfo) {
|
||||
Configuration.out.println(trainDb.simpleDescribe(String.format("%s:training set", this.name)));
|
||||
Configuration.out.println(validDb == null ? "null" : validDb.simpleDescribe(String.format("%s:validation set", this.name)));
|
||||
Configuration.out.println(testDb.simpleDescribe(String.format("%s:test set", this.name)));
|
||||
}
|
||||
// do validation
|
||||
Map<String, List<int[]>> truePredLabelPairsMap = validation(foldIndex, trainDb, validDb, testDb);
|
||||
for (Map.Entry<String, List<int[]>> entry : truePredLabelPairsMap.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
List<int[]> truePredLabelPairs = entry.getValue();
|
||||
List<int[]> result = tmp.get(key);
|
||||
if (result == null) {
|
||||
result = new ArrayList<int[]>();
|
||||
}
|
||||
result.addAll(truePredLabelPairs);
|
||||
tmp.put(key, result);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
System.out.println("init data from scratch");
|
||||
// load dataset
|
||||
TimeDatabase db = DataSource.initDatabase();
|
||||
// log
|
||||
if (this.isPrintInfo) {
|
||||
Configuration.out.println(db.describe(this.name + ":dataset"));
|
||||
}
|
||||
// split training and test sets
|
||||
int tpNum = db.getNumberOfTimePoints();
|
||||
Assertion.assertPositive(tpNum);
|
||||
int[][][] trainAndTestFolds = DataSource.splitTrainAndTestByKFold(tpNum, this.fold);
|
||||
// cross validation
|
||||
for (int foldIndex = 0; foldIndex < trainAndTestFolds.length; foldIndex++) {
|
||||
Configuration.out.printf(" > %s:fold %d\n", this.name, foldIndex);
|
||||
// get training and test sets
|
||||
int[] trainIndices = trainAndTestFolds[foldIndex][0];
|
||||
int[] testIndices = trainAndTestFolds[foldIndex][1];
|
||||
TimeDatabase[] trainValidAndTestDb = prepareTrainValidAndTestSets(db.slice(trainIndices), db.slice(testIndices));
|
||||
TimeDatabase trainDb = trainValidAndTestDb[0];
|
||||
TimeDatabase validDb = trainValidAndTestDb[1];
|
||||
TimeDatabase testDb = trainValidAndTestDb[2];
|
||||
// initial classes
|
||||
if (classes == null) {
|
||||
classes = trainDb.getClasses();
|
||||
}
|
||||
// serialize
|
||||
TimeDatabase.serialize(trainDb, root + File.separator + "fold_" + foldIndex + File.separator + "train.json");
|
||||
TimeDatabase.serialize(testDb, root + File.separator + "fold_" + foldIndex + File.separator + "test.json");
|
||||
if (validDb != null) {
|
||||
TimeDatabase.serialize(validDb, root + File.separator + "fold_" + foldIndex + File.separator + "valid.json");
|
||||
}
|
||||
// log
|
||||
if (this.isPrintInfo) {
|
||||
Configuration.out.println(trainDb.simpleDescribe(String.format("%s:training set", this.name)));
|
||||
Configuration.out.println(validDb == null ? "null" : validDb.simpleDescribe(String.format("%s:validation set", this.name)));
|
||||
Configuration.out.println(testDb.simpleDescribe(String.format("%s:test set", this.name)));
|
||||
}
|
||||
// do validation
|
||||
Map<String, List<int[]>> truePredLabelPairsMap = validation(foldIndex, trainDb, validDb, testDb);
|
||||
for (Map.Entry<String, List<int[]>> entry : truePredLabelPairsMap.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
List<int[]> truePredLabelPairs = entry.getValue();
|
||||
List<int[]> result = tmp.get(key);
|
||||
if (result == null) {
|
||||
result = new ArrayList<int[]>();
|
||||
}
|
||||
result.addAll(truePredLabelPairs);
|
||||
tmp.put(key, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<String> keys = new ArrayList<String>(tmp.keySet());
|
||||
Collections.sort(keys);
|
||||
Map<String, Evaluation> result = new HashMap<String, Evaluation>();
|
||||
for (String key : keys) {
|
||||
List<int[]> value = tmp.get(key);
|
||||
Evaluation evaluation = Evaluation.evaluate(key, classes, value, true);
|
||||
result.put(key, evaluation);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
protected abstract TimeDatabase[] prepareTrainValidAndTestSets(TimeDatabase trainDb, TimeDatabase testDb);
|
||||
|
||||
protected abstract Map<String, List<int[]>> validation(int foldIndex, TimeDatabase trainDb, TimeDatabase validDb, TimeDatabase testDb) throws Exception;
|
||||
}
|
||||
|
|
@ -0,0 +1,301 @@
|
|||
package edu.nju.ics.frontier.learning;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileFilter;
|
||||
|
||||
public class ValidationResultAnalysis {
|
||||
public static void main(String[] args) {
|
||||
String coarseRegex = "[0-9]{8}_[0-9]{6}_erase_rate_0[.][0-9]0[.]txt";
|
||||
String coarseCvFilename = "cv_erase_rate_from_0.1_to_0.9_in_0.1_step.csv";
|
||||
String coarseTvFilename = "tv_erase_rate_from_0.1_to_0.9_in_0.1_step.csv";
|
||||
String fineRegex = "[0-9]{8}_[0-9]{6}_erase_rate_0[.][0-9][1-9][.]txt";
|
||||
String fineCvFilename = "cv_erase_rate_from_0.91_to_0.99_in_0.01_step.csv";
|
||||
String fineTvFilename = "tv_erase_rate_from_0.91_to_0.99_in_0.01_step.csv";
|
||||
parseCrossValidationResult(coarseRegex, coarseCvFilename);
|
||||
parseCrossValidationResult(fineRegex, fineCvFilename);
|
||||
parseTrainAndValidResult(coarseRegex, coarseTvFilename);
|
||||
parseTrainAndValidResult(fineRegex, fineTvFilename);
|
||||
}
|
||||
|
||||
private static void parseTrainAndValidResult(final String regex, final String filename) {
|
||||
String root = "C:\\Users\\zzw\\Desktop\\fujitsu\\log\\";
|
||||
File[] files = new File(root).listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isFile() &&
|
||||
pathname.getName().matches(regex);
|
||||
}
|
||||
});
|
||||
Configuration.out.open(root + filename);
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
String name = files[i].getName();
|
||||
double eraseRate = Double.parseDouble(name.substring(name.lastIndexOf("_") + 1, name.lastIndexOf(".")));
|
||||
// Configuration.out.open(String.format("%stv_erase_rate_%.2f.csv", root, eraseRate));
|
||||
System.out.printf("[%2d] %.2f %s\n", i + 1, eraseRate, files[i].getName());
|
||||
parseTrainAndValidResult(files[i].getAbsolutePath(), eraseRate, i == 0);
|
||||
}
|
||||
Configuration.out.close();
|
||||
}
|
||||
|
||||
private static void parseTrainAndValidResult(String path, double eraseRate, boolean hasHeader) {
|
||||
int foldIndex = -1;
|
||||
String model = null;
|
||||
String strategy = null;
|
||||
int epoch = -1;
|
||||
Dataset trainSet = null;
|
||||
Dataset validSet = null;
|
||||
double distance = -1;
|
||||
Metrics finalMetrics = null;
|
||||
Metrics interMetrics = null;
|
||||
Metrics intraMetrics = null;
|
||||
Metrics obserMetrics = null;
|
||||
int rowCount = 0;
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(path);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
// parse fold
|
||||
if (line.startsWith(" >")) {
|
||||
foldIndex = Integer.parseInt(line.substring(line.length() - 1));
|
||||
}
|
||||
// parse model+strategy+epoch{
|
||||
if (line.endsWith("{")) {
|
||||
Assertion.assertNotNegative(foldIndex);
|
||||
String[] info = line.substring(0, line.length() - 1).split("[+]");
|
||||
model = info[0];
|
||||
strategy = info[1];
|
||||
if ("init".equals(info[2])) {
|
||||
epoch = 0;
|
||||
} else {
|
||||
epoch = Integer.parseInt(info[2]) + 1;
|
||||
}
|
||||
}
|
||||
// parse train{all:[0-9]+,lbl:[0-9]+,unl:[0-9]+,emp:[0-9]+,cls\[[0-9]+,[0-9]+,[0-9]+\]}
|
||||
// parse valid{all:[0-9]+,lbl:[0-9]+,unl:[0-9]+,emp:[0-9]+,cls\[[0-9]+,[0-9]+,[0-9]+\]}
|
||||
if (line.startsWith("train") || line.startsWith("valid")) {
|
||||
if (line.equals("valid{null}")) {
|
||||
continue;
|
||||
}
|
||||
int labeledNum = Integer.parseInt(line.substring(line.indexOf("lbl") + 4, line.indexOf("unl") - 1));
|
||||
int unlabeledNum = Integer.parseInt(line.substring(line.indexOf("unl") + 4, line.indexOf("emp") - 1));
|
||||
String[] classNumStrs = line.substring(line.indexOf("cls") + 4, line.length() - 2).split(",");
|
||||
int[] classNums = new int[classNumStrs.length];
|
||||
for (int i = 0; i < classNums.length; i++) {
|
||||
classNums[i] = Integer.parseInt(classNumStrs[i]);
|
||||
}
|
||||
Dataset dataset = new Dataset(labeledNum, unlabeledNum, classNums);
|
||||
if (line.startsWith("train")) {
|
||||
Assertion.assertNotNull(model);
|
||||
Assertion.assertNotNull(strategy);
|
||||
Assertion.assertNotNegative(epoch);
|
||||
trainSet = dataset;
|
||||
} else {
|
||||
Assertion.assertNotNull(trainSet);
|
||||
validSet = dataset;
|
||||
}
|
||||
}
|
||||
// parse change:9363.3297839872539043426513671875 or retain
|
||||
if (line.startsWith("change") || line.startsWith("retain")) {
|
||||
if (line.startsWith("change")) {
|
||||
distance = Double.parseDouble(line.substring(line.indexOf(":") + 1));
|
||||
} else {
|
||||
distance = 0.0;
|
||||
}
|
||||
}
|
||||
// parse final{Accuracy:0.155,Precision:0.948,Recall:0.155,F1 Score:0.184,Matrix:[[2,1,67][0,10,14][0,0,3]]}
|
||||
// parse inter{Accuracy:0.206,Precision:0.831,Recall:0.206,F1 Score:0.239,Matrix:[[7,13,50][0,10,14][0,0,3]]}
|
||||
// parse intra{Accuracy:0.113,Precision:0.929,Recall:0.113,F1 Score:0.144,Matrix:[[3,1,66][0,5,19][0,0,3]]}
|
||||
// parse obser{Accuracy:0.227,Precision:0.814,Recall:0.227,F1 Score:0.252,Matrix:[[8,19,43][0,11,13][0,0,3]]}
|
||||
if (line.startsWith("final") || line.startsWith("inter") || line.startsWith("intra") || line.startsWith("obser")) {
|
||||
double accuracy = Double.parseDouble(line.substring(line.indexOf("Accuracy:") + 9, line.indexOf("Precision:") - 1));
|
||||
double precision = Double.parseDouble(line.substring(line.indexOf("Precision:") + 10, line.indexOf("Recall:") - 1));
|
||||
double recall = Double.parseDouble(line.substring(line.indexOf("Recall:") + 7, line.indexOf("F1 Score:") - 1));
|
||||
double f1Score = Double.parseDouble(line.substring(line.indexOf("F1 Score:") + 9, line.indexOf("Matrix:") - 1));
|
||||
Metrics metrics = new Metrics(accuracy, precision, recall, f1Score);
|
||||
if (line.startsWith("final")) {
|
||||
finalMetrics = metrics;
|
||||
} else if (line.startsWith("inter")) {
|
||||
interMetrics = metrics;
|
||||
} else if (line.startsWith("intra")) {
|
||||
intraMetrics = metrics;
|
||||
} else {
|
||||
obserMetrics = metrics;
|
||||
}
|
||||
}
|
||||
// parse }
|
||||
if (line.equals("}")) {
|
||||
if (hasHeader && rowCount == 0) {
|
||||
Configuration.out.print("EraseRate,Fold,Models,Strategies,Combinations,Epoch,TrainLabeledNum,TrainUnlabeledNum,TrainClassNum,TrainLabeledPro");
|
||||
if (validSet != null) {
|
||||
Configuration.out.print(",ValidLabeledNum,ValidUnlabeledNum,ValidClassNum,ValidLabeledPro");
|
||||
}
|
||||
Configuration.out.print(",Distance");
|
||||
if (obserMetrics != null) {
|
||||
Configuration.out.print(",obserAccuracy,obserPrecision,obserRecall,obserF1Score");
|
||||
}
|
||||
if (intraMetrics != null) {
|
||||
Configuration.out.print(",intraAccuracy,intraPrecision,intraRecall,intraF1Score");
|
||||
}
|
||||
if (interMetrics != null) {
|
||||
Configuration.out.print(",interAccuracy,interPrecision,interRecall,interF1Score");
|
||||
}
|
||||
if (finalMetrics != null) {
|
||||
Configuration.out.print(",finalAccuracy,finalPrecision,finalRecall,finalF1Score");
|
||||
}
|
||||
Configuration.out.println();
|
||||
}
|
||||
Configuration.out.printf("%.2f,%d,%s,%s,%s,%d,%s", eraseRate, foldIndex, model, strategy, model + "+" + strategy, epoch, trainSet);
|
||||
model = null;
|
||||
strategy = null;
|
||||
epoch = -1;
|
||||
trainSet = null;
|
||||
if (validSet != null) {
|
||||
Configuration.out.printf(",%s", validSet);
|
||||
validSet = null;
|
||||
}
|
||||
Configuration.out.printf(",%f", distance);
|
||||
distance = -1;
|
||||
if (obserMetrics != null) {
|
||||
Configuration.out.printf(",%s", obserMetrics);
|
||||
obserMetrics = null;
|
||||
}
|
||||
if (intraMetrics != null) {
|
||||
Configuration.out.printf(",%s", intraMetrics);
|
||||
intraMetrics = null;
|
||||
}
|
||||
if (interMetrics != null) {
|
||||
Configuration.out.printf(",%s", interMetrics);
|
||||
interMetrics = null;
|
||||
}
|
||||
if (finalMetrics != null) {
|
||||
Configuration.out.printf(",%s", finalMetrics);
|
||||
finalMetrics = null;
|
||||
}
|
||||
Configuration.out.println();
|
||||
rowCount++;
|
||||
}
|
||||
}
|
||||
reader.close();
|
||||
}
|
||||
|
||||
private static void parseCrossValidationResult(final String regex, final String filename) {
|
||||
String root = "C:\\Users\\zzw\\Desktop\\fujitsu\\log\\";
|
||||
File[] files = new File(root).listFiles(new FileFilter() {
|
||||
public boolean accept(File pathname) {
|
||||
return pathname.isFile() &&
|
||||
pathname.getName().matches(regex);
|
||||
}
|
||||
});
|
||||
Configuration.out.open(root + filename);
|
||||
Configuration.out.println("EraseRate,Models,Strategies,Combinations,Phases,Metrics,Value");
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
String name = files[i].getName();
|
||||
double eraseRate = Double.parseDouble(name.substring(name.lastIndexOf("_") + 1, name.lastIndexOf(".")));
|
||||
System.out.printf("[%2d] %.2f %s\n", i + 1, eraseRate, files[i].getName());
|
||||
parseCrossValidationResult(files[i].getAbsolutePath(), eraseRate);
|
||||
}
|
||||
Configuration.out.close();
|
||||
}
|
||||
|
||||
private static void parseCrossValidationResult(String path, final double eraseRate) {
|
||||
String model = null;
|
||||
String strategy = null;
|
||||
String phase = null;
|
||||
double accuracy = -1.0;
|
||||
double precision = -1.0;
|
||||
double recall = -1.0;
|
||||
double f1Score = -1.0;
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(path);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.startsWith("[")) {
|
||||
Assertion.assertNull(model);
|
||||
Assertion.assertNull(strategy);
|
||||
Assertion.assertNull(phase);
|
||||
Assertion.assertEqual(accuracy, -1.0);
|
||||
Assertion.assertEqual(precision, -1.0);
|
||||
Assertion.assertEqual(recall, -1.0);
|
||||
Assertion.assertEqual(f1Score, -1.0);
|
||||
String[] info = line.substring(1, line.length() - 1).split(":");
|
||||
model = info[0];
|
||||
strategy = info[1];
|
||||
phase = info[2];
|
||||
} else if (line.startsWith("Correctly")) {
|
||||
Assertion.assertNotNull(model);
|
||||
Assertion.assertNotNull(strategy);
|
||||
Assertion.assertNotNull(phase);
|
||||
accuracy = Double.parseDouble(line.substring(line.length() - 9, line.length() - 2));
|
||||
accuracy /= 100.0;
|
||||
} else if (line.startsWith("Weighted")) {
|
||||
Assertion.assertNotNegative(accuracy);
|
||||
precision = Double.parseDouble(line.substring(35, 40));
|
||||
recall = Double.parseDouble(line.substring(46, 51));
|
||||
f1Score = Double.parseDouble(line.substring(55, 60));
|
||||
Configuration.out.printf("%.2f,%s,%s,%s,%s,Accuracy,%.3f\n", eraseRate, model, strategy, model + "+" + strategy, phase, accuracy);
|
||||
Configuration.out.printf("%.2f,%s,%s,%s,%s,Precision,%.3f\n", eraseRate, model, strategy, model + "+" + strategy, phase, precision);
|
||||
Configuration.out.printf("%.2f,%s,%s,%s,%s,Recall,%.3f\n", eraseRate, model, strategy, model + "+" + strategy, phase, recall);
|
||||
Configuration.out.printf("%.2f,%s,%s,%s,%s,F1 Score,%.3f\n", eraseRate, model, strategy, model + "+" + strategy, phase, f1Score);
|
||||
model = null;
|
||||
strategy = null;
|
||||
phase = null;
|
||||
accuracy = -1.0;
|
||||
precision = -1.0;
|
||||
recall = -1.0;
|
||||
f1Score = -1.0;
|
||||
}
|
||||
|
||||
}
|
||||
reader.close();
|
||||
}
|
||||
|
||||
private static class Dataset {
|
||||
public final int labeledNum;
|
||||
public final int unlabeledNum;
|
||||
public final int[] classNums;
|
||||
public final double labeledProportion;
|
||||
|
||||
public Dataset(int labeledNum, int unlabeledNum, int[] classNums) {
|
||||
this.labeledNum = labeledNum;
|
||||
this.unlabeledNum = unlabeledNum;
|
||||
this.classNums = classNums;
|
||||
this.labeledProportion = ((double) this.labeledNum) / ((double) (this.labeledNum + this.unlabeledNum));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(labeledNum).append(",").append(unlabeledNum).append(",");
|
||||
builder.append(classNums[0]);
|
||||
for (int i = 1; i < classNums.length; i++) {
|
||||
builder.append(":").append(classNums[i]);
|
||||
}
|
||||
builder.append(",").append(labeledProportion);
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
||||
private static class Metrics {
|
||||
public final double accuracy;
|
||||
public final double precision;
|
||||
public final double recall;
|
||||
public final double f1Score;
|
||||
|
||||
public Metrics(double accuracy, double precision, double recall, double f1Score) {
|
||||
this.accuracy = accuracy;
|
||||
this.precision = precision;
|
||||
this.recall = recall;
|
||||
this.f1Score = f1Score;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("%.3f,%.3f,%.3f,%.3f", accuracy, precision, recall, f1Score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
package edu.nju.ics.frontier.motivating_example;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.List;
|
||||
|
||||
public class DataPreparation {
|
||||
public static void main(String[] args) {
|
||||
String root = "C:\\Users\\zzw\\Desktop\\fujitsu\\motivating_example\\";
|
||||
List<String[]> srcData = loadCsvData(root + "representative1.csv");
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
writer.open(root + "representative2.csv");
|
||||
writer.println("team,user,date,timestamp,timedelta,addedLocSum,deletedLocSum,locGainSum,addedOrDeletedLocSum,label");
|
||||
int offset = 1;
|
||||
List<String[]> srcSession = getNextSession(srcData, offset);
|
||||
while (srcSession != null) {
|
||||
List<String[]> destSession = downSampling(srcSession, 2, false);
|
||||
if (destSession != null) {
|
||||
List<String[]> variables = extractVariables(srcData.get(0), destSession);
|
||||
saveCsvData(variables, writer);
|
||||
}
|
||||
offset += srcSession.size();
|
||||
srcSession = getNextSession(srcData, offset);
|
||||
}
|
||||
writer.close();
|
||||
}
|
||||
|
||||
private static List<String[]> loadCsvData(String path) {
|
||||
List<String[]> result = new ArrayList<String[]>();
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(path);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
result.add(line.split(","));
|
||||
}
|
||||
reader.close();
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static List<String[]> extractVariables(String[] head, List<String[]> session) {
|
||||
List<String[]> result = new ArrayList<String[]>();
|
||||
int teamIndex = argWhere(head, "team");
|
||||
int userIndex = argWhere(head, "user");
|
||||
int dateIndex = argWhere(head, "date");
|
||||
int sliceBegTimeIndex = argWhere(head, "sliceBegTime");
|
||||
int addedLocSumIndex = argWhere(head, "addedLocSum");
|
||||
int deletedLocSumIndex = argWhere(head, "deletedLocSum");
|
||||
int locGainSumIndex = argWhere(head, "locGainSum");
|
||||
int addedOrDeletedLocSumIndex = argWhere(head, "addedOrDeletedLocSum");
|
||||
int internalScoreIndex = argWhere(head, "internalScore");
|
||||
for (String[] row : session) {
|
||||
int internalScore = Integer.parseInt(row[internalScoreIndex]);
|
||||
// Assertion.assertPositive(internalScore);
|
||||
int label;
|
||||
if (internalScore == -1) {
|
||||
label = 0;
|
||||
} else if (internalScore < 3) {
|
||||
label = 1;
|
||||
} else if (internalScore == 3) {
|
||||
label = 2;
|
||||
} else {
|
||||
label = 3;
|
||||
}
|
||||
long timestamp = Long.parseLong(row[sliceBegTimeIndex]);
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.setTimeInMillis(timestamp);
|
||||
calendar.set(Calendar.MILLISECOND, 0);
|
||||
calendar.set(Calendar.SECOND, 0);
|
||||
calendar.set(Calendar.MINUTE, 0);
|
||||
calendar.set(Calendar.HOUR_OF_DAY, 0);
|
||||
long timedelta = timestamp - calendar.getTimeInMillis();
|
||||
double second = ((double) timedelta) / 1000.0;
|
||||
double minute = second / 60.0;
|
||||
double hour = minute / 60.0;
|
||||
result.add(new String[]{
|
||||
row[teamIndex],
|
||||
row[userIndex],
|
||||
row[dateIndex],
|
||||
row[sliceBegTimeIndex],
|
||||
String.valueOf(hour),
|
||||
row[addedLocSumIndex],
|
||||
row[deletedLocSumIndex],
|
||||
row[locGainSumIndex],
|
||||
row[addedOrDeletedLocSumIndex],
|
||||
String.valueOf(label),
|
||||
});
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static void saveCsvData(List<String[]> variables, OkTextWriter writer) {
|
||||
if (variables != null) {
|
||||
for (String[] row : variables) {
|
||||
for (int i = 0; i < row.length - 1; i++) {
|
||||
writer.print(row[i] + ",");
|
||||
}
|
||||
writer.println(row[row.length - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String[]> getNextSession(List<String[]> data, int offset) {
|
||||
if (data == null || data.isEmpty() || offset < 0 || offset >= data.size()) {
|
||||
return null;
|
||||
}
|
||||
List<String[]> result = new ArrayList<String[]>();
|
||||
String[] row = data.get(offset);
|
||||
int sessionId = Integer.parseInt(row[7]);
|
||||
result.add(row);
|
||||
for (int i = offset + 1; i < data.size(); i++) {
|
||||
row = data.get(i);
|
||||
int id = Integer.parseInt(row[7]);
|
||||
if (sessionId == id) {
|
||||
result.add(row);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static List<String[]> downSampling(List<String[]> session, int stride, boolean isOnlyLabeled) {
|
||||
List<String[]> result = new ArrayList<String[]>();
|
||||
|
||||
long sliceBegTime = Long.parseLong(session.get(0)[3]);
|
||||
long thirtyMinute = 30 * 60 * 1000;
|
||||
int i;
|
||||
if (sliceBegTime % thirtyMinute == 0) {
|
||||
i = 0;
|
||||
} else {
|
||||
i = 1;
|
||||
}
|
||||
for (; i < session.size(); i += stride) {
|
||||
String[] row = session.get(i);
|
||||
if (isOnlyLabeled) {
|
||||
if (Integer.parseInt(row[row.length - 2]) == -1) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
// int prevIndex = -1;
|
||||
// int nextIndex = getIndexOfNextLabeledItem(session, 0);
|
||||
// while (nextIndex != -1) {
|
||||
// if (prevIndex == -1) {
|
||||
// result.add(session.get(nextIndex));
|
||||
// prevIndex = nextIndex;
|
||||
// nextIndex = getIndexOfNextLabeledItem(session, nextIndex + 1);
|
||||
// } else {
|
||||
// if (nextIndex - prevIndex >= stride) {
|
||||
// result.add(session.get(nextIndex));
|
||||
// prevIndex = nextIndex;
|
||||
// nextIndex = getIndexOfNextLabeledItem(session, nextIndex + 1);
|
||||
// } else {
|
||||
// nextIndex = getIndexOfNextLabeledItem(session, nextIndex + 1);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// for (int i = 0; i < session.size(); i += stride) {
|
||||
// String[] row = session.get(i);
|
||||
// if (isOnlyLabeled) {
|
||||
// if (Integer.parseInt(row[row.length - 2]) == -1) {
|
||||
// continue;
|
||||
// }
|
||||
// }
|
||||
// result.add(row);
|
||||
// }
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static int getIndexOfNextLabeledItem(List<String[]> session, int offset) {
|
||||
if (session == null || session.isEmpty() || offset < 0 || offset >= session.size()) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = offset; i < session.size(); i++) {
|
||||
String[] row = session.get(i);
|
||||
if (Integer.parseInt(row[row.length - 2]) != -1) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static <T> int argWhere(T[] array, T value) {
|
||||
for (int i = 0; i < array.length; i++) {
|
||||
if (array[i].equals(value)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class AbstractEventLabelTask implements Executor.Task {
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File dateDir = new File(FileUtil.combine(userDir.getAbsolutePath(), dateStr));
|
||||
if (!dateDir.exists()) {
|
||||
return;
|
||||
}
|
||||
|
||||
File[] eventOrLabelFiles = FileUtil.listSubFiles(dateDir, null, ".json");
|
||||
if (eventOrLabelFiles == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<File> eventFiles = new ArrayList<File>();
|
||||
List<File> labelFiles = new ArrayList<File>();
|
||||
for (File eventOrLabelFile : eventOrLabelFiles) {
|
||||
String fileName = eventOrLabelFile.getName();
|
||||
if (fileName.startsWith("event")) {
|
||||
eventFiles.add(eventOrLabelFile);
|
||||
} else if (fileName.startsWith("label")) {
|
||||
labelFiles.add(eventOrLabelFile);
|
||||
}
|
||||
}
|
||||
|
||||
List<IntelliJEvent> events = null;
|
||||
if (!eventFiles.isEmpty()) {
|
||||
events = new ArrayList<IntelliJEvent>();
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> tmp = DataSource.loadEvents(eventFile);
|
||||
if (tmp != null) {
|
||||
events.addAll(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<Label> labels = null;
|
||||
if (!labelFiles.isEmpty()) {
|
||||
labels = new ArrayList<Label>();
|
||||
for (File labelFile : labelFiles) {
|
||||
List<Label> tmp = DataSource.loadLabels(labelFile);
|
||||
if (tmp != null) {
|
||||
labels.addAll(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process(srcRoot, destRoot, teamDir, userDir, dateDir, events, labels);
|
||||
}
|
||||
|
||||
protected abstract void process(String srcRoot, String destRoot, File teamDir, File userDir, File dateDir, List<IntelliJEvent> events, List<Label> labels);
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.WindowEvent;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class CursorContextTask implements Executor.Task {
|
||||
private Map<String, String> mFrame2Project;
|
||||
private Map<String, Set<String>> mFrame2Files;
|
||||
private Map<String, String> mFrame2FocusedFile;
|
||||
private String mFocusedFrame;
|
||||
private String mFocusedProject;
|
||||
private String mFocusedFile;
|
||||
|
||||
public CursorContextTask() {
|
||||
this.mFrame2Project = new HashMap<String, String>();
|
||||
this.mFrame2Files = new HashMap<String, Set<String>>();
|
||||
this.mFrame2FocusedFile = new HashMap<String, String>();
|
||||
this.mFocusedFrame = null;
|
||||
this.mFocusedProject = null;
|
||||
this.mFocusedFile = null;
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
List<IntelliJEvent> events = ParseUtil.loadWorkdayEvents(dateDir);
|
||||
if (events == null) {
|
||||
continue;
|
||||
}
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isWindowEvent(event)) {
|
||||
WindowEvent windowEvent = IntelliJEvent.toWindowEvent(event);
|
||||
if (windowEvent.isWhenWindowActivated()) {
|
||||
String winName = windowEvent.getThisWindowName();
|
||||
if (winName.startsWith("frame")) {
|
||||
if (!winName.equals(mFocusedFrame)) {
|
||||
mFocusedFrame = winName;
|
||||
mFocusedProject = mFrame2Project.get(mFocusedFrame);
|
||||
mFocusedFile = mFrame2FocusedFile.get(mFocusedFrame);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.bean.intellij.CursorEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.FileEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.Logger;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public class IntelliJEventCleanTask implements Executor.Task {
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File intellijDir = new File(FileUtil.combine(userDir.getAbsolutePath(), "interaction_traces", "intellij"));
|
||||
if (!intellijDir.exists()) {
|
||||
return;
|
||||
}
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(getClass().getName()).append(" {").
|
||||
append("\n arguments:\n intellijDir=").
|
||||
append(intellijDir.getAbsolutePath()).
|
||||
append("\n date=").append(dateStr);
|
||||
|
||||
File dateDir = new File(FileUtil.combine(intellijDir.getAbsolutePath(), dateStr));
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, null, ".json");
|
||||
if (eventFiles != null) {
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> data = loadData(eventFile, userDir.getName());
|
||||
File dest = saveData(eventFile.getParentFile(), data);
|
||||
eventFile.delete();
|
||||
|
||||
builder.append("\n data cleaned:\n src=").
|
||||
append(eventFile.getAbsolutePath()).
|
||||
append("\n dest=").
|
||||
append(dest == null ? "null" : dest.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
|
||||
builder.append("\n}");
|
||||
Logger.println(builder.toString());
|
||||
}
|
||||
|
||||
private List<IntelliJEvent> loadData(File file, String user) {
|
||||
List<IntelliJEvent> result = new ArrayList<IntelliJEvent>();
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
Gson gson = new Gson();
|
||||
|
||||
reader.open(file);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
IntelliJEvent event = gson.fromJson(line, IntelliJEvent.class);
|
||||
if (isFileContentChangedEvent(event) ||
|
||||
isAnonymousFileChanged(event) ||
|
||||
isFileCreatedOrDeletedEvent(event) ||
|
||||
isCursorAutoChangedInConsoleEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IntelliJEvent.isFileEvent(event)) {
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
String oldPath = fileEvent.getOldPathField();
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
if (oldPath != null) {
|
||||
if (!oldPath.equals(newPath)) {
|
||||
result.add(event);
|
||||
}
|
||||
} else if (newPath != null) {
|
||||
result.add(event);
|
||||
}
|
||||
} else {
|
||||
String path = fileEvent.getPathField();
|
||||
String project = ProjectFileMapping.getProjectByFile4User(user, path);
|
||||
if (project != null) {
|
||||
event.addExtraData("project", project);
|
||||
result.add(event);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
result.add(event);
|
||||
}
|
||||
}
|
||||
reader.close();
|
||||
|
||||
if (!result.isEmpty()) {
|
||||
boolean isCursorChangedFollowingFileChangedPattern = true;
|
||||
for (int i = 0; i < result.size(); i++) {
|
||||
IntelliJEvent event = result.get(i);
|
||||
if (!(isFileChangedEvent(event) || isCursorChangedInConsoleEvent(event))) {
|
||||
isCursorChangedFollowingFileChangedPattern = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (isCursorChangedFollowingFileChangedPattern) {
|
||||
result.clear();
|
||||
}
|
||||
}
|
||||
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private boolean isFileContentChangedEvent(IntelliJEvent event) {
|
||||
return "File".equals(event.getType()) && "content_changed".equals(event.getWhen());
|
||||
}
|
||||
|
||||
private boolean isAnonymousFileChanged(IntelliJEvent event) {
|
||||
return "File".equals(event.getType()) && "changed".equals(event.getWhen()) &&
|
||||
(event.getExtraData("path") == null);
|
||||
}
|
||||
|
||||
private boolean isFileChangedEvent(IntelliJEvent event) {
|
||||
return "File".equals(event.getType()) && "changed".equals(event.getWhen());
|
||||
}
|
||||
|
||||
private boolean isCursorAutoChangedInConsoleEvent(IntelliJEvent event) {
|
||||
if (!IntelliJEvent.isCursorEvent(event)) {
|
||||
return false;
|
||||
}
|
||||
CursorEvent cursorEvent = IntelliJEvent.toCursorEvent(event);
|
||||
if (!cursorEvent.isWhenCursorChanged()) {
|
||||
return false;
|
||||
}
|
||||
if (!cursorEvent.isConsoleEditor()) {
|
||||
return false;
|
||||
}
|
||||
long[] oldPos = CursorEvent.getLineAndColumn(cursorEvent.getOldPosField());
|
||||
long[] newPos = CursorEvent.getLineAndColumn(cursorEvent.getNewPosField());
|
||||
return oldPos[1] == 0 && newPos[1] == 0;
|
||||
}
|
||||
|
||||
private boolean isCursorChangedInConsoleEvent(IntelliJEvent event) {
|
||||
if (!"Cursor".equals(event.getType())) {
|
||||
return false;
|
||||
}
|
||||
if (!"cursor_changed".equals(event.getWhen())) {
|
||||
return false;
|
||||
}
|
||||
if (!event.hasData()) {
|
||||
return false;
|
||||
}
|
||||
String editor = event.getData().get("editor");
|
||||
return (editor != null && "CONSOLE".equals(editor));
|
||||
}
|
||||
|
||||
private boolean isFileCreatedOrDeletedEvent(IntelliJEvent event) {
|
||||
if (!IntelliJEvent.isFileEvent(event)) {
|
||||
return false;
|
||||
}
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
return fileEvent.isWhenCreated() || fileEvent.isWhenDeleted();
|
||||
}
|
||||
|
||||
private File saveData(File dateDir, List<IntelliJEvent> data) {
|
||||
if (data == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
|
||||
String begTimeStr = sdf.format(new Date(data.get(0).getTime()));
|
||||
String endTimeStr = sdf.format(new Date(data.get(data.size() - 1).getTime()));
|
||||
String fileName = begTimeStr + "-" + endTimeStr + ".json";
|
||||
String path = FileUtil.combine(dateDir.getAbsolutePath(), fileName);
|
||||
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
Gson gson = new Gson();
|
||||
|
||||
writer.open(path);
|
||||
for (int i = 0; i < data.size(); i++) {
|
||||
String json = gson.toJson(data.get(i));
|
||||
writer.println(json);
|
||||
}
|
||||
writer.close();
|
||||
|
||||
return new File(path);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.Logger;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public class IntelliJEventParseTask implements Executor.Task {
|
||||
private static final long ONE_DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File intellijDir = new File(FileUtil.combine(userDir.getAbsolutePath(), "interaction_traces", "intellij"));
|
||||
if (!intellijDir.exists()) {
|
||||
return;
|
||||
}
|
||||
File destDir = new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName()));
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(getClass().getName()).append(" {").
|
||||
append("\n arguments:\n intellijDir=").
|
||||
append(intellijDir.getAbsolutePath()).
|
||||
append("\n destDir=").append(destDir.getAbsolutePath()).
|
||||
append("\n date=").append(dateStr);
|
||||
|
||||
List<IntelliJEvent> intellijData = parseDataFromIntellijDirectory(intellijDir, dateStr);
|
||||
builder.append("\n intellij parsed: #data=").
|
||||
append((intellijData == null) ? 0 : intellijData.size());
|
||||
|
||||
List<List<IntelliJEvent>> dividedData = divideData(intellijData);
|
||||
builder.append("\n data divided: #day=").
|
||||
append((dividedData == null) ? 0 : dividedData.size());
|
||||
|
||||
saveData(dividedData, destDir);
|
||||
builder.append("\n data saved\n}");
|
||||
|
||||
Logger.println(builder.toString());
|
||||
|
||||
}
|
||||
|
||||
private List<IntelliJEvent> parseDataFromIntellijDirectory(File intellijDir, String dateStr) {
|
||||
OkTextReader reader = new OkTextReader();
|
||||
Gson gson = new Gson();
|
||||
List<IntelliJEvent> events = new ArrayList<IntelliJEvent>();
|
||||
|
||||
File dateDir = new File(intellijDir.getAbsolutePath() + File.separator + dateStr);
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, null, ".json");
|
||||
if (eventFiles == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (File eventFile : eventFiles) {
|
||||
reader.open(eventFile);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
IntelliJEvent event = gson.fromJson(line, IntelliJEvent.class);
|
||||
events.add(event);
|
||||
}
|
||||
reader.close();
|
||||
}
|
||||
|
||||
return events.isEmpty() ? null : events;
|
||||
}
|
||||
|
||||
private List<List<IntelliJEvent>> divideData(List<IntelliJEvent> mergedData) {
|
||||
if (mergedData == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<IntelliJEvent> copiedMergedData = new ArrayList<IntelliJEvent>(mergedData);
|
||||
List<List<IntelliJEvent>> result = new ArrayList<List<IntelliJEvent>>();
|
||||
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.setTimeInMillis(copiedMergedData.get(0).getTime());
|
||||
calendar.set(Calendar.HOUR_OF_DAY, 6);
|
||||
calendar.set(Calendar.MINUTE, 0);
|
||||
calendar.set(Calendar.SECOND, 0);
|
||||
calendar.set(Calendar.MILLISECOND, 0);
|
||||
long endTime = calendar.getTimeInMillis() + ONE_DAY;
|
||||
|
||||
List<IntelliJEvent> tmp = new ArrayList<IntelliJEvent>();
|
||||
|
||||
while (!copiedMergedData.isEmpty()) {
|
||||
if (copiedMergedData.get(0).getTime() < endTime) {
|
||||
tmp.add(copiedMergedData.remove(0));
|
||||
} else {
|
||||
if (!tmp.isEmpty()) {
|
||||
result.add(tmp);
|
||||
}
|
||||
|
||||
// prepare next day
|
||||
endTime += ONE_DAY;
|
||||
tmp = new ArrayList<IntelliJEvent>();
|
||||
}
|
||||
}
|
||||
|
||||
if (!tmp.isEmpty()) {
|
||||
result.add(tmp);
|
||||
}
|
||||
|
||||
return (result.isEmpty() ? null : result);
|
||||
}
|
||||
|
||||
private void saveData(List<List<IntelliJEvent>> dividedData, File destDir) {
|
||||
if (dividedData == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
|
||||
SimpleDateFormat timeFormat = new SimpleDateFormat("HHmmss");
|
||||
Gson gson = new Gson();
|
||||
|
||||
for (int i = 0; i < dividedData.size(); i++) {
|
||||
List<IntelliJEvent> data = dividedData.get(i);
|
||||
|
||||
String dateStr = dateFormat.format(new Date(data.get(0).getTime()));
|
||||
String begTimeStr = timeFormat.format(new Date(data.get(0).getTime()));
|
||||
String endTimeStr = timeFormat.format(new Date(data.get(data.size() - 1).getTime()));
|
||||
String path = destDir.getAbsolutePath() + File.separator + dateStr + File.separator +
|
||||
"event-" + begTimeStr + "-" + endTimeStr + ".json";
|
||||
FileUtil.toJson(gson, data, new File(path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,217 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import edu.nju.ics.frontier.bean.label.*;
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.Logger;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
public class LabelParseTask implements Executor.Task {
|
||||
private static final long ONE_DAY = 24 * 60 * 60 * 1000;
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File destDir = new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName()));
|
||||
|
||||
File scaleDir = new File(FileUtil.combine(userDir.getAbsolutePath(), "interaction_traces", "scale"));
|
||||
if (!scaleDir.exists()) {
|
||||
return;
|
||||
}
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(getClass().getName()).append(" {").
|
||||
append("\n arguments:\n scale=").
|
||||
append(scaleDir.getAbsolutePath()).
|
||||
append("\n dest=").append(destDir.getAbsolutePath()).
|
||||
append("\n date=").append(dateStr);
|
||||
|
||||
List<Label> scaleData = parseData(scaleDir, dateStr);
|
||||
builder.append("\n scale parsed: #data=").
|
||||
append((scaleData == null) ? 0 : scaleData.size());
|
||||
|
||||
Map<String, List<Label>> dividedData = divideData(scaleData);
|
||||
builder.append("\n data divided: #day=").
|
||||
append((dividedData == null) ? 0 : dividedData.size());
|
||||
|
||||
saveData(dividedData, destDir);
|
||||
builder.append("\n data saved\n}");
|
||||
Logger.println(builder.toString());
|
||||
}
|
||||
|
||||
private Label parseLabel(File file) {
|
||||
long begTime = -1;
|
||||
long endTime = -1;
|
||||
List<Action> actions = new ArrayList<Action>();
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(file);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
int commaIndex1 = line.indexOf(",");
|
||||
int commaIndex2 = line.indexOf(",", commaIndex1 + 1);
|
||||
|
||||
long time = Long.parseLong(line.substring(0, commaIndex1));
|
||||
String action = line.substring(commaIndex1 + 1, commaIndex2);
|
||||
|
||||
if ("begin".equals(action)) {
|
||||
begTime = time;
|
||||
} else if ("end".equals(action)) {
|
||||
endTime = time;
|
||||
} else if ("select".equals(action)) {
|
||||
AnswerModel model = new Gson().fromJson(
|
||||
line.substring(commaIndex2 + 1), AnswerModel.class);
|
||||
SelectOrSubmitAction.Dimension dimension = "engagement".equals(model.title) ?
|
||||
SelectOrSubmitAction.Dimension.ENGAGEMENT :
|
||||
SelectOrSubmitAction.Dimension.PRODUCTIVITY;
|
||||
actions.add(new SelectAction(time, dimension, model.answer, model.optionScore));
|
||||
} else if ("submit".equals(action)) {
|
||||
AnswerModel model = new Gson().fromJson(
|
||||
line.substring(commaIndex2 + 1), AnswerModel.class);
|
||||
SelectOrSubmitAction.Dimension dimension = "engagement".equals(model.title) ?
|
||||
SelectOrSubmitAction.Dimension.ENGAGEMENT :
|
||||
SelectOrSubmitAction.Dimension.PRODUCTIVITY;
|
||||
actions.add(new SubmitAction(time, dimension, model.answer, model.optionScore));
|
||||
} else if ("refuse".equals(action)) {
|
||||
String option = line.substring(commaIndex2 + 1);
|
||||
RefuseAction.OutOfIDEActivity activity = null;
|
||||
if ("meeting".equals(option)) {
|
||||
activity = RefuseAction.OutOfIDEActivity.MEETING;
|
||||
} else if ("rest".equals(option)) {
|
||||
activity = RefuseAction.OutOfIDEActivity.REST;
|
||||
} else if ("talk".equals(option)) {
|
||||
activity = RefuseAction.OutOfIDEActivity.TALK;
|
||||
} else if ("other".equals(option)) {
|
||||
activity = RefuseAction.OutOfIDEActivity.OTHER;
|
||||
}
|
||||
if (activity != null) {
|
||||
actions.add(new RefuseAction(time, activity));
|
||||
}
|
||||
}
|
||||
}
|
||||
reader.close();
|
||||
|
||||
if (begTime == -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (endTime == -1) {
|
||||
if (!actions.isEmpty()) {
|
||||
endTime = actions.get(actions.size() - 1).getTime();
|
||||
} else {
|
||||
endTime = begTime;
|
||||
}
|
||||
}
|
||||
|
||||
return new Label(begTime, endTime, actions);
|
||||
}
|
||||
|
||||
private List<Label> parseData(File scaleDir, String dateStr) {
|
||||
List<Label> result = new ArrayList<Label>();
|
||||
|
||||
File dateDir = new File(FileUtil.combine(scaleDir.getAbsolutePath(), dateStr));
|
||||
File[] labelFiles = FileUtil.listSubFiles(dateDir, null, ".json");
|
||||
if (labelFiles == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (File labelFile : labelFiles) {
|
||||
Label label = parseLabel(labelFile);
|
||||
if (label != null) {
|
||||
result.add(label);
|
||||
}
|
||||
}
|
||||
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private Map<String, List<Label>> divideData(List<Label> scaleData) {
|
||||
if (scaleData == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Map<String, List<Label>> result = new HashMap<String, List<Label>>();
|
||||
for (Label label : scaleData) {
|
||||
String k = getLabelDate(label);
|
||||
if (k == null) {
|
||||
continue;
|
||||
}
|
||||
List<Label> v = result.get(k);
|
||||
if (v == null) {
|
||||
v = new ArrayList<Label>();
|
||||
}
|
||||
v.add(label);
|
||||
result.put(k, v);
|
||||
}
|
||||
return (result.isEmpty() ? null : result);
|
||||
}
|
||||
|
||||
private void saveData(Map<String, List<Label>> dividedData, File destDir) {
|
||||
if (dividedData == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");
|
||||
SimpleDateFormat timeFormat = new SimpleDateFormat("HHmmss");
|
||||
|
||||
for (Map.Entry<String, List<Label>> entry : dividedData.entrySet()) {
|
||||
String dateStr = entry.getKey();
|
||||
List<Label> labels = entry.getValue();
|
||||
String begTimeStr = timeFormat.format(new Date(labels.get(0).getBegTime()));
|
||||
String endTimeStr = timeFormat.format(new Date(labels.get(labels.size() - 1).getEndTime()));
|
||||
String path = FileUtil.combine(destDir.getAbsolutePath(), dateStr,
|
||||
"label-" + begTimeStr + "-" + endTimeStr + ".json");
|
||||
DataSource.saveLabels(labels, new File(path));
|
||||
}
|
||||
}
|
||||
|
||||
private String getLabelDate(Label label) {
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.setTimeInMillis(label.getEndTime());
|
||||
calendar.set(Calendar.HOUR_OF_DAY, 6);
|
||||
calendar.set(Calendar.MINUTE, 0);
|
||||
calendar.set(Calendar.SECOND, 0);
|
||||
calendar.set(Calendar.MILLISECOND, 0);
|
||||
long begTime = calendar.getTimeInMillis();
|
||||
long endTime = begTime + ONE_DAY;
|
||||
|
||||
if (begTime <= label.getBegTime() && label.getEndTime() <= endTime) {
|
||||
return new SimpleDateFormat("yyyyMMdd").format(new Date(begTime));
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AnswerModel {
|
||||
private String title;
|
||||
private int answer;
|
||||
private int optionScore;
|
||||
|
||||
public AnswerModel(String title, int answer, int optionScore) {
|
||||
this.title = title;
|
||||
this.answer = answer;
|
||||
this.optionScore = optionScore;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public int getAnswer() {
|
||||
return answer;
|
||||
}
|
||||
|
||||
public int getOptionScore() {
|
||||
return optionScore;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("{title:%s,answer:%d,optionScore:%d}", title, answer, optionScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,223 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.FileEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.intellij.ProjectEvent;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.CollectionUtil;
|
||||
import edu.nju.ics.frontier.util.Constants;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
public class ParseMain {
|
||||
public static void main(String[] args) {
|
||||
cleanIntelliJEvent();
|
||||
parseIntelliJEvent();
|
||||
parseLabel();
|
||||
}
|
||||
|
||||
public static void cleanIntelliJEvent() {
|
||||
String root = "C:\\Users\\zzw\\Desktop\\Research\\dataset\\flow_tracker_plugin\\fujitsu";
|
||||
Executor.executeToDateDir(root, root, new IntelliJEventCleanTask());
|
||||
}
|
||||
|
||||
public static void parseIntelliJEvent() {
|
||||
String srcRoot = "C:\\Users\\zzw\\Desktop\\Research\\dataset\\flow_tracker_plugin\\fujitsu";
|
||||
String destRoot = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
Executor.executeToDateDir(srcRoot, destRoot, new IntelliJEventParseTask());
|
||||
}
|
||||
|
||||
public static void parseLabel() {
|
||||
String srcRoot = "C:\\Users\\zzw\\Desktop\\Research\\dataset\\flow_tracker_plugin\\fujitsu";
|
||||
String destRoot = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
Executor.executeToDateDir(srcRoot, destRoot, new LabelParseTask());
|
||||
}
|
||||
|
||||
public static void enumerateProjects() {
|
||||
String srcRoot = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
Executor.executeToUserDir(srcRoot, "", new Executor.Task() {
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<String> projects = new HashSet<String>();
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
List<IntelliJEvent> events = ParseUtil.loadWorkdayEvents(dateDir);
|
||||
if (events == null) {
|
||||
continue;
|
||||
}
|
||||
for (IntelliJEvent event : events) {
|
||||
if (!IntelliJEvent.isProjectEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
ProjectEvent projectEvent = IntelliJEvent.toProjectEvent(event);
|
||||
CollectionUtil.updateSet(projects, projectEvent.getNameField());
|
||||
}
|
||||
}
|
||||
|
||||
System.out.printf("%s/%s/%d\n%s\n", teamDir.getName(), userDir.getName(), projects.size(), projects);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static void enumerateFiles() {
|
||||
String srcRoot = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
Executor.executeToUserDir(srcRoot, "", new Executor.Task() {
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<String> globalFiles = new HashSet<String>();
|
||||
Set<String> switchFiles = new HashSet<String>();
|
||||
Set<String> selectFiles = new HashSet<String>();
|
||||
List<Set<String>> fileSwitches = new ArrayList<Set<String>>();
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
List<IntelliJEvent> events = ParseUtil.loadWorkdayEvents(dateDir);
|
||||
if (events == null) {
|
||||
continue;
|
||||
}
|
||||
for (IntelliJEvent event : events) {
|
||||
if (!IntelliJEvent.isFileEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
String oldPath = fileEvent.getOldPathField();
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
CollectionUtil.updateSet(globalFiles, oldPath);
|
||||
CollectionUtil.updateSet(globalFiles, newPath);
|
||||
CollectionUtil.updateSet(selectFiles, oldPath);
|
||||
CollectionUtil.updateSet(selectFiles, newPath);
|
||||
if (oldPath != null && newPath != null && (!oldPath.equals(newPath))) {
|
||||
CollectionUtil.updateSet(switchFiles, oldPath);
|
||||
CollectionUtil.updateSet(switchFiles, newPath);
|
||||
Set<String> tmp = new HashSet<String>();
|
||||
tmp.add(oldPath);
|
||||
tmp.add(newPath);
|
||||
fileSwitches.add(tmp);
|
||||
}
|
||||
} else if (fileEvent.isWhenChanged()) {
|
||||
String path = fileEvent.getPathField();
|
||||
CollectionUtil.updateSet(globalFiles, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
System.out.printf("%s/%s\n", teamDir.getName(), userDir.getName());
|
||||
|
||||
// connected components
|
||||
List<Set<String>> fileConnectedComponents = getConnectedComponents(fileSwitches);
|
||||
System.out.println("connected components:");
|
||||
Set<String> usedFiles = new HashSet<String>();
|
||||
for (Set<String> fileConnectedComponent : fileConnectedComponents) {
|
||||
for (String item : fileConnectedComponent) {
|
||||
Assertion.assertTrue(usedFiles.add(item));
|
||||
}
|
||||
System.out.println(fileConnectedComponent);
|
||||
}
|
||||
|
||||
// files selected but not switched
|
||||
System.out.println("files selected but not switched:");
|
||||
Set<String> selectButNotSwitch = new HashSet<String>();
|
||||
for (String file : selectFiles) {
|
||||
if (!usedFiles.contains(file)) {
|
||||
selectButNotSwitch.add(file);
|
||||
}
|
||||
}
|
||||
System.out.println(selectButNotSwitch);
|
||||
|
||||
// files changed but not selected
|
||||
System.out.println("files changed but not selected:");
|
||||
Set<String> changedButNotSelect = new HashSet<String>();
|
||||
for (String file : globalFiles) {
|
||||
if (!selectFiles.contains(file)) {
|
||||
changedButNotSelect.add(file);
|
||||
}
|
||||
}
|
||||
System.out.println(changedButNotSelect);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static List<Set<String>> getConnectedComponents(List<Set<String>> fileSwitches) {
|
||||
Map<String, Set<Integer>> clusterIdsForEachFile = getClusterIdsForEachFile(fileSwitches);
|
||||
int[] conflictClusterIds = getConflictClusterIds(clusterIdsForEachFile);
|
||||
while (conflictClusterIds != null) {
|
||||
resolveClusterConflict(clusterIdsForEachFile, conflictClusterIds[1], conflictClusterIds[0]);
|
||||
conflictClusterIds = getConflictClusterIds(clusterIdsForEachFile);
|
||||
}
|
||||
|
||||
Map<Integer, Set<String>> fileClusters = new HashMap<Integer, Set<String>>();
|
||||
for (Map.Entry<String, Set<Integer>> entry : clusterIdsForEachFile.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
Object[] value = entry.getValue().toArray();
|
||||
Assertion.assertTrue(value.length == 1);
|
||||
CollectionUtil.updateSetMap(fileClusters, (Integer) value[0], key);
|
||||
}
|
||||
|
||||
List<Set<String>> result = new ArrayList<Set<String>>();
|
||||
for (Map.Entry<Integer, Set<String>> entry : fileClusters.entrySet()) {
|
||||
result.add(entry.getValue());
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static Map<String, Set<Integer>> getClusterIdsForEachFile(List<Set<String>> fileSwitches) {
|
||||
if (fileSwitches == null || fileSwitches.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Map<String, Set<Integer>> result = new HashMap<String, Set<Integer>>();
|
||||
for (int i = 0, l = fileSwitches.size(); i < l; i++) {
|
||||
Set<String> tmp = fileSwitches.get(i);
|
||||
for (String item : tmp) {
|
||||
CollectionUtil.updateSetMap(result, item, i);
|
||||
}
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private static int[] getConflictClusterIds(Map<String, Set<Integer>> clusterIdsForEachFile) {
|
||||
if (clusterIdsForEachFile == null) {
|
||||
return null;
|
||||
}
|
||||
for (Map.Entry<String, Set<Integer>> entry : clusterIdsForEachFile.entrySet()) {
|
||||
Set<Integer> value = entry.getValue();
|
||||
if (value.size() > 1) {
|
||||
int min = Integer.MAX_VALUE;
|
||||
int max = Integer.MIN_VALUE;
|
||||
for (int item : value) {
|
||||
if (item < min) {
|
||||
min = item;
|
||||
}
|
||||
if (item > max) {
|
||||
max = item;
|
||||
}
|
||||
}
|
||||
return new int[]{min, max};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static void resolveClusterConflict(Map<String, Set<Integer>> clusterIdsForEachFile, int from, int to) {
|
||||
if (clusterIdsForEachFile == null) {
|
||||
return;
|
||||
}
|
||||
for (Map.Entry<String, Set<Integer>> entry : clusterIdsForEachFile.entrySet()) {
|
||||
Set<Integer> value = entry.getValue();
|
||||
if (value.contains(from)) {
|
||||
value.remove(from);
|
||||
value.add(to);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class ParseUtil {
|
||||
public static List<IntelliJEvent> loadWorkdayEvents(File dateDir) {
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, "event", ".json");
|
||||
if (eventFiles == null) {
|
||||
return null;
|
||||
}
|
||||
List<IntelliJEvent> events = new ArrayList<IntelliJEvent>();
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> tmp = DataSource.loadEvents(eventFile);
|
||||
if (tmp != null) {
|
||||
events.addAll(tmp);
|
||||
}
|
||||
}
|
||||
return events.isEmpty() ? null : events;
|
||||
}
|
||||
|
||||
public static List<Label> loadWorkdayLabels(File dateDir) {
|
||||
File[] labelFiles = FileUtil.listSubFiles(dateDir, "label", ".json");
|
||||
if (labelFiles == null) {
|
||||
return null;
|
||||
}
|
||||
List<Label> labels = new ArrayList<Label>();
|
||||
for (File labelFile : labelFiles) {
|
||||
List<Label> tmp = DataSource.loadLabels(labelFile);
|
||||
if (tmp != null) {
|
||||
labels.addAll(tmp);
|
||||
}
|
||||
}
|
||||
return labels.isEmpty() ? null : labels;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
package edu.nju.ics.frontier.parse;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.Constants;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class ProjectFileMapping {
|
||||
private static Map<String, Map<String, String>> file2Project4EachUser = new HashMap<String, Map<String, String>>();
|
||||
|
||||
static {
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open(FileUtil.combine(Constants.RESOURCE_ROOT, "file_2_project_4_each_user.txt"));
|
||||
String line;
|
||||
String user = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (line.contains("/")) {
|
||||
user = line.split("/")[1];
|
||||
} else if (line.startsWith("project:")) {
|
||||
Assertion.assertNotNull(user);
|
||||
String project = line.substring(line.indexOf(":") + 1, line.indexOf(","));
|
||||
String[] files = line.substring(line.indexOf("[") + 1, line.indexOf("]")).split(",");
|
||||
|
||||
Map<String, String> value = file2Project4EachUser.get(user);
|
||||
if (value == null) {
|
||||
value = new HashMap<String, String>();
|
||||
}
|
||||
for (String file : files) {
|
||||
Assertion.assertNull(value.put(file, project));
|
||||
}
|
||||
file2Project4EachUser.put(user, value);
|
||||
}
|
||||
}
|
||||
reader.close();
|
||||
}
|
||||
|
||||
public static Map<String, Map<String, String>> getFile2Project4EachUser() {
|
||||
return file2Project4EachUser;
|
||||
}
|
||||
|
||||
public static Map<String, String> getFile2ProjectByUser(String user) {
|
||||
if (user == null) {
|
||||
return null;
|
||||
}
|
||||
return file2Project4EachUser.get(user);
|
||||
}
|
||||
|
||||
public static String getProjectByFile4User(String user, String file) {
|
||||
if (user == null || file == null) {
|
||||
return null;
|
||||
}
|
||||
Map<String, String> file2Project = getFile2ProjectByUser(user);
|
||||
return file2Project == null ? null : file2Project.get(file);
|
||||
}
|
||||
|
||||
public static void printProjectFileMapping() {
|
||||
for (Map.Entry<String, Map<String, String>> entry1 : file2Project4EachUser.entrySet()) {
|
||||
String k1 = entry1.getKey();
|
||||
Map<String, String> v1 = entry1.getValue();
|
||||
System.out.printf("------------------------------------------------------------------\n%s\n", k1);
|
||||
for (Map.Entry<String, String> entry2 : v1.entrySet()) {
|
||||
System.out.printf("%s->%s\n", entry2.getKey(), entry2.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
package edu.nju.ics.frontier.recognition.feature;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Max;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Min;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class FeatureEngineeringTask implements Executor.Task {
|
||||
private boolean mStandardize;
|
||||
private boolean mNormalize;
|
||||
private OkTextReader mReader;
|
||||
private OkTextWriter mWriter;
|
||||
|
||||
public FeatureEngineeringTask(boolean standardize, boolean normalize) {
|
||||
mStandardize = standardize;
|
||||
mNormalize = normalize;
|
||||
mReader = new OkTextReader();
|
||||
mWriter = new OkTextWriter();
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File sampleFile = new File(FileUtil.combine(userDir.getAbsolutePath(), "samples.csv"));
|
||||
if (!sampleFile.exists()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// load
|
||||
List<String[]> tmp = new ArrayList<String[]>();
|
||||
mReader.open(sampleFile);
|
||||
String line;
|
||||
while ((line = mReader.readLine()) != null) {
|
||||
tmp.add(line.split(","));
|
||||
}
|
||||
mReader.close();
|
||||
|
||||
// process
|
||||
Object[][] table = new Object[tmp.size()][tmp.get(0).length];
|
||||
int offset = 0;
|
||||
// for team, user, date, sliceBegTime, sliceEndTime, sessionBegTime, sessionEndTime and sessionId
|
||||
for (; offset < 8; offset++) {
|
||||
for (int i = 0, m = tmp.size(); i < m; i++) {
|
||||
table[i][offset] = tmp.get(i)[offset];
|
||||
}
|
||||
}
|
||||
// standardization: for features
|
||||
for (int n = tmp.get(0).length - 2; offset < n; offset++) {
|
||||
double[] values = new double[tmp.size()];
|
||||
for (int i = 0, m = values.length; i < m; i++) {
|
||||
values[i] = Double.parseDouble(tmp.get(i)[offset]);
|
||||
}
|
||||
// standardization (Z-score)
|
||||
if (mStandardize) {
|
||||
double mean = new Mean().evaluate(values);
|
||||
double sd = new StandardDeviation().evaluate(values);
|
||||
for (int i = 0, m = values.length; i < m; i++) {
|
||||
table[i][offset] = (sd == 0.0) ? 0.0 : (values[i] - mean) / sd;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// normalization (0, 1)
|
||||
if (mNormalize) {
|
||||
double min = new Min().evaluate(values);
|
||||
double max = new Max().evaluate(values);
|
||||
double range = max - min;
|
||||
for (int i = 0, m = values.length; i < m; i++) {
|
||||
table[i][offset] = (range == 0.0) ? 0.0 : (values[i] - min) / range;
|
||||
Assertion.assertNotNegative((Double) table[i][offset]);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (int i = 0, m = values.length; i < m; i++) {
|
||||
table[i][offset] = values[i];
|
||||
}
|
||||
}
|
||||
// for engagement and productivity labels
|
||||
for (int n = tmp.get(0).length; offset < n; offset++) {
|
||||
for (int i = 0, m = tmp.size(); i < m; i++) {
|
||||
table[i][offset] = Integer.parseInt(tmp.get(i)[offset]);
|
||||
}
|
||||
}
|
||||
|
||||
// restore
|
||||
mWriter.open(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName(), "instances.csv"));
|
||||
for (Object[] row : table) {
|
||||
for (int j = 0, n = row.length - 1; j < n; j++) {
|
||||
mWriter.print(row[j] + ",");
|
||||
}
|
||||
mWriter.println(row[row.length - 1]);
|
||||
}
|
||||
mWriter.close();
|
||||
|
||||
System.out.printf("%s/%s -> %d instances\n", teamDir.getName(), userDir.getName(), table.length);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,819 @@
|
|||
package edu.nju.ics.frontier.recognition.feature;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import edu.nju.ics.frontier.bean.intellij.*;
|
||||
import edu.nju.ics.frontier.bean.label.SelectOrSubmitAction;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.recognition.segment.Slice;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
public class FeatureExtractionTask implements Executor.Task {
|
||||
private Gson mGson;
|
||||
private OkTextWriter mWriter;
|
||||
|
||||
public FeatureExtractionTask() {
|
||||
mGson = DataSource.getGson();
|
||||
mWriter = new OkTextWriter();
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
FileUtil.delete(new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName())));
|
||||
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int sampleCount = 0;
|
||||
File sampleFile = new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName(), "samples.csv"));
|
||||
mWriter.open(sampleFile);
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
File sliceFile = new File(FileUtil.combine(dateDir.getAbsolutePath(), "slices.json"));
|
||||
if (!sliceFile.exists()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Slice> slices = FileUtil.fromJson(mGson, new TypeToken<ArrayList<Slice>>(){}.getType(), sliceFile);
|
||||
if (slices == null || slices.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Slice slice : slices) {
|
||||
Object[] sample = createSample(slice);
|
||||
StringBuilder builder = new StringBuilder();
|
||||
for (Object value : sample) {
|
||||
builder.append(value).append(",");
|
||||
}
|
||||
String row = builder.toString();
|
||||
row = row.substring(0, row.length() - 1);
|
||||
mWriter.println(row);
|
||||
sampleCount++;
|
||||
}
|
||||
}
|
||||
|
||||
mWriter.close();
|
||||
|
||||
System.out.printf("%s/%s -> %d samples\n", teamDir.getName(), userDir.getName(), sampleCount);
|
||||
}
|
||||
|
||||
private Object[] createSample(Slice slice) {
|
||||
String team = slice.getTeam();
|
||||
String user = slice.getUser();
|
||||
String date = slice.getDate();
|
||||
long sliceBegTime = slice.getSliceBegTime();
|
||||
long sliceEndTime = slice.getSliceEndTime();
|
||||
long sessionBegTime = slice.getSessionBegTime();
|
||||
long sessionEndTime = slice.getSessionEndTime();
|
||||
int sessionId = slice.getSessionId();
|
||||
List<IntelliJEvent> events = slice.getEvents();
|
||||
Label label = slice.getLabel();
|
||||
|
||||
long begTime = sliceBegTime;
|
||||
long endTime = sliceEndTime;
|
||||
if (sliceBegTime < sessionBegTime) {
|
||||
begTime = sessionBegTime;
|
||||
endTime = begTime + (sliceEndTime - sliceBegTime);
|
||||
}
|
||||
if (sliceEndTime > sessionEndTime) {
|
||||
endTime = sessionEndTime;
|
||||
begTime = endTime - (sliceEndTime - sliceBegTime);
|
||||
}
|
||||
Assertion.assertTrue(sessionBegTime <= begTime && endTime <= sessionEndTime);
|
||||
|
||||
// TODO: feature extraction
|
||||
// for team, user, date, sliceBegTime, sliceEndTime, sessionBegTime, sessionEndTime,
|
||||
// sessionId, engagement label, and productivity label
|
||||
int featureCount = 10;
|
||||
double[] part1 = aggregateAppEvents(events, begTime, endTime);
|
||||
featureCount += part1.length;
|
||||
double[] part2 = aggregateWindowFileCursorEvents(events);
|
||||
featureCount += part2.length;
|
||||
double[] part3 = aggregateExecuteEvents(events);
|
||||
featureCount += part3.length;
|
||||
double[] part4 = aggregateActionEvents(events);
|
||||
featureCount += part4.length;
|
||||
double[] part5 = aggregateKeyMouseEvents(events, begTime, endTime);
|
||||
featureCount += part5.length;
|
||||
|
||||
Object[] result = new Object[featureCount];
|
||||
int offset = 0;
|
||||
result[offset++] = team;
|
||||
result[offset++] = user;
|
||||
result[offset++] = date;
|
||||
result[offset++] = sliceBegTime;
|
||||
result[offset++] = sliceEndTime;
|
||||
result[offset++] = sessionBegTime;
|
||||
result[offset++] = sessionEndTime;
|
||||
result[offset++] = sessionId;
|
||||
offset += copyValues(result, part1, offset);
|
||||
offset += copyValues(result, part2, offset);
|
||||
offset += copyValues(result, part3, offset);
|
||||
offset += copyValues(result, part4, offset);
|
||||
offset += copyValues(result, part5, offset);
|
||||
result[offset++] = label == null ? -1 : label.getLabelScore(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
result[offset] = label == null ? -1 : label.getLabelScore(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private double[] aggregateAppEvents(List<IntelliJEvent> events, long begTime, long endTime) {
|
||||
List<AppEvent> appEvents = MeasureUtil.getAppEventSequence(events);
|
||||
int[] enterAndExitIdeCount = MeasureUtil.getEnterAndExitIdeCount(appEvents);
|
||||
long[] inAndOutsideIdeTime = MeasureUtil.getInAndOutsideIdeTime(appEvents, begTime, endTime);
|
||||
long inactiveTime = MeasureUtil.getInactiveTimeInIDE(events, appEvents, begTime, endTime, 300000);
|
||||
double oneMinute = 60 * 1000;
|
||||
|
||||
Assertion.assertEqual(inAndOutsideIdeTime[0] + inAndOutsideIdeTime[1], endTime - begTime);
|
||||
|
||||
long timeInterval = Math.round(safeDivision(endTime - begTime, oneMinute));
|
||||
int enterIdeCount = enterAndExitIdeCount[0];
|
||||
int exitIdeCount = enterAndExitIdeCount[1];
|
||||
long inIdeTime = Math.round(safeDivision(inAndOutsideIdeTime[0], oneMinute));
|
||||
long outsideIdeTime = timeInterval - inIdeTime;
|
||||
long inIdeInactiveTime = Math.round(safeDivision(inactiveTime, oneMinute));
|
||||
long inIdeActiveTime = inIdeTime - inIdeInactiveTime;
|
||||
|
||||
// ratios of time spent in and outside the IDE
|
||||
double inIdeTimeRatio = safeDivision(inIdeTime, timeInterval);
|
||||
double outsideIdeTimeRatio = safeDivision(outsideIdeTime, timeInterval);
|
||||
|
||||
// ratios of active time interval in the IDE related to duration in IDE
|
||||
// and duration of slice, respectively
|
||||
double inIdeActiveTimeRatio = safeDivision(inIdeActiveTime, inIdeTime);
|
||||
double activeTimeRatio = safeDivision(inIdeActiveTime, timeInterval);
|
||||
Assertion.assertTrue(activeTimeRatio <= inIdeActiveTimeRatio);
|
||||
double inIdeInactiveTimeRatio = safeDivision(inIdeInactiveTime, inIdeTime);
|
||||
double inactiveTimeRatio = safeDivision(outsideIdeTime + inIdeInactiveTime, timeInterval);
|
||||
|
||||
return new double[]{
|
||||
// number of switches from outside to IDE
|
||||
// Assertion.assertNotNegative(enterIdeCount),
|
||||
// number of switches from IDE to outside
|
||||
Assertion.assertNotNegative(exitIdeCount),
|
||||
// minutes spent in the IDE
|
||||
Assertion.assertNotNegative(inIdeTime),
|
||||
// fraction of in-IDE time and total time
|
||||
// Assertion.assertInClosedIntervalFrom0To1(inIdeTimeRatio),
|
||||
// minutes spent outside the IDE
|
||||
Assertion.assertNotNegative(outsideIdeTime),
|
||||
// fraction of outside-IDE time and total time
|
||||
// Assertion.assertInClosedIntervalFrom0To1(outsideIdeTimeRatio),
|
||||
// active time spent in the IDE
|
||||
Assertion.assertByCondition(inIdeActiveTime, 0 <= inIdeActiveTime && inIdeActiveTime <= inIdeTime),
|
||||
// inactive time spent in the IDE
|
||||
Assertion.assertByCondition(inIdeInactiveTime, 0 <= inIdeInactiveTime && inIdeInactiveTime <= inIdeTime),
|
||||
// ratio of active time interval in the IDE related to duration of slice
|
||||
// Assertion.assertInClosedIntervalFrom0To1(activeTimeRatio),
|
||||
// ratio of active time interval in the IDE related to duration in IDE
|
||||
// Assertion.assertInClosedIntervalFrom0To1(inIdeActiveTimeRatio),
|
||||
// ratio of the sum of inactive time interval in the IDE and outside the IDE related to duration of slice
|
||||
// Assertion.assertInClosedIntervalFrom0To1(inactiveTimeRatio),
|
||||
// ratio of inactive time interval in the IDE related to duration in IDE
|
||||
// Assertion.assertInClosedIntervalFrom0To1(inIdeInactiveTimeRatio),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* in a comprehensible way, we define several concept:
|
||||
* <ul>
|
||||
* <li><strong>project switching sequence</strong>: a sequence consists of N times
|
||||
* project switches, denotes as project_1, project_2, ..., project_N and project_i
|
||||
* is not equal to project_{i+1}. There are k projects appear in the sequence, we
|
||||
* assume project i appears n_i times, so \Sigma_{i=1}^k n_i = N.</li>
|
||||
* <li><strong>file switching sequence</strong>: a sequence consists of N times file
|
||||
* switches, denotes as file_1, file_2, ..., file_N and file_i is not equal to
|
||||
* file_{i+1}. There are k files appear in the sequence, we assume file i appears n_i
|
||||
* times, so \Sigma_{i=1}^k n_i = N.</li>
|
||||
* <li><strong>file modifying sequence</strong>: a sequence consists of N times file
|
||||
* modifies, denotes as file_1, file_2, ..., file_N and file_i is not equal to
|
||||
* file_{i+1}. There are k files appear in the sequence, we assume file i appears n_i
|
||||
* times, so \Sigma_{i=1}^k n_i = N.</li>
|
||||
* <li><strong>file modifying history</strong>: given a file i, the modifying history
|
||||
* is a sequence of line of code.</li>
|
||||
* <li><strong>cursor changing history</strong>: given a file i, the changing history
|
||||
* is a sequence of line of cursor.</li>
|
||||
* </ul>
|
||||
* @param events
|
||||
* @return
|
||||
*/
|
||||
private double[] aggregateWindowFileCursorEvents(List<IntelliJEvent> events) {
|
||||
List<IntelliJEvent> windowFileCursorEvents = MeasureUtil.getWindowFileCursorEventSequence(events);
|
||||
|
||||
List<String> projectSwitchingSequence = new ArrayList<String>();
|
||||
List<String> fileSwitchingSequence = new ArrayList<String>();
|
||||
List<String> fileModifyingSequence = new ArrayList<String>();
|
||||
Map<String, List<long[]>> fileModifyingHistory = new HashMap<String, List<long[]>>();
|
||||
Map<String, List<long[]>> cursorChangingHistory = new HashMap<String, List<long[]>>();
|
||||
Set<String> browsedProjects = new HashSet<String>();
|
||||
Set<String> modifiedProjects = new HashSet<String>();
|
||||
Set<String> browsedFiles = new HashSet<String>();
|
||||
Set<String> modifiedFiles = new HashSet<String>();
|
||||
|
||||
if (windowFileCursorEvents != null) {
|
||||
for (IntelliJEvent event : windowFileCursorEvents) {
|
||||
if (event instanceof WindowEvent) {
|
||||
WindowEvent windowEvent = (WindowEvent) event;
|
||||
Assertion.assertTrue(windowEvent.isWhenWindowActivated());
|
||||
String thisName = windowEvent.getThisWindowName();
|
||||
Assertion.assertNotNull(thisName);
|
||||
Assertion.assertTrue(thisName.startsWith("frame"));
|
||||
String oppoName = windowEvent.getOppositeWindowName();
|
||||
Assertion.assertTrue(oppoName == null || oppoName.startsWith("frame"));
|
||||
updateSequence(projectSwitchingSequence, thisName);
|
||||
browsedProjects.add(thisName);
|
||||
} else if (event instanceof FileEvent) {
|
||||
FileEvent fileEvent = (FileEvent) event;
|
||||
Assertion.assertTrue(fileEvent.isWhenSelected() || fileEvent.isWhenChanged());
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
Assertion.assertNotNull(newPath);
|
||||
updateSequence(fileSwitchingSequence, newPath);
|
||||
String frame = fileEvent.getExtraData("frame");
|
||||
if (frame != null) {
|
||||
browsedProjects.add(frame);
|
||||
}
|
||||
browsedFiles.add(newPath);
|
||||
} else {
|
||||
String path = fileEvent.getPathField();
|
||||
Assertion.assertNotNull(path);
|
||||
updateSequence(fileModifyingSequence, path);
|
||||
MeasureUtil.updateListMap(fileModifyingHistory, path,
|
||||
new long[]{fileEvent.getTime(), Long.parseLong(fileEvent.getLineField())});
|
||||
String frame = fileEvent.getExtraData("frame");
|
||||
if (frame != null) {
|
||||
browsedProjects.add(frame);
|
||||
modifiedProjects.add(frame);
|
||||
}
|
||||
browsedFiles.add(path);
|
||||
modifiedFiles.add(path);
|
||||
}
|
||||
} else {
|
||||
CursorEvent cursorEvent = (CursorEvent) event;
|
||||
Assertion.assertTrue(cursorEvent.isWhenCursorChanged() || cursorEvent.isWhenSelectChanged());
|
||||
String file = cursorEvent.getExtraData("file");
|
||||
if (file == null) {
|
||||
file = "anonymous";
|
||||
}
|
||||
if (cursorEvent.isWhenCursorChanged()) {
|
||||
Assertion.assertTrue(cursorEvent.isUntypedEditor());
|
||||
long newLine = CursorEvent.getLineAndColumn(cursorEvent.getNewPosField())[0];
|
||||
long oldLine = CursorEvent.getLineAndColumn(cursorEvent.getOldPosField())[0];
|
||||
MeasureUtil.updateListMap(cursorChangingHistory, file,
|
||||
new long[]{cursorEvent.getTime(), newLine, oldLine});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// count
|
||||
int browsedProjectCount = browsedProjects.size(); // number of browsed projects
|
||||
int modifiedProjectCount = modifiedProjects.size(); // number of modified projects
|
||||
double projectModifiedRatio = safeDivision(modifiedProjectCount, browsedProjectCount);
|
||||
int browsedFileCount = browsedFiles.size(); // number of browsed files
|
||||
int modifiedFileCount = modifiedFiles.size(); // number of modified files
|
||||
double fileModifiedRatio = safeDivision(modifiedFileCount, browsedFileCount);
|
||||
|
||||
// focus switches
|
||||
int projectSwitchingCount = projectSwitchingSequence.size();
|
||||
int fileSwitchingCount = fileSwitchingSequence.size();
|
||||
int fileModifyingCount = fileModifyingSequence.size();
|
||||
double averageProjectSwitchingCount = safeDivision(projectSwitchingCount, new HashSet<String>(projectSwitchingSequence).size());
|
||||
double averageFileSwitchingCount = safeDivision(fileSwitchingCount, new HashSet<String>(fileSwitchingSequence).size());
|
||||
double averageFileModifyingCount = safeDivision(fileModifyingCount, new HashSet<String>(fileModifyingSequence).size());
|
||||
double projectSwitchingEntropy = MeasureUtil.shannonEntropy(projectSwitchingSequence);
|
||||
double fileSwitchingEntropy = MeasureUtil.shannonEntropy(fileSwitchingSequence);
|
||||
double fileModifyingEntropy = MeasureUtil.shannonEntropy(fileModifyingSequence);
|
||||
|
||||
// region of interest & walking distance
|
||||
Map<String, Long> regionOfInterest = new HashMap<String, Long>();
|
||||
Map<String, Long> walkingDistance = new HashMap<String, Long>();
|
||||
for (Map.Entry<String, List<long[]>> entry : cursorChangingHistory.entrySet()) {
|
||||
List<long[]> positions = entry.getValue();
|
||||
MeasureUtil.removeOutliers(positions);
|
||||
if (positions.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
long prevLine = positions.get(0)[1];
|
||||
long top = prevLine;
|
||||
long bottom = prevLine;
|
||||
long distance = 0;
|
||||
for (int i = 1, l = positions.size(); i < l; i++) {
|
||||
long nextLine = positions.get(i)[1];
|
||||
if (nextLine < top) {
|
||||
top = nextLine;
|
||||
}
|
||||
if (nextLine > bottom) {
|
||||
bottom = nextLine;
|
||||
}
|
||||
distance += Math.abs(nextLine - prevLine);
|
||||
prevLine = nextLine;
|
||||
}
|
||||
long region = bottom - top;
|
||||
Assertion.assertTrue(region >= 0);
|
||||
Assertion.assertTrue(region <= distance);
|
||||
if (region > 0) {
|
||||
regionOfInterest.put(entry.getKey(), region);
|
||||
walkingDistance.put(entry.getKey(), distance);
|
||||
}
|
||||
}
|
||||
|
||||
double regionOfInterestSum = MeasureUtil.countMap(regionOfInterest);
|
||||
double regionOfInterestPerFile = safeDivision(regionOfInterestSum, regionOfInterest.size());
|
||||
double walkingDistanceSum = MeasureUtil.countMap(walkingDistance);
|
||||
double walkingDistancePerFile = safeDivision(walkingDistanceSum, walkingDistance.size());
|
||||
double regionOfInterestEntropy = MeasureUtil.shannonEntropy(regionOfInterest, regionOfInterestSum);
|
||||
double walkingDistanceEntropy = MeasureUtil.shannonEntropy(walkingDistance, walkingDistanceSum);
|
||||
double duplicateRatio = safeDivision(walkingDistanceSum, regionOfInterestSum);
|
||||
|
||||
// line of code
|
||||
Map<String, long[]> lineOfCode = new HashMap<String, long[]>();
|
||||
for (Map.Entry<String, List<long[]>> entry : fileModifyingHistory.entrySet()) {
|
||||
long addedLoc = 0, deletedLoc = 0;
|
||||
List<long[]> lines = entry.getValue();
|
||||
long prevLine = lines.get(0)[1];
|
||||
for (int i = 1, l = lines.size(); i < l; i++) {
|
||||
long nextLine = lines.get(i)[1];
|
||||
long deltaLine = nextLine - prevLine;
|
||||
if (deltaLine > 0) {
|
||||
addedLoc += deltaLine;
|
||||
} else if (deltaLine < 0) {
|
||||
deletedLoc -= deltaLine;
|
||||
}
|
||||
prevLine = nextLine;
|
||||
}
|
||||
Assertion.assertNotNegative(addedLoc);
|
||||
Assertion.assertNotNegative(deletedLoc);
|
||||
if (addedLoc > 0 || deletedLoc > 0) {
|
||||
lineOfCode.put(entry.getKey(), new long[]{addedLoc, deletedLoc});
|
||||
}
|
||||
}
|
||||
|
||||
double addedLocSum = 0;
|
||||
double deletedLocSum = 0;
|
||||
for (Map.Entry<String, long[]> entry : lineOfCode.entrySet()) {
|
||||
long[] value = entry.getValue();
|
||||
addedLocSum += value[0];
|
||||
deletedLocSum += value[1];
|
||||
}
|
||||
double addedOrDeletedLocSum = addedLocSum + deletedLocSum;
|
||||
double locGainSum = addedLocSum - deletedLocSum;
|
||||
|
||||
double addedLocPerFile = safeDivision(addedLocSum, lineOfCode.size());
|
||||
double deletedLocPerFile = safeDivision(deletedLocSum, lineOfCode.size());
|
||||
double addedOrDeletedLocPerFile = safeDivision(addedOrDeletedLocSum, lineOfCode.size());
|
||||
double locGainPerFile = safeDivision(locGainSum, lineOfCode.size());
|
||||
|
||||
return new double[]{
|
||||
// number of browsed projects
|
||||
Assertion.assertNotNegative(browsedProjectCount),
|
||||
// number of modified projects
|
||||
Assertion.assertNotNegative(modifiedProjectCount),
|
||||
// ratio of modified related to browsed projects
|
||||
// Assertion.assertInClosedIntervalFrom0To1(projectModifiedRatio),
|
||||
// number of browsed files
|
||||
Assertion.assertNotNegative(browsedFileCount),
|
||||
// number of modified files
|
||||
Assertion.assertNotNegative(modifiedFileCount),
|
||||
// ratio of modified related to browsed files
|
||||
// Assertion.assertInClosedIntervalFrom0To1(fileModifiedRatio),
|
||||
// number of project switching
|
||||
Assertion.assertNotNegative(projectSwitchingCount),
|
||||
// number of file switching
|
||||
Assertion.assertNotNegative(fileSwitchingCount),
|
||||
// number of file modifying
|
||||
Assertion.assertNotNegative(fileModifyingCount),
|
||||
// average number of project switching
|
||||
// Assertion.assertNotNegative(averageProjectSwitchingCount),
|
||||
// average number of file switching
|
||||
// Assertion.assertNotNegative(averageFileSwitchingCount),
|
||||
// average number of file modifying
|
||||
// Assertion.assertNotNegative(averageFileModifyingCount),
|
||||
// entropy of focus switches at project level
|
||||
Assertion.assertNotNegative(projectSwitchingEntropy),
|
||||
// entropy of focus switches at browsed file level
|
||||
Assertion.assertNotNegative(fileSwitchingEntropy),
|
||||
// entropy of focus switches at modified file level
|
||||
Assertion.assertNotNegative(fileModifyingEntropy),
|
||||
// sum of region of interest of each file
|
||||
Assertion.assertNotNegative(regionOfInterestSum),
|
||||
// region of interest per file
|
||||
// Assertion.assertNotNegative(regionOfInterestPerFile),
|
||||
// sum of walking distance of each file
|
||||
Assertion.assertByCondition(walkingDistanceSum, walkingDistanceSum >= regionOfInterestSum),
|
||||
// walking distance per file
|
||||
// Assertion.assertByCondition(walkingDistancePerFile, walkingDistancePerFile >= regionOfInterestPerFile),
|
||||
// entropy of region of interest
|
||||
// Assertion.assertNotNegative(regionOfInterestEntropy),
|
||||
// entropy of walking distance
|
||||
// Assertion.assertNotNegative(walkingDistanceEntropy),
|
||||
// sum of walking distance divided by sum of region of interest
|
||||
Assertion.assertByCondition(duplicateRatio, duplicateRatio == 0 || duplicateRatio >= 1),
|
||||
// sum of added line of code
|
||||
Assertion.assertNotNegative(addedLocSum),
|
||||
// sum of deleted line of code
|
||||
Assertion.assertNotNegative(deletedLocSum),
|
||||
// gain of line of code
|
||||
Assertion.assertNotNaN(locGainSum),
|
||||
// sum of added/deleted line of code
|
||||
Assertion.assertNotNegative(addedOrDeletedLocSum),
|
||||
// added line of code per file
|
||||
// Assertion.assertNotNegative(addedLocPerFile),
|
||||
// deleted line of code per file
|
||||
// Assertion.assertNotNegative(deletedLocPerFile),
|
||||
// gain of line of code per file
|
||||
// Assertion.assertNotNaN(locGainPerFile),
|
||||
// added/deleted line of code per file
|
||||
// Assertion.assertNotNegative(addedOrDeletedLocPerFile),
|
||||
};
|
||||
}
|
||||
|
||||
private double[] aggregateExecuteEvents(List<IntelliJEvent> events) {
|
||||
int scheduledRunCount = 0;
|
||||
int cancelledRunCount = 0;
|
||||
int scheduledDebugCount = 0;
|
||||
int cancelledDebugCount = 0;
|
||||
int scheduledCoverageCount = 0;
|
||||
int cancelledCoverageCount = 0;
|
||||
|
||||
if (!(events == null || events.isEmpty())) {
|
||||
for (IntelliJEvent event : events) {
|
||||
if (!IntelliJEvent.isExecuteEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
ExecuteEvent executeEvent = IntelliJEvent.toExecuteEvent(event);
|
||||
if (executeEvent.isWhenScheduled()) {
|
||||
String id = executeEvent.getIdField();
|
||||
if ("Run".equals(id)) {
|
||||
scheduledRunCount++;
|
||||
} else if ("Debug".equals(id)) {
|
||||
scheduledDebugCount++;
|
||||
} else if ("Coverage".equals(id)) {
|
||||
scheduledCoverageCount++;
|
||||
} else {
|
||||
throw new IllegalArgumentException(executeEvent.toString());
|
||||
}
|
||||
} else if (executeEvent.isWhenCancelled()) {
|
||||
String id = executeEvent.getIdField();
|
||||
if ("Run".equals(id)) {
|
||||
cancelledRunCount++;
|
||||
} else if ("Debug".equals(id)) {
|
||||
cancelledDebugCount++;
|
||||
} else if ("Coverage".equals(id)) {
|
||||
cancelledCoverageCount++;
|
||||
} else {
|
||||
throw new IllegalArgumentException(executeEvent.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assertion.assertNotNegative(scheduledRunCount);
|
||||
Assertion.assertNotNegative(scheduledDebugCount);
|
||||
Assertion.assertNotNegative(scheduledCoverageCount);
|
||||
Assertion.assertNotNegative(cancelledRunCount);
|
||||
Assertion.assertNotNegative(cancelledDebugCount);
|
||||
Assertion.assertNotNegative(cancelledCoverageCount);
|
||||
|
||||
int scheduledExecuteCount = scheduledRunCount + scheduledDebugCount + scheduledCoverageCount;
|
||||
int cancelledExecuteCount = cancelledRunCount + cancelledDebugCount + cancelledCoverageCount;
|
||||
|
||||
return new double[]{
|
||||
// number of all terminated running
|
||||
// Math.max(0, scheduledRunCount - cancelledRunCount),
|
||||
// number of all cancelled running
|
||||
// cancelledRunCount,
|
||||
// number of all terminated debugging
|
||||
// Math.max(0, scheduledDebugCount - cancelledDebugCount),
|
||||
// number of all cancelled debugging
|
||||
// cancelledDebugCount,
|
||||
// number of all terminated coverage
|
||||
// Math.max(0, scheduledCoverageCount - cancelledCoverageCount),
|
||||
// number of all cancelled coverage
|
||||
// cancelledCoverageCount,
|
||||
// number of all terminated executing
|
||||
// Math.max(0, scheduledExecuteCount - cancelledExecuteCount),
|
||||
// number of all cancelled executing
|
||||
// cancelledExecuteCount,
|
||||
};
|
||||
}
|
||||
|
||||
private double[] aggregateActionEvents(List<IntelliJEvent> events) {
|
||||
int editActionCount = 0, codeActionCount = 0, refactorActionCount = 0;
|
||||
int findActionCount = 0, navigateActionCount = 0, debugActionCount = 0;
|
||||
int buildActionCount = 0, runActionCount = 0, testActionCount = 0, analyzeActionCount = 0;
|
||||
int fileActionCount = 0, vcsActionCount = 0, windowActionCount = 0;
|
||||
int otherActionCount = 0, actionCount = 0;
|
||||
|
||||
if (!(events == null || events.isEmpty())) {
|
||||
for (IntelliJEvent event : events) {
|
||||
if (!IntelliJEvent.isActionEvent(event)) {
|
||||
continue;
|
||||
}
|
||||
ActionEvent actionEvent = IntelliJEvent.toActionEvent(event);
|
||||
ActionCategory.Category category = ActionCategory.getCategory(actionEvent);
|
||||
if (category == null) {
|
||||
otherActionCount++;
|
||||
} else {
|
||||
switch (category) {
|
||||
case Edit:
|
||||
editActionCount++;
|
||||
break;
|
||||
case Code:
|
||||
codeActionCount++;
|
||||
break;
|
||||
case Refactor:
|
||||
refactorActionCount++;
|
||||
break;
|
||||
case Find:
|
||||
findActionCount++;
|
||||
break;
|
||||
case Navigate:
|
||||
navigateActionCount++;
|
||||
break;
|
||||
case Debug:
|
||||
debugActionCount++;
|
||||
break;
|
||||
case Build:
|
||||
buildActionCount++;
|
||||
break;
|
||||
case Run:
|
||||
runActionCount++;
|
||||
break;
|
||||
case Test:
|
||||
testActionCount++;
|
||||
break;
|
||||
case Analyze:
|
||||
analyzeActionCount++;
|
||||
break;
|
||||
case File:
|
||||
fileActionCount++;
|
||||
break;
|
||||
case VCS:
|
||||
vcsActionCount++;
|
||||
break;
|
||||
case Window:
|
||||
windowActionCount++;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
actionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
Assertion.assertEqual(actionCount, editActionCount + codeActionCount + refactorActionCount +
|
||||
findActionCount + navigateActionCount + debugActionCount +
|
||||
buildActionCount + runActionCount + testActionCount + analyzeActionCount +
|
||||
fileActionCount + vcsActionCount + windowActionCount + otherActionCount);
|
||||
|
||||
int editingActionCount = editActionCount + codeActionCount + refactorActionCount;
|
||||
int understandingActionCount = findActionCount + navigateActionCount + debugActionCount;
|
||||
int executingActionCount = buildActionCount + runActionCount + testActionCount + analyzeActionCount;
|
||||
int generalActionCount = fileActionCount + vcsActionCount + windowActionCount;
|
||||
|
||||
Assertion.assertEqual(actionCount, editingActionCount + understandingActionCount +
|
||||
executingActionCount + generalActionCount + otherActionCount);
|
||||
|
||||
double editActionRatio = safeDivision(editActionCount, actionCount);
|
||||
double codeActionRatio = safeDivision(codeActionCount, actionCount);
|
||||
double refactorActionRatio = safeDivision(refactorActionCount, actionCount);
|
||||
double findActionRatio = safeDivision(findActionCount, actionCount);
|
||||
double navigateActionRatio = safeDivision(navigateActionCount, actionCount);
|
||||
double debugActionRatio = safeDivision(debugActionCount, actionCount);
|
||||
double buildActionRatio = safeDivision(buildActionCount, actionCount);
|
||||
double runActionRatio = safeDivision(runActionCount, actionCount);
|
||||
double testActionRatio = safeDivision(testActionCount, actionCount);
|
||||
double analyzeActionRatio = safeDivision(analyzeActionCount, actionCount);
|
||||
double fileActionRatio = safeDivision(fileActionCount, actionCount);
|
||||
double vcsActionRatio = safeDivision(vcsActionCount, actionCount);
|
||||
double windowActionRatio = safeDivision(windowActionCount, actionCount);
|
||||
double otherActionRatio = safeDivision(otherActionCount, actionCount);
|
||||
|
||||
double editingActionRatio = safeDivision(editingActionCount, actionCount);
|
||||
double understandingActionRatio = safeDivision(understandingActionCount, actionCount);
|
||||
double executingActionRatio = safeDivision(executingActionCount, actionCount);
|
||||
double generalActionRatio = safeDivision(generalActionCount, actionCount);
|
||||
|
||||
return new double[]{
|
||||
Assertion.assertNotNegative(actionCount),
|
||||
Assertion.assertNotNegative(editActionCount),
|
||||
Assertion.assertNotNegative(codeActionCount),
|
||||
Assertion.assertNotNegative(refactorActionCount),
|
||||
Assertion.assertNotNegative(findActionCount),
|
||||
Assertion.assertNotNegative(navigateActionCount),
|
||||
Assertion.assertNotNegative(debugActionCount),
|
||||
Assertion.assertNotNegative(buildActionCount),
|
||||
Assertion.assertNotNegative(runActionCount),
|
||||
Assertion.assertNotNegative(testActionCount),
|
||||
Assertion.assertNotNegative(analyzeActionCount),
|
||||
Assertion.assertNotNegative(fileActionCount),
|
||||
Assertion.assertNotNegative(vcsActionCount),
|
||||
Assertion.assertNotNegative(windowActionCount),
|
||||
Assertion.assertNotNegative(otherActionCount),
|
||||
Assertion.assertNotNegative(editingActionCount),
|
||||
Assertion.assertNotNegative(understandingActionCount),
|
||||
Assertion.assertNotNegative(executingActionCount),
|
||||
Assertion.assertNotNegative(generalActionCount),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(editActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(codeActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(refactorActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(findActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(navigateActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(debugActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(buildActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(runActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(testActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(analyzeActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(fileActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(vcsActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(windowActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(otherActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(editingActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(understandingActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(executingActionRatio),
|
||||
// Assertion.assertInClosedIntervalFrom0To1(generalActionRatio),
|
||||
};
|
||||
}
|
||||
|
||||
private double[] aggregateKeyMouseEvents(List<IntelliJEvent> events, long begTime, long endTime) {
|
||||
int letterKeyCount = 0, numberKeyCount = 0, punctuationKeyCount = 0;
|
||||
int enterKeyCount = 0, backspaceAndDeleteKeyCount = 0, escapeKeyCount = 0;
|
||||
int directionKeyCount = 0, functionKeyCount = 0, otherKeyCount = 0;
|
||||
int keyTypeCount = 0;
|
||||
|
||||
int mouseClickCount = 0, mouseDragCount = 0;
|
||||
int mouseWheelCount = 0, mouseMoveCount = 0;
|
||||
int mouseEnterCount = 0, mouseExitCount = 0;
|
||||
MouseEvent prevMouseEvent = null;
|
||||
List<MouseEvent> mouseMovingHistory = new ArrayList<MouseEvent>();
|
||||
|
||||
if (!(events == null || events.isEmpty())) {
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isKeyEvent(event)) {
|
||||
KeyEvent keyEvent = IntelliJEvent.toKeyEvent(event);
|
||||
if (keyEvent.isWhenKeyPressed() && keyEvent.getTextField() != null) {
|
||||
keyTypeCount++;
|
||||
if (keyEvent.isPureOrShiftLetter()) {
|
||||
letterKeyCount++;
|
||||
} else if (keyEvent.isPureOrShiftNumber()) {
|
||||
numberKeyCount++;
|
||||
} else if (keyEvent.isPureOrShiftPunctuation()) {
|
||||
punctuationKeyCount++;
|
||||
} else if (keyEvent.isEnter()) {
|
||||
enterKeyCount++;
|
||||
} else if (keyEvent.isBackspace() || keyEvent.isDelete()) {
|
||||
backspaceAndDeleteKeyCount++;
|
||||
} else if (keyEvent.isEscape()) {
|
||||
escapeKeyCount++;
|
||||
} else if (keyEvent.isDirection()) {
|
||||
directionKeyCount++;
|
||||
} else if (keyEvent.isFunction()) {
|
||||
functionKeyCount++;
|
||||
} else {
|
||||
otherKeyCount++;
|
||||
}
|
||||
}
|
||||
} else if (IntelliJEvent.isMouseEvent(event)) {
|
||||
MouseEvent mouseEvent = IntelliJEvent.toMouseEvent(event);
|
||||
if (mouseEvent.isWhenMouseClicked()) {
|
||||
mouseClickCount++;
|
||||
} else if (mouseEvent.isWhenMouseDragged()) {
|
||||
if (prevMouseEvent != null && (!prevMouseEvent.isWhenMouseDragged())) {
|
||||
mouseDragCount++;
|
||||
}
|
||||
} else if (mouseEvent.isWhenMouseMoved()) {
|
||||
if (prevMouseEvent != null && (!prevMouseEvent.isWhenMouseMoved())) {
|
||||
mouseMoveCount++;
|
||||
}
|
||||
mouseMovingHistory.add(mouseEvent);
|
||||
} else if (mouseEvent.isWhenMouseWheel()) {
|
||||
if (prevMouseEvent != null && (!prevMouseEvent.isWhenMouseWheel())) {
|
||||
mouseWheelCount++;
|
||||
}
|
||||
} else if (mouseEvent.isWhenMouseEntered()) {
|
||||
mouseEnterCount++;
|
||||
} else if (mouseEvent.isWhenMouseExited()) {
|
||||
mouseExitCount++;
|
||||
}
|
||||
prevMouseEvent = mouseEvent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Assertion.assertTrue(keyTypeCount == letterKeyCount + numberKeyCount + punctuationKeyCount +
|
||||
enterKeyCount + backspaceAndDeleteKeyCount + escapeKeyCount +
|
||||
directionKeyCount + functionKeyCount + otherKeyCount);
|
||||
|
||||
int oneMinute = 60 * 1000;
|
||||
|
||||
double letterKeyRatio = safeDivision(letterKeyCount, keyTypeCount);
|
||||
double numberKeyRatio = safeDivision(numberKeyCount, keyTypeCount);
|
||||
double punctuationKeyRatio = safeDivision(punctuationKeyCount, keyTypeCount);
|
||||
double enterKeyRatio = safeDivision(enterKeyCount, keyTypeCount);
|
||||
double backspaceAndDeleteKeyRatio = safeDivision(backspaceAndDeleteKeyCount, keyTypeCount);
|
||||
double escapeKeyRatio = safeDivision(escapeKeyCount, keyTypeCount);
|
||||
double directionKeyRatio = safeDivision(directionKeyCount, keyTypeCount);
|
||||
double functionKeyRatio = safeDivision(functionKeyCount, keyTypeCount);
|
||||
double otherKeyRatio = safeDivision(otherKeyCount, keyTypeCount);
|
||||
double keyTypeSpeed = safeDivision(keyTypeCount, safeDivision(endTime - begTime, oneMinute));
|
||||
|
||||
double mouseMoveDistance = 0;
|
||||
if (!mouseMovingHistory.isEmpty()) {
|
||||
int[] prevPoint = mouseMovingHistory.get(0).getAbsoluteCoordinate();
|
||||
for (int i = 1, l = mouseMovingHistory.size(); i < l; i++) {
|
||||
int[] nextPoint = mouseMovingHistory.get(i).getAbsoluteCoordinate();
|
||||
double deltaX = nextPoint[0] - prevPoint[0];
|
||||
double deltaY = nextPoint[1] - prevPoint[1];
|
||||
mouseMoveDistance += Math.sqrt(deltaX * deltaX + deltaY * deltaY);
|
||||
prevPoint = nextPoint;
|
||||
}
|
||||
}
|
||||
double mouseMoveSpeed = safeDivision(mouseMoveDistance, safeDivision(endTime - begTime, oneMinute));
|
||||
|
||||
return new double[]{
|
||||
// #(keystrokes)
|
||||
Assertion.assertNotNegative(keyTypeCount),
|
||||
// #(letter)
|
||||
Assertion.assertNotNegative(letterKeyCount),
|
||||
// #(number)
|
||||
Assertion.assertNotNegative(numberKeyCount),
|
||||
// #(punctuation)
|
||||
Assertion.assertNotNegative(punctuationKeyCount),
|
||||
// #(enter)
|
||||
Assertion.assertNotNegative(enterKeyCount),
|
||||
// #(backspace & delete)
|
||||
Assertion.assertNotNegative(backspaceAndDeleteKeyCount),
|
||||
// #(escape)
|
||||
Assertion.assertNotNegative(escapeKeyCount),
|
||||
// #(Up & Down & Left & Right & Home & End & PgUp & PgDn)
|
||||
Assertion.assertNotNegative(directionKeyCount),
|
||||
// #(Fn family)
|
||||
Assertion.assertNotNegative(functionKeyCount),
|
||||
// #(others)
|
||||
Assertion.assertNotNegative(otherKeyCount),
|
||||
// #(letter) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(letterKeyRatio),
|
||||
// #(number) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(numberKeyRatio),
|
||||
// #(punctuation) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(punctuationKeyRatio),
|
||||
// #(enter) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(enterKeyRatio),
|
||||
// #(backspace & delete) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(backspaceAndDeleteKeyRatio),
|
||||
// #(escape) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(escapeKeyRatio),
|
||||
// #(Up & Down & Left & Right & Home & End & PgUp & PgDn) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(directionKeyRatio),
|
||||
// #(Fn family) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(functionKeyRatio),
|
||||
// #(others) / #(keystrokes)
|
||||
// Assertion.assertInClosedIntervalFrom0To1(otherKeyRatio),
|
||||
// #(keystrokes) per minute
|
||||
// Assertion.assertNotNegative(keyTypeSpeed),
|
||||
// #(mouse clicked)
|
||||
Assertion.assertNotNegative(mouseClickCount),
|
||||
// #(mouse dragged)
|
||||
Assertion.assertNotNegative(mouseDragCount),
|
||||
// #(mouse moved)
|
||||
Assertion.assertNotNegative(mouseMoveCount),
|
||||
// #(mouse wheel)
|
||||
Assertion.assertNotNegative(mouseWheelCount),
|
||||
// #(mouse entered)
|
||||
// Assertion.assertNotNegative(mouseEnterCount),
|
||||
// #(mouse exited)
|
||||
// Assertion.assertNotNegative(mouseExitCount),
|
||||
// distance of mouse moved per minute
|
||||
Assertion.assertNotNegative(mouseMoveSpeed),
|
||||
};
|
||||
}
|
||||
|
||||
private int copyValues(Object[] dest, double[] src, int offset) {
|
||||
for (int i = 0, l = src.length; i < l; i++) {
|
||||
dest[i + offset] = src[i];
|
||||
}
|
||||
return src.length;
|
||||
}
|
||||
|
||||
private void updateSequence(List<String> list, String item) {
|
||||
if (list.isEmpty()) {
|
||||
list.add(item);
|
||||
} else if (!list.get(list.size() - 1).equals(item)) {
|
||||
list.add(item);
|
||||
}
|
||||
}
|
||||
|
||||
private double safeDivision(double numerator, double denominator) {
|
||||
return denominator == 0 ? 0 : numerator / denominator;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
package edu.nju.ics.frontier.recognition.feature;
|
||||
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.common.io.OkTextWriter;
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Constants;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class FeatureMain {
|
||||
public static void main(String[] args) {
|
||||
String prefix = "time_aligned_";
|
||||
|
||||
String srcRoot1 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", prefix + "slice");
|
||||
String destRoot1 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", prefix + "sample");
|
||||
Executor.executeToUserDir(srcRoot1, destRoot1, new FeatureExtractionTask());
|
||||
|
||||
String srcRoot2 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", prefix + "sample");
|
||||
String destRoot2 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", prefix + "instance");
|
||||
Executor.executeToUserDir(srcRoot2, destRoot2, new FeatureEngineeringTask(
|
||||
Configuration.STANDARDIZE, Configuration.NORMALIZE));
|
||||
|
||||
String srcRoot3 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", prefix + "instance");
|
||||
String destRoot3 = "C:\\Users\\zzw\\Desktop\\fujitsu\\" + prefix + "instances.csv";
|
||||
mergeInstances(srcRoot3, destRoot3);
|
||||
}
|
||||
|
||||
private static void mergeInstances(String srcRoot, String destPath) {
|
||||
File destFile = new File(destPath);
|
||||
FileUtil.delete(destFile);
|
||||
|
||||
OkTextReader reader = new OkTextReader();
|
||||
OkTextWriter writer = new OkTextWriter();
|
||||
|
||||
File[] teamDirs = FileUtil.listSubDirectories(new File(srcRoot));
|
||||
if (teamDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
writer.open(destFile);
|
||||
int instanceCount = 0;
|
||||
|
||||
// write header
|
||||
for (int i = 0, l = HEADER.length - 1; i < l; i++) {
|
||||
writer.print(HEADER[i] + ",");
|
||||
}
|
||||
writer.println(HEADER[HEADER.length - 1]);
|
||||
|
||||
for (File teamDir : teamDirs) {
|
||||
File[] userDirs = FileUtil.listSubDirectories(teamDir);
|
||||
if (userDirs == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (File userDir : userDirs) {
|
||||
File instanceFile = new File(FileUtil.combine(userDir.getAbsolutePath(), "instances.csv"));
|
||||
if (!instanceFile.exists()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
reader.open(instanceFile);
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
writer.println(line);
|
||||
instanceCount++;
|
||||
}
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
writer.close();
|
||||
|
||||
if (instanceCount == 0) {
|
||||
FileUtil.delete(destFile);
|
||||
System.out.println("no instance found.");
|
||||
} else {
|
||||
System.out.printf("%d instances are merged into %s.\n", instanceCount, destFile);
|
||||
}
|
||||
}
|
||||
|
||||
public static final String[] HEADER = new String[]{
|
||||
// id & time
|
||||
"team",
|
||||
"user",
|
||||
"date",
|
||||
"sliceBegTime",
|
||||
"sliceEndTime",
|
||||
"sessionBegTime",
|
||||
"sessionEndTime",
|
||||
"sessionId",
|
||||
// app features
|
||||
// "enterIdeCount",
|
||||
"exitIdeCount",
|
||||
"inIdeTime",
|
||||
// "inIdeTimeRatio",
|
||||
"outsideIdeTime",
|
||||
// "outsideIdeTimeRatio",
|
||||
"inIdeActiveTime",
|
||||
"inIdeInactiveTime",
|
||||
// "activeTimeRatio",
|
||||
// "inIdeActiveTimeRatio",
|
||||
// "inactiveTimeRatio",
|
||||
// "inIdeInactiveTimeRatio",
|
||||
// window & file & cursor features
|
||||
"browsedProjectCount",
|
||||
"modifiedProjectCount",
|
||||
// "projectModifiedRatio",
|
||||
"browsedFileCount",
|
||||
"modifiedFileCount",
|
||||
// "fileModifiedRatio",
|
||||
"projectSwitchingCount",
|
||||
"fileSwitchingCount",
|
||||
"fileModifyingCount",
|
||||
// "averageProjectSwitchingCount",
|
||||
// "averageFileSwitchingCount",
|
||||
// "averageFileModifyingCount",
|
||||
"projectSwitchingEntropy",
|
||||
"fileSwitchingEntropy",
|
||||
"fileModifyingEntropy",
|
||||
"regionOfInterestSum",
|
||||
// "regionOfInterestPerFile",
|
||||
"walkingDistanceSum",
|
||||
// "walkingDistancePerFile",
|
||||
// "regionOfInterestEntropy",
|
||||
// "walkingDistanceEntropy",
|
||||
"duplicateRatio",
|
||||
"addedLocSum",
|
||||
"deletedLocSum",
|
||||
"locGainSum",
|
||||
"addedOrDeletedLocSum",
|
||||
// "addedLocPerFile",
|
||||
// "deletedLocPerFile",
|
||||
// "locGainPerFile",
|
||||
// "addedOrDeletedLocPerFile",
|
||||
// execute features
|
||||
// "scheduledRunCount",
|
||||
// "cancelledRunCount",
|
||||
// "scheduledDebugCount",
|
||||
// "cancelledDebugCount",
|
||||
// "scheduledCoverageCount",
|
||||
// "cancelledCoverageCount",
|
||||
// "scheduledExecuteCount",
|
||||
// "cancelledExecuteCount",
|
||||
// action features
|
||||
"actionCount",
|
||||
"editActionCount",
|
||||
"codeActionCount",
|
||||
"refactorActionCount",
|
||||
"findActionCount",
|
||||
"navigateActionCount",
|
||||
"debugActionCount",
|
||||
"buildActionCount",
|
||||
"runActionCount",
|
||||
"testActionCount",
|
||||
"analyzeActionCount",
|
||||
"fileActionCount",
|
||||
"vcsActionCount",
|
||||
"windowActionCount",
|
||||
"otherActionCount",
|
||||
"editingActionCount",
|
||||
"understandingActionCount",
|
||||
"executingActionCount",
|
||||
"generalActionCount",
|
||||
// "editActionRatio",
|
||||
// "codeActionRatio",
|
||||
// "refactorActionRatio",
|
||||
// "findActionRatio",
|
||||
// "navigateActionRatio",
|
||||
// "debugActionRatio",
|
||||
// "buildActionRatio",
|
||||
// "runActionRatio",
|
||||
// "testActionRatio",
|
||||
// "analyzeActionRatio",
|
||||
// "fileActionRatio",
|
||||
// "vcsActionRatio",
|
||||
// "windowActionRatio",
|
||||
// "otherActionRatio",
|
||||
// "editingActionRatio",
|
||||
// "understandingActionRatio",
|
||||
// "executingActionRatio",
|
||||
// "generalActionRatio",
|
||||
// key & mouse features
|
||||
"keyTypeCount",
|
||||
"letterKeyCount",
|
||||
"numberKeyCount",
|
||||
"punctuationKeyCount",
|
||||
"enterKeyCount",
|
||||
"backspaceAndDeleteKeyCount",
|
||||
"escapeKeyCount",
|
||||
"directionKeyCount",
|
||||
"functionKeyCount",
|
||||
"otherKeyCount",
|
||||
// "letterKeyRatio",
|
||||
// "numberKeyRatio",
|
||||
// "punctuationKeyRatio",
|
||||
// "enterKeyRatio",
|
||||
// "backspaceAndDeleteKeyRatio",
|
||||
// "escapeKeyRatio",
|
||||
// "directionKeyRatio",
|
||||
// "functionKeyRatio",
|
||||
// "otherKeyRatio",
|
||||
// "keyTypeSpeed",
|
||||
"mouseClickCount",
|
||||
"mouseDragCount",
|
||||
"mouseMoveCount",
|
||||
"mouseWheelCount",
|
||||
// "mouseEnterCount",
|
||||
// "mouseExitCount",
|
||||
"mouseMoveSpeed",
|
||||
// labels
|
||||
"internalScore",
|
||||
"externalScore",
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,591 @@
|
|||
package edu.nju.ics.frontier.recognition.feature;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.*;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class MeasureUtil {
|
||||
/**
|
||||
* select out app events out of a given event sequence. It worth noting that the
|
||||
* selected app event sequence is alternating <strong>app activated event</strong>
|
||||
* and <storng>app deactivated event</storng>.
|
||||
* @param events the given event sequence
|
||||
* @return the selected app event sequence
|
||||
*/
|
||||
public static List<AppEvent> getAppEventSequence(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<AppEvent> appEvents = new ArrayList<AppEvent>();
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isAppEvent(event)) {
|
||||
AppEvent appEvent = IntelliJEvent.toAppEvent(event);
|
||||
if (appEvent.isWhenActivated()) {
|
||||
if (appEvents.isEmpty()) {
|
||||
appEvents.add(appEvent);
|
||||
} else if (appEvents.get(appEvents.size() - 1).isWhenDeactivated()) {
|
||||
appEvents.add(appEvent);
|
||||
}
|
||||
} else if (appEvent.isWhenDeactivated()) {
|
||||
if (appEvents.isEmpty()) {
|
||||
appEvents.add(appEvent);
|
||||
} else if (appEvents.get(appEvents.size() - 1).isWhenActivated()) {
|
||||
appEvents.add(appEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return appEvents.isEmpty() ? null : appEvents;
|
||||
}
|
||||
|
||||
/**
|
||||
* separate durations in the IDE of the given event sequence, and the durations
|
||||
* outside the IDE are ignored.
|
||||
* @param events the given event sequence
|
||||
* @param appEvents the selected app event sequence of <code>events</code>
|
||||
* @param begTime begin time of <code>events</code>
|
||||
* @param endTime end time of <code>events</code>
|
||||
* @return durations in the IDE
|
||||
*/
|
||||
public static List<List<IntelliJEvent>> getDurationInIdes(List<IntelliJEvent> events, List<AppEvent> appEvents, long begTime, long endTime) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<List<IntelliJEvent>> durationInIDEs = new ArrayList<List<IntelliJEvent>>();
|
||||
if (appEvents == null || appEvents.isEmpty()) {
|
||||
durationInIDEs.add(events);
|
||||
} else {
|
||||
AppEvent prevAppEvent = appEvents.get(0);
|
||||
if (prevAppEvent.isWhenDeactivated()) {
|
||||
List<IntelliJEvent> tmp = DataSource.sliceEvents(events, begTime, prevAppEvent.getTime());
|
||||
if (tmp != null) {
|
||||
durationInIDEs.add(tmp);
|
||||
}
|
||||
}
|
||||
for (int i = 1, l = appEvents.size(); i < l; i++) {
|
||||
AppEvent nextAppEvent = appEvents.get(i);
|
||||
if (nextAppEvent.isWhenDeactivated()) {
|
||||
List<IntelliJEvent> tmp = DataSource.sliceEvents(events, prevAppEvent.getTime(), nextAppEvent.getTime());
|
||||
if (tmp != null) {
|
||||
durationInIDEs.add(tmp);
|
||||
}
|
||||
}
|
||||
prevAppEvent = nextAppEvent;
|
||||
}
|
||||
if (prevAppEvent.isWhenActivated()) {
|
||||
List<IntelliJEvent> tmp = DataSource.sliceEvents(events, prevAppEvent.getTime(), endTime);
|
||||
if (tmp != null) {
|
||||
durationInIDEs.add(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
return durationInIDEs;
|
||||
}
|
||||
|
||||
/**
|
||||
* @see MeasureUtil#getDurationInIdes(List, List, long, long)
|
||||
* @param events
|
||||
* @param begTime
|
||||
* @param endTime
|
||||
* @return
|
||||
*/
|
||||
public static List<List<IntelliJEvent>> getDurationInIdes(List<IntelliJEvent> events, long begTime, long endTime) {
|
||||
List<AppEvent> appEvents = getAppEventSequence(events);
|
||||
return getDurationInIdes(events, appEvents, begTime, endTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the number of entering IDE and the number of exiting IDE.
|
||||
* @param appEvents selected app events
|
||||
* @return the number of entering IDE and the number of exiting IDE
|
||||
*/
|
||||
public static int[] getEnterAndExitIdeCount(List<AppEvent> appEvents) {
|
||||
int appEventCount = (appEvents == null) ? 0 : appEvents.size();
|
||||
|
||||
int exitIdeCount;
|
||||
if (appEventCount % 2 == 0) { // 0, 2, 4, 6, 8, ...
|
||||
exitIdeCount = appEventCount / 2;
|
||||
} else { // 1, 3, 5, 7, 9, ...
|
||||
exitIdeCount = appEvents.get(0).isWhenActivated() ? ((appEventCount - 1) / 2) : ((appEventCount + 1) / 2);
|
||||
}
|
||||
int enterIdeCount = appEventCount - exitIdeCount;
|
||||
return new int[]{enterIdeCount, exitIdeCount};
|
||||
}
|
||||
|
||||
/**
|
||||
* get the time interval stay in the IDE and the time interval outside the IDE.
|
||||
* @param appEvents selected app events
|
||||
* @param begTime begin time
|
||||
* @param endTime end time
|
||||
* @return the time interval stay in the IDE and the time interval outside the IDE
|
||||
*/
|
||||
public static long[] getInAndOutsideIdeTime(List<AppEvent> appEvents, long begTime, long endTime) {
|
||||
long inIdeTime; // time spent in the IDE
|
||||
long outsideIdeTime;// time spent outside the IDE
|
||||
if (appEvents == null || appEvents.isEmpty()) {
|
||||
// this means that developers always stays in the IDE.
|
||||
inIdeTime = endTime - begTime;
|
||||
outsideIdeTime = 0;
|
||||
} else {
|
||||
inIdeTime = 0;
|
||||
outsideIdeTime = 0;
|
||||
AppEvent prevAppEvent = appEvents.get(0);
|
||||
if (prevAppEvent.isWhenActivated()) {
|
||||
outsideIdeTime += (prevAppEvent.getTime() - begTime);
|
||||
} else {
|
||||
inIdeTime += (prevAppEvent.getTime() - begTime);
|
||||
}
|
||||
for (int i = 1, l = appEvents.size(); i < l; i++) {
|
||||
AppEvent nextAppEvent = appEvents.get(i);
|
||||
if (nextAppEvent.isWhenActivated()) {
|
||||
outsideIdeTime += (nextAppEvent.getTime() - prevAppEvent.getTime());
|
||||
} else {
|
||||
inIdeTime += (nextAppEvent.getTime() - prevAppEvent.getTime());
|
||||
}
|
||||
prevAppEvent = nextAppEvent;
|
||||
}
|
||||
if (prevAppEvent.isWhenActivated()) {
|
||||
inIdeTime += (endTime - prevAppEvent.getTime());
|
||||
} else {
|
||||
outsideIdeTime += (endTime - prevAppEvent.getTime());
|
||||
}
|
||||
}
|
||||
return new long[]{inIdeTime, outsideIdeTime};
|
||||
}
|
||||
|
||||
/**
|
||||
* get the time interval of inactivity in the IDE.
|
||||
* @param events event sequence
|
||||
* @param appEvents selected app events
|
||||
* @param begTime begin time
|
||||
* @param endTime end time
|
||||
* @return the number and the time interval of inactivity in the IDE
|
||||
*/
|
||||
public static long getInactiveTimeInIDE(List<IntelliJEvent> events, List<AppEvent> appEvents, long begTime, long endTime, long threshold) {
|
||||
long inactiveTime = 0;
|
||||
List<List<IntelliJEvent>> durationInIdes = getDurationInIdes(events, appEvents, begTime, endTime);
|
||||
if (durationInIdes == null) {
|
||||
return endTime - begTime;
|
||||
}
|
||||
for (List<IntelliJEvent> durationInIde : durationInIdes) {
|
||||
IntelliJEvent prevEvent = durationInIde.get(0);
|
||||
for (int i = 1, l = durationInIde.size(); i < l; i++) {
|
||||
IntelliJEvent nextEvent = durationInIde.get(i);
|
||||
long deltaTime = nextEvent.getTime() - prevEvent.getTime();
|
||||
if (deltaTime >= threshold) {
|
||||
Assertion.assertTrue(deltaTime < 900000); // 15 minutes
|
||||
inactiveTime += deltaTime;
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
}
|
||||
}
|
||||
return inactiveTime;
|
||||
}
|
||||
|
||||
/**
|
||||
* select out context-related events including:
|
||||
* <ul>
|
||||
* <li><strong>Window activated events</strong>: we require ones whose <i>this</i> field denotes an IDE frame;</li>
|
||||
* <li><strong>File selected events</strong>: we require ones whose <i>newPath</i> field is not null;</li>
|
||||
* <li><strong>File changed events</strong>: we require ones whose <i>path</i> field is not null;</li>
|
||||
* <li><strong>Cursor changed events</strong>: we require ones whose <i>editor</i> field denotes a main editor;</li>
|
||||
* <li><strong>Cursor select changed events</strong>: we require all these.</li>
|
||||
* </ul>
|
||||
* @param events the given event sequence
|
||||
* @return context-related events
|
||||
*/
|
||||
public static List<IntelliJEvent> getWindowFileCursorEventSequence(List<IntelliJEvent> events) {
|
||||
if (events == null || events.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<WindowEvent> windowEvents = new ArrayList<WindowEvent>();
|
||||
List<FileEvent> fileEvents = new ArrayList<FileEvent>();
|
||||
List<CursorEvent> cursorEvents = new ArrayList<CursorEvent>();
|
||||
for (IntelliJEvent event : events) {
|
||||
if (IntelliJEvent.isWindowEvent(event)) {
|
||||
WindowEvent windowEvent = IntelliJEvent.toWindowEvent(event);
|
||||
if (windowEvent.isWhenWindowActivated()) {
|
||||
String thisName = windowEvent.getThisWindowName();
|
||||
if (thisName != null && thisName.startsWith("frame")) {
|
||||
windowEvents.add(windowEvent);
|
||||
}
|
||||
}
|
||||
} else if (IntelliJEvent.isFileEvent(event)) {
|
||||
FileEvent fileEvent = IntelliJEvent.toFileEvent(event);
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
if (!(fileEvent.getNewPathField() == null && fileEvent.getOldPathField() == null)) {
|
||||
fileEvents.add(fileEvent);
|
||||
}
|
||||
} else if (fileEvent.isWhenChanged()) {
|
||||
if (fileEvent.getPathField() != null) {
|
||||
fileEvents.add(fileEvent);
|
||||
}
|
||||
}
|
||||
} else if (IntelliJEvent.isCursorEvent(event)) {
|
||||
CursorEvent cursorEvent = IntelliJEvent.toCursorEvent(event);
|
||||
if (cursorEvent.isWhenCursorChanged()) {
|
||||
if (cursorEvent.isUntypedEditor() && (!"(0,0)".equals(cursorEvent.getOldPosField()))) {
|
||||
cursorEvents.add(cursorEvent);
|
||||
}
|
||||
} else if (cursorEvent.isWhenSelectChanged()) {
|
||||
cursorEvents.add(cursorEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<WindowEvent> selectedWindowEvents = new ArrayList<WindowEvent>();
|
||||
String initFrame = selectWindowEvents(windowEvents, selectedWindowEvents);
|
||||
List<FileEvent> selectedFileEvents = new ArrayList<FileEvent>();
|
||||
String initFile = selectFileEvents(fileEvents, selectedFileEvents, selectedWindowEvents, initFrame);
|
||||
|
||||
List<IntelliJEvent> selectedEvents = DataSource.mergeEvents(DataSource.mergeEvents(
|
||||
selectedWindowEvents, selectedFileEvents), cursorEvents);
|
||||
if (selectedEvents == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String focusedFrame = initFrame;
|
||||
Map<String, String> frame2FocusedFile = new HashMap<String, String>();
|
||||
updateStringMap(frame2FocusedFile, initFrame, initFile);
|
||||
for (IntelliJEvent event : selectedEvents) {
|
||||
if (event instanceof WindowEvent) {
|
||||
WindowEvent windowEvent = (WindowEvent) event;
|
||||
focusedFrame = windowEvent.getThisWindowName();
|
||||
} else if (event instanceof FileEvent) {
|
||||
FileEvent fileEvent = (FileEvent) event;
|
||||
// Assertion.assertEqual(focusedFrame, fileEvent.getExtraData("frame"));
|
||||
String frame = fileEvent.getExtraData("frame");
|
||||
if (frame != null) {
|
||||
focusedFrame = frame;
|
||||
}
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
updateStringMap(frame2FocusedFile, focusedFrame, fileEvent.getNewPathField());
|
||||
} else {
|
||||
updateStringMap(frame2FocusedFile, focusedFrame, fileEvent.getPathField());
|
||||
}
|
||||
} else {
|
||||
CursorEvent cursorEvent = (CursorEvent) event;
|
||||
cursorEvent.addExtraData("file", frame2FocusedFile.get(focusedFrame));
|
||||
cursorEvent.addExtraData("frame", focusedFrame);
|
||||
}
|
||||
}
|
||||
|
||||
return selectedEvents;
|
||||
}
|
||||
|
||||
private static String selectWindowEvents(List<WindowEvent> src, List<WindowEvent> dest) {
|
||||
String initFrame = null;
|
||||
String currFrame = null;
|
||||
for (int i = 0, l = src.size(); i < l; i++) {
|
||||
WindowEvent windowEvent = src.get(i);
|
||||
Assertion.assertTrue(windowEvent.isWhenWindowActivated());
|
||||
String thisName = windowEvent.getThisWindowName();
|
||||
Assertion.assertNotNull(thisName);
|
||||
Assertion.assertTrue(thisName.startsWith("frame"));
|
||||
String oppoName = windowEvent.getOppositeWindowName();
|
||||
|
||||
if (i == 0) {
|
||||
if (oppoName == null || oppoName.startsWith("frame")) {
|
||||
initFrame = oppoName;
|
||||
} else {
|
||||
initFrame = thisName;
|
||||
}
|
||||
}
|
||||
|
||||
if (!thisName.equals(currFrame)) {
|
||||
currFrame = thisName;
|
||||
if (oppoName == null || oppoName.startsWith("frame")) {
|
||||
dest.add(windowEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
return initFrame;
|
||||
}
|
||||
|
||||
private static String selectFileEvents(List<FileEvent> src, List<FileEvent> dest, List<WindowEvent> selectedWindowEvents, String initFrame) {
|
||||
Map<String, Set<String>> file2Frames = new HashMap<String, Set<String>>();
|
||||
Map<String, Integer> fileFrame2Count = new HashMap<String, Integer>();
|
||||
for (FileEvent fileEvent : src) {
|
||||
Assertion.assertTrue(fileEvent.isWhenSelected() || fileEvent.isWhenChanged());
|
||||
|
||||
String frame = initFrame;
|
||||
for (WindowEvent windowEvent : selectedWindowEvents) {
|
||||
if (windowEvent.getTime() <= fileEvent.getTime()) {
|
||||
frame = windowEvent.getThisWindowName();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (frame == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
String oldPath = fileEvent.getOldPathField();
|
||||
String newPath = fileEvent.getNewPathField();
|
||||
Assertion.assertNotAllNull(oldPath, newPath);
|
||||
updateSetMap(file2Frames, oldPath, frame);
|
||||
updateSetMap(file2Frames, newPath, frame);
|
||||
updateCountMap(fileFrame2Count, String.format("%s:%s", frame, oldPath));
|
||||
updateCountMap(fileFrame2Count, String.format("%s:%s", frame, newPath));
|
||||
} else {
|
||||
String path = fileEvent.getPathField();
|
||||
Assertion.assertNotNull(path);
|
||||
updateSetMap(file2Frames, path, frame);
|
||||
updateCountMap(fileFrame2Count, String.format("%s:%s", frame, path));
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, String> file2Frame = new HashMap<String, String>();
|
||||
for (Map.Entry<String, Set<String>> entry : file2Frames.entrySet()) {
|
||||
String frame = null;
|
||||
int count = 0;
|
||||
for (String s : entry.getValue()) {
|
||||
int v = fileFrame2Count.get(String.format("%s:%s", s, entry.getKey()));
|
||||
if (v > count) {
|
||||
frame = s;
|
||||
count = v;
|
||||
}
|
||||
}
|
||||
Assertion.assertNotNull(frame);
|
||||
file2Frame.put(entry.getKey(), frame);
|
||||
}
|
||||
|
||||
String initFile = null;
|
||||
boolean isFileInitialized = false;
|
||||
for (FileEvent fileEvent : src) {
|
||||
String path = fileEvent.isWhenSelected() ? fileEvent.getNewPathField() : fileEvent.getPathField();
|
||||
String frame = file2Frame.get(path);
|
||||
fileEvent.addExtraData("frame", frame);
|
||||
if (path != null) {
|
||||
dest.add(fileEvent);
|
||||
}
|
||||
|
||||
if ((!isFileInitialized) && frame != null && frame.equals(initFrame)) {
|
||||
if (fileEvent.isWhenSelected()) {
|
||||
initFile = fileEvent.getOldPathField();
|
||||
} else {
|
||||
initFile = fileEvent.getPathField();
|
||||
}
|
||||
isFileInitialized = true;
|
||||
}
|
||||
}
|
||||
return initFile;
|
||||
}
|
||||
|
||||
// private static void selectCursorEvents(List<CursorEvent> src, List<IntelliJEvent> windowFileEvents) {
|
||||
// for (CursorEvent cursorEvent : src) {
|
||||
// Assertion.assertTrue(cursorEvent.isWhenCursorChanged() || cursorEvent.isWhenSelectChanged());
|
||||
// if (cursorEvent.isWhenCursorChanged()) {
|
||||
// Assertion.assertTrue(cursorEvent.isUntypedEditor());
|
||||
// Assertion.assertNotEqual(cursorEvent.getOldPosField(), "(0,0)");
|
||||
// }
|
||||
//
|
||||
// String frame = null;
|
||||
// String file = null;
|
||||
// for (FileEvent fileEvent : selectedFileEvents) {
|
||||
// if (fileEvent.getTime() <= cursorEvent.getTime()) {
|
||||
// file = fileEvent.isWhenSelected() ? fileEvent.getNewPathField() : fileEvent.getPathField();
|
||||
// frame = fileEvent.getExtraData("frame");
|
||||
// } else {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (file == null) {
|
||||
// frame = initFrame;
|
||||
// for (WindowEvent windowEvent : selectedWindowEvents) {
|
||||
// if (windowEvent.getTime() <= cursorEvent.getTime()) {
|
||||
// frame = windowEvent.getThisWindowName();
|
||||
// } else {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// cursorEvent.addExtraData("file", file);
|
||||
// cursorEvent.addExtraData("frame", frame);
|
||||
// }
|
||||
// }
|
||||
|
||||
public static <T> void updateListMap(Map<String, List<T>> map, String k, T v) {
|
||||
if (k == null || v == null) {
|
||||
return;
|
||||
}
|
||||
List<T> list = map.get(k);
|
||||
if (list == null) {
|
||||
list = new ArrayList<T>();
|
||||
}
|
||||
list.add(v);
|
||||
map.put(k, list);
|
||||
}
|
||||
|
||||
public static <T> void updateSetMap(Map<String, Set<T>> map, String k, T v) {
|
||||
if (k == null || v == null) {
|
||||
return;
|
||||
}
|
||||
Set<T> set = map.get(k);
|
||||
if (set == null) {
|
||||
set = new HashSet<T>();
|
||||
}
|
||||
set.add(v);
|
||||
map.put(k, set);
|
||||
}
|
||||
|
||||
public static void updateStringMap(Map<String, String> map, String k, String v) {
|
||||
if (k == null || v == null) {
|
||||
return;
|
||||
}
|
||||
map.put(k, v);
|
||||
}
|
||||
|
||||
public static void updateCountMap(Map<String, Integer> map, String k) {
|
||||
if (k == null) {
|
||||
return;
|
||||
}
|
||||
Integer v = map.get(k);
|
||||
if (v == null) {
|
||||
v = 0;
|
||||
}
|
||||
v++;
|
||||
map.put(k, v);
|
||||
}
|
||||
|
||||
public static <T> Map<T, Integer> countList(List<T> list) {
|
||||
if (list == null || list.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
Map<T, Integer> map = new HashMap<T, Integer>();
|
||||
for (T item : list) {
|
||||
Integer count = map.get(item);
|
||||
if (count == null) {
|
||||
count = 0;
|
||||
}
|
||||
count++;
|
||||
map.put(item, count);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
public static <K, V extends Number> double countMap(Map<K, V> map) {
|
||||
if (map == null || map.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
double count = 0;
|
||||
for (Map.Entry<K, V> entry : map.entrySet()) {
|
||||
count += entry.getValue().doubleValue();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static final double LOG_10_2 = Math.log(2);
|
||||
|
||||
/**
|
||||
* calculate Shannon entropy: -\Sigma_{i=1}^n p_i * log_2 p_i, where p_i equals to frequency / count.
|
||||
* @param frequencies frequency of each item
|
||||
* @param count \Sigma_{i=1}^n frequency_i
|
||||
* @return Shannon entropy
|
||||
*/
|
||||
public static <K, V extends Number> double shannonEntropy(Map<K, V> frequencies, double count) {
|
||||
if (frequencies == null || frequencies.isEmpty() || count <= 0) {
|
||||
return 0;
|
||||
}
|
||||
double result = 0;
|
||||
for (Map.Entry<K, V> entry : frequencies.entrySet()) {
|
||||
double frequency = entry.getValue().doubleValue();
|
||||
Assertion.assertPositive(frequency);
|
||||
double probability = frequency / count;
|
||||
result -= (probability * Math.log(probability) / LOG_10_2);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @see MeasureUtil#shannonEntropy(Map, double)
|
||||
* @param frequencies
|
||||
* @param <K>
|
||||
* @param <V>
|
||||
* @return
|
||||
*/
|
||||
public static <K, V extends Number> double shannonEntropy(Map<K, V> frequencies) {
|
||||
if (frequencies == null || frequencies.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
double count = countMap(frequencies);
|
||||
return shannonEntropy(frequencies, count);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see MeasureUtil#shannonEntropy(Map, double)
|
||||
* @param sequence
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T> double shannonEntropy(List<T> sequence) {
|
||||
if (sequence == null || sequence.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
Map<T, Integer> frequencies = countList(sequence);
|
||||
return shannonEntropy(frequencies, sequence.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* @see MeasureUtil#removeOutliers(List, int, int)
|
||||
*/
|
||||
public static void removeOutliers(List<long[]> cursorChangingHistory) {
|
||||
if (cursorChangingHistory == null || cursorChangingHistory.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
int maxDis = 5;
|
||||
int minHop = Math.min(cursorChangingHistory.size(), Math.max(5, cursorChangingHistory.size() / 10));
|
||||
removeOutliers(cursorChangingHistory, maxDis, minHop);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removing outliers in the cursor changed sequence.
|
||||
* @param cursorChangingHistory the history of changing the position (line) of the cursor.
|
||||
* @param maxDis for any two adjacent cursor changed events, the difference between
|
||||
* the former's {@code newPos} and the latter's {@code oldPos} must be
|
||||
* less than the given {@code maxDis}.
|
||||
* @param minHop the continuous sub cursor changed sequence is valid
|
||||
* if and only if its hops exceed {@code minHop}.
|
||||
*/
|
||||
public static void removeOutliers(List<long[]> cursorChangingHistory, int maxDis, int minHop) {
|
||||
if (cursorChangingHistory == null || cursorChangingHistory.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<List<long[]>> traces = new ArrayList<List<long[]>>();
|
||||
List<long[]> tmp = null;
|
||||
|
||||
for (long[] item : cursorChangingHistory) {
|
||||
if (tmp == null) {
|
||||
tmp = new ArrayList<long[]>();
|
||||
tmp.add(item);
|
||||
traces.add(tmp);
|
||||
} else {
|
||||
long lastNewLine = tmp.get(tmp.size() - 1)[1];
|
||||
long currOldLine = item[2];
|
||||
|
||||
if (Math.abs(currOldLine - lastNewLine) < maxDis) {
|
||||
tmp.add(item);
|
||||
} else {
|
||||
tmp = new ArrayList<long[]>();
|
||||
tmp.add(item);
|
||||
traces.add(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (List<long[]> trace : traces) {
|
||||
if (trace.size() < minHop) {
|
||||
cursorChangingHistory.removeAll(trace);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.reflect.TypeToken;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class AbstractSliceSeparateTask implements Executor.Task {
|
||||
protected long mWindowSize;
|
||||
protected double mOverlap;
|
||||
protected Gson mGson;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
* @param windowSize window size in minute.
|
||||
*/
|
||||
public AbstractSliceSeparateTask(long windowSize, double overlap) {
|
||||
mWindowSize = windowSize * 60 * 1000;
|
||||
mOverlap = overlap;
|
||||
mGson = DataSource.getGson();
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
FileUtil.delete(new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName())));
|
||||
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int sliceCount = 0;
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
File sessionFile = new File(FileUtil.combine(dateDir.getAbsolutePath(), "sessions.json"));
|
||||
if (!sessionFile.exists()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Session> sessions = FileUtil.fromJson(mGson, new TypeToken<ArrayList<Session>>(){}.getType(), sessionFile);
|
||||
if (sessions == null || sessions.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<Slice> slices = new ArrayList<Slice>();
|
||||
for (Session session : sessions) {
|
||||
List<Slice> tmp = createSlice(session);
|
||||
if (tmp != null) {
|
||||
slices.addAll(tmp);
|
||||
}
|
||||
}
|
||||
if (!slices.isEmpty()) {
|
||||
FileUtil.toJson(mGson, slices, new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName(), dateDir.getName(), "slices.json")));
|
||||
sliceCount += slices.size();
|
||||
}
|
||||
}
|
||||
|
||||
System.out.printf("%s/%s -> %d slices\n", teamDir.getName(), userDir.getName(), sliceCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Separate slices out of a session.
|
||||
* @param session session
|
||||
* @return new slices, or null if no slice meets all the conditions.
|
||||
*/
|
||||
protected abstract List<Slice> createSlice(Session session);
|
||||
|
||||
protected List<Label> filterLabels(List<Label> labels, long sessionBegTime, long sessionEndTime) {
|
||||
if (labels == null || labels.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<Label> result = new ArrayList<Label>();
|
||||
for (Label label : labels) {
|
||||
// check whether the developer answer all of the questions
|
||||
Label.Type type = label.getCanonicalType();
|
||||
if (type == Label.Type.REFUSE_WITHOUT_SELECT || type == Label.Type.CLOSE_WITHOUT_SELECT) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check is there sufficient events for the label
|
||||
long sliceEndTime = label.getEndTime();
|
||||
long sliceBegTime = sliceEndTime - mWindowSize;
|
||||
if (!(sessionBegTime <= sliceBegTime && sliceEndTime <= sessionEndTime)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check whether submit repeatedly in a short time
|
||||
if (!result.isEmpty()) {
|
||||
long t = result.get(result.size() - 1).getEndTime();
|
||||
if (sliceBegTime <= t && t <= sliceEndTime) {
|
||||
double ratio = ((double) (t - sliceBegTime)) / ((double) mWindowSize);
|
||||
Assertion.assertInClosedIntervalFrom0To1(ratio);
|
||||
if (ratio > mOverlap) {
|
||||
result.remove(result.size() - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// usable label
|
||||
result.add(label);
|
||||
}
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class EventBasedSliceSeparateTask extends AbstractSliceSeparateTask {
|
||||
public EventBasedSliceSeparateTask(long windowSize, double overlap) {
|
||||
super(windowSize, overlap);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Slice> createSlice(Session session) {
|
||||
String team = session.getTeam();
|
||||
String user = session.getUser();
|
||||
String date = session.getDate();
|
||||
long sessionBegTime = session.getBegTime();
|
||||
long sessionEndTime = session.getEndTime();
|
||||
int sessionId = session.getSessionId();
|
||||
List<IntelliJEvent> events = session.getEvents();
|
||||
List<Label> labels = filterLabels(session.getLabels(), sessionBegTime, sessionEndTime);
|
||||
if (events == null || labels == null) {
|
||||
return null;
|
||||
}
|
||||
Assertion.assertTrue(sessionEndTime - sessionBegTime >= mWindowSize);
|
||||
|
||||
List<Slice> result = new ArrayList<Slice>();
|
||||
|
||||
int labelOffset = 0;
|
||||
long sliceOffset = (long) ((1 - mOverlap) * mWindowSize);
|
||||
for (long sliceBegTime = sessionBegTime; sliceBegTime < sessionEndTime; sliceBegTime += sliceOffset) {
|
||||
long sliceEndTime = sliceBegTime + mWindowSize;
|
||||
if (sliceEndTime <= sessionEndTime) {
|
||||
List<IntelliJEvent> eventCache = DataSource.sliceEvents(events, sliceBegTime, sliceEndTime);
|
||||
Label label = getLabel(labels, labelOffset, sliceBegTime, sliceEndTime);
|
||||
if (label != null) {
|
||||
labelOffset++;
|
||||
}
|
||||
result.add(new Slice(team, user, date, sliceBegTime, sliceEndTime,
|
||||
sessionBegTime, sessionEndTime, sessionId, eventCache, label));
|
||||
} else {
|
||||
Assertion.assertFalse(result.isEmpty());
|
||||
long extraTime = sliceEndTime - sessionEndTime;
|
||||
Assertion.assertTrue(0 < extraTime && extraTime <= sliceOffset);
|
||||
|
||||
Label newLabel = getLabel(labels, labelOffset, sliceBegTime, sliceEndTime);
|
||||
if (newLabel != null) {
|
||||
labelOffset++;
|
||||
Slice slice = result.get(result.size() - 1);
|
||||
Label oldLabel = slice.getLabel();
|
||||
if (oldLabel != null) {
|
||||
Assertion.assertTrue(newLabel.getEndTime() - oldLabel.getEndTime() >= sliceOffset);
|
||||
}
|
||||
slice.setLabel(newLabel);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
Assertion.assertEqual(labelOffset, labels.size());
|
||||
return result.isEmpty() ? null : result;
|
||||
}
|
||||
|
||||
private Label getLabel(List<Label> labels, int offset, long sliceBegTime, long sliceEndTime) {
|
||||
if (labels == null || labels.isEmpty() || offset < 0 || offset >= labels.size()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Label label = labels.get(offset);
|
||||
long time = label.getEndTime();
|
||||
if (time < sliceBegTime) {
|
||||
throw new IllegalArgumentException("this means there is a missing label!");
|
||||
} else {
|
||||
if (time <= sliceEndTime) {
|
||||
return label;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class InputEventIterator {
|
||||
private List<IntelliJEvent> mEvents;
|
||||
private int mOffset;
|
||||
private int mBreakpoint;
|
||||
private boolean mIsOffsetUpdated;
|
||||
|
||||
public InputEventIterator(List<IntelliJEvent> events) {
|
||||
mEvents = events;
|
||||
mOffset = 0;
|
||||
mBreakpoint = 0;
|
||||
mIsOffsetUpdated = false;
|
||||
}
|
||||
|
||||
public IntelliJEvent nextInputEvent() {
|
||||
if (mEvents == null ||mEvents.isEmpty() || mOffset >= mEvents.size()) {
|
||||
return null;
|
||||
}
|
||||
for (int i = mOffset, l = mEvents.size(); i < l; i++) {
|
||||
IntelliJEvent event = mEvents.get(i);
|
||||
if (IntelliJEvent.isKeyEvent(event) || IntelliJEvent.isMouseEvent(event)) {
|
||||
mOffset = i + 1;
|
||||
mIsOffsetUpdated = true;
|
||||
return event;
|
||||
}
|
||||
}
|
||||
mOffset = mEvents.size();
|
||||
mIsOffsetUpdated = true;
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<IntelliJEvent> separate() {
|
||||
if (mEvents == null || mEvents.isEmpty() || (!mIsOffsetUpdated) || mBreakpoint == mOffset - 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int begIndex = mBreakpoint;
|
||||
int endIndex = mOffset - 2;
|
||||
if (begIndex < 0 || endIndex >= mEvents.size() || begIndex > endIndex) {
|
||||
throw new IllegalArgumentException(String.format("begin index=%d, end index=%d, interval=[0, %d]",
|
||||
begIndex, endIndex, mEvents.size()));
|
||||
}
|
||||
|
||||
List<IntelliJEvent> result = new ArrayList<IntelliJEvent>();
|
||||
for (int i = begIndex; i <= endIndex; i++) {
|
||||
result.add(mEvents.get(i));
|
||||
}
|
||||
mBreakpoint = mOffset - 1;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
public class LabelBasedSliceSeparateTask extends AbstractSliceSeparateTask {
|
||||
public LabelBasedSliceSeparateTask(long windowSize, double overlap) {
|
||||
super(windowSize, overlap);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Slice> createSlice(Session session) {
|
||||
String team = session.getTeam();
|
||||
String user = session.getUser();
|
||||
String date = session.getDate();
|
||||
long sessionBegTime = session.getBegTime();
|
||||
long sessionEndTime = session.getEndTime();
|
||||
int sessionId = session.getSessionId();
|
||||
List<IntelliJEvent> events = session.getEvents();
|
||||
List<Label> labels = filterLabels(session.getLabels(), sessionBegTime, sessionEndTime);
|
||||
if (events == null || labels == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<Slice> slices = new LinkedList<Slice>();
|
||||
for (Label label : labels) {
|
||||
long sliceEndTime = label.getEndTime();
|
||||
long sliceBegTime = sliceEndTime - mWindowSize;
|
||||
List<IntelliJEvent> eventCache = DataSource.sliceEvents(events, sliceBegTime, sliceEndTime);
|
||||
boolean hasAnyInputEvent = false;
|
||||
for (IntelliJEvent event : eventCache) {
|
||||
if (IntelliJEvent.isKeyEvent(event) || IntelliJEvent.isMouseEvent(event)) {
|
||||
hasAnyInputEvent = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (hasAnyInputEvent) {
|
||||
slices.add(new Slice(team, user, date, sliceBegTime, sliceEndTime,
|
||||
sessionBegTime, sessionEndTime, sessionId, eventCache, label));
|
||||
} else {
|
||||
throw new IllegalArgumentException("no any input event in the slice!");
|
||||
}
|
||||
}
|
||||
return slices.isEmpty() ? null : slices;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,282 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A Study of Visual Studio Usage in Practice.
|
||||
* Since we cannot directly determine the kind of inactivity from our events, we heuristically
|
||||
* separate inactivity intervals by their duration, reasoning that longer inactivity is more
|
||||
* likely to be an actual interruption of a developer’s work in the IDE. We separate short
|
||||
* inactivity from long inactivity using a threshold t.
|
||||
*/
|
||||
public class SaturationEffectTask implements Executor.Task {
|
||||
private long[] mShortInactivityCounts;
|
||||
private long[] mShortInactivityTimes;
|
||||
private long[] mLongInactivityCounts;
|
||||
private long[] mLongInactivityTimes;
|
||||
private boolean mIsOnlyInput;
|
||||
|
||||
public SaturationEffectTask(boolean isOnlyInput) {
|
||||
this.mShortInactivityCounts = initLongArray(30, 0);
|
||||
this.mShortInactivityTimes = initLongArray(30, 0);
|
||||
this.mLongInactivityCounts = initLongArray(30, 0);
|
||||
this.mLongInactivityTimes = initLongArray(30, 0);
|
||||
this.mIsOnlyInput = isOnlyInput;
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int workdayCount = 0;
|
||||
|
||||
long[] shortInactivityCounts = initLongArray(30, 0);
|
||||
long[] shortInactivityTimes = initLongArray(30, 0);
|
||||
long[] longInactivityCounts = initLongArray(30, 0);
|
||||
long[] longInactivityTimes = initLongArray(30, 0);
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, "event", ".json");
|
||||
if (eventFiles == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<IntelliJEvent> events = new ArrayList<IntelliJEvent>();
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> tmp = DataSource.loadEvents(eventFile);
|
||||
if (tmp != null) {
|
||||
events.addAll(tmp);
|
||||
}
|
||||
}
|
||||
if (events.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mIsOnlyInput) {
|
||||
calculateDeltaTimeBetweenInputEvents(events, shortInactivityCounts, shortInactivityTimes, longInactivityCounts, longInactivityTimes);
|
||||
} else {
|
||||
calculateDeltaTimeBetweenEvents(events, shortInactivityCounts, shortInactivityTimes, longInactivityCounts, longInactivityTimes);
|
||||
}
|
||||
|
||||
workdayCount++;
|
||||
}
|
||||
|
||||
addArray(mShortInactivityCounts, shortInactivityCounts);
|
||||
addArray(mShortInactivityTimes, shortInactivityTimes);
|
||||
addArray(mLongInactivityCounts, longInactivityCounts);
|
||||
addArray(mLongInactivityTimes, longInactivityTimes);
|
||||
|
||||
System.out.printf("%s/%s: #workday=%d\n", teamDir.getName(), userDir.getName(), workdayCount);
|
||||
determineSaturationEffect(shortInactivityCounts, shortInactivityTimes, longInactivityCounts, longInactivityTimes, true);
|
||||
}
|
||||
|
||||
private void calculateDeltaTimeBetweenInputEvents(
|
||||
List<IntelliJEvent> events, long[] shortInactivityCounts, long[] shortInactivityTimes,
|
||||
long[] longInactivityCounts, long[] longInactivityTimes) {
|
||||
InputEventIterator iterator = new InputEventIterator(events);
|
||||
IntelliJEvent prevEvent = iterator.nextInputEvent();
|
||||
if (prevEvent == null) {
|
||||
return;
|
||||
}
|
||||
IntelliJEvent nextEvent;
|
||||
while ((nextEvent = iterator.nextInputEvent()) != null) {
|
||||
long deltaTime = nextEvent.getTime() - prevEvent.getTime();
|
||||
for (int j = 0, n = shortInactivityCounts.length; j < n; j++) {
|
||||
long threshold = 60000 * (j + 1);
|
||||
if (deltaTime < threshold) {
|
||||
shortInactivityCounts[j]++;
|
||||
shortInactivityTimes[j] += deltaTime;
|
||||
} else {
|
||||
longInactivityCounts[j]++;
|
||||
longInactivityTimes[j] += deltaTime;
|
||||
}
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
}
|
||||
}
|
||||
|
||||
private void calculateDeltaTimeBetweenEvents(
|
||||
List<IntelliJEvent> events, long[] shortInactivityCounts, long[] shortInactivityTimes,
|
||||
long[] longInactivityCounts, long[] longInactivityTimes) {
|
||||
IntelliJEvent prevEvent = events.get(0);
|
||||
for (int i = 1, m = events.size(); i < m; i++) {
|
||||
IntelliJEvent nextEvent = events.get(i);
|
||||
long deltaTime = nextEvent.getTime() - prevEvent.getTime();
|
||||
for (int j = 0, n = shortInactivityCounts.length; j < n; j++) {
|
||||
long threshold = 60000 * (j + 1);
|
||||
if (deltaTime < threshold) {
|
||||
shortInactivityCounts[j]++;
|
||||
shortInactivityTimes[j] += deltaTime;
|
||||
} else {
|
||||
longInactivityCounts[j]++;
|
||||
longInactivityTimes[j] += deltaTime;
|
||||
}
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
}
|
||||
}
|
||||
|
||||
private double[][] percentageAssignedToLongInactivity(long[] shortInactivityCounts, long[] shortInactivityTimes,
|
||||
long[] longInactivityCounts, long[] longInactivityTimes) {
|
||||
double[][] result = new double[longInactivityCounts.length][2];
|
||||
for (int i = 0, l = result.length; i < l; i++) {
|
||||
result[i][0] = ((double) longInactivityCounts[i]) / ((double) (shortInactivityCounts[i] + longInactivityCounts[i]));
|
||||
result[i][1] = ((double) longInactivityTimes[i]) / ((double) (shortInactivityTimes[i] + longInactivityTimes[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public void determineSaturationEffect(boolean isShowDetails) {
|
||||
determineSaturationEffect(mShortInactivityCounts, mShortInactivityTimes, mLongInactivityCounts, mLongInactivityTimes, isShowDetails);
|
||||
}
|
||||
|
||||
private void determineSaturationEffect(long[] shortInactivityCounts, long[] shortInactivityTimes,
|
||||
long[] longInactivityCounts, long[] longInactivityTimes,
|
||||
boolean isShowDetails) {
|
||||
double[][] percentages = percentageAssignedToLongInactivity(
|
||||
shortInactivityCounts, shortInactivityTimes,
|
||||
longInactivityCounts, longInactivityTimes);
|
||||
|
||||
int deltaSICountT = -1, deltaLICountT = -1, deltaSICountPercentageT = -1;
|
||||
int deltaSITimeT = -1, deltaLITimeT = -1, deltaSITimePercentageT = -1;
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(" t | #SI(i-1,i+1) #LI(i-1,i+1) #LI%(i-1,i+1) | △SI(i-1,i+1) △LI(i-1,i+1) △LI%(i-1,i+1)\n");
|
||||
builder.append("--------------------------------------------------------------------------------------\n");
|
||||
|
||||
double deltaSICount1 = Double.NaN;
|
||||
double deltaSICount2 = calculateRelativeDelta(shortInactivityCounts[1], shortInactivityCounts[0]);
|
||||
double deltaLICount1 = Double.NaN;
|
||||
double deltaLICount2 = calculateRelativeDelta(longInactivityCounts[1], longInactivityCounts[0]);
|
||||
double deltaLICountPercentage1 = Double.NaN;
|
||||
double deltaLICountPercentage2 = calculateRelativeDelta(percentages[1][0], percentages[0][0]);
|
||||
|
||||
double deltaSITime1 = Double.NaN;
|
||||
double deltaSITime2 = calculateRelativeDelta(shortInactivityTimes[1], shortInactivityTimes[0]);
|
||||
double deltaLITime1 = Double.NaN;
|
||||
double deltaLITime2 = calculateRelativeDelta(longInactivityTimes[1], longInactivityTimes[0]);
|
||||
double deltaLITimePercentage1 = Double.NaN;
|
||||
double deltaLITimePercentage2 = calculateRelativeDelta(percentages[1][1], percentages[0][1]);
|
||||
|
||||
builder.append(String.format("%2d | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f\n",
|
||||
1,
|
||||
deltaSICount1, deltaSICount2, deltaLICount1, deltaLICount2, deltaLICountPercentage1, deltaLICountPercentage2,
|
||||
deltaSITime1, deltaSITime2, deltaLITime1, deltaLITime2, deltaLITimePercentage1, deltaLITimePercentage2));
|
||||
|
||||
for (int i = 1; i < percentages.length - 1; i++) {
|
||||
deltaSICount1 = calculateRelativeDelta(shortInactivityCounts[i], shortInactivityCounts[i - 1]);
|
||||
deltaSICount2 = calculateRelativeDelta(shortInactivityCounts[i + 1], shortInactivityCounts[i]);
|
||||
deltaLICount1 = calculateRelativeDelta(longInactivityCounts[i], longInactivityCounts[i - 1]);
|
||||
deltaLICount2 = calculateRelativeDelta(longInactivityCounts[i + 1], longInactivityCounts[i]);
|
||||
deltaLICountPercentage1 = calculateRelativeDelta(percentages[i][0], percentages[i - 1][0]);
|
||||
deltaLICountPercentage2 = calculateRelativeDelta(percentages[i + 1][0], percentages[i][0]);
|
||||
|
||||
deltaSITime1 = calculateRelativeDelta(shortInactivityTimes[i], shortInactivityTimes[i - 1]);
|
||||
deltaSITime2 = calculateRelativeDelta(shortInactivityTimes[i + 1], shortInactivityTimes[i]);
|
||||
deltaLITime1 = calculateRelativeDelta(longInactivityTimes[i], longInactivityTimes[i - 1]);
|
||||
deltaLITime2 = calculateRelativeDelta(longInactivityTimes[i + 1], longInactivityTimes[i]);
|
||||
deltaLITimePercentage1 = calculateRelativeDelta(percentages[i][1], percentages[i - 1][1]);
|
||||
deltaLITimePercentage2 = calculateRelativeDelta(percentages[i + 1][1], percentages[i][1]);
|
||||
|
||||
builder.append(String.format("%2d | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f\n",
|
||||
i + 1,
|
||||
deltaSICount1, deltaSICount2, deltaLICount1, deltaLICount2, deltaLICountPercentage1, deltaLICountPercentage2,
|
||||
deltaSITime1, deltaSITime2, deltaLITime1, deltaLITime2, deltaLITimePercentage1, deltaLITimePercentage2));
|
||||
|
||||
if (deltaSICountT == -1) {
|
||||
if (Math.abs(deltaSICount1) <= 5.0 && Math.abs(deltaSICount2) <= 5.0) {
|
||||
deltaSICountT = i + 1;
|
||||
}
|
||||
}
|
||||
if (deltaLICountT == -1) {
|
||||
if (Math.abs(deltaLICount1) <= 5.0 && Math.abs(deltaLICount2) <= 5.0) {
|
||||
deltaLICountT = i + 1;
|
||||
}
|
||||
}
|
||||
if (deltaSICountPercentageT == -1) {
|
||||
if (Math.abs(deltaLICountPercentage1) <= 5.0 && Math.abs(deltaLICountPercentage2) <= 5.0) {
|
||||
deltaSICountPercentageT = i + 1;
|
||||
}
|
||||
}
|
||||
if (deltaSITimeT == -1) {
|
||||
if (Math.abs(deltaSITime1) <= 5.0 && Math.abs(deltaSITime2) <= 5.0) {
|
||||
deltaSITimeT = i + 1;
|
||||
}
|
||||
}
|
||||
if (deltaLITimeT == -1) {
|
||||
if (Math.abs(deltaLITime1) <= 5.0 && Math.abs(deltaLITime2) <= 5.0) {
|
||||
deltaLITimeT = i + 1;
|
||||
}
|
||||
}
|
||||
if (deltaSITimePercentageT == -1) {
|
||||
if (Math.abs(deltaLITimePercentage1) <= 5.0 && Math.abs(deltaLITimePercentage2) <= 5.0) {
|
||||
deltaSITimePercentageT = i + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deltaSICount1 = calculateRelativeDelta(shortInactivityCounts[shortInactivityCounts.length - 1], shortInactivityCounts[shortInactivityCounts.length - 2]);
|
||||
deltaSICount2 = Double.NaN;
|
||||
deltaLICount1 = calculateRelativeDelta(longInactivityCounts[longInactivityCounts.length - 1], longInactivityCounts[longInactivityCounts.length - 2]);
|
||||
deltaLICount2 = Double.NaN;
|
||||
deltaLICountPercentage1 = calculateRelativeDelta(percentages[percentages.length - 1][0], percentages[percentages.length - 2][0]);
|
||||
deltaLICountPercentage2 = Double.NaN;
|
||||
|
||||
deltaSITime1 = calculateRelativeDelta(shortInactivityTimes[shortInactivityTimes.length - 1], shortInactivityTimes[shortInactivityTimes.length - 2]);
|
||||
deltaSITime2 = Double.NaN;
|
||||
deltaLITime1 = calculateRelativeDelta(longInactivityTimes[longInactivityTimes.length - 1], longInactivityTimes[longInactivityTimes.length - 2]);
|
||||
deltaLITime2 = Double.NaN;
|
||||
deltaLITimePercentage1 = calculateRelativeDelta(percentages[percentages.length - 1][1], percentages[percentages.length - 2][1]);
|
||||
deltaLITimePercentage2 = Double.NaN;
|
||||
|
||||
builder.append(String.format("%2d | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f | %5.1f,%5.1f %5.1f,%5.1f %5.1f,%5.1f\n",
|
||||
shortInactivityCounts.length,
|
||||
deltaSICount1, deltaSICount2, deltaLICount1, deltaLICount2, deltaLICountPercentage1, deltaLICountPercentage2,
|
||||
deltaSITime1, deltaSITime2, deltaLITime1, deltaLITime2, deltaLITimePercentage1, deltaLITimePercentage2));
|
||||
if (isShowDetails) {
|
||||
System.out.print(builder.toString());
|
||||
}
|
||||
|
||||
System.out.printf("deltaSICountT:%2d, deltaLICountT:%2d, deltaLICountPercentageT:%2d, deltaSITimeT:%2d, deltaLITimeT:%2d, deltaLITimePercentageT:%2d\n",
|
||||
deltaSICountT, deltaLICountT, deltaSICountPercentageT, deltaSITimeT, deltaLITimeT, deltaSITimePercentageT);
|
||||
}
|
||||
|
||||
private long[] initLongArray(int length, long value) {
|
||||
if (length <= 0) {
|
||||
return null;
|
||||
}
|
||||
long[] array = new long[length];
|
||||
for (int i = 0; i < length; i++) {
|
||||
array[i] = value;
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
/**
|
||||
* Change of p relative to q.
|
||||
* @param p
|
||||
* @param q
|
||||
* @return
|
||||
*/
|
||||
private double calculateRelativeDelta(double p, double q) {
|
||||
return Double.isNaN(p) || Double.isNaN(q) ? Double.NaN : 100 * (p - q) / q;
|
||||
}
|
||||
|
||||
private void addArray(long[] dest, long[] src) {
|
||||
if (dest == null || src == null || dest.length != src.length) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0, l = dest.length; i < l; i++) {
|
||||
dest[i] += src[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.configuration.Configuration;
|
||||
import edu.nju.ics.frontier.util.Constants;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class SegmentMain {
|
||||
public static void main(String[] args) {
|
||||
// String srcRoot1 = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
// ThresholdDetermineTask thresholdDetermineTask = new ThresholdDetermineTask(false);
|
||||
// Executor.executeToUserDir(srcRoot1, "", thresholdDetermineTask);
|
||||
// thresholdDetermineTask.determineSuitableLambda(true);
|
||||
// SaturationEffectTask saturationEffectTask = new SaturationEffectTask(false);
|
||||
// Executor.executeToUserDir(srcRoot1, "", saturationEffectTask);
|
||||
// saturationEffectTask.determineSaturationEffect(true);
|
||||
|
||||
// String srcRoot2 = FileUtil.combine(Constants.FUJITSU_ROOT, "parse");
|
||||
// String destRoot2 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "session");
|
||||
// Executor.executeToUserDir(srcRoot2, destRoot2, new SessionSeparateTask(15, false));
|
||||
// final Map<String, Integer> lambdas = getLambdaForIndividual();
|
||||
// Executor.executeToUserDir(srcRoot2, destRoot2, new Executor.TaskFactory() {
|
||||
// public Executor.Task newTask(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
// Integer lambda = lambdas.get(userDir.getName());
|
||||
// if (lambda == null) {
|
||||
// throw new IllegalArgumentException("no lambda for " + userDir.getName());
|
||||
// }
|
||||
// return new SessionSeparateTask(lambda, 0, false);
|
||||
// }
|
||||
// });
|
||||
|
||||
// String srcRoot3 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "session");
|
||||
// String destRoot3 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "label_based_slice");
|
||||
// Executor.executeToUserDir(srcRoot3, destRoot3, new LabelBasedSliceSeparateTask(30, 0.5));
|
||||
|
||||
// String srcRoot4 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "session");
|
||||
// String destRoot4 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "event_based_slice");
|
||||
// Executor.executeToUserDir(srcRoot4, destRoot4, new EventBasedSliceSeparateTask(30, 0.5));
|
||||
|
||||
String srcRoot5 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "session");
|
||||
String destRoot5 = FileUtil.combine(Constants.FUJITSU_ROOT, "recognition", "time_aligned_slice");
|
||||
TimeAlignedSliceSeparateTask timeAlignedSliceSeparateTask = new TimeAlignedSliceSeparateTask(
|
||||
Configuration.WINDOW_SIZE_IN_MINUTE, Configuration.OVERLAP);
|
||||
Executor.executeToUserDir(srcRoot5, destRoot5, timeAlignedSliceSeparateTask);
|
||||
timeAlignedSliceSeparateTask.showLabelProperties();
|
||||
}
|
||||
|
||||
private static Map<String, Integer> getLambdaForIndividual() {
|
||||
Map<String, Integer> result = new HashMap<String, Integer>();
|
||||
result.put("张瑶", 5);
|
||||
result.put("徐敏敏", 5);
|
||||
result.put("成小林", 12);
|
||||
result.put("闾爽", 9);
|
||||
result.put("刘亚辉", 7);
|
||||
result.put("卢正华", 10);
|
||||
result.put("司维", 10);
|
||||
result.put("吴凡", 8);
|
||||
result.put("夏国仟", 15);
|
||||
result.put("孙昭", 12);
|
||||
result.put("张怡文", 12);
|
||||
result.put("彭钰", 19);
|
||||
result.put("毛文婷", 8);
|
||||
result.put("江超", 9);
|
||||
result.put("王海飞", 15);
|
||||
result.put("邵乃钊", 17);
|
||||
result.put("陈键", 12);
|
||||
result.put("伏秀秀", 8);
|
||||
result.put("叶坤坤", 7);
|
||||
result.put("周静", 7);
|
||||
result.put("孙海明", 17);
|
||||
result.put("孙荣荣", 14);
|
||||
result.put("廖清", 15);
|
||||
result.put("张学刚", 17);
|
||||
result.put("张馨月", 10);
|
||||
result.put("曹玺", 15);
|
||||
result.put("李政", 12);
|
||||
result.put("林翠平", 9);
|
||||
result.put("王鑫", 25);
|
||||
result.put("缪忍忍", 26);
|
||||
result.put("钱佳峰", 11);
|
||||
result.put("陈乐", 15);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Output of ThresholdDetermineTask:
|
||||
lambda #segment change(i-1,i+1)
|
||||
1 22584 NaN,-39.2
|
||||
2 13726 -39.2,-27.0
|
||||
3 10021 -27.0,-20.7
|
||||
4 7942 -20.7,-15.7
|
||||
5 6699 -15.7,-25.4
|
||||
6 4999 -25.4,-10.7
|
||||
7 4462 -10.7, -8.7
|
||||
8 4073 -8.7, -8.0
|
||||
9 3746 -8.0, -7.6
|
||||
10 3463 -7.6, -6.5
|
||||
11 3238 -6.5, -6.0
|
||||
12 3043 -6.0, -5.0
|
||||
13 2891 -5.0, -5.7
|
||||
14 2726 -5.7, -4.9
|
||||
15 2592 -4.9, -4.8
|
||||
16 2468 -4.8, -4.1
|
||||
17 2367 -4.1, -3.8
|
||||
18 2276 -3.8, -4.5
|
||||
19 2173 -4.5, -3.5
|
||||
20 2098 -3.5, -4.0
|
||||
21 2014 -4.0, -4.0
|
||||
22 1933 -4.0, -3.2
|
||||
23 1872 -3.2, -2.5
|
||||
24 1825 -2.5, -3.5
|
||||
25 1761 -3.5, -2.4
|
||||
26 1718 -2.4, -2.4
|
||||
27 1677 -2.4, -2.7
|
||||
28 1631 -2.7, -2.3
|
||||
29 1593 -2.3, -2.3
|
||||
30 1556 -2.3, -3.6
|
||||
31 1500 -3.6, -3.1
|
||||
32 1454 -3.1, -2.3
|
||||
33 1421 -2.3, -2.6
|
||||
34 1384 -2.6, -2.6
|
||||
35 1348 -2.6, -2.2
|
||||
36 1319 -2.2, -1.9
|
||||
37 1294 -1.9, -2.0
|
||||
38 1268 -2.0, -2.4
|
||||
39 1238 -2.4, -2.2
|
||||
40 1211 -2.2, -1.2
|
||||
41 1196 -1.2, -2.3
|
||||
42 1168 -2.3, -2.7
|
||||
43 1136 -2.7, -1.7
|
||||
44 1117 -1.7, -1.5
|
||||
45 1100 -1.5, -0.8
|
||||
46 1091 -0.8, -1.6
|
||||
47 1074 -1.6, -1.8
|
||||
48 1055 -1.8, -1.1
|
||||
49 1043 -1.1, -1.2
|
||||
50 1031 -1.2, -1.8
|
||||
51 1012 -1.8, -1.6
|
||||
52 996 -1.6, -1.2
|
||||
53 984 -1.2, -1.2
|
||||
54 972 -1.2, -2.2
|
||||
55 951 -2.2, -2.2
|
||||
56 930 -2.2, -2.4
|
||||
57 908 -2.4, -2.3
|
||||
58 887 -2.3, -1.8
|
||||
59 871 -1.8, -2.0
|
||||
60 854 -2.0, -1.5
|
||||
61 841 -1.5, -2.3
|
||||
62 822 -2.3, -2.4
|
||||
63 802 -2.4, -2.4
|
||||
64 783 -2.4, -2.2
|
||||
65 766 -2.2, -3.9
|
||||
66 736 -3.9, -3.5
|
||||
67 710 -3.5, -2.8
|
||||
68 690 -2.8, -2.5
|
||||
69 673 -2.5, -1.8
|
||||
70 661 -1.8, -1.8
|
||||
71 649 -1.8, -3.5
|
||||
72 626 -3.5, -2.1
|
||||
73 613 -2.1, -2.4
|
||||
74 598 -2.4, -2.2
|
||||
75 585 -2.2, -2.6
|
||||
76 570 -2.6, -1.8
|
||||
77 560 -1.8, -1.6
|
||||
78 551 -1.6, -1.5
|
||||
79 543 -1.5, -2.0
|
||||
80 532 -2.0, -1.9
|
||||
81 522 -1.9, -1.0
|
||||
82 517 -1.0, -1.2
|
||||
83 511 -1.2, -1.8
|
||||
84 502 -1.8, -2.2
|
||||
85 491 -2.2, -1.6
|
||||
86 483 -1.6, -2.1
|
||||
87 473 -2.1, -1.9
|
||||
88 464 -1.9, -1.7
|
||||
89 456 -1.7, -1.8
|
||||
90 448 -1.8, -1.1
|
||||
91 443 -1.1, -1.8
|
||||
92 435 -1.8, -1.8
|
||||
93 427 -1.8, -0.9
|
||||
94 423 -0.9, -0.5
|
||||
95 421 -0.5, -1.9
|
||||
96 413 -1.9, -0.5
|
||||
97 411 -0.5, -1.0
|
||||
98 407 -1.0, -2.5
|
||||
99 397 -2.5, -3.3
|
||||
100 384 -3.3, -2.9
|
||||
101 373 -2.9, -1.3
|
||||
102 368 -1.3, -1.9
|
||||
103 361 -1.9, -0.8
|
||||
104 358 -0.8, -0.3
|
||||
105 357 -0.3, -1.4
|
||||
106 352 -1.4, -1.7
|
||||
107 346 -1.7, -1.7
|
||||
108 340 -1.7, -1.2
|
||||
109 336 -1.2, -1.8
|
||||
110 330 -1.8, -2.1
|
||||
111 323 -2.1, -0.6
|
||||
112 321 -0.6, -1.2
|
||||
113 317 -1.2, -1.3
|
||||
114 313 -1.3, -1.6
|
||||
115 308 -1.6, -0.6
|
||||
116 306 -0.6, -1.3
|
||||
117 302 -1.3, -1.7
|
||||
118 297 -1.7, -0.7
|
||||
119 295 -0.7, -1.0
|
||||
120 292 -1.0, NaN
|
||||
-------------------------------
|
||||
minimum suitable lambda: 15
|
||||
*/
|
||||
|
||||
/*
|
||||
Output of SaturationEffectTask:
|
||||
t | #SI(i-1,i+1) #LI(i-1,i+1) #LI%(i-1,i+1) | △SI(i-1,i+1) △LI(i-1,i+1) △LI%(i-1,i+1)
|
||||
--------------------------------------------------------------------------------------
|
||||
1 | NaN, 0.0 NaN,-39.2 NaN,-39.2 | NaN, 23.8 NaN, -5.7 NaN, -5.7
|
||||
2 | 0.0, 0.0 -39.2,-27.0 -39.2,-27.0 | 23.8, 14.0 -5.7, -4.4 -5.7, -4.4
|
||||
3 | 0.0, 0.0 -27.0,-20.7 -27.0,-20.7 | 14.0, 9.8 -4.4, -3.6 -4.4, -3.6
|
||||
4 | 0.0, 0.0 -20.7,-15.7 -20.7,-15.7 | 9.8, 6.9 -3.6, -2.9 -3.6, -2.9
|
||||
5 | 0.0, 0.0 -15.7,-25.4 -15.7,-25.4 | 6.9, 10.2 -2.9, -4.8 -2.9, -4.8
|
||||
6 | 0.0, 0.0 -25.4,-10.7 -25.4,-10.7 | 10.2, 3.6 -4.8, -2.0 -4.8, -2.0
|
||||
7 | 0.0, 0.0 -10.7, -8.7 -10.7, -8.7 | 3.6, 2.9 -2.0, -1.7 -2.0, -1.7
|
||||
8 | 0.0, 0.0 -8.7, -8.0 -8.7, -8.0 | 2.9, 2.7 -1.7, -1.6 -1.7, -1.6
|
||||
9 | 0.0, 0.0 -8.0, -7.6 -8.0, -7.6 | 2.7, 2.6 -1.6, -1.6 -1.6, -1.6
|
||||
10 | 0.0, 0.0 -7.6, -6.5 -7.6, -6.5 | 2.6, 2.2 -1.6, -1.4 -1.6, -1.4
|
||||
11 | 0.0, 0.0 -6.5, -6.0 -6.5, -6.0 | 2.2, 2.0 -1.4, -1.4 -1.4, -1.4
|
||||
12 | 0.0, 0.0 -6.0, -5.0 -6.0, -5.0 | 2.0, 1.7 -1.4, -1.2 -1.4, -1.2
|
||||
13 | 0.0, 0.0 -5.0, -5.7 -5.0, -5.7 | 1.7, 2.0 -1.2, -1.4 -1.2, -1.4
|
||||
14 | 0.0, 0.0 -5.7, -4.9 -5.7, -4.9 | 2.0, 1.7 -1.4, -1.2 -1.4, -1.2
|
||||
15 | 0.0, 0.0 -4.9, -4.8 -4.9, -4.8 | 1.7, 1.6 -1.2, -1.3 -1.2, -1.3
|
||||
16 | 0.0, 0.0 -4.8, -4.1 -4.8, -4.1 | 1.6, 1.4 -1.3, -1.1 -1.3, -1.1
|
||||
17 | 0.0, 0.0 -4.1, -3.8 -4.1, -3.8 | 1.4, 1.3 -1.1, -1.1 -1.1, -1.1
|
||||
18 | 0.0, 0.0 -3.8, -4.5 -3.8, -4.5 | 1.3, 1.5 -1.1, -1.3 -1.1, -1.3
|
||||
19 | 0.0, 0.0 -4.5, -3.5 -4.5, -3.5 | 1.5, 1.2 -1.3, -1.0 -1.3, -1.0
|
||||
20 | 0.0, 0.0 -3.5, -4.0 -3.5, -4.0 | 1.2, 1.4 -1.0, -1.2 -1.0, -1.2
|
||||
21 | 0.0, 0.0 -4.0, -4.0 -4.0, -4.0 | 1.4, 1.4 -1.2, -1.2 -1.2, -1.2
|
||||
22 | 0.0, 0.0 -4.0, -3.2 -4.0, -3.2 | 1.4, 1.1 -1.2, -1.0 -1.2, -1.0
|
||||
23 | 0.0, 0.0 -3.2, -2.5 -3.2, -2.5 | 1.1, 0.8 -1.0, -0.8 -1.0, -0.8
|
||||
24 | 0.0, 0.0 -2.5, -3.5 -2.5, -3.5 | 0.8, 1.2 -0.8, -1.1 -0.8, -1.1
|
||||
25 | 0.0, 0.0 -3.5, -2.4 -3.5, -2.4 | 1.2, 0.8 -1.1, -0.8 -1.1, -0.8
|
||||
26 | 0.0, 0.0 -2.4, -2.4 -2.4, -2.4 | 0.8, 0.8 -0.8, -0.8 -0.8, -0.8
|
||||
27 | 0.0, 0.0 -2.4, -2.7 -2.4, -2.7 | 0.8, 0.9 -0.8, -0.9 -0.8, -0.9
|
||||
28 | 0.0, 0.0 -2.7, -2.3 -2.7, -2.3 | 0.9, 0.8 -0.9, -0.8 -0.9, -0.8
|
||||
29 | 0.0, 0.0 -2.3, -2.3 -2.3, -2.3 | 0.8, 0.8 -0.8, -0.8 -0.8, -0.8
|
||||
30 | 0.0, NaN -2.3, NaN -2.3, NaN | 0.8, NaN -0.8, NaN -0.8, NaN
|
||||
deltaSICountT: 2, deltaLICountT:15, deltaLICountPercentageT:15, deltaSITimeT: 7, deltaLITimeT: 3, deltaLITimePercentageT: 3
|
||||
*/
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Session {
|
||||
private String team;
|
||||
private String user;
|
||||
private String date;
|
||||
private long begTime;
|
||||
private long endTime;
|
||||
private int sessionId;
|
||||
private List<IntelliJEvent> events;
|
||||
private List<Label> labels;
|
||||
|
||||
public Session(String team, String user, String date, int sessionId, List<IntelliJEvent> events, List<Label> labels) {
|
||||
this.team = team;
|
||||
this.user = user;
|
||||
this.date = date;
|
||||
this.begTime = events.get(0).getTime();
|
||||
this.endTime = events.get(events.size() - 1).getTime();
|
||||
this.sessionId = sessionId;
|
||||
this.events = events;
|
||||
this.labels = labels;
|
||||
}
|
||||
|
||||
public String getTeam() {
|
||||
return team;
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public String getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public long getBegTime() {
|
||||
return begTime;
|
||||
}
|
||||
|
||||
public long getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public int getSessionId() {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public List<IntelliJEvent> getEvents() {
|
||||
return events;
|
||||
}
|
||||
|
||||
public List<Label> getLabels() {
|
||||
return labels;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Action;
|
||||
import edu.nju.ics.frontier.bean.label.ActionTypeConverter;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class SessionSeparateTask implements Executor.Task {
|
||||
private long mLambda; // threshold to separate sessions
|
||||
private boolean mIsOnlyInput;
|
||||
private Gson mGson;
|
||||
private int mSessionId;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
* @param lambda threshold to separate sessions, in minute.
|
||||
*/
|
||||
public SessionSeparateTask(int lambda, boolean isOnlyInput) {
|
||||
mLambda = lambda * 60 * 1000;
|
||||
mIsOnlyInput = isOnlyInput;
|
||||
GsonBuilder builder = new GsonBuilder();
|
||||
builder.registerTypeAdapter(Action.class, new ActionTypeConverter());
|
||||
mGson = builder.create();
|
||||
mSessionId = 0;
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
// delete old directories and files
|
||||
FileUtil.delete(new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName())));
|
||||
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int sessionCount = 0;
|
||||
for (File dateDir : dateDirs) {
|
||||
// load events
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, "event", ".json");
|
||||
if (eventFiles == null) {
|
||||
continue;
|
||||
}
|
||||
List<IntelliJEvent> events = new ArrayList<IntelliJEvent>();
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> tmp = DataSource.loadEvents(eventFile);
|
||||
if (tmp != null) {
|
||||
events.addAll(tmp);
|
||||
}
|
||||
}
|
||||
if (events.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// load labels
|
||||
File[] labelFiles = FileUtil.listSubFiles(dateDir, "label", ".json");
|
||||
List<Label> labels = new ArrayList<Label>();
|
||||
if (labelFiles != null) {
|
||||
for (File labelFile : labelFiles) {
|
||||
List<Label> tmp = DataSource.loadLabels(labelFile);
|
||||
if (tmp != null) {
|
||||
labels.addAll(tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// segment
|
||||
List<Session> sessions = new ArrayList<Session>();
|
||||
|
||||
if (mIsOnlyInput) {
|
||||
separateSessionByInputEvent(sessions, events, labels, teamDir, userDir, dateDir);
|
||||
} else {
|
||||
separateSessionByEvent(sessions, events, labels, teamDir, userDir, dateDir);
|
||||
}
|
||||
|
||||
// restore sessions
|
||||
if (!sessions.isEmpty()) {
|
||||
FileUtil.toJson(mGson, sessions, new File(FileUtil.combine(destRoot, teamDir.getName(), userDir.getName(), dateDir.getName(), "sessions.json")));
|
||||
sessionCount += sessions.size();
|
||||
}
|
||||
}
|
||||
|
||||
System.out.printf("%s/%s -> %d sessions\n",
|
||||
teamDir.getName(), userDir.getName(), sessionCount);
|
||||
}
|
||||
|
||||
private void separateSessionByInputEvent(List<Session> sessions, List<IntelliJEvent> events, List<Label> labels,
|
||||
File teamDir, File userDir, File dateDir) {
|
||||
InputEventIterator iterator = new InputEventIterator(events);
|
||||
IntelliJEvent prevEvent = iterator.nextInputEvent();
|
||||
if (prevEvent == null) {
|
||||
System.out.printf("%s/%s/%s: no any input event in the %d events.\n",
|
||||
teamDir.getName(), userDir.getName(), dateDir.getName(), events.size());
|
||||
return;
|
||||
}
|
||||
IntelliJEvent nextEvent;
|
||||
while ((nextEvent = iterator.nextInputEvent()) != null) {
|
||||
if (nextEvent.getTime() - prevEvent.getTime() >= mLambda) {
|
||||
Session session = createSession(teamDir, userDir, dateDir, iterator.separate(), labels);
|
||||
if (session != null) {
|
||||
sessions.add(session);
|
||||
}
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
}
|
||||
Session session = createSession(teamDir, userDir, dateDir, iterator.separate(), labels);
|
||||
if (session != null) {
|
||||
sessions.add(session);
|
||||
}
|
||||
}
|
||||
|
||||
private void separateSessionByEvent(List<Session> sessions, List<IntelliJEvent> events, List<Label> labels,
|
||||
File teamDir, File userDir, File dateDir) {
|
||||
List<IntelliJEvent> eventCache = new ArrayList<IntelliJEvent>();
|
||||
IntelliJEvent prevEvent = events.get(0);
|
||||
eventCache.add(prevEvent);
|
||||
for (int i = 1, l = events.size(); i < l; i++) {
|
||||
IntelliJEvent nextEvent = events.get(i);
|
||||
if (nextEvent.getTime() - prevEvent.getTime() >= mLambda) {
|
||||
Session session = createSession(teamDir, userDir, dateDir, eventCache, labels);
|
||||
if (session != null) {
|
||||
sessions.add(session);
|
||||
}
|
||||
eventCache = new ArrayList<IntelliJEvent>();
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
eventCache.add(prevEvent);
|
||||
}
|
||||
Session session = createSession(teamDir, userDir, dateDir, eventCache, labels);
|
||||
if (session != null) {
|
||||
sessions.add(session);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create a session given the events and labels. A session is successful created
|
||||
* if all the below conditions are met:
|
||||
* <ul>
|
||||
* <li>The time interval between any pair of subsequent events is less than
|
||||
* <code>mLambda</code>.</li>
|
||||
* <li>The duration of the session, which is the time interval between the
|
||||
* first and the last events, is not less than <code>mLength</code>;</li>
|
||||
* <li>There is at least one label is located in the session's duration.</li>
|
||||
* </ul>
|
||||
* @param teamDir team
|
||||
* @param userDir user
|
||||
* @param dateDir date
|
||||
* @param eventCache events
|
||||
* @param labels labels
|
||||
* @return An new session, or null if any condition is not met.
|
||||
*/
|
||||
private Session createSession(File teamDir, File userDir, File dateDir,
|
||||
List<IntelliJEvent> eventCache, List<Label> labels) {
|
||||
if (eventCache == null || eventCache.isEmpty()) {
|
||||
throw new IllegalArgumentException("no any event in the session!");
|
||||
}
|
||||
long begTime = eventCache.get(0).getTime();
|
||||
long endTime = eventCache.get(eventCache.size() - 1).getTime();
|
||||
List<Label> labelCache = DataSource.sliceLabels(labels, begTime, endTime, false);
|
||||
return new Session(teamDir.getName(), userDir.getName(), dateDir.getName(), ++mSessionId,
|
||||
eventCache, labelCache);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class Slice {
|
||||
private String team;
|
||||
private String user;
|
||||
private String date;
|
||||
private long sliceBegTime;
|
||||
private long sliceEndTime;
|
||||
private long sessionBegTime;
|
||||
private long sessionEndTime;
|
||||
private int sessionId;
|
||||
private List<IntelliJEvent> events;
|
||||
private Label label;
|
||||
|
||||
public Slice(String team, String user, String date, long sliceBegTime, long sliceEndTime, long sessionBegTime,
|
||||
long sessionEndTime, int sessionId, List<IntelliJEvent> events, Label label) {
|
||||
this.team = team;
|
||||
this.user = user;
|
||||
this.date = date;
|
||||
this.sliceBegTime = sliceBegTime;
|
||||
this.sliceEndTime = sliceEndTime;
|
||||
this.sessionBegTime = sessionBegTime;
|
||||
this.sessionEndTime = sessionEndTime;
|
||||
this.sessionId = sessionId;
|
||||
this.events = events;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public String getTeam() {
|
||||
return team;
|
||||
}
|
||||
|
||||
public String getUser() {
|
||||
return user;
|
||||
}
|
||||
|
||||
public String getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public int getSessionId() {
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
public long getSliceBegTime() {
|
||||
return sliceBegTime;
|
||||
}
|
||||
|
||||
public long getSliceEndTime() {
|
||||
return sliceEndTime;
|
||||
}
|
||||
|
||||
public long getSessionBegTime() {
|
||||
return sessionBegTime;
|
||||
}
|
||||
|
||||
public long getSessionEndTime() {
|
||||
return sessionEndTime;
|
||||
}
|
||||
|
||||
public List<IntelliJEvent> getEvents() {
|
||||
return events;
|
||||
}
|
||||
|
||||
public Label getLabel() {
|
||||
return label;
|
||||
}
|
||||
|
||||
public void setLabel(Label label) {
|
||||
this.label = label;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import edu.nju.ics.frontier.util.Executor;
|
||||
import edu.nju.ics.frontier.util.FileUtil;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* [17_JSS_CCF-B_Patterns of developers behaviour: A 1000-hour industrial study] defined
|
||||
* a <b>working segment</b> as a sequence of events captured on the same workstation in
|
||||
* which two subsequent events occur in less than <i>lambda</i> minutes.
|
||||
* With a suitable <i>lambda</i>_i value, the number of sessions obtained by applying
|
||||
* <i>lambda</i>_i as splitting threshold does not change of more than 5% with respect
|
||||
* to <i>lambda</i>_{i-1} and <i>lambda</i>_{i+1}, where <i>lambda</i>_{i-1} + 1 =
|
||||
* <i>lambda</i>_i = <i>lambda</i>_{i+1} - 1.
|
||||
*/
|
||||
public class ThresholdDetermineTask implements Executor.Task {
|
||||
/**
|
||||
* We varied lambda between 1 and 120 minute at steps of 1 minute,
|
||||
* and count the number of sessions with the specified lambda.
|
||||
*/
|
||||
private long[] mSessionNums;
|
||||
private boolean mIsOnlyInput;
|
||||
|
||||
public ThresholdDetermineTask(boolean isOnlyInput) {
|
||||
this.mSessionNums = initLongArray(120, 0);
|
||||
this.mIsOnlyInput = isOnlyInput;
|
||||
}
|
||||
|
||||
public void run(String srcRoot, String destRoot, File teamDir, File userDir, String dateStr) {
|
||||
File[] dateDirs = FileUtil.listSubDirectories(userDir);
|
||||
if (dateDirs == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
long[] sessionNums = initLongArray(120, 0);
|
||||
int workdayCount = 0;
|
||||
|
||||
for (File dateDir : dateDirs) {
|
||||
File[] eventFiles = FileUtil.listSubFiles(dateDir, "event", ".json");
|
||||
if (eventFiles == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
List<IntelliJEvent> events = new ArrayList<IntelliJEvent>();
|
||||
for (File eventFile : eventFiles) {
|
||||
List<IntelliJEvent> tmp = DataSource.loadEvents(eventFile);
|
||||
if (tmp != null) {
|
||||
events.addAll(tmp);
|
||||
}
|
||||
}
|
||||
if (events.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (mIsOnlyInput) {
|
||||
calculateDeltaTimeBetweenInputEvents(events, sessionNums);
|
||||
} else {
|
||||
calculateDeltaTimeBetweenEvents(events, sessionNums);
|
||||
}
|
||||
workdayCount++;
|
||||
}
|
||||
|
||||
addArray(mSessionNums, sessionNums);
|
||||
|
||||
System.out.printf("%s/%s: #workday=%d\n", teamDir.getName(), userDir.getName(), workdayCount);
|
||||
determineSuitableLambda(sessionNums, false);
|
||||
}
|
||||
|
||||
private void calculateDeltaTimeBetweenInputEvents(List<IntelliJEvent> events, long[] sessionNums) {
|
||||
InputEventIterator iterator = new InputEventIterator(events);
|
||||
IntelliJEvent prevEvent = iterator.nextInputEvent();
|
||||
if (prevEvent == null) {
|
||||
return;
|
||||
}
|
||||
IntelliJEvent nextEvent;
|
||||
while ((nextEvent = iterator.nextInputEvent()) != null) {
|
||||
int diffMin = (int) ((nextEvent.getTime() - prevEvent.getTime()) / 60000);
|
||||
for (int j = 0, n = sessionNums.length; j < n; j++) {
|
||||
if (diffMin >= (j + 1)) {
|
||||
sessionNums[j]++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
prevEvent = nextEvent;
|
||||
}
|
||||
}
|
||||
|
||||
private void calculateDeltaTimeBetweenEvents(List<IntelliJEvent> events, long[] sessionNums) {
|
||||
IntelliJEvent prevEvent = events.get(0);
|
||||
for (int i = 1, m = events.size(); i < m; i++) {
|
||||
IntelliJEvent currEvent = events.get(i);
|
||||
int diffMin = (int) ((currEvent.getTime() - prevEvent.getTime()) / 60000);
|
||||
for (int j = 0, n = sessionNums.length; j < n; j++) {
|
||||
if (diffMin >= (j + 1)) {
|
||||
sessionNums[j]++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
prevEvent = currEvent;
|
||||
}
|
||||
}
|
||||
|
||||
public int determineSuitableLambda(boolean isShowDetails) {
|
||||
return determineSuitableLambda(mSessionNums, isShowDetails);
|
||||
}
|
||||
|
||||
private int determineSuitableLambda(long[] sessionNums, boolean isShowDetails) {
|
||||
List<Integer> candidates = new ArrayList<Integer>();
|
||||
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("lambda #segment change(i-1,i+1)\n");
|
||||
builder.append(String.format("%6d %8d %5.1f,%5.1f\n",
|
||||
1, sessionNums[0], Double.NaN,
|
||||
(((double) (sessionNums[1] - sessionNums[0])) / ((double) sessionNums[0])) * 100.0));
|
||||
|
||||
for (int i = 1; i < sessionNums.length - 1; i++) {
|
||||
long lastNum = sessionNums[i - 1];
|
||||
long currNum = sessionNums[i];
|
||||
long nextNum = sessionNums[i + 1];
|
||||
|
||||
double delta1 = (((double) (currNum - lastNum)) / ((double) lastNum)) * 100.0;
|
||||
double delta2 = (((double) (nextNum - currNum)) / ((double) currNum)) * 100.0;
|
||||
builder.append(String.format("%6d %8d %5.1f,%5.1f\n", (i + 1), sessionNums[i], delta1, delta2));
|
||||
|
||||
if (Math.abs(delta1) <= 5.0 && Math.abs(delta2) <= 5.0) {
|
||||
candidates.add(i + 1);
|
||||
}
|
||||
}
|
||||
|
||||
builder.append(String.format("%6d %8d %5.1f,%5.1f\n",
|
||||
sessionNums.length, sessionNums[sessionNums.length - 1],
|
||||
(((double) (sessionNums[sessionNums.length - 1] - sessionNums[sessionNums.length - 2])) / ((double) sessionNums[sessionNums.length - 2])) * 100.0, Double.NaN));
|
||||
builder.append("-------------------------------\n");
|
||||
|
||||
if (isShowDetails) {
|
||||
System.out.print(builder.toString());
|
||||
}
|
||||
System.out.print("minimum suitable lambda: ");
|
||||
if (candidates.isEmpty()) {
|
||||
System.out.println("null");
|
||||
return -1;
|
||||
} else {
|
||||
System.out.println(candidates.get(0));
|
||||
return candidates.get(0);
|
||||
}
|
||||
}
|
||||
|
||||
private long[] initLongArray(int length, long value) {
|
||||
if (length <= 0) {
|
||||
return null;
|
||||
}
|
||||
long[] array = new long[length];
|
||||
for (int i = 0; i < length; i++) {
|
||||
array[i] = value;
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
private void addArray(long[] dest, long[] src) {
|
||||
if (dest == null || src == null || dest.length != src.length) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0, l = dest.length; i < l; i++) {
|
||||
dest[i] += src[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,326 @@
|
|||
package edu.nju.ics.frontier.recognition.segment;
|
||||
|
||||
import edu.nju.ics.frontier.bean.label.SelectOrSubmitAction;
|
||||
import edu.nju.ics.frontier.common.io.OkTextReader;
|
||||
import edu.nju.ics.frontier.bean.intellij.IntelliJEvent;
|
||||
import edu.nju.ics.frontier.bean.label.Label;
|
||||
import edu.nju.ics.frontier.bean.label.SelectAction;
|
||||
import edu.nju.ics.frontier.recognition.feature.MeasureUtil;
|
||||
import edu.nju.ics.frontier.util.Assertion;
|
||||
import edu.nju.ics.frontier.util.DataSource;
|
||||
import org.apache.commons.math3.stat.StatUtils;
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Max;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Median;
|
||||
import org.apache.commons.math3.stat.descriptive.rank.Min;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
public class TimeAlignedSliceSeparateTask extends AbstractSliceSeparateTask {
|
||||
public TimeAlignedSliceSeparateTask(long windowSize, double overlap) {
|
||||
super(windowSize, overlap);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected List<Slice> createSlice(Session session) {
|
||||
long sessionBegTime = session.getBegTime();
|
||||
long sessionEndTime = session.getEndTime();
|
||||
if (sessionEndTime - sessionBegTime < mWindowSize) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String team = session.getTeam();
|
||||
String user = session.getUser();
|
||||
String date = session.getDate();
|
||||
int sessionId = session.getSessionId();
|
||||
List<IntelliJEvent> events = session.getEvents();
|
||||
List<Label> labels = session.getLabels();
|
||||
|
||||
long step = (long) ((1 - mOverlap) * mWindowSize);
|
||||
|
||||
List<Slice> result = new ArrayList<Slice>();
|
||||
Slice slice;
|
||||
|
||||
long sliceBegTime = findMaximumAlignedTimeBeforeSessionBegTime(sessionBegTime);
|
||||
long sliceEndTime = sliceBegTime + mWindowSize;
|
||||
List<IntelliJEvent> eventCache = DataSource.sliceEvents(events, sessionBegTime, sessionBegTime + mWindowSize);
|
||||
List<Label> labelCache = DataSource.sliceLabels(labels, sessionBegTime, sessionBegTime + mWindowSize, true);
|
||||
Label label = labelCache == null ? null : labelCache.get(labelCache.size() - 1);
|
||||
label = eraseLabel(label);
|
||||
slice = new Slice(team, user, date, sliceBegTime, sliceEndTime, sessionBegTime, sessionEndTime, sessionId, eventCache, label);
|
||||
result.add(slice);
|
||||
// TODO: check
|
||||
recordLabelProperties(slice);
|
||||
|
||||
if (sessionEndTime - sessionBegTime == mWindowSize) {
|
||||
return result;
|
||||
}
|
||||
|
||||
for (sliceBegTime += step; sliceBegTime + mWindowSize < sessionEndTime; sliceBegTime += step) {
|
||||
sliceEndTime = sliceBegTime + mWindowSize;
|
||||
eventCache = DataSource.sliceEvents(events, sliceBegTime, sliceEndTime);
|
||||
labelCache = DataSource.sliceLabels(labels, sliceBegTime, sliceEndTime, true);
|
||||
label = labelCache == null ? null : labelCache.get(labelCache.size() - 1);
|
||||
label = eraseLabel(label);
|
||||
slice = new Slice(team, user, date, sliceBegTime, sliceEndTime, sessionBegTime, sessionEndTime, sessionId, eventCache, label);
|
||||
result.add(slice);
|
||||
// TODO: check
|
||||
recordLabelProperties(slice);
|
||||
}
|
||||
|
||||
sliceEndTime = sliceBegTime + mWindowSize;
|
||||
eventCache = DataSource.sliceEvents(events, sessionEndTime - mWindowSize, sessionEndTime);
|
||||
labelCache = DataSource.sliceLabels(labels, sessionEndTime - mWindowSize, sessionEndTime, true);
|
||||
label = labelCache == null ? null : labelCache.get(labelCache.size() - 1);
|
||||
label = eraseLabel(label);
|
||||
slice = new Slice(team, user, date, sliceBegTime, sliceEndTime, sessionBegTime, sessionEndTime, sessionId, eventCache, label);
|
||||
result.add(slice);
|
||||
// TODO: check
|
||||
recordLabelProperties(slice);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private long aligningTime(long millisecond) {
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.setTimeInMillis(millisecond);
|
||||
calendar.set(Calendar.MINUTE, 0);
|
||||
calendar.set(Calendar.SECOND, 0);
|
||||
calendar.set(Calendar.MILLISECOND, 0);
|
||||
return calendar.getTimeInMillis();
|
||||
}
|
||||
|
||||
private long findMaximumAlignedTimeBeforeSessionBegTime(long sessionBegTime) {
|
||||
long sliceBegTime = aligningTime(sessionBegTime);
|
||||
Assertion.assertTrue(sliceBegTime <= sessionBegTime);
|
||||
if (sliceBegTime == sessionBegTime) {
|
||||
return sliceBegTime;
|
||||
}
|
||||
long step = (long) ((1 - mOverlap) * mWindowSize);
|
||||
Assertion.assertPositive(step);
|
||||
while (sliceBegTime < sessionBegTime) {
|
||||
if (sliceBegTime + step < sessionBegTime) {
|
||||
sliceBegTime += step;
|
||||
} else if (sliceBegTime + step > sessionBegTime) {
|
||||
break;
|
||||
} else {
|
||||
sliceBegTime += step;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Assertion.assertTrue(sliceBegTime <= sessionBegTime && sliceBegTime + step > sessionBegTime);
|
||||
return sliceBegTime;
|
||||
}
|
||||
|
||||
private static final SimpleDateFormat DATE_SDF = new SimpleDateFormat("yyyyMMdd");
|
||||
private static final Map<String, Set<Long>> BAD_LABEL = new HashMap<String, Set<Long>>();
|
||||
private static final int BAD_LABEL_NUM;
|
||||
|
||||
static {
|
||||
int badDataNum = 0;
|
||||
OkTextReader reader = new OkTextReader();
|
||||
reader.open("C:\\Users\\zzw\\Desktop\\fujitsu\\bad_data.csv");
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
String[] info = line.split(",");
|
||||
Assertion.assertEqual(info.length, 2);
|
||||
String user = info[0];
|
||||
long timestamp = Long.parseLong(info[1]);
|
||||
MeasureUtil.updateSetMap(BAD_LABEL, user, timestamp);
|
||||
badDataNum++;
|
||||
}
|
||||
reader.close();
|
||||
BAD_LABEL_NUM = badDataNum;
|
||||
System.out.println("load bad label: " + BAD_LABEL_NUM);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private List<Object[]>[][] labelProperties = new ArrayList[][]{
|
||||
{
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
},
|
||||
{
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
new ArrayList<Object[]>(),
|
||||
},
|
||||
};
|
||||
|
||||
public void showLabelProperties() {
|
||||
for (int j = 0; j < labelProperties[0].length; j++) {
|
||||
showLabelProperties("B_" + (j + 1), labelProperties[0][j]);
|
||||
showLabelProperties("G_" + (j + 1), labelProperties[1][j]);
|
||||
}
|
||||
}
|
||||
|
||||
private Label eraseLabel(Label label) {
|
||||
if (label == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int engScore = label.getLabelScore(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
int proScore = label.getLabelScore(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
|
||||
long deltaFormTime = label.getEndTime() - label.getBegTime();
|
||||
Assertion.assertNotNegative(deltaFormTime);
|
||||
|
||||
List<SelectAction> engActions = label.getSelectActions(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
List<SelectAction> proActions = label.getSelectActions(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
long lastEngSelectTime = engActions.get(engActions.size() - 1).getTime();
|
||||
long lastProSelectTime = proActions.get(proActions.size() - 1).getTime();
|
||||
long deltaItemTime = Math.abs(lastProSelectTime - lastEngSelectTime);
|
||||
|
||||
switch (engScore) {
|
||||
case 1: {
|
||||
if (proScore != 1) {
|
||||
return null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
if (proScore != 2) {
|
||||
return null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
if (proScore != 3 || deltaItemTime > 1000) {
|
||||
return null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
if (proScore != 4 || deltaFormTime > 7000) {
|
||||
return null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 5: {
|
||||
if (proScore != 5) {
|
||||
return null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return label;
|
||||
}
|
||||
|
||||
private void recordLabelProperties(Slice slice) {
|
||||
Label label = slice.getLabel();
|
||||
if (label == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
long labelBegTime = label.getBegTime();
|
||||
long labelEndTime = label.getEndTime();
|
||||
|
||||
String type = label.getCanonicalType().name();
|
||||
String date = DATE_SDF.format(new Date(labelEndTime));
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
calendar.setTimeInMillis(labelEndTime);
|
||||
String hour = String.valueOf(calendar.get(Calendar.HOUR_OF_DAY));
|
||||
|
||||
int engScore = label.getLabelScore(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
int proScore = label.getLabelScore(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
int difScore = engScore - proScore;
|
||||
|
||||
long formDiffTime = labelEndTime - labelBegTime;
|
||||
Assertion.assertNotNegative(formDiffTime);
|
||||
List<SelectAction> engActions = label.getSelectActions(SelectOrSubmitAction.Dimension.ENGAGEMENT);
|
||||
List<SelectAction> proActions = label.getSelectActions(SelectOrSubmitAction.Dimension.PRODUCTIVITY);
|
||||
long lastEngSelectTime = engActions.get(engActions.size() - 1).getTime();
|
||||
long lastProSelectTime = proActions.get(proActions.size() - 1).getTime();
|
||||
long itemDiffTime = Math.abs(lastProSelectTime - lastEngSelectTime);
|
||||
long sessDiffTime = labelEndTime - slice.getSessionBegTime();
|
||||
Assertion.assertNotNegative(sessDiffTime);
|
||||
|
||||
Object[] tmp = new Object[]{
|
||||
type, date, hour, proScore, difScore,
|
||||
formDiffTime, itemDiffTime, sessDiffTime,
|
||||
};
|
||||
Set<Long> timestamps = BAD_LABEL.get(slice.getUser());
|
||||
if (timestamps != null && timestamps.contains(slice.getSliceBegTime())) {
|
||||
labelProperties[0][engScore - 1].add(tmp);
|
||||
} else {
|
||||
labelProperties[1][engScore - 1].add(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
private void showLabelProperties(String title, List<Object[]> dataList) {
|
||||
if (dataList.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
System.out.println("==================================================");
|
||||
System.out.println(title);
|
||||
|
||||
// type, proScore, difScore, formDiffTime,
|
||||
// itemDiffTime, sessDiffTime, date, hour,
|
||||
Map<String, Integer> type = new HashMap<String, Integer>();
|
||||
Map<String, Integer> date = new HashMap<String, Integer>();
|
||||
Map<String, Integer> hour = new HashMap<String, Integer>();
|
||||
double[] proScore = new double[dataList.size()];
|
||||
double[] difScore = new double[dataList.size()];
|
||||
double[] formDiffTime = new double[dataList.size()];
|
||||
double[] itemDiffTime = new double[dataList.size()];
|
||||
double[] sessDiffTime = new double[dataList.size()];
|
||||
for (int i = 0; i < dataList.size(); i++) {
|
||||
Object[] item = dataList.get(i);
|
||||
MeasureUtil.updateCountMap(type, ((String) item[0]));
|
||||
MeasureUtil.updateCountMap(date, ((String) item[1]));
|
||||
MeasureUtil.updateCountMap(hour, ((String) item[2]));
|
||||
proScore[i] = ((Integer) item[3]);
|
||||
difScore[i] = ((Integer) item[4]);
|
||||
formDiffTime[i] = ((Long) item[5]);
|
||||
itemDiffTime[i] = ((Long) item[6]);
|
||||
sessDiffTime[i] = ((Long) item[7]);
|
||||
}
|
||||
|
||||
showCountMap("type", type);
|
||||
showCountMap("date", date);
|
||||
showCountMap("hour", hour);
|
||||
boxPlot(proScore, "proScore");
|
||||
boxPlot(difScore, "difScore");
|
||||
boxPlot(formDiffTime, "formDiffTime");
|
||||
boxPlot(itemDiffTime, "itemDiffTime");
|
||||
boxPlot(sessDiffTime, "sessDiffTime");
|
||||
}
|
||||
|
||||
private void showCountMap(String title, Map<String, Integer> map) {
|
||||
List<String> keys = new ArrayList<String>(map.keySet());
|
||||
Collections.sort(keys, new Comparator<String>() {
|
||||
public int compare(String o1, String o2) {
|
||||
return o1.compareTo(o2);
|
||||
}
|
||||
});
|
||||
System.out.printf("%s{", title);
|
||||
int sum = 0;
|
||||
for (String key : keys) {
|
||||
int value = map.get(key);
|
||||
System.out.printf("#(%s)=%4d, ", key, value);
|
||||
sum += value;
|
||||
}
|
||||
System.out.printf("#(SUM)=%4d}\n", sum);
|
||||
}
|
||||
|
||||
private void boxPlot(double[] vector, String tag) {
|
||||
double min = new Min().evaluate(vector);
|
||||
double max = new Max().evaluate(vector);
|
||||
double q1 = StatUtils.percentile(vector, 25);
|
||||
double median = new Median().evaluate(vector);
|
||||
double q3 = StatUtils.percentile(vector, 75);
|
||||
double mean = new Mean().evaluate(vector);
|
||||
System.out.printf("%s:{avg:%f,min:%f,max:%f,q1:%f,q2:%f,q3:%f}\n",
|
||||
tag, mean, min, max, q1, median, q3);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue