Skip to content

P
physical
Commit d3dde44a authored by 段英荣's avatar 段英荣
Browse files
Options

Merge branch 'test' into 'master' 

# Conflicts:
#   search/utils/topic.py
parents d56866c2 8675f182
Hide whitespace changes
Inline Side-by-side
Showing with 1399 additions and 51 deletions
.idea/codeStyles/Project.xml 0 → 100644
View file @ d3dde44a
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
<DBN-PSQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false" />
</DBN-PSQL>
<DBN-SQL>
<case-options enabled="false">
<option name="KEYWORD_CASE" value="lower" />
<option name="FUNCTION_CASE" value="lower" />
<option name="PARAMETER_CASE" value="lower" />
<option name="DATATYPE_CASE" value="lower" />
<option name="OBJECT_CASE" value="preserve" />
</case-options>
<formatting-settings enabled="false">
<option name="STATEMENT_SPACING" value="one_line" />
<option name="CLAUSE_CHOP_DOWN" value="chop_down_if_statement_long" />
<option name="ITERATION_ELEMENTS_WRAPPING" value="chop_down_if_not_single" />
</formatting-settings>
</DBN-SQL>
</code_scheme>
</component>
\ No newline at end of file
.idea/dbnavigator.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DBNavigator.Project.DataEditorManager">
<record-view-column-sorting-type value="BY_INDEX" />
<value-preview-text-wrapping value="true" />
<value-preview-pinned value="false" />
</component>
<component name="DBNavigator.Project.DataExportManager">
<export-instructions>
<create-header value="true" />
<quote-values-containing-separator value="true" />
<quote-all-values value="false" />
<value-separator value="" />
<file-name value="" />
<file-location value="" />
<scope value="GLOBAL" />
<destination value="FILE" />
<format value="EXCEL" />
<charset value="UTF-8" />
</export-instructions>
</component>
<component name="DBNavigator.Project.DatabaseBrowserManager">
<autoscroll-to-editor value="false" />
<autoscroll-from-editor value="true" />
<show-object-properties value="true" />
<loaded-nodes />
</component>
<component name="DBNavigator.Project.EditorStateManager">
<last-used-providers />
</component>
<component name="DBNavigator.Project.MethodExecutionManager">
<method-browser />
<execution-history>
<group-entries value="true" />
<execution-inputs />
</execution-history>
<argument-values-cache />
</component>
<component name="DBNavigator.Project.ObjectDependencyManager">
<last-used-dependency-type value="INCOMING" />
</component>
<component name="DBNavigator.Project.ObjectQuickFilterManager">
<last-used-operator value="EQUAL" />
<filters />
</component>
<component name="DBNavigator.Project.ScriptExecutionManager" clear-outputs="true">
<recently-used-interfaces />
</component>
<component name="DBNavigator.Project.Settings">
<connections />
<browser-settings>
<general>
<display-mode value="TABBED" />
<navigation-history-size value="100" />
<show-object-details value="false" />
</general>
<filters>
<object-type-filter>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="true" />
<object-type name="ROLE" enabled="true" />
<object-type name="PRIVILEGE" enabled="true" />
<object-type name="CHARSET" enabled="true" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED_VIEW" enabled="true" />
<object-type name="NESTED_TABLE" enabled="true" />
<object-type name="COLUMN" enabled="true" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET_TRIGGER" enabled="true" />
<object-type name="DATABASE_TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="true" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="TYPE_ATTRIBUTE" enabled="true" />
<object-type name="ARGUMENT" enabled="true" />
<object-type name="DIMENSION" enabled="true" />
<object-type name="CLUSTER" enabled="true" />
<object-type name="DBLINK" enabled="true" />
</object-type-filter>
</filters>
<sorting>
<object-type name="COLUMN" sorting-type="NAME" />
<object-type name="FUNCTION" sorting-type="NAME" />
<object-type name="PROCEDURE" sorting-type="NAME" />
<object-type name="ARGUMENT" sorting-type="POSITION" />
</sorting>
<default-editors>
<object-type name="VIEW" editor-type="SELECTION" />
<object-type name="PACKAGE" editor-type="SELECTION" />
<object-type name="TYPE" editor-type="SELECTION" />
</default-editors>
</browser-settings>
<navigation-settings>
<lookup-filters>
<lookup-objects>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="false" />
<object-type name="ROLE" enabled="false" />
<object-type name="PRIVILEGE" enabled="false" />
<object-type name="CHARSET" enabled="false" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED VIEW" enabled="true" />
<object-type name="NESTED TABLE" enabled="false" />
<object-type name="COLUMN" enabled="false" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET TRIGGER" enabled="true" />
<object-type name="DATABASE TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="false" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="TYPE ATTRIBUTE" enabled="false" />
<object-type name="ARGUMENT" enabled="false" />
<object-type name="DIMENSION" enabled="false" />
<object-type name="CLUSTER" enabled="false" />
<object-type name="DBLINK" enabled="true" />
</lookup-objects>
<force-database-load value="false" />
<prompt-connection-selection value="true" />
<prompt-schema-selection value="true" />
</lookup-filters>
</navigation-settings>
<dataset-grid-settings>
<general>
<enable-zooming value="true" />
<enable-column-tooltip value="true" />
</general>
<sorting>
<nulls-first value="true" />
<max-sorting-columns value="4" />
</sorting>
<tracking-columns>
<columnNames value="" />
<visible value="true" />
<editable value="false" />
</tracking-columns>
</dataset-grid-settings>
<dataset-editor-settings>
<text-editor-popup>
<active value="false" />
<active-if-empty value="false" />
<data-length-threshold value="100" />
<popup-delay value="1000" />
</text-editor-popup>
<values-list-popup>
<show-popup-button value="true" />
<element-count-threshold value="1000" />
<data-length-threshold value="250" />
</values-list-popup>
<general>
<fetch-block-size value="100" />
<fetch-timeout value="30" />
<trim-whitespaces value="true" />
<convert-empty-strings-to-null value="true" />
<select-content-on-cell-edit value="true" />
<large-value-preview-active value="true" />
</general>
<filters>
<prompt-filter-dialog value="true" />
<default-filter-type value="BASIC" />
</filters>
<qualified-text-editor text-length-threshold="300">
<content-types>
<content-type name="Text" enabled="true" />
<content-type name="Properties" enabled="true" />
<content-type name="XML" enabled="true" />
<content-type name="DTD" enabled="true" />
<content-type name="HTML" enabled="true" />
<content-type name="XHTML" enabled="true" />
<content-type name="SQL" enabled="true" />
<content-type name="PL/SQL" enabled="true" />
<content-type name="YAML" enabled="true" />
</content-types>
</qualified-text-editor>
<record-navigation>
<navigation-target value="VIEWER" />
</record-navigation>
</dataset-editor-settings>
<code-editor-settings>
<general>
<show-object-navigation-gutter value="false" />
<show-spec-declaration-navigation-gutter value="true" />
<enable-spellchecking value="true" />
<enable-reference-spellchecking value="false" />
</general>
<confirmations>
<save-changes value="false" />
<revert-changes value="true" />
</confirmations>
</code-editor-settings>
<code-completion-settings>
<filters>
<basic-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="false" />
<filter-element type="OBJECT" id="view" selected="false" />
<filter-element type="OBJECT" id="materialized view" selected="false" />
<filter-element type="OBJECT" id="index" selected="false" />
<filter-element type="OBJECT" id="constraint" selected="false" />
<filter-element type="OBJECT" id="trigger" selected="false" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="false" />
<filter-element type="OBJECT" id="procedure" selected="false" />
<filter-element type="OBJECT" id="function" selected="false" />
<filter-element type="OBJECT" id="package" selected="false" />
<filter-element type="OBJECT" id="type" selected="false" />
<filter-element type="OBJECT" id="dimension" selected="false" />
<filter-element type="OBJECT" id="cluster" selected="false" />
<filter-element type="OBJECT" id="dblink" selected="false" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</basic-filter>
<extended-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</extended-filter>
</filters>
<sorting enabled="true">
<sorting-element type="RESERVED_WORD" id="keyword" />
<sorting-element type="RESERVED_WORD" id="datatype" />
<sorting-element type="OBJECT" id="column" />
<sorting-element type="OBJECT" id="table" />
<sorting-element type="OBJECT" id="view" />
<sorting-element type="OBJECT" id="materialized view" />
<sorting-element type="OBJECT" id="index" />
<sorting-element type="OBJECT" id="constraint" />
<sorting-element type="OBJECT" id="trigger" />
<sorting-element type="OBJECT" id="synonym" />
<sorting-element type="OBJECT" id="sequence" />
<sorting-element type="OBJECT" id="procedure" />
<sorting-element type="OBJECT" id="function" />
<sorting-element type="OBJECT" id="package" />
<sorting-element type="OBJECT" id="type" />
<sorting-element type="OBJECT" id="dimension" />
<sorting-element type="OBJECT" id="cluster" />
<sorting-element type="OBJECT" id="dblink" />
<sorting-element type="OBJECT" id="schema" />
<sorting-element type="OBJECT" id="role" />
<sorting-element type="OBJECT" id="user" />
<sorting-element type="RESERVED_WORD" id="function" />
<sorting-element type="RESERVED_WORD" id="parameter" />
</sorting>
<format>
<enforce-code-style-case value="true" />
</format>
</code-completion-settings>
<execution-engine-settings>
<statement-execution>
<fetch-block-size value="100" />
<execution-timeout value="20" />
<debug-execution-timeout value="600" />
<focus-result value="false" />
<prompt-execution value="false" />
</statement-execution>
<script-execution>
<command-line-interfaces />
<execution-timeout value="300" />
</script-execution>
<method-execution>
<execution-timeout value="30" />
<debug-execution-timeout value="600" />
<parameter-history-size value="10" />
</method-execution>
</execution-engine-settings>
<operation-settings>
<transactions>
<uncommitted-changes>
<on-project-close value="ASK" />
<on-disconnect value="ASK" />
<on-autocommit-toggle value="ASK" />
</uncommitted-changes>
<multiple-uncommitted-changes>
<on-commit value="ASK" />
<on-rollback value="ASK" />
</multiple-uncommitted-changes>
</transactions>
<session-browser>
<disconnect-session value="ASK" />
<kill-session value="ASK" />
<reload-on-filter-change value="false" />
</session-browser>
<compiler>
<compile-type value="KEEP" />
<compile-dependencies value="ASK" />
<always-show-controls value="false" />
</compiler>
<debugger>
<debugger-type value="JDBC" />
<use-generic-runners value="true" />
</debugger>
</operation-settings>
<ddl-file-settings>
<extensions>
<mapping file-type-id="VIEW" extensions="vw" />
<mapping file-type-id="TRIGGER" extensions="trg" />
<mapping file-type-id="PROCEDURE" extensions="prc" />
<mapping file-type-id="FUNCTION" extensions="fnc" />
<mapping file-type-id="PACKAGE" extensions="pkg" />
<mapping file-type-id="PACKAGE_SPEC" extensions="pks" />
<mapping file-type-id="PACKAGE_BODY" extensions="pkb" />
<mapping file-type-id="TYPE" extensions="tpe" />
<mapping file-type-id="TYPE_SPEC" extensions="tps" />
<mapping file-type-id="TYPE_BODY" extensions="tpb" />
</extensions>
<general>
<lookup-ddl-files value="true" />
<create-ddl-files value="false" />
<synchronize-ddl-files value="true" />
<use-qualified-names value="false" />
<make-scripts-rerunnable value="true" />
</general>
</ddl-file-settings>
<general-settings>
<regional-settings>
<date-format value="MEDIUM" />
<number-format value="UNGROUPED" />
<locale value="SYSTEM_DEFAULT" />
<use-custom-formats value="false" />
</regional-settings>
<environment>
<environment-types>
<environment-type id="development" name="Development" description="Development environment" color="-2430209/-12296320" readonly-code="false" readonly-data="false" />
<environment-type id="integration" name="Integration" description="Integration environment" color="-2621494/-12163514" readonly-code="true" readonly-data="false" />
<environment-type id="production" name="Production" description="Productive environment" color="-11574/-10271420" readonly-code="true" readonly-data="true" />
<environment-type id="other" name="Other" description="" color="-1576/-10724543" readonly-code="false" readonly-data="false" />
</environment-types>
<visibility-settings>
<connection-tabs value="true" />
<dialog-headers value="true" />
<object-editor-tabs value="true" />
<script-editor-tabs value="false" />
<execution-result-tabs value="true" />
</visibility-settings>
</environment>
</general-settings>
</component>
<component name="DBNavigator.Project.StatementExecutionManager">
<execution-variables />
</component>
</project>
\ No newline at end of file
.idea/dictionaries/lxrent.xml 0 → 100644
View file @ d3dde44a
<component name="ProjectDictionaryState">
<dictionary name="lxrent" />
</component>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="PROJECT" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/misc.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (venv)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>
</project>
\ No newline at end of file
.idea/modules.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/physical.iml" filepath="$PROJECT_DIR$/.idea/physical.iml" />
</modules>
</component>
</project>
\ No newline at end of file
.idea/physical.iml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml 0 → 100644
View file @ d3dde44a
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="d7dd36ca-85ef-4a59-9db5-8b1ee4993a4e" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FUSProjectUsageTrigger">
<session id="-127591351">
<usages-collector id="statistics.lifecycle.project">
<counts>
<entry key="project.open.time.1" value="1" />
<entry key="project.opened" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="5" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="Python" value="5" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="py" value="28" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="Python" value="28" />
</counts>
</usages-collector>
</session>
<session id="1569122105">
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="py" value="2" />
<entry key="template" value="1" />
<entry key="xml" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="PLAIN_TEXT" value="1" />
<entry key="Python" value="2" />
<entry key="XML" value="1" />
</counts>
</usages-collector>
</session>
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="209">
<caret line="85" column="46" selection-start-line="85" selection-start-column="46" selection-end-line="85" selection-end-column="46" />
<folding>
<element signature="e#463#1573#0" />
<element signature="e#6049#6240#0" />
<element signature="e#7051#7614#0" />
<element signature="e#8492#10033#0" />
<element signature="e#10162#10820#0" />
<element signature="e#10934#11363#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1760">
<caret line="293" column="66" selection-start-line="293" selection-start-column="66" selection-end-line="293" selection-end-column="66" />
<folding>
<element signature="e#47#61#0" expanded="true" />
<element signature="e#363#1043#0" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/trans2es/management/commands/trans2es_data2es_parallel.py" />
<option value="$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py" />
<option value="$PROJECT_DIR$/search/views/topic.py" />
<option value="$PROJECT_DIR$/search/utils/topic.py" />
<option value="$PROJECT_DIR$/physical/settings.py" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="279" />
<option name="y" value="23" />
<option name="width" value="1280" />
<option name="height" value="738" />
</component>
<component name="ProjectInspectionProfilesVisibleTreeState">
<entry key="Project Default">
<profile-state>
<expanded-state>
<State />
<State>
<id>General</id>
</State>
</expanded-state>
<selected-state>
<State>
<id>Buildout</id>
</State>
</selected-state>
</profile-state>
</entry>
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="linucb" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="physical" type="b2602c69:ProjectViewProjectNode" />
<item name="physical" type="462c0819:PsiDirectoryNode" />
<item name="search" type="462c0819:PsiDirectoryNode" />
<item name="views" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
</panes>
</component>
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/search/views" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="d7dd36ca-85ef-4a59-9db5-8b1ee4993a4e" name="Default Changelist" comment="" />
<created>1548319196437</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1548319196437</updated>
</task>
<servers />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1280" height="724" extended-state="6" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.22160445" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="DB Browser" order="2" />
<window_info id="Favorites" order="3" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="DB Execution Console" order="8" />
<window_info active="true" anchor="bottom" id="Terminal" order="9" visible="true" weight="0.44663382" />
<window_info anchor="bottom" id="Python Console" order="10" />
<window_info anchor="bottom" id="Event Log" order="11" side_tool="true" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
</layout>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/trans2es/models/face_user_contrast_similar.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="21" column="26" selection-start-line="21" selection-start-column="26" selection-end-line="21" selection-end-column="26" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/trans2es/management/commands/trans2es_data2es_parallel.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="364">
<caret line="129" column="63" selection-start-line="129" selection-start-column="54" selection-end-line="129" selection-end-column="63" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/utils/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1760">
<caret line="293" column="66" selection-start-line="293" selection-start-column="66" selection-end-line="293" selection-end-column="66" />
<folding>
<element signature="e#47#61#0" expanded="true" />
<element signature="e#363#1043#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/app_conf.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings_local.py.template">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1987" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/search/views/contrast_similar.py">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/search/views/topic.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="209">
<caret line="85" column="46" selection-start-line="85" selection-start-column="46" selection-end-line="85" selection-end-column="46" />
<folding>
<element signature="e#463#1573#0" />
<element signature="e#6049#6240#0" />
<element signature="e#7051#7614#0" />
<element signature="e#8492#10033#0" />
<element signature="e#10162#10820#0" />
<element signature="e#10934#11363#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/physical/settings.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-2448">
<caret line="12" column="9" lean-forward="true" selection-start-line="12" selection-start-column="9" selection-end-line="12" selection-end-column="9" />
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ScopeChooserConfigurable.UI">
<settings>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
\ No newline at end of file
app_conf.xml
View file @ d3dde44a
...@@ -9,10 +9,12 @@ ...@@ -9,10 +9,12 @@
<config name="initializer_list"> <config name="initializer_list">
<element value="physical.django_init"/> <element value="physical.django_init"/>
<element value="search.views.topic"/> <element value="search.views.topic"/>
<element value="search.views.business_topic"/>
<element value="search.views.pick"/> <element value="search.views.pick"/>
<element value="search.views.group"/> <element value="search.views.group"/>
<element value="search.views.user"/> <element value="search.views.user"/>
<element value="search.views.tag"/> <element value="search.views.tag"/>
<element value="search.views.contrast_similar"/>
<element value="injection.data_sync.tasks"/> <element value="injection.data_sync.tasks"/>
<element value="search.views.contrast_similar"/> <element value="search.views.contrast_similar"/>
</config> </config>
......
libs/es.py
View file @ d3dde44a
...@@ -207,4 +207,4 @@ class ESPerform(object): ...@@ -207,4 +207,4 @@ class ESPerform(object):
return res return res
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]} return {"total_count":0,"hits":[]}
\ No newline at end of file
physical/DBRouter.py 0 → 100644
View file @ d3dde44a
# !/usr/bin/env python
# -*- coding: utf-8 -*-
class DBRouter:
"""
A router to control all database operations on models in the
user application.
"""
def db_for_read(self, model, **hints):
"""
Attempts to read user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
return None
def db_for_write(self, model, **hints):
"""
Attempts to write user models go to users_db.
"""
if model._meta.app_label == 'face':
return 'face'
return None
def allow_relation(self, obj1, obj2, **hints):
"""
Allow relations if a model in the user app is involved.
"""
if obj1._meta.app_label == 'face' or \
obj2._meta.app_label == 'face':
return True
return None
def allow_migrate(self, db, app_label, model_name=None, **hints):
"""
Make sure the auth app only appears in the 'users_db'
database.
"""
if app_label == 'face':
return db == 'face'
return None
physical/settings.py
View file @ d3dde44a
...@@ -14,4 +14,189 @@ import os ...@@ -14,4 +14,189 @@ import os
from .log_settings import * from .log_settings import *
from datetime import timedelta from datetime import timedelta
from celery.schedules import crontab from celery.schedules import crontab
from .settings_local import *
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '^j3sg)sj8rc@du74%fb$c2926tv!!4g(kp-=rx1)c5!1&1(dq='
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = []
# Application definition
SENTRY_CELERY_ENDPOINT = "http://60b0004c8884420f8067fb32fc3ed244:20f97fc73ffa4aad9735d0e6542a6d78@sentry.igengmei.com/140"
BROKER_URL = "redis://127.0.0.1:6379/8"
# CELERY_SEND_EVENTS = True
# CELERY_SEND_TASK_SENT_EVENT = True
#
# CELERY_DEFAULT_EXCHANGE = 'celery'
# CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
# CELERY_DEFAULT_ROUTING_KEY = 'celery'
#
# CELERY_QUEUES = {
# 'celery': {
# 'exchange': CELERY_DEFAULT_EXCHANGE,
# 'routing_key': CELERY_DEFAULT_ROUTING_KEY,
# },
# 'order': {
# 'exchange': 'order',
# 'routing_key': 'order',
# },
# }
CELERY_BROKER_URL = "redis://127.0.0.1:6379/8"
CELERY_TIMEZONE = 'Asia/Shanghai'
CELERY_ROUTES = ['physical.celery_task_router.CeleryTaskRouter']
INSTALLED_APPS = (
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'trans2es',
'search',
'injection.data_sync',
)
CELERYBEAT_SCHEDULE = {
'sync_face_similar_data_to_redis': {
'task': 'injection.data_sync.tasks.sync_face_similar_data_to_redis',
'schedule': timedelta(seconds=120),
'args': ()
},
}
"""
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
"""
ROOT_URLCONF = 'physical.urls'
WSGI_APPLICATION = 'physical.wsgi.application'
REDIS_URL = "redis://127.0.0.1:6379"
# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
DATABASE_ROUTERS = ['physical.DBRouter.DBRouter']
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'alpha',
'USER': 'work',
'PASSWORD': 'Gengmei123',
# 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
"HOST": 'rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com',
#'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
'PORT': '3306',
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4",
},
},
'face': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'face',
'USER': 'work',
'PASSWORD': 'Gengmei123',
# 'HOST': 'rm-2ze5k2we69904787l.mysql.rds.aliyuncs.com',
"HOST": "rm-2zeaut61u9sm21m0bjo.mysql.rds.aliyuncs.com",
# 'HOST': 'rm-2zeaut61u9sm21m0b.mysql.rds.aliyuncs.com',
'PORT': '3306',
# 'CONN_MAX_AGE': None,
'OPTIONS': {
"init_command": "SET foreign_key_checks = 0;",
"charset": "utf8mb4",
},
}
}
ES_INFO_LIST = [
{
"host": "10.29.130.141",
"port": 9200
}
]
ES_INDEX_PREFIX = "gm-dbmw"
MIDDLEWARE_CLASSES = (
'gm_tracer.middleware.TracerMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'helios.DjangoL5dMiddleware',
)
# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/
# LANGUAGE_CODE = 'en-us'
#
TIME_ZONE = 'Asia/Shanghai'
#
USE_I18N = True
USE_L10N = True
#
# USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/
STATIC_URL = '/static/'
search/utils/topic.py
View file @ d3dde44a
...@@ -41,7 +41,6 @@ class TopicUtils(object): ...@@ -41,7 +41,6 @@ class TopicUtils(object):
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return {"total_count":0,"hits":[]} return {"total_count":0,"hits":[]}
@classmethod @classmethod
def analyze_related_user_id_list(cls,related_user_id_list): def analyze_related_user_id_list(cls,related_user_id_list):
""" """
...@@ -574,21 +573,43 @@ class TopicUtils(object): ...@@ -574,21 +573,43 @@ class TopicUtils(object):
return list() return list()
@classmethod @classmethod
def process_filters(cls, filters): def process_filters(cls, filters, filter_online=True):
"""处理过滤器部分。""" """处理过滤器部分。"""
f = [ f = [
{"term": {"is_online": True}},
{"term": {"is_deleted": False}}, {"term": {"is_deleted": False}},
] ]
if not filters: if not filters:
return f return f
for k, v in filters.items(): for k, v in filters.items():
if k == "group_id":
if k in ["create_time_gte", "create_time_lte"]:
if k == "create_time_gte":
op = "gte"
elif k == "create_time_lte":
op = "lte"
f.append({ f.append({
"term": {"group_id": v}, "range": {
"create_time_val": {
op: v,
}
}
}) })
else:
if isinstance(k, list):
f.append({
"terms": {k: v},
})
else:
f.append({
"term": {k: v},
})
if filter_online:
f.append({"term": {"is_online": True}})
return f return f
...@@ -620,22 +641,29 @@ class TopicUtils(object): ...@@ -620,22 +641,29 @@ class TopicUtils(object):
}, },
}) })
elif sorts_by == TOPIC_SEARCH_SORT.VOTE_NUM:
pass
elif sorts_by == '2':
sort_rule.append({
"id":{
"order":"desc"
},
})
return sort_rule return sort_rule
@classmethod @classmethod
def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10): def list_topic_ids(cls, filters, nfilters, sorts_by, offset=0, size=10, index_name="topic", filter_online=True):
must = cls.process_filters(filters, filter_online=True)
q = { q = {
"query": { "query": {
"bool": { "bool": {
"must": cls.process_filters(filters), "must": must,
"must_not": cls.process_nfilters(nfilters), "must_not": cls.process_nfilters(nfilters),
} }
}, }
"_source": {
"includes":["id"]
},
"sort": [],
} }
if sorts_by: if sorts_by:
...@@ -643,10 +671,18 @@ class TopicUtils(object): ...@@ -643,10 +671,18 @@ class TopicUtils(object):
q["sort"] = sorts q["sort"] = sorts
try: try:
result_dict = ESPerform.get_search_results(ESPerform.get_cli(), sub_index_name="topic", result_dict = ESPerform.get_search_results(
query_body=q, offset=offset, size=size) ESPerform.get_cli(), sub_index_name=index_name,
query_body=q, offset=offset, size=size
return result_dict["hits"] )
return {
"hits": result_dict["hits"],
"total_count": result_dict["total_count"]
}
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return [] return {
"hits": [],
"total_count": 0
}
search/views/business_topic.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import, print_function
from gm_rpcd.all import bind
import logging
import traceback
import json
from search.utils.topic import TopicUtils
from libs.es import ESPerform
from libs.cache import redis_client
from search.utils.common import *
from libs.es import ESPerform
@bind("physical/search/business/topic")
def business_topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
"""帖子搜索。"""
try:
result_list = TopicUtils.list_topic_ids(
filters=filters, nfilters=nfilters, sorts_by=sorts_by,
offset=offset, size=size, filter_online=False,
index_name="topic"
)
topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids, "total_count": result_list["total_count"]}
except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": [], "total_count": 0}
search/views/topic.py
View file @ d3dde44a
...@@ -302,11 +302,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10): ...@@ -302,11 +302,14 @@ def topic_search(filters, nfilters=None, sorts_by=None, offset=0, size=10):
try: try:
result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters, result_list = TopicUtils.list_topic_ids(filters=filters, nfilters=nfilters,
sorts_by=sorts_by, offset=offset, size=size) sorts_by=sorts_by, offset=offset, size=size)
topic_ids = [item["_source"]["id"] for item in result_list] topic_ids = [item["_source"]["id"] for item in result_list["hits"]]
return {"topic_ids": topic_ids} return {
"topic_ids": topic_ids,
"total_count": result_list["total_count"]
}
except: except:
logging.error("catch exception, err_msg:%s" % traceback.format_exc()) logging.error("catch exception, err_msg:%s" % traceback.format_exc())
return {"topic_ids": []} return {"topic_ids": [], "total_count": 0}
@bind("physical/search/query_topic_by_user_similarity") @bind("physical/search/query_topic_by_user_similarity")
...@@ -319,6 +322,7 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1 ...@@ -319,6 +322,7 @@ def query_topic_by_user_similarity(topic_similarity_score_dict, offset=0, size=1
:return: :return:
""" """
try: try:
must_topic_id_list = list(topic_similarity_score_dict.keys()) must_topic_id_list = list(topic_similarity_score_dict.keys())
topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size, topic_id_list = TopicUtils.get_recommend_topic_ids(tag_id=0, user_id=-1, offset=offset, size=size,single_size=size,
must_topic_id_list=must_topic_id_list) must_topic_id_list=must_topic_id_list)
......
trans2es/management/commands/trans2es_mapping2es.py
View file @ d3dde44a
...@@ -6,29 +6,29 @@ from django.core.management.base import BaseCommand, CommandError ...@@ -6,29 +6,29 @@ from django.core.management.base import BaseCommand, CommandError
import traceback import traceback
import logging import logging
from libs.es import ESPerform from libs.es import ESPerform
from trans2es.type_info import get_type_info_map,TypeInfo from trans2es.type_info import get_type_info_map, TypeInfo
class Command(BaseCommand): class Command(BaseCommand):
args = '' args = ''
help = 'dump mapping to elasticsearch' help = 'dump mapping to elasticsearch'
from optparse import make_option from optparse import make_option
option_list = BaseCommand.option_list + ( option_list = BaseCommand.option_list + (
make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',default=''), make_option('-t', '--type', dest='type', help='type name to dump data to elasticsearch', metavar='TYPE',
make_option('-T', '--indices_template', dest='indices_template', help='index template name to dump data to elasticsearch', metavar='TYPE',
default=''), default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch', metavar='INDEX_PREFIX'), make_option('-T', '--indices_template', dest='indices_template',
help='index template name to dump data to elasticsearch', metavar='TYPE',
default=''),
make_option('-i', '--index-prefix', dest='index_prefix', help='index name to dump data to elasticsearch',
metavar='INDEX_PREFIX'),
make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'), make_option('-p', '--parallel', dest='parallel', help='parallel process count', metavar='PARALLEL'),
make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''), make_option('-s', '--pks', dest='pks', help='specify sync pks, comma separated', metavar='PKS', default=''),
make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True), make_option('--streaming-slicing', dest='streaming_slicing', action='store_true', default=True),
make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True), make_option('--no-streaming-slicing', dest='streaming_slicing', action='store_false', default=True),
) )
def handle(self, *args, **options): def handle(self, *args, **options):
try: try:
es_cli = ESPerform.get_cli() es_cli = ESPerform.get_cli()
...@@ -37,23 +37,21 @@ class Command(BaseCommand): ...@@ -37,23 +37,21 @@ class Command(BaseCommand):
for type_name in type_name_list: for type_name in type_name_list:
if len(options["type"]): if len(options["type"]):
if type_name==options["type"]: if options["type"] == "all" or type_name == options["type"]:
# ESPerform.put_index_mapping(es_cli, type_name, force_sync=True)
official_index_name = ESPerform.get_official_index_name(type_name) official_index_name = ESPerform.get_official_index_name(type_name)
index_exists = es_cli.indices.exists(official_index_name) index_exists = es_cli.indices.exists(official_index_name)
if not index_exists: if not index_exists:
logging.info("begin create [%s] index!" % type_name) logging.info("begin create [%s] index and mapping!" % type_name)
ESPerform.create_index(es_cli,type_name) ESPerform.create_index(es_cli, type_name)
ESPerform.put_index_mapping(es_cli, type_name)
logging.info("begin create [%s] mapping!" % type_name) else:
ESPerform.put_index_mapping(es_cli,type_name,force_sync=True) logging.warning("index:[%s] has already existing!" % type_name)
if len(options["indices_template"]): if len(options["indices_template"]):
template_file_name = options["indices_template"] template_file_name = options["indices_template"]
if ESPerform.put_indices_template(es_cli=es_cli,template_file_name=template_file_name,template_name=template_file_name): if ESPerform.put_indices_template(es_cli=es_cli, template_file_name=template_file_name,
template_name=template_file_name):
logging.info("put indices template suc!") logging.info("put indices template suc!")
else: else:
logging.error("put indices template err!") logging.error("put indices template err!")
......
trans2es/mapping/contrast_similar.json 0 → 100644
View file @ d3dde44a
{
"dynamic":"strict",
"properties": {
"id":{"type":"long"},
"is_online":{"type":"boolean"},//上线
"create_time":{"type":"date", "format":"date_time_no_millis"},
"update_time":{"type":"date", "format":"date_time_no_millis"},
"is_deleted":{"type":"boolean"},//是否被删除
"participant_user_id":{"type":"long"},//参与者ID
"contrast_user_id_list":{"type":"long"},//对照者ID
"similarity":{"type":"double"},//相似度
}
}
\ No newline at end of file
trans2es/mapping/topic.json
View file @ d3dde44a
...@@ -25,6 +25,29 @@ ...@@ -25,6 +25,29 @@
"update_time":{"type":"date", "format":"date_time_no_millis"}, "update_time":{"type":"date", "format":"date_time_no_millis"},
"create_time_val":{"type":"long"}, "create_time_val":{"type":"long"},
"update_time_val":{"type":"long"}, "update_time_val":{"type":"long"},
"language_type":{"type":"long"} "language_type":{"type":"long"},
"is_shadow": {"type": "boolean"},
"is_recommend": {"type": "boolean"},
"is_complaint": {"type": "boolean"}, // 是否被举报
"virtual_content_level":{"type": "text"},
"like_num_crawl": {"type": "long"}, // 爬取点赞数
"comment_num_crawl": {"type": "long"}, // 爬取评论数
"is_crawl": {"type": "boolean"},
"platform": {"type": "long"},
"platform_id": {"type": "long"},
"drop_score":{"type": "double"}, // 人工降分
"sort_score":{"type": "double"}, // 排序分
"pictorial_id":{"type": "long"}, //所在组ID
"pictorial_name":{ // 所在组名称
"type": "text",
"analyzer": "gm_default_index",
"search_analyzer": "gm_default_index"
}
} }
} }
\ No newline at end of file
trans2es/models/__init__.py
View file @ d3dde44a
...@@ -13,3 +13,4 @@ from .tag import AccountUserTag ...@@ -13,3 +13,4 @@ from .tag import AccountUserTag
from .user import User from .user import User
from .group import Group from .group import Group
from .topic import Topic from .topic import Topic
# from .pictorial import PictorialTopic
trans2es/models/contrast_similar.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import traceback
import logging
from django.db import models
import datetime
from .topic import Topic
manager = lambda: models.Manager().db_manager(using='face')
class ContrastSimilar(models.Model):
class Meta:
verbose_name = u"用户脸相似度表"
db_table = "face_user_contrast_similar"
objects = manager()
id = models.IntegerField(verbose_name="主键ID", primary_key=True)
is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name="是否删除")
participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID')
contrast_user_id = models.IntegerField(verbose_name="对照者用户ID")
similarity = models.FloatField(verbose_name=u'相似度', default=0)
create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
def get_similar_user_id(self):
try:
similar_user_id_list = list()
if self.similarity > 0.3:
query_list = Topic.objects.filter(user_id=self.participant_user_id, is_deleted=False, is_online=True,
content_level__in=("3", "4", "5"))
for item in query_list:
if item.user_id != self.participant_user_id:
similar_user_id_list.append(item.id)
return similar_user_id_list
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return []
trans2es/models/face_user_contrast_similar.py
View file @ d3dde44a
...@@ -10,19 +10,21 @@ import datetime ...@@ -10,19 +10,21 @@ import datetime
manager = lambda: models.Manager().db_manager(using='face') manager = lambda: models.Manager().db_manager(using='face')
class FaceUserContrastSimilar(models.Model): class FaceUserContrastSimilar(models.Model):
class Meta: class Meta:
verbose_name=u"用户脸相似表" app_label = "face"
db_table="face_user_contrast_similar" verbose_name = u"用户脸相似表"
db_table = "face_user_contrast_similar"
objects = manager() objects = manager()
id = models.IntegerField(verbose_name="主键ID",primary_key=True) id = models.IntegerField(verbose_name="主键ID", primary_key=True)
is_online = models.BooleanField(verbose_name="是否上线") is_online = models.BooleanField(verbose_name="是否上线")
is_deleted = models.BooleanField(verbose_name="是否删除") is_deleted = models.BooleanField(verbose_name="是否删除")
participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID') participant_user_id = models.IntegerField(verbose_name=u'参与者用户ID')
similarity = models.FloatField(verbose_name=u'相似度',default=0) similarity = models.FloatField(verbose_name=u'相似度', default=0)
create_time = models.DateTimeField(verbose_name=u'创建时间',default=datetime.datetime.fromtimestamp(0)) create_time = models.DateTimeField(verbose_name=u'创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0)) update_time = models.DateTimeField(verbose_name=u'更新时间', default=datetime.datetime.fromtimestamp(0))
contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID') contrast_user_id = models.IntegerField(verbose_name=u'对照者用户ID')
......
trans2es/models/pictorial.py 0 → 100644
View file @ d3dde44a
# from django.db import models
# class PictorialTopic(models.Model):
# """画报帖子关系"""
# class Meta:
# verbose_name = u'画报帖子关系'
# app_label = 'community'
# db_table = 'community_pictorial_topic'
# id = models.IntegerField(verbose_name=u'日记ID', primary_key=True)
# pictorial_id = models.BigIntegerField(verbose_name=u'画报ID')
# topic_id = models.BigIntegerField(verbose_name=u'帖子ID')
# user_id = models.BigIntegerField(verbose_name=u'用户ID')
# is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
# is_online = models.BooleanField(verbose_name=u'是否上线')
# is_deleted = models.BooleanField(verbose_name=u'是否删除')
trans2es/models/topic.py
View file @ d3dde44a
...@@ -10,6 +10,8 @@ from libs.es import ESPerform ...@@ -10,6 +10,8 @@ from libs.es import ESPerform
from django.db import models from django.db import models
import datetime import datetime
from alpha_types.venus import GRAP_PLATFORM
from .pick_topic import PickTopic from .pick_topic import PickTopic
from .tag import TopicTag, Tag from .tag import TopicTag, Tag
from .user_extra import UserExtra from .user_extra import UserExtra
...@@ -70,6 +72,26 @@ class Topic(models.Model): ...@@ -70,6 +72,26 @@ class Topic(models.Model):
create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0)) create_time = models.DateTimeField(verbose_name=u'日记创建时间', default=datetime.datetime.fromtimestamp(0))
update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0)) update_time = models.DateTimeField(verbose_name=u'日记更新时间', default=datetime.datetime.fromtimestamp(0))
is_crawl = models.BooleanField(verbose_name="是否是爬回的数据", default=False)
has_image = models.BooleanField(verbose_name=u"是否有图片", default=False)
virtual_content_level = models.CharField(verbose_name=u'模拟内容等级', max_length=100, default='')
like_num_crawl = models.IntegerField(verbose_name=u'爬取点赞数', default=0)
comment_num_crawl = models.IntegerField(verbose_name=u'爬取评论数', default=0)
is_recommend = models.IntegerField(verbose_name=u'是否推荐 0 非推荐 1 推荐', default=0)
sort_score = models.FloatField(verbose_name='排序分', default=0)
is_shadow = models.BooleanField(verbose_name="是否是马甲账户", default=False)
platform = models.IntegerField(verbose_name=u'平台来源', choices=GRAP_PLATFORM, default=GRAP_PLATFORM.ALPHA)
platform_id = models.BigIntegerField(verbose_name='用平台ID', null=True)
@property
def is_complaint(self):
"""是否被举报"""
if TopicComplaint.objects.filter(topic_id=self.id, is_online=True).exists():
return True
return False
def topic_has_image(self): def topic_has_image(self):
try: try:
has_image = False has_image = False
...@@ -173,3 +195,19 @@ class Topic(models.Model): ...@@ -173,3 +195,19 @@ class Topic(models.Model):
except: except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc()) logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return 0.0 return 0.0
class TopicComplaint(models.Model):
"""帖子举报"""
class Meta:
verbose_name = u'帖子举报'
app_label = 'community'
db_table = 'topic_complaint'
id = models.IntegerField(verbose_name='日记图片ID', primary_key=True)
user_id = models.BigIntegerField(verbose_name=u'用户ID', db_index=True)
topic = models.ForeignKey(
Topic, verbose_name=u"关联的帖子", null=True, blank=True, default=None, on_delete=models.CASCADE, related_name='complaints')
is_online = models.BooleanField(verbose_name=u"是否有效", default=True)
\ No newline at end of file
trans2es/type_info.py
View file @ d3dde44a
...@@ -12,13 +12,14 @@ import elasticsearch ...@@ -12,13 +12,14 @@ import elasticsearch
import elasticsearch.helpers import elasticsearch.helpers
import sys import sys
from trans2es.models import topic,user,pick_celebrity,group,celebrity,tag from trans2es.models import topic, user, pick_celebrity, group, celebrity, tag, contrast_similar
from trans2es.utils.user_transfer import UserTransfer from trans2es.utils.user_transfer import UserTransfer
from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer from trans2es.utils.pick_celebrity_transfer import PickCelebrityTransfer
from trans2es.utils.group_transfer import GroupTransfer from trans2es.utils.group_transfer import GroupTransfer
from trans2es.utils.topic_transfer import TopicTransfer from trans2es.utils.topic_transfer import TopicTransfer
from trans2es.utils.celebrity_transfer import CelebrityTransfer from trans2es.utils.celebrity_transfer import CelebrityTransfer
from trans2es.utils.tag_transfer import TagTransfer from trans2es.utils.tag_transfer import TagTransfer
from trans2es.utils.contrast_similar_transfer import Contrast_Similar_Transfer
from libs.es import ESPerform from libs.es import ESPerform
__es = None __es = None
...@@ -160,7 +161,7 @@ class TypeInfo(object): ...@@ -160,7 +161,7 @@ class TypeInfo(object):
# traceback.print_exc() # traceback.print_exc()
# es_result = 'error' # es_result = 'error'
return ESPerform.es_helpers_bulk(es,data_list,sub_index_name,True) return ESPerform.es_helpers_bulk(es, data_list, sub_index_name, True)
def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None): def elasticsearch_bulk_insert(self, sub_index_name, instance_iterable, es=None):
data_list = self.bulk_get_data(instance_iterable) data_list = self.bulk_get_data(instance_iterable)
...@@ -266,11 +267,11 @@ def get_type_info_map(): ...@@ -266,11 +267,11 @@ def get_type_info_map():
round_insert_period=2, round_insert_period=2,
), ),
TypeInfo( TypeInfo(
name = "user", #用户 name="user", # 用户
type="user", type="user",
model=user.User, model=user.User,
#query_deferred=lambda:user.User.objects.all().query, # query_deferred=lambda:user.User.objects.all().query,
#query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query, # query_deferred=lambda: user.User.objects.prefetch_related('myuserfollow').query,
query_deferred=lambda: user.User.objects.all().query, query_deferred=lambda: user.User.objects.all().query,
get_data_func=UserTransfer.get_user_data, get_data_func=UserTransfer.get_user_data,
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
...@@ -319,6 +320,16 @@ def get_type_info_map(): ...@@ -319,6 +320,16 @@ def get_type_info_map():
bulk_insert_chunk_size=100, bulk_insert_chunk_size=100,
round_insert_chunk_size=5, round_insert_chunk_size=5,
round_insert_period=2, round_insert_period=2,
),
TypeInfo(
name='contrast_similar', # facesimilar
type='contrast_similar',
model=contrast_similar.ContrastSimilar,
query_deferred=lambda: contrast_similar.ContrastSimilar.objects.all().query,
get_data_func=Contrast_Similar_Transfer.get_contrast_similar_data,
bulk_insert_chunk_size=100,
round_insert_chunk_size=5,
round_insert_period=2
) )
] ]
......
trans2es/utils/contrast_similar_transfer.py 0 → 100644
View file @ d3dde44a
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import logging
import traceback
from libs.tools import tzlc
import time
from trans2es.models.contrast_similar import ContrastSimilar
class Contrast_Similar_Transfer(object):
@classmethod
def get_contrast_similar_data(cls, instance):
try:
res = dict()
res["id"] = instance.id
res["is_online"] = instance.is_online
res["is_deleted"] = instance.is_deleted
res["participant_user_id"] = instance.participant_user_id
create_time = instance.create_time
tzlc_create_time = tzlc(create_time)
res["create_time"] = tzlc_create_time
update_time = instance.update_time
tzlc_update_time = tzlc(update_time)
res["update_time"] = tzlc_update_time
res["similarity"] = instance.similarity
res["contrast_user_id_list"] = ContrastSimilar.get_similar_user_id()
return res
except:
logging.error("catch exception,err_msg:%s" % traceback.format_exc())
return None
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment