Skip to content

B
bi-report
Commit e388a661 authored by yindanlei's avatar yindanlei
Browse files
Options

update codes

parents 6e9fff6e ab609662
Show whitespace changes
Inline Side-by-side
Showing with 2752 additions and 3985 deletions
.idea/dbnavigator.xml 0 → 100644
View file @ e388a661
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DBNavigator.Project.DataEditorManager">
<record-view-column-sorting-type value="BY_INDEX" />
<value-preview-text-wrapping value="true" />
<value-preview-pinned value="false" />
</component>
<component name="DBNavigator.Project.DataExportManager">
<export-instructions>
<create-header value="true" />
<quote-values-containing-separator value="true" />
<quote-all-values value="false" />
<value-separator value="" />
<file-name value="" />
<file-location value="" />
<scope value="GLOBAL" />
<destination value="FILE" />
<format value="EXCEL" />
<charset value="UTF-8" />
</export-instructions>
</component>
<component name="DBNavigator.Project.DatabaseBrowserManager">
<autoscroll-to-editor value="false" />
<autoscroll-from-editor value="true" />
<show-object-properties value="true" />
<loaded-nodes />
</component>
<component name="DBNavigator.Project.DatabaseFileManager">
<open-files />
</component>
<component name="DBNavigator.Project.EditorStateManager">
<last-used-providers />
</component>
<component name="DBNavigator.Project.MethodExecutionManager">
<method-browser />
<execution-history>
<group-entries value="true" />
<execution-inputs />
</execution-history>
<argument-values-cache />
</component>
<component name="DBNavigator.Project.ObjectDependencyManager">
<last-used-dependency-type value="INCOMING" />
</component>
<component name="DBNavigator.Project.ObjectQuickFilterManager">
<last-used-operator value="EQUAL" />
<filters />
</component>
<component name="DBNavigator.Project.ScriptExecutionManager" clear-outputs="true">
<recently-used-interfaces />
</component>
<component name="DBNavigator.Project.Settings">
<connections />
<browser-settings>
<general>
<display-mode value="TABBED" />
<navigation-history-size value="100" />
<show-object-details value="false" />
</general>
<filters>
<object-type-filter>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="true" />
<object-type name="ROLE" enabled="true" />
<object-type name="PRIVILEGE" enabled="true" />
<object-type name="CHARSET" enabled="true" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED_VIEW" enabled="true" />
<object-type name="NESTED_TABLE" enabled="true" />
<object-type name="COLUMN" enabled="true" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET_TRIGGER" enabled="true" />
<object-type name="DATABASE_TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="true" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="TYPE_ATTRIBUTE" enabled="true" />
<object-type name="ARGUMENT" enabled="true" />
<object-type name="DIMENSION" enabled="true" />
<object-type name="CLUSTER" enabled="true" />
<object-type name="DBLINK" enabled="true" />
</object-type-filter>
</filters>
<sorting>
<object-type name="COLUMN" sorting-type="NAME" />
<object-type name="FUNCTION" sorting-type="NAME" />
<object-type name="PROCEDURE" sorting-type="NAME" />
<object-type name="ARGUMENT" sorting-type="POSITION" />
</sorting>
<default-editors>
<object-type name="VIEW" editor-type="SELECTION" />
<object-type name="PACKAGE" editor-type="SELECTION" />
<object-type name="TYPE" editor-type="SELECTION" />
</default-editors>
</browser-settings>
<navigation-settings>
<lookup-filters>
<lookup-objects>
<object-type name="SCHEMA" enabled="true" />
<object-type name="USER" enabled="false" />
<object-type name="ROLE" enabled="false" />
<object-type name="PRIVILEGE" enabled="false" />
<object-type name="CHARSET" enabled="false" />
<object-type name="TABLE" enabled="true" />
<object-type name="VIEW" enabled="true" />
<object-type name="MATERIALIZED VIEW" enabled="true" />
<object-type name="NESTED TABLE" enabled="false" />
<object-type name="COLUMN" enabled="false" />
<object-type name="INDEX" enabled="true" />
<object-type name="CONSTRAINT" enabled="true" />
<object-type name="DATASET TRIGGER" enabled="true" />
<object-type name="DATABASE TRIGGER" enabled="true" />
<object-type name="SYNONYM" enabled="false" />
<object-type name="SEQUENCE" enabled="true" />
<object-type name="PROCEDURE" enabled="true" />
<object-type name="FUNCTION" enabled="true" />
<object-type name="PACKAGE" enabled="true" />
<object-type name="TYPE" enabled="true" />
<object-type name="TYPE ATTRIBUTE" enabled="false" />
<object-type name="ARGUMENT" enabled="false" />
<object-type name="DIMENSION" enabled="false" />
<object-type name="CLUSTER" enabled="false" />
<object-type name="DBLINK" enabled="true" />
</lookup-objects>
<force-database-load value="false" />
<prompt-connection-selection value="true" />
<prompt-schema-selection value="true" />
</lookup-filters>
</navigation-settings>
<dataset-grid-settings>
<general>
<enable-zooming value="true" />
<enable-column-tooltip value="true" />
</general>
<sorting>
<nulls-first value="true" />
<max-sorting-columns value="4" />
</sorting>
<tracking-columns>
<columnNames value="" />
<visible value="true" />
<editable value="false" />
</tracking-columns>
</dataset-grid-settings>
<dataset-editor-settings>
<text-editor-popup>
<active value="false" />
<active-if-empty value="false" />
<data-length-threshold value="100" />
<popup-delay value="1000" />
</text-editor-popup>
<values-actions-popup>
<show-popup-button value="true" />
<element-count-threshold value="1000" />
<data-length-threshold value="250" />
</values-actions-popup>
<general>
<fetch-block-size value="100" />
<fetch-timeout value="30" />
<trim-whitespaces value="true" />
<convert-empty-strings-to-null value="true" />
<select-content-on-cell-edit value="true" />
<large-value-preview-active value="true" />
</general>
<filters>
<prompt-filter-dialog value="true" />
<default-filter-type value="BASIC" />
</filters>
<qualified-text-editor text-length-threshold="300">
<content-types>
<content-type name="Text" enabled="true" />
<content-type name="Properties" enabled="true" />
<content-type name="XML" enabled="true" />
<content-type name="DTD" enabled="true" />
<content-type name="HTML" enabled="true" />
<content-type name="XHTML" enabled="true" />
<content-type name="Java" enabled="true" />
<content-type name="SQL" enabled="true" />
<content-type name="PL/SQL" enabled="true" />
<content-type name="JSON" enabled="true" />
<content-type name="JSON5" enabled="true" />
<content-type name="Groovy" enabled="true" />
<content-type name="AIDL" enabled="true" />
<content-type name="YAML" enabled="true" />
<content-type name="Manifest" enabled="true" />
</content-types>
</qualified-text-editor>
<record-navigation>
<navigation-target value="VIEWER" />
</record-navigation>
</dataset-editor-settings>
<code-editor-settings>
<general>
<show-object-navigation-gutter value="false" />
<show-spec-declaration-navigation-gutter value="true" />
<enable-spellchecking value="true" />
<enable-reference-spellchecking value="false" />
</general>
<confirmations>
<save-changes value="false" />
<revert-changes value="true" />
</confirmations>
</code-editor-settings>
<code-completion-settings>
<filters>
<basic-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="false" />
<filter-element type="OBJECT" id="view" selected="false" />
<filter-element type="OBJECT" id="materialized view" selected="false" />
<filter-element type="OBJECT" id="index" selected="false" />
<filter-element type="OBJECT" id="constraint" selected="false" />
<filter-element type="OBJECT" id="trigger" selected="false" />
<filter-element type="OBJECT" id="synonym" selected="false" />
<filter-element type="OBJECT" id="sequence" selected="false" />
<filter-element type="OBJECT" id="procedure" selected="false" />
<filter-element type="OBJECT" id="function" selected="false" />
<filter-element type="OBJECT" id="package" selected="false" />
<filter-element type="OBJECT" id="type" selected="false" />
<filter-element type="OBJECT" id="dimension" selected="false" />
<filter-element type="OBJECT" id="cluster" selected="false" />
<filter-element type="OBJECT" id="dblink" selected="false" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</basic-filter>
<extended-filter>
<filter-element type="RESERVED_WORD" id="keyword" selected="true" />
<filter-element type="RESERVED_WORD" id="function" selected="true" />
<filter-element type="RESERVED_WORD" id="parameter" selected="true" />
<filter-element type="RESERVED_WORD" id="datatype" selected="true" />
<filter-element type="RESERVED_WORD" id="exception" selected="true" />
<filter-element type="OBJECT" id="schema" selected="true" />
<filter-element type="OBJECT" id="user" selected="true" />
<filter-element type="OBJECT" id="role" selected="true" />
<filter-element type="OBJECT" id="privilege" selected="true" />
<user-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</user-schema>
<public-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</public-schema>
<any-schema>
<filter-element type="OBJECT" id="table" selected="true" />
<filter-element type="OBJECT" id="view" selected="true" />
<filter-element type="OBJECT" id="materialized view" selected="true" />
<filter-element type="OBJECT" id="index" selected="true" />
<filter-element type="OBJECT" id="constraint" selected="true" />
<filter-element type="OBJECT" id="trigger" selected="true" />
<filter-element type="OBJECT" id="synonym" selected="true" />
<filter-element type="OBJECT" id="sequence" selected="true" />
<filter-element type="OBJECT" id="procedure" selected="true" />
<filter-element type="OBJECT" id="function" selected="true" />
<filter-element type="OBJECT" id="package" selected="true" />
<filter-element type="OBJECT" id="type" selected="true" />
<filter-element type="OBJECT" id="dimension" selected="true" />
<filter-element type="OBJECT" id="cluster" selected="true" />
<filter-element type="OBJECT" id="dblink" selected="true" />
</any-schema>
</extended-filter>
</filters>
<sorting enabled="true">
<sorting-element type="RESERVED_WORD" id="keyword" />
<sorting-element type="RESERVED_WORD" id="datatype" />
<sorting-element type="OBJECT" id="column" />
<sorting-element type="OBJECT" id="table" />
<sorting-element type="OBJECT" id="view" />
<sorting-element type="OBJECT" id="materialized view" />
<sorting-element type="OBJECT" id="index" />
<sorting-element type="OBJECT" id="constraint" />
<sorting-element type="OBJECT" id="trigger" />
<sorting-element type="OBJECT" id="synonym" />
<sorting-element type="OBJECT" id="sequence" />
<sorting-element type="OBJECT" id="procedure" />
<sorting-element type="OBJECT" id="function" />
<sorting-element type="OBJECT" id="package" />
<sorting-element type="OBJECT" id="type" />
<sorting-element type="OBJECT" id="dimension" />
<sorting-element type="OBJECT" id="cluster" />
<sorting-element type="OBJECT" id="dblink" />
<sorting-element type="OBJECT" id="schema" />
<sorting-element type="OBJECT" id="role" />
<sorting-element type="OBJECT" id="user" />
<sorting-element type="RESERVED_WORD" id="function" />
<sorting-element type="RESERVED_WORD" id="parameter" />
</sorting>
<format>
<enforce-code-style-case value="true" />
</format>
</code-completion-settings>
<execution-engine-settings>
<statement-execution>
<fetch-block-size value="100" />
<execution-timeout value="20" />
<debug-execution-timeout value="600" />
<focus-result value="false" />
<prompt-execution value="false" />
</statement-execution>
<script-execution>
<command-line-interfaces />
<execution-timeout value="300" />
</script-execution>
<method-execution>
<execution-timeout value="30" />
<debug-execution-timeout value="600" />
<parameter-history-size value="10" />
</method-execution>
</execution-engine-settings>
<operation-settings>
<transactions>
<uncommitted-changes>
<on-project-close value="ASK" />
<on-disconnect value="ASK" />
<on-autocommit-toggle value="ASK" />
</uncommitted-changes>
<multiple-uncommitted-changes>
<on-commit value="ASK" />
<on-rollback value="ASK" />
</multiple-uncommitted-changes>
</transactions>
<session-browser>
<disconnect-session value="ASK" />
<kill-session value="ASK" />
<reload-on-filter-change value="false" />
</session-browser>
<compiler>
<compile-type value="KEEP" />
<compile-dependencies value="ASK" />
<always-show-controls value="false" />
</compiler>
<debugger>
<debugger-type value="ASK" />
<use-generic-runners value="true" />
</debugger>
</operation-settings>
<ddl-file-settings>
<extensions>
<mapping file-type-id="VIEW" extensions="vw" />
<mapping file-type-id="TRIGGER" extensions="trg" />
<mapping file-type-id="PROCEDURE" extensions="prc" />
<mapping file-type-id="FUNCTION" extensions="fnc" />
<mapping file-type-id="PACKAGE" extensions="pkg" />
<mapping file-type-id="PACKAGE_SPEC" extensions="pks" />
<mapping file-type-id="PACKAGE_BODY" extensions="pkb" />
<mapping file-type-id="TYPE" extensions="tpe" />
<mapping file-type-id="TYPE_SPEC" extensions="tps" />
<mapping file-type-id="TYPE_BODY" extensions="tpb" />
</extensions>
<general>
<lookup-ddl-files value="true" />
<create-ddl-files value="false" />
<synchronize-ddl-files value="true" />
<use-qualified-names value="false" />
<make-scripts-rerunnable value="true" />
</general>
</ddl-file-settings>
<general-settings>
<regional-settings>
<date-format value="MEDIUM" />
<number-format value="UNGROUPED" />
<locale value="SYSTEM_DEFAULT" />
<use-custom-formats value="false" />
</regional-settings>
<environment>
<environment-types>
<environment-type id="development" name="Development" description="Development environment" color="-2430209/-12296320" readonly-code="false" readonly-data="false" />
<environment-type id="integration" name="Integration" description="Integration environment" color="-2621494/-12163514" readonly-code="true" readonly-data="false" />
<environment-type id="production" name="Production" description="Productive environment" color="-11574/-10271420" readonly-code="true" readonly-data="true" />
<environment-type id="other" name="Other" description="" color="-1576/-10724543" readonly-code="false" readonly-data="false" />
</environment-types>
<visibility-settings>
<connection-tabs value="true" />
<dialog-headers value="true" />
<object-editor-tabs value="true" />
<script-editor-tabs value="false" />
<execution-result-tabs value="true" />
</visibility-settings>
</environment>
</general-settings>
</component>
<component name="DBNavigator.Project.StatementExecutionManager">
<execution-variables />
</component>
</project>
\ No newline at end of file
.idea/sqldialects.xml deleted 100644 → 0
View file @ 6e9fff6e
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/sqls/bak/demo_spark_param/demo_spark_param.sql" dialect="SQL92" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml deleted 100644 → 0
View file @ 6e9fff6e
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<<<<<<< HEAD
<list default="true" id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" />
=======
<list default="true" id="034b59f7-f608-4144-b0af-f6951647bd51" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/lib/shell/hive.sh" beforeDir="false" afterPath="$PROJECT_DIR$/lib/shell/hive.sh" afterDir="false" />
</list>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<<<<<<< HEAD
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectId" id="1dfVlMQlC7SI7HHCgC0U76ZyFWG" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
=======
<component name="CodeStyleSettingsInfer">
<option name="done" value="true" />
</component>
<component name="ComposerSettings">
<execution>
<executable />
</execution>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectCodeStyleSettingsMigration">
<option name="version" value="1" />
</component>
<component name="ProjectId" id="1dfQ0VEii7hX7vb7p4D7taxyaw0" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showExcludedFiles" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
</component>
<component name="RunManager">
<configuration default="true" type="ClojureREPL" factoryName="Local" activateToolWindowBeforeRun="false">
<setting name="replType" value="NREPL" />
<setting name="execution" value="LEININGEN" />
<setting name="jvmArgs" value="" />
<setting name="parameters" value="" />
<setting name="workingDir" value="" />
<setting name="profiles" value="" />
<setting name="aliases" value="" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
<configuration default="true" type="ClojureREPL" factoryName="Remote" activateToolWindowBeforeRun="false">
<setting name="host" value="" />
<setting name="port" value="0" />
<setting name="replType" value="SOCKET" />
<setting name="configType" value="SPECIFY" />
<setting name="fixLineNumbers" value="false" />
<method v="2" />
</configuration>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<<<<<<< HEAD
<changelist id="765e09a6-fc0b-4ec9-8de8-88bb721f9a89" name="Default Changelist" comment="" />
<created>1592816285414</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1592816285414</updated>
</task>
<servers />
</component>
=======
<changelist id="034b59f7-f608-4144-b0af-f6951647bd51" name="Default Changelist" comment="" />
<created>1592813444854</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1592813444854</updated>
<workItem from="1592813446154" duration="2153000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
>>>>>>> 0a1e615beb0ca00f6a1a43796de5b7488c9dd780
</project>
\ No newline at end of file
bin/pull_master.sh
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout master
git pull origin master
\ No newline at end of file
bin/pull_wy.sh 0 → 100755
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout wangyan
git pull origin wangyan
\ No newline at end of file
bin/pull_wz.sh 0 → 100755
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout wangzhuo
git pull origin wangzhuo
\ No newline at end of file
bin/pull_ydl.sh
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout yindanlei
git pull origin yindanlei
\ No newline at end of file
bin/pull_zf.sh 0 → 100755
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout zhaofei
git pull origin zhaofei
\ No newline at end of file
bin/pull_zjw.sh
View file @ e388a661
#!/bin/bash
set -e
set -x
git checkout zhaojianwei
git pull origin zhaojianwei
\ No newline at end of file
conf/bak/auth.cf deleted 100644 → 0
View file @ 6e9fff6e
mazhimo
bi
sjxuwei
zhaojianwei
zhaofei
weiyimin
yindanlei
zhanghuixin
wangzhuo
wangyan
root
hadoop
\ No newline at end of file
conf/en-cn.properties deleted 100644 → 0
View file @ 6e9fff6e
demo=示例程序-demo
demo_bak=示例程序-demo_bak
demo_param=带参数的示例程序
demo_java=JAVA实例程序
meigou-detail-page=美购详情页
meigou-detail-page-dispense-pv=美购详情页分发pv
meigou-detail-page-dispense-uv=美购详情页分发uv
meigou-detail-page-source-pv=美购详情页来源pv
meigou-detail-page-source-uv=美购详情页来源uv
daily_content_data=内容日报-新
search_click_path=分搜索入口的搜索框点击数据
search_result_click_rate=分搜索词转化率数据
search_input_type=分搜索方式近7天数据
daily_recommend_strategy=首页推荐策略日报
docs/deploy.txt → doc/readme.txt
View file @ e388a661
......@@ -5,9 +5,6 @@ BI report project init.
部署节点:
bj-gm-prod-cos-dataweb005
临时文件目录
/data/bi-report/tmp
结果目录
/data/bi-report/result
......@@ -22,3 +19,11 @@ BI report project init.
mkdir -p /data/bi-report/logs
chown -R bi:bi /data/bi-report
开发过程中需要注意的事项：
1.在en-cn.properties中维护映射关系，则显示中文名，否则显示英文名
2.在azkaban中替换项目内容不需要重新配置定时任务；删除后重新上传，则需要重新配置定时任务
规范约定：
1.job文件的命名
必须以job1_01的方式命名，以便在azkaban中可以有序查看
docs/readme.txt deleted 100644 → 0
View file @ 6e9fff6e
开发过程中需要注意的事项：
1.在en-cn.properties中维护映射关系，则显示中文名，否则显示英文名
2.在编写类似insert into tableA select * from tableB语句时，不需要添加set role admin以及set mapreduce.job.queuename=data语句，而且不能带有";"符号
3.在azkaban中替换项目内容不需要重新配置定时任务；删除后重新上传，则需要重新配置定时任务
需求v1.0：
1.附件改为xlsx格式 【metabase支持csv、xlsx两种格式】 python代码实现 ok
2.指定发送人 优先级比较低【metabase默认为doctor】 重新写邮件发送代码 ok
3.更改附件的名 指定 中文 带账期【metabase默认取的是问题的名称】 指定附件的名称为中文，重新转换或者直接指定 ok
4.优化邮件内容，wps打开去掉 样式格式化【metabase中包含内容简略视图，而且有metabaselogo】 邮件内容格式化 python实现 ok
5.一个附件多个sheet 或者 一封邮件 多个附件【metabase默认支持指定多个问题、多个附件】 python发送邮件实现，一个文件多个sheet实现起来相对比较难 发送邮件时需要指定附件名称(带账期) ok
6.excel文件内容格式化 优先级比较低 【指定每个字段的最大长度】 使用python实现，尝试 ok
jobs/bak/demo_hive/step1.job deleted 100644 → 0
View file @ 6e9fff6e
#step1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/bak/demo_hive/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1
command=sh /home/bi/bi-report/lib/shell/hive2csv.sh demo_hive
\ No newline at end of file
jobs/bak/demo_java/step1_3.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/bak/demo_java/step1_4.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/bak/demo_java/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4
command=curl -X GET http://localhost:8553/api/report/sendEmail/zhaojianwei@igengmei.com/jianweizhao@yeah.net/demo_java
\ No newline at end of file
jobs/bak/demo_java_gen/step1_4.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/bak/demo_java_gen/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4
command=curl -X GET http://localhost:8553/api/report/genExcel?name=demo_java
\ No newline at end of file
jobs/bak/demo_spark_param/step1_1.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_3_0_view
\ No newline at end of file
jobs/bak/demo_spark_param/step1_2.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_aggregate_view
\ No newline at end of file
jobs/bak/demo_spark_param/step1_4.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_attr_view
\ No newline at end of file
jobs/bak/demo_spark_param/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6
command=sh /home/bi/bi-report/lib/shell/sparksql2email.sh demo_spark_param zhaojianwei zhaojianwei
\ No newline at end of file
jobs/bak/hive_insert/step1.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
command=curl -X GET http://localhost:8553/api/report/execSql?name=hive_insert
\ No newline at end of file
jobs/bak/meigou-detail-page/step1_1.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
jobs/bak/meigou-detail-page/step1_3.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/bak/meigou-detail-page/step1_4.job deleted 100644 → 0
View file @ 6e9fff6e
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_hospital_spam_pv_day
\ No newline at end of file
jobs/bak/meigou-detail-page/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4
command=curl -X GET http://localhost:8553/api/report/sendEmail/zhanghuixin@igengmei.com/mazhimo@igengmei.com,zhaojianwei@igengmei.com/meigou-detail-page
\ No newline at end of file
jobs/daily_content_data/step4.job deleted 100644 → 0
View file @ 6e9fff6e
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/liudi@igengmei.com,wangxin@igengmei.com,malinxi@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaojianwei@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com/daily_content_data
\ No newline at end of file
jobs/daily_recommend_strategy/step4.job deleted 100644 → 0
View file @ 6e9fff6e
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/weiyimin@igengmei.com/zhaojianwei@igengmei.com/daily_recommend_strategy
\ No newline at end of file
jobs/search_related/step2.job deleted 100644 → 0
View file @ 6e9fff6e
#step2.job
type=command
dependencies=step1_1,step1_2
command=curl -X GET http://localhost:8553/api/report/execSql?name=search_related_insert
\ No newline at end of file
jobs/search_related/step4.job deleted 100644 → 0
View file @ 6e9fff6e
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/sendEmail/weiyimin@igengmei.com/search_related
\ No newline at end of file
lib/java/bi-report-service/.idea/workspace.xml
View file @ e388a661
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/../../shell/hive.sh" afterDir="false" />
</list>
<list default="true" id="4ef7a995-babe-48ad-a058-4897dd73703a" name="Default Changelist" comment="" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
......@@ -17,12 +15,40 @@
<executable />
</execution>
</component>
<component name="DatabaseView">
<option name="SHOW_INTERMEDIATE" value="true" />
<option name="GROUP_DATA_SOURCES" value="true" />
<option name="GROUP_SCHEMA" value="true" />
<option name="GROUP_CONTENTS" value="false" />
<option name="SORT_POSITIONED" value="false" />
<option name="SHOW_EMPTY_GROUPS" value="false" />
<option name="AUTO_SCROLL_FROM_SOURCE" value="false" />
<option name="HIDDEN_KINDS">
<set />
</option>
<expand />
<select />
</component>
<component name="FavoritesManager">
<favorites_list name="bi-report-service" />
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Class" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/../../.." />
</component>
<component name="MavenImportPreferences">
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="ProjectCodeStyleSettingsMigration">
<option name="version" value="1" />
</component>
......@@ -36,6 +62,7 @@
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="ASKED_ADD_EXTERNAL_FILES" value="true" />
<property name="RequestMappingsPanelOrder0" value="0" />
<property name="RequestMappingsPanelOrder1" value="1" />
<property name="RequestMappingsPanelWidth0" value="75" />
......@@ -45,7 +72,18 @@
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunManager" selected="JUnit.ExcelServiceTest.testGenExcel">
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/service-client/shs" />
</key>
</component>
<component name="RunAnythingCache">
<option name="myCommands">
<command value="mvn clean" />
<command value="mvn clean install -DskipTests" />
</option>
</component>
<component name="RunManager" selected="JUnit.EmailServiceTest.testSendAttachmentsMail">
<configuration name="ExcelServiceImpl" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.gmei.data.gateway.server.service.impl.ExcelServiceImpl" />
<module name="service-server" />
......@@ -59,6 +97,19 @@
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration name="Test" type="Application" factoryName="Application" temporary="true" nameIsGenerated="true">
<option name="MAIN_CLASS_NAME" value="com.gmei.data.gateway.server.Test" />
<module name="service-server" />
<extension name="coverage">
<pattern>
<option name="PATTERN" value="com.gmei.data.gateway.server.*" />
<option name="ENABLED" value="true" />
</pattern>
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration default="true" type="ClojureREPL" factoryName="Local" activateToolWindowBeforeRun="false">
<setting name="replType" value="NREPL" />
<setting name="execution" value="LEININGEN" />
......@@ -125,9 +176,10 @@
</configuration>
<recent_temporary>
<list>
<item itemvalue="JUnit.EmailServiceTest.testSendAttachmentsMail" />
<item itemvalue="Application.Test" />
<item itemvalue="JUnit.ExcelServiceTest.testGenExcel" />
<item itemvalue="Application.ExcelServiceImpl" />
<item itemvalue="JUnit.EmailServiceTest.testSendAttachmentsMail" />
<item itemvalue="Spring Boot.ServiceServerApplication" />
</list>
</recent_temporary>
......@@ -146,48 +198,77 @@
<workItem from="1591668349091" duration="1455000" />
<workItem from="1592214013610" duration="2379000" />
<workItem from="1592568026557" duration="772000" />
<workItem from="1593835574199" duration="4438000" />
<workItem from="1594091429524" duration="3648000" />
<workItem from="1594173992446" duration="1574000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsManagerConfiguration">
<option name="ADD_EXTERNAL_FILES_SILENTLY" value="true" />
</component>
<component name="WindowStateProjectService">
<state width="1878" height="313" key="GridCell.Tab.0.bottom" timestamp="1592276447004">
<state x="773" y="303" key="#com.intellij.ide.util.MemberChooser" timestamp="1593835959317">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447004" />
<state x="773" y="303" key="#com.intellij.ide.util.MemberChooser/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1593835959317" />
<state width="1878" height="693" key="GridCell.Tab.0.bottom" timestamp="1594459289785">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="693" key="GridCell.Tab.0.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459289785" />
<state width="1878" height="243" key="GridCell.Tab.0.bottom/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520783" />
<state width="1878" height="313" key="GridCell.Tab.0.center" timestamp="1592276447002">
<state width="1878" height="693" key="GridCell.Tab.0.center" timestamp="1594459289783">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447002" />
<state width="1878" height="693" key="GridCell.Tab.0.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459289783" />
<state width="1878" height="243" key="GridCell.Tab.0.center/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.left" timestamp="1592276447001">
<state width="1878" height="693" key="GridCell.Tab.0.left" timestamp="1594459289782">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447001" />
<state width="1878" height="693" key="GridCell.Tab.0.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459289782" />
<state width="1878" height="243" key="GridCell.Tab.0.left/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520781" />
<state width="1878" height="313" key="GridCell.Tab.0.right" timestamp="1592276447003">
<state width="1878" height="693" key="GridCell.Tab.0.right" timestamp="1594459289784">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="313" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1592276447003" />
<state width="1878" height="693" key="GridCell.Tab.0.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459289784" />
<state width="1878" height="243" key="GridCell.Tab.0.right/1752.1200.1920.1151/0.22.1920.1178/1920.120.1920.1080@1752.1200.1920.1151" timestamp="1590742520782" />
<state width="1878" height="386" key="GridCell.Tab.1.bottom" timestamp="1591671347527">
<state width="1878" height="386" key="GridCell.Tab.1.bottom" timestamp="1594459250977">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459250977" />
<state width="1878" height="386" key="GridCell.Tab.1.center" timestamp="1594459250975">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.bottom/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347527" />
<state width="1878" height="386" key="GridCell.Tab.1.center" timestamp="1591671347526">
<state width="1878" height="386" key="GridCell.Tab.1.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459250975" />
<state width="1878" height="386" key="GridCell.Tab.1.left" timestamp="1594459250975">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.center/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347526" />
<state width="1878" height="386" key="GridCell.Tab.1.left" timestamp="1591671347525">
<state width="1878" height="386" key="GridCell.Tab.1.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459250975" />
<state width="1878" height="386" key="GridCell.Tab.1.right" timestamp="1594459250976">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.left/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347525" />
<state width="1878" height="386" key="GridCell.Tab.1.right" timestamp="1591671347526">
<state width="1878" height="386" key="GridCell.Tab.1.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459250976" />
<state x="626" y="343" width="667" height="486" key="find.popup" timestamp="1594455372378">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state width="1878" height="386" key="GridCell.Tab.1.right/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1591671347526" />
<state x="626" y="343" width="667" height="486" key="find.popup/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594455372378" />
<state x="625" y="273" width="670" height="675" key="run.anything.popup" timestamp="1594459250950">
<screen x="0" y="22" width="1920" height="1129" />
</state>
<state x="625" y="273" width="670" height="675" key="run.anything.popup/0.22.1920.1129/-981.-1200.1920.1200/939.-1080.1920.1080@0.22.1920.1129" timestamp="1594459250950" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" type="java-line">
<url>file://$PROJECT_DIR$/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/EmailServiceImpl.java</url>
<line>125</line>
<option name="timeStamp" value="9" />
</line-breakpoint>
</breakpoints>
</breakpoint-manager>
</component>
</project>
\ No newline at end of file
lib/java/bi-report-service/pom.xml
View file @ e388a661
......@@ -3,17 +3,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<groupId>com.gmei.data</groupId>
<artifactId>bi-report-service</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>bi-report-service</name>
<description>Demo project for Spring Boot</description>
<modules>
<module>service-server</module>
</modules>
<!--定义属性值-->
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
......@@ -21,7 +18,6 @@
<java.version>1.8</java.version>
<spring.boot.version>2.0.0.RELEASE</spring.boot.version>
</properties>
<!--Maven远程仓库设置 非必要，可以删除-->
<repositories>
<repository>
......@@ -47,7 +43,6 @@
</snapshots>
</pluginRepository>
</pluginRepositories>
<!-- 依赖声明 -->
<dependencyManagement>
<dependencies>
......
lib/java/bi-report-service/service-client/shs/email.sh 0 → 100755
View file @ e388a661
#!/usr/bin/env bash
curl -X GET http://localhost:8553/api/report/email/demo/zhaojianwei@igengmei.com/jianweizhao@yeah.net
\ No newline at end of file
lib/java/bi-report-service/service-client/shs/start-finance.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
exit `curl -X GET http://172.16.44.8:8970/api/oozie/task/finance`
\ No newline at end of file
lib/java/bi-report-service/service-client/shs/start-oozie-finance.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
function replace_date(){
#old date
old_start_date=`cat job.properties | grep start_date | awk -F"=" '{print $2}'`
old_end_date=`cat job.properties | grep end_date | awk -F"=" '{print $2}'`
#new date
new_start_date=`date +"%Y-%m-%d"`"T00:00+0800"
new_end_date=`date +"%Y-%m-%d"`"T12:00+0800"
#replace
sed -i "s/$old_start_date/$new_start_date/g" job.properties
sed -i "s/$old_end_date/$new_end_date/g" job.properties
}
function start_job(){
export OOZIE_URL=http://bj-gm-prod-cos-datacenter005:11000/oozie
/opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run
#echo "/opt/oozie-4.1.0-cdh5.16.1/bin/oozie job -config job.properties -run"
}
cd /opt/warehouse/workflow/tl_finance_financeaccount_view/
replace_date
start_job
cd ../tl_finance_financeexpand_view/
replace_date
start_job
cd ../tl_finance_financerevenue_view/
replace_date
start_job
\ No newline at end of file
lib/java/bi-report-service/service-server/pom.xml
View file @ e388a661
......@@ -7,16 +7,13 @@
<groupId>com.gmei.data</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>report-service-server</artifactId>
<properties>
<java.version>1.8</java.version>
<scala.version>2.11</scala.version>
<spark.version>2.4.5</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/endpoint/BiReportEndpoint.java
View file @ e388a661
......@@ -34,4 +34,9 @@ public class BiReportEndpoint {
public int sendEmail(@PathVariable("to") String to,@PathVariable("cc") String cc,@PathVariable("name") String name){
return biReportService.sendEmail(to,cc,name);
}
@RequestMapping(value = "/email/{name}/{to}/{cc}",method = RequestMethod.GET)
public void email(@PathVariable("name") String name,@PathVariable("to") String to,@PathVariable("cc") String cc) throws Exception{
biReportService.email(name,to,cc);
}
}
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/BiReportService.java
View file @ e388a661
......@@ -2,7 +2,8 @@ package com.gmei.data.gateway.server.service;
public interface BiReportService {
int execSql(String name);
int execSqlByHive(String name);
int genExcel(String name);
int execSqlByHive(String name);
void email(String name, String to, String cc) throws Exception;
int sendEmail(String to,String cc,String name);
}
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/SparksqlService.java
View file @ e388a661
......@@ -4,5 +4,6 @@ import com.gmei.data.gateway.server.dto.ExcelGenDto;
public interface SparksqlService {
int executeSql(String name);
ExcelGenDto getResult(String name);
ExcelGenDto getResultDto(String name);
}
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/BiReportServiceImpl.java
View file @ e388a661
......@@ -34,6 +34,22 @@ public class BiReportServiceImpl implements BiReportService {
return hivesqlService.executeSql(name);
}
@Override
public void email(String name, String to, String cc) throws Exception{
ExcelGenDto resultDto = sparksqlService.getResult(name);
if(null == resultDto){
throw new Exception("获取查询结果失败，请核实!");
}else{
int genExcelRs = excelService.genExcel(resultDto,name);
if(Constants.SUCCESS == genExcelRs){
int rs = emailService.sendAttachmentsMail(to, cc, name);
if(rs == Constants.FAIL){
throw new Exception("收件人或抄送人邮件地址有误，未完全发送成功，请核实!");
}
}
}
}
@Override
public int genExcel(String name){
ExcelGenDto resultDto = sparksqlService.getResultDto(name);
......@@ -53,7 +69,11 @@ public class BiReportServiceImpl implements BiReportService {
}else{
int genExcelRs = excelService.genExcel(resultDto,name);
if(Constants.SUCCESS == genExcelRs){
rs = emailService.sendAttachmentsMail(to,cc,name);
try{
emailService.sendAttachmentsMail(to,cc,name);
}catch (Exception e){
rs = Constants.FAIL;
}
}
}
return rs;
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/EmailServiceImpl.java
View file @ e388a661
......@@ -4,17 +4,24 @@ import com.gmei.data.gateway.server.common.Constants;
import com.gmei.data.gateway.server.service.EmailService;
import com.gmei.data.gateway.server.utils.DateUtils;
import com.gmei.data.gateway.server.utils.ProperUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.FileSystemResource;
import org.springframework.mail.MailSendException;
import org.springframework.mail.javamail.JavaMailSender;
import org.springframework.mail.javamail.MimeMessageHelper;
import org.springframework.stereotype.Component;
import javax.mail.Address;
import javax.mail.SendFailedException;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@Component
......@@ -29,35 +36,63 @@ public class EmailServiceImpl implements EmailService {
private String filePath;
@Value("${proper.file.path}")
private String properFilePath;
@Value("${proper.base.path}")
private String properBasePath;
@Autowired
private JavaMailSender mailSender;
private static final Logger logger = LoggerFactory.getLogger(EmailServiceImpl.class);
private List<String> getInvalidAddresses(Throwable throwable) {
List<String> rsList = new ArrayList<String>();
if (throwable == null) {
return rsList;
}
if (throwable instanceof MailSendException) {
Exception[] exceptions = ((MailSendException) throwable).getMessageExceptions();
for (Exception exception : exceptions) {
if (exception instanceof SendFailedException) {
rsList = getStringAddress(((SendFailedException) exception).getInvalidAddresses());
}
}
}
if (throwable instanceof SendFailedException) {
rsList = getStringAddress(((SendFailedException) throwable).getInvalidAddresses());
}
return rsList;
}
private List<String> getStringAddress(Address[] addresses) {
List<String> invalid = new ArrayList<String>();
for (Address a : addresses) {
String address = ((InternetAddress) a).getAddress();
if (address != null && StringUtils.isNotEmpty(address)) {
invalid.add(address);
}
}
return invalid;
}
@Override
public int sendAttachmentsMail(String to,String cc,String name){
int rs = Constants.SUCCESS;
String[] toUsers = to.split(",");
String[] ccUsers = cc.split(",");
try {
String subject = "";
String value = ProperUtils.getValue(properFilePath, name);
String properPath = properBasePath + "pm/" + name + "/en-cn.properties";
logger.info("邮件主题属性文件目录为：{}",properPath);
String value = ProperUtils.getValue(properPath , name);
if(null != value){
subject = value;
}else{
subject = name;
}
MimeMessage message = mailSender.createMimeMessage();
try {
//MimeMessageHelper helper = new MimeMessageHelper(message, true);
MimeMessageHelper helper = new MimeMessageHelper(message, true,"UTF-8");
helper.setFrom(from);
String[] toUsers = to.split(",");
helper.setTo(toUsers);
helper.setSubject(subject);
helper.setText(content, true);
if (cc != null && !"".equals(cc.trim())) {
String[] ccUsers = cc.split(",");
helper.setCc(ccUsers);
}
if (bcc != null && !"".equals(bcc.trim())) {
String[] bccUsers = bcc.split(",");
helper.setBcc(bccUsers);
......@@ -66,14 +101,33 @@ public class EmailServiceImpl implements EmailService {
String fileNameSrc = name + "-" + DateUtils.getYesterdayDateSimpleStr() + ".xlsx";
FileSystemResource file = new FileSystemResource(new File(filePath + fileNameSrc));
helper.addAttachment(fileName, file);
//helper.addAttachment(MimeUtility.encodeWord(file.getFilename(),"utf-8","B"), file);
mailSender.send(message);
logger.info("带附件的邮件已经发送。");
} catch (Exception e) {
logger.error("发送带附件的邮件时发生异常！", e);
rs = Constants.FAIL;
}catch(Throwable e){
String validToUsers = "";
List<String> invalid = getInvalidAddresses(e);
for(String toUser : toUsers){
if(invalid.contains(toUser)){
logger.error("剔除无效接收人邮箱地址：{}",toUser);
continue;
}
return rs;
validToUsers = validToUsers + toUser + ",";
}
String validCcUsers = "";
for(String ccUser : ccUsers){
if(invalid.contains(ccUser)){
logger.error("剔除无效抄送人邮箱地址：{}",ccUser);
continue;
}
validCcUsers = validCcUsers + ccUser + ",";
}
if (invalid.size() > 0) {
validToUsers = validToUsers.substring(0, validToUsers.length() - 1);
validCcUsers = validCcUsers.substring(0, validCcUsers.length() - 1);
sendAttachmentsMail(validToUsers,validCcUsers,name);
return Constants.FAIL;
}
}
return Constants.SUCCESS;
}
@Override
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/ExcelServiceImpl.java
View file @ e388a661
......@@ -37,8 +37,6 @@ public class ExcelServiceImpl implements ExcelService {
@Value("${excel.output.path}")
private String filePath;
@Value("${proper.file.path}")
private String properFilePath;
private static final Logger logger = LoggerFactory.getLogger(ExcelServiceImpl.class);
/**
......@@ -64,13 +62,13 @@ public class ExcelServiceImpl implements ExcelService {
rs = Constants.FAIL;
logger.error("Error!");
}
// String subject = "";
// String value = ProperUtils.getValue(properFilePath, name);
// if(null != value){
//String subject = "";
//String value = ProperUtils.getValue(properFilePath, name);
//if(null != value){
// subject = value;
// }else{
//}else{
// subject = name;
// }
//}
FileOutputStream out = new FileOutputStream(filePath + name + "-" + DateUtils.getYesterdayDateSimpleStr() + ".xlsx");
wb.write(out);
out.close();
......@@ -158,7 +156,7 @@ public class ExcelServiceImpl implements ExcelService {
}
}else{
cell.setCellValue("");
logger.error("the value is null!");
logger.warn("the value is null!");
}
}
......@@ -172,9 +170,7 @@ public class ExcelServiceImpl implements ExcelService {
HSSFCellStyle style = workbook.createCellStyle();
style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
style.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER);
HSSFSheet sheet = workbook.createSheet("sheet");
HSSFRow row0 = sheet.createRow(0);
HSSFCell cell_00 = row0.createCell(0);
cell_00.setCellStyle(style);
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/HivesqlServiceImpl.java
View file @ e388a661
......@@ -62,7 +62,6 @@ public class HivesqlServiceImpl implements HivesqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(hiveSqlJdbcUrl,"bi","");
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/service/impl/SparksqlServiceImpl.java
View file @ e388a661
......@@ -29,10 +29,14 @@ public class SparksqlServiceImpl implements SparksqlService {
@Value("${sql.input.path}")
private String sqlPath;
@Value("${sql.base.path}")
private String sqlBasePath;
@Value("${datasource.sparkSql.jdbcUrl}")
private String sparkSqlJdbcUrl;
@Value("${proper.file.path}")
private String properFilePath;
@Value("${proper.base.path}")
private String properBasePath;
private static final Logger logger = LoggerFactory.getLogger(SparksqlServiceImpl.class);
@Override
......@@ -67,13 +71,12 @@ public class SparksqlServiceImpl implements SparksqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
Statement statement = conn.createStatement();
statement.executeUpdate("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar");
statement.executeUpdate("CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'");
//statement.executeUpdate("ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar");
//statement.executeUpdate("CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade'");
int updateCount = statement.executeUpdate(hql);
if(updateCount > 0){
logger.info("*** Execute By Sparksql success, hql: {}",hql);
......@@ -90,17 +93,96 @@ public class SparksqlServiceImpl implements SparksqlService {
}
@Override
public ExcelGenDto getResultDto(String name){
public ExcelGenDto getResult(String name) {
List<String> sheetNameList = new ArrayList<String>();
List<List<String>> headersList = new ArrayList<List<String>>();
List<List<List>> rowsList = new ArrayList<List<List>>();
File sqlDir = new File(sqlBasePath + "pm/" + name + "/report");
File[] files = sqlDir.listFiles();
for(File file : files){
String fileName = file.getName();
String nameStr =fileName.replace(".sql","");
String subject = "";
String properPath = properBasePath + "pm/" + name + "/en-cn.properties";
logger.info("Sheets属性文件目录为：{}",properPath);
String value = ProperUtils.getValue(properPath , nameStr);
if(null != value){
subject = value;
}else{
subject = nameStr;
}
sheetNameList.add(subject);
List<String> headers = new ArrayList<String>();
List<List> rows = new ArrayList<List>();
BufferedReader reader = null;
StringBuffer sbf = new StringBuffer();
try {
reader = new BufferedReader(new FileReader(file));
String tempStr;
while ((tempStr = reader.readLine()) != null) {
sbf.append("\n").append(tempStr);
}
reader.close();
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage());
continue;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
e1.printStackTrace();
logger.error(e1.getMessage());
continue;
}
}
}
String hql = sbf.toString();
hql = hql.replace("${partition_date}", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replace("${partition_day}", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replace("$partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replace("$partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replace(";","");
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
Statement statement = conn.createStatement();
ResultSet rs = statement.executeQuery(hql);
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
for(int i=1;i<= columnCount;i++){
String columnName = metaData.getColumnName(i);
headers.add(columnName);
}
while(rs.next()){
List<String> rowVals = new ArrayList<String>();
for(int i=1;i<= columnCount;i++){
rowVals.add(rs.getString(i));
}
rows.add(rowVals);
}
}catch (Exception e){
e.printStackTrace();
logger.error(e.getMessage());
return null;
}
headersList.add(headers);
rowsList.add(rows);
}
return new ExcelGenDto(sheetNameList,headersList,rowsList);
}
@Override
public ExcelGenDto getResultDto(String name){
List<String> sheetNameList = new ArrayList<String>();
List<List<String>> headersList = new ArrayList<List<String>>();
List<List<List>> rowsList = new ArrayList<List<List>>();
File sqlDir = new File(sqlPath + name);
File[] files = sqlDir.listFiles();
for(File file : files){
String fileName = file.getName();
String nameStr =fileName.replace(".sql","");
String subject = "";
String value = ProperUtils.getValue(properFilePath, nameStr);
if(null != value){
......@@ -109,10 +191,8 @@ public class SparksqlServiceImpl implements SparksqlService {
subject = nameStr;
}
sheetNameList.add(subject);
List<String> headers = new ArrayList<String>();
List<List> rows = new ArrayList<List>();
BufferedReader reader = null;
StringBuffer sbf = new StringBuffer();
try {
......@@ -140,7 +220,6 @@ public class SparksqlServiceImpl implements SparksqlService {
String hql = sbf.toString();
hql = hql.replaceAll("#partition_date", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()))
.replaceAll("#partition_day", String.format("'%s'", DateUtils.getYesterdayDateSimpleStr()));
Connection conn = null;
try{
conn = DriverManager.getConnection(sparkSqlJdbcUrl,"data","");
......
lib/java/bi-report-service/service-server/src/main/java/com/gmei/data/gateway/server/utils/DateUtils.java
View file @ e388a661
......@@ -8,7 +8,6 @@ import java.util.Date;
* @author apple
*/
public class DateUtils {
private static final String DATE_FORMATE_YMDHMS = "yyyy-MM-dd HH:mm:ss";
private static final String DATE_FORMATE_YMD_SIMPLE = "yyyyMMdd";
private static final String DATE_FORMATE_YMD = "yyyy-MM-dd";
......@@ -29,14 +28,6 @@ public class DateUtils {
return new SimpleDateFormat(DATE_FORMATE_YMD).format(new Date());
}
/**
* 获取当前时间字符串
* @return
*/
// public static String getCurrentDateSimpleStr() {
// return new SimpleDateFormat(DATE_FORMATE_YMD_SIMPLE).format(new Date());
// }
/**
* 获取昨天的日期字符串
* @return
......
lib/java/bi-report-service/service-server/src/main/resources/application.yml
View file @ e388a661
......@@ -27,6 +27,8 @@ mail:
sql:
input:
path: /home/bi/bi-report/sqls/
base:
path: /home/bi/bi-report/
#---excel config---
excel:
......@@ -41,3 +43,5 @@ proper:
file:
path: /home/bi/bi-report/conf/en-cn.properties
#path: /Users/apple/git-repo/bi-report/conf/en-cn.properties
base:
path: /home/bi/bi-report/
lib/java/bi-report-service/service-server/src/main/resources/logback.xml
View file @ e388a661
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">
<contextName>logback</contextName>
<property name="log.path" value="/data/bi-report/logs/bi-report-service.log"/>
<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.path}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
......@@ -20,10 +17,8 @@
</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="console"/>
<appender-ref ref="file"/>
</root>
</configuration>
\ No newline at end of file
lib/java/bi-report-service/service-server/src/test/html/demo.html deleted 100644 → 0
View file @ 6e9fff6e
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<h1 style="text-align:center;">北极星监控预警信息</h1>
<h2 style="color:red;">北极星监控预警</h2>
<h3>北极星监控预警</h3>
<h4>北极星监控预警</h4>
<h5>北极星监控预警</h5>
<h6>北极星监控预警</h6>
\ No newline at end of file
lib/java/bi-report-service/service-server/src/test/java/com/gmei/data/gateway/server/service/EmailServiceTest.java
View file @ e388a661
......@@ -21,7 +21,7 @@ public class EmailServiceTest {
// @Test
// public void testSendAttachmentsMail() {
// emailService.sendAttachmentsMail("zhaojianwei@igengmei.com","jianweizhao@yeah.net","demo");
// emailService.sendAttachmentsMail("zhaojianwei@igengmei.com,zjw@igengmei.com","jianweizhao@yeah.net,aa@bb.com","demo");
// }
//
// @Test
......
lib/python/sendmail.py deleted 100644 → 0
View file @ 6e9fff6e
#coding: utf-8
#author:
#date:
#desc:send email with attach
import smtplib
import sys
import getopt
import os
import getpass
import time
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.header import Header
#email info
to_address = ""
subject = ""
content = ""
attach = ""
memail = ""
htable = ""
#userinfo
authfile = "/home/bi/bi-report/conf/auth.cf"
def readallow(filepath):
usernames = []
f = open(filepath,"r")
lines = f.readlines()
for line in lines:
usernames.append(line.strip())
return usernames
def checkauth():
global authfile
usernames = readallow(authfile)
username = getpass.getuser()
return username in usernames
def isfile(filestr):
if os.path.isfile(filestr):
file = open(filestr,'r')
lines = file.readlines()
if len(lines) >= 2:
return True
write2file("%s file is less than 2 lines or does not exist...." % getfilename(filestr))
print "--->%s file is less than 2 lines or does not exist...." % getfilename(filestr)
return False
def getfilename(filestr):
return os.path.basename(filestr)
def makeaddress():
global to_address
address_str = ""
for address in to_address.split(","):
if address.find("@") > 0:
address_str += address + ";"
else:
address_str += "%s@igengmei.com;" % address
return address_str[:-1]
def addattch(msgRoot):
global attach
if attach.strip() == "":
return 0
for item in attach.split(","):
if isfile(item):
att = MIMEText(open('%s' % item, 'rb').read(), 'base64', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
att["Content-Disposition"] = 'attachment; filename="%s"'% getfilename(item)
msgRoot.attach(att)
else:
print "--->send mail failed, %s file is not exist or file row num less than 2" % getfilename(item)
return -1
def makecc():
global memail
address_str = ""
for address in memail.split(","):
if address.find("@") > 0:
address_str += address + ";"
else:
address_str += "%s@igengmei.com;" % address
print address_str
return address_str[:-1]
def addsubject(msgRoot):
global subject
global content
global htable
msgRoot['Subject'] = Header(subject, 'utf-8')
msgRoot['To'] = makeaddress()
msgRoot['Cc'] = makecc()
if htable != "":
for ff in htable.split(","):
if not isfile(ff):
return -1
hcontent = trans2table(ff)
content += hcontent
content += "<br><br><br><br><font color=red size='3'>这是一封自动发出的邮件，邮件发送自大数据BI组。<br>请使用wps打开，appstore地址:https://appsto.re/cn/MAqWJ.i</font>"
msg = MIMEText(content,'html','utf-8')
msgRoot.attach(msg)
return True
def trans2table(path):
for ff in path.split(","):
index = 0
result = "<br><br><table border='1' style='border-collapse:collapse' bordercolor='#9D9D9D'>\n"
for eachline in open(ff,'r'):
if len(eachline.strip()) == 0:
continue
result += " <tr>"
for item in eachline.strip().split(","):
if index == 0:
result += "<td align='left' bgcolor='#8EA9DB' style='font-family:Microsoft YaHei;padding:2px 5px'>%s</td>" % item
else:
result += "<td align='left' style='font-family:Microsoft YaHei;padding:2px 5px'>%s</td>" % item
index += 1
result += "</tr>\n"
result += "</table>"
return result
def sendmail():
global to_address
#account info
sender = 'youjiantongji@wanmeizhensuo.com'
username = 'youjiantongji@wanmeizhensuo.com'
password = 'mq37tgk8nGi6eg1Hgq388oro3itvdmPl'
smtpserver = 'smtp.exmail.qq.com'
server_port = 25
if not checkparam():
return 0
msgRoot = MIMEMultipart()
if addsubject(msgRoot) == -1:
return 0
if addattch(msgRoot) == -1:
return 0
smtp = smtplib.SMTP(smtpserver, server_port)
smtp.ehlo()
smtp.starttls()
smtp.login(username, password)
to_receiver = makeaddress()
to_cc = makecc()
smtp.sendmail(sender, to_receiver.split(";") + to_cc.split(";"), msgRoot.as_string())
smtp.quit()
print "send mail success...."
write2file("send mail success")
write2file("--------------------------------------------------------------------------------------")
def processcmd(cmd):
global to_address
global subject
global content
global attach
global memail
global htable
try:
options, args = getopt.getopt(cmd, "c:a:t:s:m:f:")
for name,value in options:
if name in ("-t"):
to_address = value
elif name in ("-s"):
subject = value
elif name in ("-c"):
content = value
elif name in ("-a"):
attach = value
elif name in ("-m"):
memail = value
elif name in ("-f"):
htable = value
except Exception, e:
print "--->%s, remove this param" % str(e)
usage()
return -1
def checkparam():
global to_address
global subject
if to_address.strip() == "" or subject.strip() == "":
print "--->must input -t to_address -s subject, and the value can not be empty"
usage()
return False
else:
return True
def loginlog(command):
day = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
user = getpass.getuser()
logstr = "%s\tuser:%s\tcmd:%s" % (day, user, command)
write2file(logstr)
def processlog():
global to_address
global subject
global content
global attach
day = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
user = getpass.getuser()
logstr = "%s\tuser:%s\ttoaddredd:%s\tsubject:%s\tcontent:%s\tattach:%s" % (day, user, to_address, subject, content, attach)
write2file(logstr)
write2file("--------------------------------------------------------------------------------------")
def write2file(sss):
logfile = "/data/bi-report/logs/sendmail.log"
output=open(logfile,'a')
output.write(str(sss) + "\n")
output.close()
def usage():
print 'sendmail usage:'
print ' -t: mail to (Required)'
print ' -s: subject (Required)'
print ' -m: cc address (Required)'
print ' -c: mail content (Optional)'
print ' -a: attach (Optional)'
print ' -f: trans file content to html table(Optional)'
if __name__ == '__main__':
loginlog(sys.argv)
if not checkauth():
print "--->You do not have permission to send email, please contact fangguoqing@igengmei.com"
sys.exit(0)
if len(sys.argv) >= 3:
if processcmd(sys.argv[1:]) != -1:
sendmail()
else:
usage()
\ No newline at end of file
lib/shell/hive 0 → 100755
View file @ e388a661
#!/bin/bash
# 定义变量方便修改
hive="/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter005:10000 -n data"
#至少输入一个参数 第一个参数为需求码(必选项)，第二个参数为分区日期(可选)
if [ $# -lt 1 ]; then
echo "Param project_name lost, please check again!"
exit 0
fi
#判断sql脚本是否存在
project_name=$1
sql_path="/home/bi/bi-report/pm/$project_name/etl/$project_name.sql"
if [ ! -f $sql_path ]; then
echo "Param project_name error, please check again!"
exit `$sql_path`
fi
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$2" ] ;then
partition_date=$2
else
partition_date=`date --date="-1 day" +%Y%m%d`
fi
#参数替换
tmp_sql=`cat ${sql_path}`
sql=${tmp_sql//'${partition_day}'/${partition_date}}
echo -e "${sql}"
#执行sql
$hive -e "$sql"
\ No newline at end of file
lib/shell/hive.sh deleted 100755 → 0
View file @ 6e9fff6e
#!/bin/bash
# 定义变量方便修改
hive="/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online -n data"
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
partition_day=$1
else
partition_day=`date --date="-1 day" +%Y%m%d`
fi
sql="
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY =$partition_day)
SELECT
t1.partition_date as day_id,
t1.device_os_type as device_os_type,
t1.active_type as active_type,
t2.card_content_type as card_content_type,
t2.recommend_type as recommend_type,
NVL(concat(round(sum(t3.session_pv)/sum(t2.session_pv)*100,2),'%'),0) as home_ctr,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t2.session_pv)*100,2),'%'),0) as second_rate,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t3.session_pv)*100,2),'%'),0) as first_rate,
NVL(sum(t3.session_pv),0) as card_click,
NVL(sum(t2.session_pv),0) as card_exposure,
NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)+NVL(sum(self_wel_pv),0)+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0) as total_second_click,
NVL(round(sum(page_stay)/count(distinct t4.cl_id)/60,2),0) as avg_page_stay,
NVL(sum(navbar_pv),0) as navbar_search,
NVL(sum(highlight_pv),0) as highlight_word,
NVL(sum(self_wel_pv),0) as self_welfare_card,
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
JOIN
(--精准曝光，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
END AS recommend_type,
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND card_content_type in ('qa','diary','user_post','answer')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
cl_id,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片' END,
card_id,
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t2
on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
LEFT JOIN
(--卡片，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
END AS recommend_type,
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片' END,
params['card_id'],
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t3
on t2.partition_date=t3.partition_date
and t2.cl_id=t3.cl_id
and t2.card_id=t3.card_id
and t2.card_content_type=t3.card_content_type
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
and t4.business_id=t3.card_id
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
and t5.business_id=t3.card_id
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
and t6.business_id=t3.card_id
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (get_json_object(params['extra_param'], '$.type')='���互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
and t7.business_id=t3.card_id
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
and t8.business_id=t3.card_id
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=t2.cl_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
"
$hive -e "$sql"
\ No newline at end of file
lib/shell/hive2csv.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
source ~/.bashrc
source /etc/profile
if [ $# -lt 1 ]; then
echo "Param error, please check again!"
exit 1
fi
#默认取输入的账期
V_DATE=$2
partition_date=`date +"%Y%m%d"`
timestamp=`date +"%Y%m%d%H%M%S"`
tmp_file_name="/data/bi-report/tmp/tmp_"$timestamp".sql"
result_file_name="/data/bi-report/result/"$1"_"$partition_date".csv"
[ $? -eq 0 ] || exit 1
#输入的账期为空，则取T+1默认账期
if [ -z ${V_DATE} ];then
V_DATE=`date --date="-1 day" +%Y-%m-%d`
fi
#默认支持的日期函数
V_PARYMD=`date --date="$V_DATE" +%Y%m%d`
V_PARY_M_D=`date --date="$V_DATE" +%Y-%m-%d`
V_1DAYMD_AGO=`date --date="$V_PARYMD -1 day" +%Y%m%d`
V_7DAYMD_AGO=`date --date="$V_PARYMD -7 day" +%Y%m%d`
V_1DAY_M_D_AGO=`date --date="$V_PARYMD -1 day" +%Y-%m-%d`
V_7DAY_M_D_AGO=`date --date="$V_PARYMD -7 day" +%Y-%m-%d`
#获取当前时间对应的年、月、日
V_PARYEAR=`date --date="$V_DATE" +%Y`
V_PARMONTH=`date --date="$V_DATE" +%m`
V_PARDAY=`date --date="$V_DATE" +%d`
#打印账期
echo ""
echo "************************************"
echo "V_PARYMD: "$V_PARYMD
echo "V_PARY_M_D: "$V_PARY_M_D
echo "V_1DAYMD_AGO: "$V_1DAYMD_AGO
echo "V_7DAYMD_AGO: "$V_7DAYMD_AGO
echo "V_1DAY_M_D_AGO: "$V_1DAY_M_D_AGO
echo "V_7DAY_M_D_AGO: "$V_7DAY_M_D_AGO
echo "************************************"
echo ""
cp "/home/bi/bi-report/sqls/"$1"/"$1".sql" $tmp_file_name
sed -i "s/\$V_PARYMD/$V_PARYMD/g" $tmp_file_name
sed -i "s/\$V_PARY_M_D/$V_PARY_M_D/g" $tmp_file_name
sed -i "s/\$V_1DAYMD_AGO/$V_1DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_1DAY_M_D_AGO/$V_1DAY_M_D_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAY_M_D_AGO/$V_7DAY_M_D_AGO/g" $tmp_file_name
[ $? -eq 0 ] || exit 1
# 新标签与新标签对应集合类型 的对应关系
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter004:10000 -n data --outputformat=csv2 --showHeader=true -e "`cat $tmp_file_name`" > $result_file_name
rm -rf $tmp_file_name
exit $?
\ No newline at end of file
lib/shell/hive2hive.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
source ~/.bashrc
source /etc/profile
if [ $# -lt 1 ]; then
echo "Param error, please check again!"
exit 1
fi
#默认取输入的账期
V_DATE=$2
timestamp=`date +"%Y%m%d%H%M%S"`
tmp_file_name="/data/bi-report/tmp/tmp_"$timestamp".sql"
[ $? -eq 0 ] || exit 1
#输入的账期为空，则取T+1默认账期
if [ -z ${V_DATE} ];then
V_DATE=`date --date="-1 day" +%Y-%m-%d`
fi
#默认支持的日期函数
V_PARYMD=`date --date="$V_DATE" +%Y%m%d`
V_PARY_M_D=`date --date="$V_DATE" +%Y-%m-%d`
V_1DAYMD_AGO=`date --date="$V_PARYMD -1 day" +%Y%m%d`
V_7DAYMD_AGO=`date --date="$V_PARYMD -7 day" +%Y%m%d`
V_1DAY_M_D_AGO=`date --date="$V_PARYMD -1 day" +%Y-%m-%d`
V_7DAY_M_D_AGO=`date --date="$V_PARYMD -7 day" +%Y-%m-%d`
#获取当前时间对应的年、月、日
V_PARYEAR=`date --date="$V_DATE" +%Y`
V_PARMONTH=`date --date="$V_DATE" +%m`
V_PARDAY=`date --date="$V_DATE" +%d`
#打印账期
echo ""
echo "************************************"
echo "V_PARYMD: "$V_PARYMD
echo "V_PARY_M_D: "$V_PARY_M_D
echo "V_1DAYMD_AGO: "$V_1DAYMD_AGO
echo "V_7DAYMD_AGO: "$V_7DAYMD_AGO
echo "V_1DAY_M_D_AGO: "$V_1DAY_M_D_AGO
echo "V_7DAY_M_D_AGO: "$V_7DAY_M_D_AGO
echo "************************************"
echo ""
cp "/home/bi/bi-report/sqls/"$1"/"$1".sql" $tmp_file_name
sed -i "s/\$V_PARYMD/$V_PARYMD/g" $tmp_file_name
sed -i "s/\$V_PARY_M_D/$V_PARY_M_D/g" $tmp_file_name
sed -i "s/\$V_1DAYMD_AGO/$V_1DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_1DAY_M_D_AGO/$V_1DAY_M_D_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAY_M_D_AGO/$V_7DAY_M_D_AGO/g" $tmp_file_name
[ $? -eq 0 ] || exit 1
# 新标签与新标签对应集合类型 的对应关系
/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter004:10000 -n data -e "`cat $tmp_file_name`"
rm -rf $tmp_file_name
exit $?
\ No newline at end of file
lib/shell/sparksql2csv.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
source ~/.bashrc
source /etc/profile
if [ $# -lt 2 ]; then
echo "Param error, please check again!"
exit 1
fi
#默认取输入的账期
V_DATE=$3
#输入的账期为空，则取T+1默认账期
if [ -z ${V_DATE} ];then
V_DATE=`date --date="-1 day" +%Y-%m-%d`
fi
#默认支持的日期函数
V_PARYMD=`date --date="$V_DATE" +%Y%m%d`
V_PARY_M_D=`date --date="$V_DATE" +%Y-%m-%d`
V_1DAYMD_AGO=`date --date="$V_PARYMD -1 day" +%Y%m%d`
V_7DAYMD_AGO=`date --date="$V_PARYMD -7 day" +%Y%m%d`
V_1DAY_M_D_AGO=`date --date="$V_PARYMD -1 day" +%Y-%m-%d`
V_7DAY_M_D_AGO=`date --date="$V_PARYMD -7 day" +%Y-%m-%d`
#获取当前时间对应的年、月、日
V_PARYEAR=`date --date="$V_DATE" +%Y`
V_PARMONTH=`date --date="$V_DATE" +%m`
V_PARDAY=`date --date="$V_DATE" +%d`
#打印账期
echo ""
echo "************************************"
echo "V_PARYMD: "$V_PARYMD
echo "V_PARY_M_D: "$V_PARY_M_D
echo "V_1DAYMD_AGO: "$V_1DAYMD_AGO
echo "V_7DAYMD_AGO: "$V_7DAYMD_AGO
echo "V_1DAY_M_D_AGO: "$V_1DAY_M_D_AGO
echo "V_7DAY_M_D_AGO: "$V_7DAY_M_D_AGO
echo "************************************"
echo ""
#cp sqls/$1".sql" tmp.sql
cp $1 tmp.sql
sed -i "s/\$V_PARYMD/$V_PARYMD/g" tmp.sql
sed -i "s/\$V_PARY_M_D/$V_PARY_M_D/g" tmp.sql
sed -i "s/\$V_1DAYMD_AGO/$V_1DAYMD_AGO/g" tmp.sql
sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" tmp.sql
sed -i "s/\$V_1DAY_M_D_AGO/$V_1DAY_M_D_AGO/g" tmp.sql
sed -i "s/\$V_7DAY_M_D_AGO/$V_7DAY_M_D_AGO/g" tmp.sql
[ $? -eq 0 ] || exit 1
# 新标签与新标签对应集合类型 的对应关系
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online -n data --outputformat=csv2 --showHeader=true -e "`cat tmp.sql`">$2
rm -rf tmp.sql
exit $?
\ No newline at end of file
lib/shell/sparksql2email.sh deleted 100644 → 0
View file @ 6e9fff6e
#!/bin/bash
source ~/.bashrc
source /etc/profile
if [ $# -lt 3 ]; then
echo "Param error, please check again!"
exit 1
fi
#默认取输入的账期
V_DATE=$4
partition_date=`date +"%Y%m%d"`
timestamp=`date +"%Y%m%d%H%M%S"`
tmp_file_name="/data/bi-report/tmp/tmp_"$timestamp".sql"
result_file_name="/data/bi-report/result/"$1"_"$partition_date".csv"
[ $? -eq 0 ] || exit 1
#输入的账期为空，则取T+1默认账期
if [ -z ${V_DATE} ];then
V_DATE=`date --date="-1 day" +%Y-%m-%d`
fi
#默认支持的日期函数
V_PARYMD=`date --date="$V_DATE" +%Y%m%d`
V_PARY_M_D=`date --date="$V_DATE" +%Y-%m-%d`
V_1DAYMD_AGO=`date --date="$V_PARYMD -1 day" +%Y%m%d`
V_7DAYMD_AGO=`date --date="$V_PARYMD -7 day" +%Y%m%d`
V_1DAY_M_D_AGO=`date --date="$V_PARYMD -1 day" +%Y-%m-%d`
V_7DAY_M_D_AGO=`date --date="$V_PARYMD -7 day" +%Y-%m-%d`
#获取当前时间对应的年、月、日
V_PARYEAR=`date --date="$V_DATE" +%Y`
V_PARMONTH=`date --date="$V_DATE" +%m`
V_PARDAY=`date --date="$V_DATE" +%d`
#打印账期
echo ""
echo "************************************"
echo "V_PARYMD: "$V_PARYMD
echo "V_PARY_M_D: "$V_PARY_M_D
echo "V_1DAYMD_AGO: "$V_1DAYMD_AGO
echo "V_7DAYMD_AGO: "$V_7DAYMD_AGO
echo "V_1DAY_M_D_AGO: "$V_1DAY_M_D_AGO
echo "V_7DAY_M_D_AGO: "$V_7DAY_M_D_AGO
echo "************************************"
echo ""
cp "/home/bi/bi-report/sqls/"$1"/"$1".sql" $tmp_file_name
sed -i "s/\$V_PARYMD/$V_PARYMD/g" $tmp_file_name
sed -i "s/\$V_PARY_M_D/$V_PARY_M_D/g" $tmp_file_name
sed -i "s/\$V_1DAYMD_AGO/$V_1DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAYMD_AGO/$V_7DAYMD_AGO/g" $tmp_file_name
sed -i "s/\$V_1DAY_M_D_AGO/$V_1DAY_M_D_AGO/g" $tmp_file_name
sed -i "s/\$V_7DAY_M_D_AGO/$V_7DAY_M_D_AGO/g" $tmp_file_name
[ $? -eq 0 ] || exit 1
# 新标签与新标签对应集合类型 的对应关系
/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010/online -n data --outputformat=csv2 --showHeader=true -e "`cat $tmp_file_name`"> $result_file_name
[ $? -eq 0 ] || exit 1
rm -rf $tmp_file_name
[ $? -eq 0 ] || exit 1
to=$2
cc=$3
echo "to: "$to
echo "cc: "$cc
python2.7 /home/bi/bi-report/lib/python/sendmail.py -t $to -s 自动发送 -m $cc -a $result_file_name
exit $?
\ No newline at end of file
pm/daily_content_data/en-cn.properties 0 → 100644
View file @ e388a661
daily_content_data=内容日报
\ No newline at end of file
tables/create_daliy_content.sql → pm/daily_content_data/etl/create_daily_content_data.sql
View file @ e388a661
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*脚本名称:
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--*更新时间:
--***************************************************************
--设置全局变量&UDF
......
sqls/daily_content_data_insert/daily_content_data_insert.sql → pm/daily_content_data/etl/daily_content_data.sql
View file @ e388a661
INSERT OVERWRITE TABLE pm.tl_pm_content_d PARTITION (PARTITION_DAY = #partition_day)
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_content_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
......@@ -9,11 +22,11 @@ SELECT
,COALESCE(T1.dau,0) AS dau
,COALESCE(T2.neirong_uv,0) AS content_uv
,COALESCE(T2.neirong_pv,0) AS content_pv
,COALESCE(ROUND(T2.neirong_uv/T1.dau,4),0) AS per_content_uv
,COALESCE(ROUND(T2.neirong_pv/T2.neirong_uv,4),0) AS per_content_pv
,COALESCE(CONCAT(ROUND(T4.retention_num1/T2.neirong_uv*100,4),'%'),0) AS retention_1
,COALESCE(CONCAT(ROUND(T4.retention_num7/T2.neirong_uv*100,4),'%'),0) AS retention_7
,COALESCE(CONCAT(ROUND(T4.retention_num30/T2.neirong_uv*100,4),'%'),0) AS retention_30
,COALESCE(ROUND(T2.neirong_uv/T1.dau,2),0) AS per_content_uv
,COALESCE(ROUND(T2.neirong_pv/T2.neirong_uv,2),0) AS per_content_pv
,COALESCE(CONCAT(ROUND(T4.retention_num1/T2.neirong_uv*100,2),'%'),0) AS retention_1
,COALESCE(CONCAT(ROUND(T4.retention_num7/T2.neirong_uv*100,2),'%'),0) AS retention_7
,COALESCE(CONCAT(ROUND(T4.retention_num30/T2.neirong_uv*100,2),'%'),0) AS retention_30
,COALESCE(T5.app_duration,0) AS avg_app_duration
,COALESCE(T3.neirong_stay,0) AS avg_content_stay
,COALESCE(T5.avg_opentimes,0) AS avg_open_times
......@@ -24,101 +37,109 @@ SELECT
,COALESCE(T9.diary_stay,0) AS content_diary_stay
,COALESCE(T9.home_stay,0) AS home_stay
,COALESCE(T9.conv_stay,0) AS conv_related_stay
,COALESCE(ROUND(T6.recommend_uv/T2.neirong_uv,4),0) AS recommend_rate
,COALESCE(ROUND(T6.recommend_pv/T6.recommend_uv,4),0) AS per_recommend_pv
,COALESCE(ROUND(T6.feeds_uv/T2.neirong_uv,4),0) AS feeds_rate
,COALESCE(ROUND(T6.feeds_pv/T6.feeds_uv,4),0) AS per_feeds_pv
,COALESCE(ROUND(T6.search_uv/T2.neirong_uv,4),0) AS search_rate
,COALESCE(ROUND(T6.search_pv/T6.search_uv,4),0) AS per_search_pv
,COALESCE(ROUND(T6.zone_uv/T2.neirong_uv,4),0) AS zone_rate
,COALESCE(ROUND(T6.zone_pv/T6.zone_uv,4),0) AS per_zone_pv
,COALESCE(ROUND(T6.content_uv/T2.neirong_uv,4),0) AS content_rate
,COALESCE(ROUND(T6.content_pv/T6.content_uv,4),0) AS per_from_content_pv
,COALESCE(ROUND(T6.blank_uv/T2.neirong_uv,4),0) AS blank_rate
,COALESCE(ROUND(T6.blank_pv/T6.blank_uv,4),0) AS per_blank_pv
,COALESCE(ROUND(T6.comment_uv/T2.neirong_uv,4),0) AS comment_rate
,COALESCE(ROUND(T6.comment_pv/T6.comment_uv,4),0) AS per_comment_pv
,COALESCE(ROUND(T6.org_uv/T2.neirong_uv,4),0) AS org_rate
,COALESCE(ROUND(T6.org_pv/T6.org_uv,4),0) AS per_org_pv
,COALESCE(ROUND(T6.category_uv/T2.neirong_uv,4),0) AS category_rate
,COALESCE(ROUND(T6.category_pv/T6.category_uv,4),0) AS per_category_pv
,COALESCE(ROUND(T6.my_diary_uv/T2.neirong_uv,4),0) AS my_diary_rate
,COALESCE(ROUND(T6.my_diary_pv/T6.my_diary_uv,4),0) AS per_my_diary_pv
,COALESCE(ROUND(T6.ai_uv/T2.neirong_uv,4),0) AS ai_rate
,COALESCE(ROUND(T6.ai_pv/T6.ai_uv,4),0) AS per_ai_pv
,COALESCE(ROUND(T6.recommend_uv/T2.neirong_uv,2),0) AS recommend_rate
,COALESCE(ROUND(T6.recommend_pv/T6.recommend_uv,2),0) AS per_recommend_pv
,COALESCE(ROUND(T6.feeds_uv/T2.neirong_uv,2),0) AS feeds_rate
,COALESCE(ROUND(T6.feeds_pv/T6.feeds_uv,2),0) AS per_feeds_pv
,COALESCE(ROUND(T6.search_uv/T2.neirong_uv,2),0) AS search_rate
,COALESCE(ROUND(T6.search_pv/T6.search_uv,2),0) AS per_search_pv
,COALESCE(ROUND(T6.zone_uv/T2.neirong_uv,2),0) AS zone_rate
,COALESCE(ROUND(T6.zone_pv/T6.zone_uv,2),0) AS per_zone_pv
,COALESCE(ROUND(T6.content_uv/T2.neirong_uv,2),0) AS content_rate
,COALESCE(ROUND(T6.content_pv/T6.content_uv,2),0) AS per_from_content_pv
,COALESCE(ROUND(T6.blank_uv/T2.neirong_uv,2),0) AS blank_rate
,COALESCE(ROUND(T6.blank_pv/T6.blank_uv,2),0) AS per_blank_pv
,COALESCE(ROUND(T6.comment_uv/T2.neirong_uv,2),0) AS comment_rate
,COALESCE(ROUND(T6.comment_pv/T6.comment_uv,2),0) AS per_comment_pv
,COALESCE(ROUND(T6.org_uv/T2.neirong_uv,2),0) AS org_rate
,COALESCE(ROUND(T6.org_pv/T6.org_uv,2),0) AS per_org_pv
,COALESCE(ROUND(T6.category_uv/T2.neirong_uv,2),0) AS category_rate
,COALESCE(ROUND(T6.category_pv/T6.category_uv,2),0) AS per_category_pv
,COALESCE(ROUND(T6.my_diary_uv/T2.neirong_uv,2),0) AS my_diary_rate
,COALESCE(ROUND(T6.my_diary_pv/T6.my_diary_uv,2),0) AS per_my_diary_pv
,COALESCE(ROUND(T6.ai_uv/T2.neirong_uv,2),0) AS ai_rate
,COALESCE(ROUND(T6.ai_pv/T6.ai_uv,2),0) AS per_ai_pv
,null AS create_topic_num
,null AS create_reply_num
,COALESCE(T2.diary_uv,0) AS diary_uv
,COALESCE(T2.diary_pv,0) AS diary_pv
,COALESCE(ROUND(T2.diary_uv/T2.neirong_uv,4),0) AS diary_rate
,COALESCE(ROUND(T2.diary_pv/T2.diary_uv,4),0) AS per_diary_pv
,COALESCE(ROUND(T2.diary_uv/T2.neirong_uv,2),0) AS diary_rate
,COALESCE(ROUND(T2.diary_pv/T2.diary_uv,2),0) AS per_diary_pv
,COALESCE(T3.diary_stay,0) AS diary_stay
,COALESCE(T2.post_uv,0) AS post_uv
,COALESCE(T2.post_pv,0) AS post_pv
,COALESCE(ROUND(T2.post_uv/T2.neirong_uv,4),0) AS post_rate
,COALESCE(ROUND(T2.post_pv/T2.post_uv,4),0) AS per_post_pv
,COALESCE(ROUND(T2.post_uv/T2.neirong_uv,2),0) AS post_rate
,COALESCE(ROUND(T2.post_pv/T2.post_uv,2),0) AS per_post_pv
,COALESCE(T3.post_stay,0) AS post_stay
,COALESCE(T2.question_uv,0) AS question_uv
,COALESCE(T2.question_pv,0) AS question_pv
,COALESCE(ROUND(T2.question_uv/T2.neirong_uv,4),0) AS question_rate
,COALESCE(ROUND(T2.question_pv/T2.question_uv,4),0) AS per_question_pv
,COALESCE(ROUND(T2.question_uv/T2.neirong_uv,2),0) AS question_rate
,COALESCE(ROUND(T2.question_pv/T2.question_uv,2),0) AS per_question_pv
,COALESCE(T3.question_stay,0) AS question_stay
,COALESCE(T2.question_answer_uv,0) AS question_answer_uv
,COALESCE(T2.question_answer_pv,0) AS question_answer_pv
,COALESCE(ROUND(T2.question_answer_uv/T2.neirong_uv,4),0) AS question_answer_rate
,COALESCE(ROUND(T2.question_answer_pv/T2.question_answer_uv,4),0) AS per_question_answer_pv
,COALESCE(ROUND(T2.question_answer_uv/T2.neirong_uv,2),0) AS question_answer_rate
,COALESCE(ROUND(T2.question_answer_pv/T2.question_answer_uv,2),0) AS per_question_answer_pv
,COALESCE(T3.question_answer_stay,0) AS question_answer_stay
,COALESCE(T2.answer_uv,0) AS answer_uv
,COALESCE(T2.answer_pv,0) AS answer_pv
,COALESCE(ROUND(T2.answer_uv/T2.neirong_uv,4),0) AS answer_rate
,COALESCE(ROUND(T2.answer_pv/T2.answer_uv,4),0) AS per_answer_pv
,COALESCE(ROUND(T2.answer_uv/T2.neirong_uv,2),0) AS answer_rate
,COALESCE(ROUND(T2.answer_pv/T2.answer_uv,2),0) AS per_answer_pv
,COALESCE(T3.answer_stay,0) AS answer_stay
,COALESCE(T2.video_uv,0) AS video_uv
,COALESCE(T2.video_pv,0) AS video_pv
,COALESCE(ROUND(T2.video_uv/T2.neirong_uv,4),0) AS video_rate
,COALESCE(ROUND(T2.video_pv/T2.video_uv,4),0) AS per_video_pv
,COALESCE(ROUND(T2.video_uv/T2.neirong_uv,2),0) AS video_rate
,COALESCE(ROUND(T2.video_pv/T2.video_uv,2),0) AS per_video_pv
,COALESCE(T3.video_stay,0) AS video_stay
,COALESCE(T2.wiki_uv,0) AS wiki_uv
,COALESCE(T2.wiki_pv,0) AS wiki_pv
,COALESCE(ROUND(T2.wiki_uv/T2.neirong_uv,4),0) AS wiki_rate
,COALESCE(ROUND(T2.wiki_pv/T2.wiki_uv,4),0) AS per_wiki_pv
,COALESCE(ROUND(T2.wiki_uv/T2.neirong_uv,2),0) AS wiki_rate
,COALESCE(ROUND(T2.wiki_pv/T2.wiki_uv,2),0) AS per_wiki_pv
,COALESCE(T3.wiki_stay,0) AS wiki_stay
,COALESCE(T2.article_uv,0) AS article_uv
,COALESCE(T2.article_pv,0) AS article_pv
,COALESCE(ROUND(T2.article_uv/T2.neirong_uv,4),0) AS article_rate
,COALESCE(ROUND(T2.article_pv/T2.article_uv,4),0) AS per_article_pv
,COALESCE(ROUND(T2.article_uv/T2.neirong_uv,2),0) AS article_rate
,COALESCE(ROUND(T2.article_pv/T2.article_uv,2),0) AS per_article_pv
,COALESCE(T3.article_stay,0) AS article_stay
FROM
(--基础维度/dau
SELECT partition_date,device_os_type,active_type,t2.channel,sum(dau) AS dau
SELECT partition_date,device_os_type,active_type,t2.channel,count(distinct device_id) AS dau
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,count(1) AS dau
FROM online.ml_device_day_active_status
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
GROUP BY partition_date,device_os_type,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END,
CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY partition_date,device_os_type,active_type,t2.channel
......@@ -152,29 +173,41 @@ LEFT JOIN
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t2.page_name
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
JOIN
(
......@@ -211,55 +244,65 @@ LEFT JOIN
,device_os_type
,active_type
,t4.channel
,round(sum(page_stay)/count(distinct cl_id)/60,4) as neirong_stay
,round(sum(page_stay)/count(distinct cl_id)/60,2) as neirong_stay
,round(sum(CASE WHEN page_name IN ('diary_detail','topic_detail') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('diary_detail','topic_detail') THEN cl_id END)/60,4) AS diary_stay
count(distinct CASE WHEN page_name IN ('diary_detail','topic_detail') THEN cl_id END)/60,2) AS diary_stay
,round(sum(CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN cl_id END)/60,4) AS post_stay
count(distinct CASE WHEN page_name IN ('post_detail','user_post_detail','doctor_post_detail') THEN cl_id END)/60,2) AS post_stay
,round(sum(CASE WHEN page_name ='question_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='question_detail' THEN cl_id END)/60,4) AS question_stay
count(distinct CASE WHEN page_name ='question_detail' THEN cl_id END)/60,2) AS question_stay
,round(sum(CASE WHEN page_name ='question_answer_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='question_answer_detail' THEN cl_id END)/60,4) AS question_answer_stay
count(distinct CASE WHEN page_name ='question_answer_detail' THEN cl_id END)/60,2) AS question_answer_stay
,round(sum(CASE WHEN page_name ='answer_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='answer_detail' THEN cl_id END)/60,4) AS answer_stay
count(distinct CASE WHEN page_name ='answer_detail' THEN cl_id END)/60,2) AS answer_stay
,round(sum(CASE WHEN page_name ='video_steep' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='video_steep' THEN cl_id END)/60,4) AS video_stay
count(distinct CASE WHEN page_name ='video_steep' THEN cl_id END)/60,2) AS video_stay
,round(sum(CASE WHEN page_name ='article_detail' THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name ='article_detail' THEN cl_id END)/60,4) AS article_stay
count(distinct CASE WHEN page_name ='article_detail' THEN cl_id END)/60,2) AS article_stay
,round(sum(CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN page_stay else 0 END)/
count(distinct CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN cl_id END)/60,4) AS wiki_stay
count(distinct CASE WHEN page_name IN ('wiki_detail','product_detail','wiki_brand','wiki_collect') THEN cl_id END)/60,2) AS wiki_stay
FROM
(
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t2.page_name,t2.page_stay
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
JOIN
(
SELECT partition_date,page_name,cl_id,page_stay
FROM online.bl_hdfs_maidian_updates
......@@ -302,29 +345,41 @@ LEFT JOIN
FROM
(
SELECT concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) AS partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
concat_ws('-',substr(partition_date,1,4),substr(partition_date,5,2),substr(partition_date,7,2)) as partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
JOIN
......@@ -374,36 +429,48 @@ LEFT JOIN
,device_os_type
,active_type
,t5.channel
,round(sum(use_duration)/count(distinct cl_id)/60,4) as app_duration
,round(sum(use_duration)/count(distinct cl_id)/60,2) as app_duration
,round(avg(open_times),4) as avg_opentimes
FROM
(
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t3.use_duration,t3.open_times
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
JOIN
......@@ -481,29 +548,41 @@ LEFT JOIN
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t2.referrer
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
LEFT JOIN
......@@ -535,7 +614,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action = 'on_click_card'
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata')
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -547,7 +626,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action in ( 'on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata')
AND params['transaction_type'] not in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr')
UNION ALL
......@@ -558,7 +637,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action = 'on_click_card'
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata')
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr')
AND params['card_content_type'] IN ('diary','diary_topic','user_post','doctor_post','question','answer','qa','live','article')
UNION ALL
......@@ -570,7 +649,7 @@ LEFT JOIN
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'home'
AND action in ('on_click_diary_card','on_click_answer_card','on_click_question_card','on_click_topic_card','on_click_live_card')
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata')
AND params['transaction_type'] in ('-1','ctr','cvr','smr','newdata','hotspot','pgc','samecity_ctr')
)t2
ON t1.partition_date=t2.partition_date
AND t1.device_id=t2.cl_id
......@@ -604,29 +683,41 @@ LEFT JOIN
SELECT t1.partition_date,device_os_type,active_type,channel,t3.id
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
LEFT JOIN
......@@ -787,29 +878,41 @@ LEFT JOIN
SELECT t1.partition_date,device_os_type,active_type,channel,t3.id,t3.type
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
LEFT JOIN
......@@ -972,29 +1075,41 @@ LEFT JOIN
SELECT t1.partition_date,device_os_type,active_type,channel,t2.cl_id,t3.page_name,t3.page_stay
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.col2 = 'AI' or (partition_date < 20200301 AND first_channel_source_type like 'promotion_toutiao_jy%') THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT
partition_date
,m.device_id, device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20200101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date>=regexp_replace(DATE_SUB(current_date,60) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
JOIN
......@@ -1044,5 +1159,5 @@ ON T1.partition_date=T9.partition_date
AND T1.device_os_type=T9.device_os_type
AND T1.active_type=T9.active_type
AND T1.channel=T9.channel
ORDER BY day_id desc,device_os_type,active_type,is_ai_channel
ORDER BY day_id desc,device_os_type,active_type,is_ai_channel;
jobs/daily_content_data/step2.job → pm/daily_content_data/job/step2.job
View file @ e388a661
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9,step1_10,step1_11,step1_12,step1_13
command=curl -X GET http://localhost:8553/api/report/execSql?name=daily_content_data_insert
\ No newline at end of file
command=/home/bi/bi-report/lib/shell/hive daily_content_data
\ No newline at end of file
pm/daily_content_data/job/step3.job 0 → 100644
View file @ e388a661
#step3.job
type=command
dependencies=step2
command=curl -X GET http://localhost:8553/api/report/email/daily_content_data/liudi@igengmei.com,wangxin@igengmei.com,dengguangyu@igengmei.com/weiyimin@igengmei.com,zhaofei@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
pm/daily_push/en-cn.properties 0 → 100644
View file @ e388a661
daily_push=push日报-策略方向
daily_push_click=push拉起设备数据
personal_push_received=push接收设备数据
\ No newline at end of file
pm/daily_push/etl/create_push_click.sql 0 → 100644
View file @ e388a661
--***************************************************************
--*脚本名称:
--*功能: push日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
dau BIGINT comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
is_open_uv BIGINT comment '{"chs_name":"push开启设备数","description":"","etl":"","value":"","remark":""}',
push_open_uv BIGINT comment '{"chs_name":"push拉起设备数","description":"","etl":"","value":"","remark":""}',
push_open_pv BIGINT comment '{"chs_name":"push拉起人次","description":"","etl":"","value":"","remark":""}',
push_open_2_uv BIGINT comment '{"chs_name":"push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
push_open_2_pv BIGINT comment '{"chs_name":"push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
op_push_open_uv BIGINT comment '{"chs_name":"运营push拉起设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_pv BIGINT comment '{"chs_name":"运营push拉起人次","description":"","etl":"","value":"","remark":""}',
op_push_open_2_uv BIGINT comment '{"chs_name":"运营push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
op_push_open_2_pv BIGINT comment '{"chs_name":"运营push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起人次","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营、个性化、签到、意愿外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
will_push_open_uv BIGINT comment '{"chs_name":"意愿探索push拉起设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_pv BIGINT comment '{"chs_name":"意愿探索push拉起人次","description":"","etl":"","value":"","remark":""}',
will_push_open_2_uv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
will_push_open_2_pv BIGINT comment '{"chs_name":"意愿探索push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_uv BIGINT comment '{"chs_name":"签到push拉起设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_pv BIGINT comment '{"chs_name":"签到push拉起人次","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_uv BIGINT comment '{"chs_name":"签到push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
sign_push_open_2_pv BIGINT comment '{"chs_name":"签到push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_uv BIGINT comment '{"chs_name":"个性化push拉起设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_pv BIGINT comment '{"chs_name":"个性化push拉起人次","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_uv BIGINT comment '{"chs_name":"个性化push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
personal_push_open_2_pv BIGINT comment '{"chs_name":"个性化push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_uv BIGINT comment '{"chs_name":"ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_pv BIGINT comment '{"chs_name":"ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_uv BIGINT comment '{"chs_name":"ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
ctr_push_open_2_pv BIGINT comment '{"chs_name":"ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_uv BIGINT comment '{"chs_name":"非ctr_push拉起设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_pv BIGINT comment '{"chs_name":"非ctr_push拉起人次","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_uv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
non_ctr_push_open_2_pv BIGINT comment '{"chs_name":"非ctr_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_uv BIGINT comment '{"chs_name":"AI_push拉起设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_pv BIGINT comment '{"chs_name":"AI_push拉起人次","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_uv BIGINT comment '{"chs_name":"AI_push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
AI_push_open_2_pv BIGINT comment '{"chs_name":"AI_push拉起有2跳人次","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_uv BIGINT comment '{"chs_name":"除运营外push拉起设备数","description":"","etl":"","value":"","remark":""}',
orignal_auto_push_open_pv BIGINT comment '{"chs_name":"除运营外push拉起人次","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_uv BIGINT comment '{"chs_name":"除运营外push拉起有2跳设备数","description":"","etl":"","value":"","remark":""}',
orginal_auto_push_open_2_pv BIGINT comment '{"chs_name":"除运营外push拉起有2跳人次","description":"","etl":"","value":"","remark":""}'
)comment 'push日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/create_push_receive.sql 0 → 100644
View file @ e388a661
--***************************************************************
--*脚本名称:
--*功能: push接收数据
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间:
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_push_receive_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
push_type BIGINT comment '{"chs_name":"push类型","description":"","etl":"","value":"","remark":""}',
received_dev_num BIGINT comment '{"chs_name":"push接收设备数","description":"","etl":"","value":"","remark":""}',
received_msg_num BIGINT comment '{"chs_name":"push接收消息数","description":"","etl":"","value":"","remark":""}',
click_dev_num BIGINT comment '{"chs_name":"push点击设备数","description":"","etl":"","value":"","remark":""}',
click_msg_num BIGINT comment '{"chs_name":"push点击消息数","description":"","etl":"","value":"","remark":""}'
)comment 'push接收数据'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
pm/daily_push/etl/daily_push.sql 0 → 100644
View file @ e388a661
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
--push点击数据
INSERT OVERWRITE TABLE pm.tl_pm_push_click_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_date as day_id
,device_os_type
,active_type
,count(distinct t1.device_id) as dau
,count(distinct is_open.cl_id) as is_open_uv --push开启设备数
,count(distinct t2.cl_id) as push_open_uv --push拉起设备数
,sum(NVL(t2.push_open_pv,0)) as push_open_pv --push拉起人次
,count(distinct t5.cl_id) as push_open_2_uv --push拉起有2跳设备数
,sum(case WHEN t5.cl_id is not null then push_open_2_pv END) as push_open_2_pv --push拉起有2跳人次
,count(distinct case when op_push_open_pv is not null and op_push_open_pv>0 then t2.cl_id end) as op_push_open_uv --运营push拉起设备数
,sum(NVL(t2.op_push_open_pv,0)) as op_push_open_pv --运营push拉起人次
,count(distinct case when op_push_open_2_pv is not null and op_push_open_2_pv>0 then t5.cl_id end) as op_push_open_2_uv --运营push拉起有2跳设备数
,sum(NVL(t5.op_push_open_2_pv,0)) as op_push_open_2_pv --运营push拉起有2跳人次
,count(distinct case when auto_push_open_pv is not null and auto_push_open_pv>0 then t2.cl_id end) as auto_push_open_uv --自动push拉起设备数
,sum(NVL(t2.auto_push_open_pv,0)) as auto_push_open_pv --自动push拉起人次
,count(distinct case when auto_push_open_2_pv is not null and auto_push_open_2_pv>0 then t5.cl_id end) as auto_push_open_2_uv --自动push拉起有2跳设备数
,sum(NVL(t5.auto_push_open_2_pv,0)) as auto_push_open_2_pv --自动push拉起有2跳人次
,count(distinct case when will_push_open_pv is not null and will_push_open_pv>0 then t2.cl_id end) as will_push_open_uv --意愿探索push拉起设备数
,sum(NVL(t2.will_push_open_pv,0)) as will_push_open_pv --意愿探索push拉起人次
,count(distinct case when will_push_open_2_pv is not null and will_push_open_2_pv>0 then t5.cl_id end) as will_push_open_2_uv --意愿探索push拉起有2跳设备数
,sum(NVL(t5.will_push_open_2_pv,0)) as will_push_open_2_pv
,count(distinct case when sign_push_open_pv is not null and sign_push_open_pv>0 then t2.cl_id end) as sign_push_open_uv --签到push拉起设备数
,sum(NVL(t2.sign_push_open_pv,0)) as sign_push_open_pv --签到push拉起人次
,count(distinct case when sign_push_open_2_pv is not null and sign_push_open_2_pv>0 then t5.cl_id end) as sign_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.sign_push_open_2_pv,0)) as sign_push_open_2_pv
,count(distinct case when t2.ctr_push_open_pv+t2.non_ctr_push_open_pv+t2.will_push_open_pv>0 then t2.cl_id end) as personal_push_open_uv --个性化push拉起设备数
,sum(NVL(t2.ctr_push_open_pv+t2.non_ctr_push_open_pv+t2.will_push_open_pv,0)) as personal_push_open_pv --个性化push拉起人次
,count(distinct case when t5.ctr_push_open_2_pv+t5.non_ctr_push_open_2_pv+t5.will_push_open_2_pv>0 then t5.cl_id end) as personal_push_open_2_uv --个性化push拉起设备数
,sum(NVL(t5.ctr_push_open_2_pv+t5.non_ctr_push_open_2_pv+t5.will_push_open_2_pv,0)) as personal_push_open_2_pv --个性化push拉起有2跳人次
,count(distinct case when ctr_push_open_pv is not null and ctr_push_open_pv>0 then t2.cl_id end) as ctr_push_open_uv --签到push拉起设备数
,sum(NVL(t2.ctr_push_open_pv,0)) as ctr_push_open_pv --签到push拉起人次
,count(distinct case when ctr_push_open_2_pv is not null and ctr_push_open_2_pv>0 then t5.cl_id end) as ctr_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.ctr_push_open_2_pv,0)) as ctr_push_open_2_pv
,count(distinct case when non_ctr_push_open_pv is not null and non_ctr_push_open_pv>0 then t2.cl_id end) as non_ctr_push_open_uv --签到push拉起设备数
,sum(NVL(t2.non_ctr_push_open_pv,0)) as non_ctr_push_open_pv --签到push拉起人次
,count(distinct case when non_ctr_push_open_2_pv is not null and non_ctr_push_open_2_pv>0 then t5.cl_id end) as non_ctr_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.non_ctr_push_open_2_pv,0)) as non_ctr_push_open_2_pv
,count(distinct case when AI_push_open_pv is not null and AI_push_open_pv>0 then t2.cl_id end) as AI_push_open_uv --签到push拉起设备数
,sum(NVL(t2.AI_push_open_pv,0)) as AI_push_open_pv --签到push拉起人次
,count(distinct case when AI_push_open_2_pv is not null and AI_push_open_2_pv>0 then t5.cl_id end) as AI_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.AI_push_open_2_pv,0)) as AI_push_open_2_pv
,count(distinct case when orignal_auto_push_open_pv is not null and orignal_auto_push_open_pv>0 then t2.cl_id end) as orignal_auto_push_open_uv --签到push拉起设备数
,sum(NVL(t2.orignal_auto_push_open_pv,0)) as orignal_auto_push_open_pv --签到push拉起人次
,count(distinct case when orginal_auto_push_open_2_pv is not null and orginal_auto_push_open_2_pv>0 then t5.cl_id end) as orginal_auto_push_open_2_uv --签到push拉起有2跳设备数
,sum(NVL(t5.orginal_auto_push_open_2_pv,0)) as orginal_auto_push_open_2_pv
FROM
(
select
partition_date
,device_id
,device_os_type
,case WHEN active_type = '4' THEN '老活' WHEN active_type in ('1','2') then '新增' END as active_type
from online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('4','1','2')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
LEFT JOIN
(
select a.partition_date,a.cl_id --开启push设备
from
(
SELECT partition_date,cl_id,max(time_str) as max_time_str
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
group by partition_date,cl_id
)a
left join
(
SELECT partition_date,
cl_id,time_str,COALESCE(params['type'],params['tyep']) as type
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='is_open_push'
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.time_str=a.max_time_str
where type=1
group by a.partition_date,a.cl_id
)is_open
on is_open.partition_date=t1.partition_date and is_open.cl_id=t1.device_id
LEFT JOIN
(
SELECT a.partition_date,a.cl_id,
count(distinct a.message_id) as push_open_pv,
count(distinct case when push_type in ('70','71') then a.message_id end) as op_push_open_pv, --运营push push_type in ('70','71')
count(distinct case when push_type in ('101','102','103','104','105') then a.message_id end) as ctr_push_open_pv, --个性化push
count(distinct case when push_type in ('111','112','113') then a.message_id end) as non_ctr_push_open_pv, --个性化push
count(distinct case when push_type in ('106','107','108') then a.message_id end) as will_push_open_pv, --意愿push
count(distinct case when push_type in ('25') then a.message_id end) as sign_push_open_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.message_id end) as auto_push_open_pv, --自动push
count(distinct case when push_type in ('93','94') then a.message_id end) as AI_push_open_pv,
count(distinct case when push_type is not null and push_type not in ('70','71') then a.message_id end) as orignal_auto_push_open_pv --自动push
FROM
(
SELECT a.partition_date,b.cl_id,a.app_session_id,b.message_id
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,app_session_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='device_opened'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),app_session_id
)a
JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
where abs(a.time_stamp-b.time_stamp)<=2
)a
LEFT JOIN
(
SELECT a.msg_id,case when b.msg_id is not null then 'ex' else push_type end as push_type
FROM
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT if(substr(jiguang_id,1,1)=',',substr(jiguang_id,2),jiguang_id) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') >= '20200629'
AND regexp_replace(substr(pushtime,1,10),'-','') <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
)b
on a.msg_id=b.msg_id
)e
on e.msg_id=a.message_id
group by a.partition_date,a.cl_id
)t2
on t2.cl_id=t1.device_id and t2.partition_date=t1.partition_date
LEFT JOIN
(
--push拉起后有2跳的设备
SELECT partition_date,
a.cl_id,
count(distinct a.app_session_id) as push_open_2_pv,
count(distinct case when push_type in ('70','71') then a.app_session_id end) as op_push_open_2_pv, --运营push push_type in ('70','71')
count(distinct case when push_type in ('101','102','103','104','105') then a.app_session_id end) as ctr_push_open_2_pv, --个性化push
count(distinct case when push_type in ('111','112','113') then a.app_session_id end) as non_ctr_push_open_2_pv, --个性化push
count(distinct case when push_type in ('106','107','108') then a.app_session_id end) as will_push_open_2_pv, --意愿push
count(distinct case when push_type in ('25') then a.app_session_id end) as sign_push_open_2_pv, --签到push
count(distinct case when push_type is not null
and push_type not in ('70','71','101','102','103','104','105','111','112','113','106','107','108','25') then a.app_session_id end) as auto_push_open_2_pv, --自动push
count(distinct case when push_type in ('93','94') then a.app_session_id end) as AI_push_open_2_pv, --签到push
count(distinct case when push_type is not null and push_type not in ('70','71') then a.app_session_id end) as orginal_auto_push_open_2_pv --自动push
FROM
(
SELECT a.partition_date,b.cl_id,a.app_session_id,b.message_id
FROM
(
--设备开启
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,app_session_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='device_opened'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),app_session_id
)a
JOIN
(
--推送点击
SELECT partition_date,cl_id,unix_timestamp(substr(time_str,1,19)) as time_stamp,params['message_id'] as message_id
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,unix_timestamp(substr(time_str,1,19)),params['message_id']
)b
on a.partition_date = b.partition_date
AND a.cl_id = b.cl_id
where abs(a.time_stamp-b.time_stamp)<=2
)a
LEFT JOIN
(
SELECT a.msg_id,case when b.msg_id is not null then 'ex' else push_type end as push_type
FROM
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date>='20190801' AND partition_date<= regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
)a
left join
(
SELECT if(substr(jiguang_id,1,1)=',',substr(jiguang_id,2),jiguang_id) as msg_id
FROM online.tl_hdfs_api_pushtask2_view --mars后台消息推送表
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND regexp_replace(substr(pushtime,1,10),'-','') >= '20200629'
AND regexp_replace(substr(pushtime,1,10),'-','') <= regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND approver_status = 'true' --审核状态，true表示审核通过
AND status = '2' --推送状态，2表示发送成功，1表示准备发送
AND creator_id in ('32018153','32719681','32264790','31779749','31436904')
)b
on a.msg_id=b.msg_id
)e
on e.msg_id=a.message_id
JOIN
(
--通过push拉起的一次会话中，有2跳，包括回退
SELECT a.partition_date,a.cl_id,a.app_session_id,a.cl_type
from
(
SELECT partition_date,cl_id,app_session_id,cl_type,if(page_name='home','home','non_home') as is_home
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
GROUP BY partition_date,cl_id,app_session_id,cl_type,if(page_name='home','home','non_home')
)a
join
(
SELECT partition_date,cl_id,app_session_id,cl_type
,count(distinct page_name) as page_num
,count(distinct array(time_str,page_name)) as page_array
FROM online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='page_view'
GROUP BY partition_date,cl_id,app_session_id,cl_type
)b
on b.partition_date=a.partition_date and b.cl_id=a.cl_id and b.app_session_id=a.app_session_id
left join
(
select device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam
on spam.device_id=a.cl_id
where spam.device_id IS NULL and
(a.cl_type='ios' and is_home='home' and page_num>=2 and page_array>=3 or
a.cl_type='ios' and is_home='non_home' and page_array>=2 or
a.cl_type='android' and page_array>=2)
GROUP BY a.partition_date,a.cl_id,a.app_session_id,a.cl_type
)c
on a.partition_date = c.partition_date
AND a.cl_id = c.cl_id
AND a.app_session_id = c.app_session_id
GROUP BY a.cl_id,a.partition_date
)t5
on t1.partition_date=t5.partition_date and t1.device_id=t5.cl_id
GROUP BY t1.partition_date,device_os_type,active_type
order by day_id,device_os_type,active_type;
--push接收数据
INSERT OVERWRITE TABLE pm.tl_pm_push_receive_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT t1.partition_day as day_id
,t1.device_os_type as device_os_type
,'老活' as active_type
,t1.push_type as push_type
,count(distinct t1.device_id) as received_dev_num
,count(t1.msg_id) as received_msg_num
,count(distinct t2.cl_id) as click_dev_num
,count(t2.msg_id) as click_msg_num
FROM
(--接收设备数
SELECT partition_day
,device_os_type
,push_type
,a.device_id
,a.msg_id
FROM
(
select partition_day
,device_id,msg_id
from bl.bl_et_bg_trackingpushlog_inc_d
where partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by partition_day,device_id,msg_id
)a
JOIN
(
select device_id,device_os_type
from online.ml_device_history_detail
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)b
on a.device_id=b.device_id
JOIN
(
select msg_id,push_type,time_stamp
from online.tl_hdfs_push2_new_view --增量表
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
group by msg_id,push_type,time_stamp
union all
SELECT msg_id,regexp_replace(labels['event'], '\\s+', '') AS push_type,time_stamp
FROM online.tl_hdfs_push2_task_view
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND labels['event_type'] = 'push'
group by msg_id,regexp_replace(labels['event'], '\\s+', ''),time_stamp
)c
on a.msg_id=c.msg_id
left join
(
select
partition_date
,device_id
from online.ml_device_day_active_status
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','3')
and device_os_type='ios'
)d
on a.partition_day=d.partition_date and a.device_id=d.device_id
where d.device_id is null
group by partition_day
,device_os_type
,push_type
,a.device_id
,a.msg_id
)t1
left join
(--点击设备数
select partition_date
,cl_id,params['message_id'] as msg_id,time_stamp
from online.bl_hdfs_maidian_updates
where partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='notification_open'
group by partition_date,cl_id,params['message_id'],time_stamp
)t2
on t2.partition_date=t1.partition_day
and t2.msg_id=t1.msg_id
and t2.cl_id=t1.device_id
group by t1.partition_day
,t1.device_os_type
,t1.push_type;
\ No newline at end of file
jobs/bak/demo_java_gen/step1_2.job → pm/daily_push/job/step1_2.job
View file @ e388a661
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_hospital_spam_pv_day
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_push2_new_view
\ No newline at end of file
jobs/bak/demo_java_gen/step1_3.job → pm/daily_push/job/step1_5.job
View file @ e388a661
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_pushtask2_view
\ No newline at end of file
jobs/bak/demo_spark_param/step1_6.job → pm/daily_push/job/step1_6.job
View file @ e388a661
#step1_6.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/daily_push/job/step1_7.job 0 → 100644
View file @ e388a661
#step1_7.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive bl bl_et_bg_trackingpushlog_inc_d
\ No newline at end of file
pm/daily_push/job/step2.job 0 → 100644
View file @ e388a661
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7
command=/home/bi/bi-report/lib/shell/hive daily_push
\ No newline at end of file
jobs/daily_content_data/step3.job → pm/daily_push/job/step3_1.job
View file @ e388a661
#step3.job
#step3_1.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_content_d
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_push_click_d
\ No newline at end of file
jobs/search_related/step3.job → pm/daily_push/job/step3_2.job
View file @ e388a661
#step3.job
#step3_2.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_click_path_d
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_push_receive_d
\ No newline at end of file
pm/daily_push/job/step4.job 0 → 100644
View file @ e388a661
#step4.job
type=command
dependencies=step3_1,step3_2
command=curl -X GET http://localhost:8553/api/report/email/daily_push/liweirui@igengmei.com,duanyingrong@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com
\ No newline at end of file
pm/daily_push/report/daily_push_click.sql 0 → 100644
View file @ e388a661
select day_id as `日期`
,device_os_type as `设备类型`
,active_type as `活跃类型`
,NVL(CONCAT(ROUND(is_open_uv/dau*100,2),'%'),0) as `push开启设备数/DAU`
,NVL(CONCAT(ROUND(op_push_open_uv/is_open_uv*100,2),'%'),0) as `运营push拉起设备/push开启设备数`
,NVL(CONCAT(ROUND(op_push_open_pv/is_open_uv*100,2),'%'),0) as `运营push拉起人次/push开启设备数`
,NVL(CONCAT(ROUND(op_push_open_2_uv/is_open_uv*100,2),'%'),0) as `运营push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(op_push_open_2_pv/is_open_uv*100,2),'%'),0) as `运营push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(op_push_open_2_pv/op_push_open_pv*100,2),'%'),0) as `运营push二跳率=2跳人次/拉起人次`
,NVL(CONCAT(ROUND(AI_push_open_uv/is_open_uv*100,2),'%'),0) as `AI push拉起设备/push开启设备数`
,NVL(CONCAT(ROUND(AI_push_open_pv/is_open_uv*100,2),'%'),0) as `AI push拉起人次/push开启设备数`
,NVL(CONCAT(ROUND(AI_push_open_2_uv/is_open_uv*100,2),'%'),0) as `AI push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(AI_push_open_2_pv/is_open_uv*100,2),'%'),0) as `AI push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(AI_push_open_2_pv/AI_push_open_pv*100,2),'%'),0) as `AIpush二跳率=2跳人次/拉起人次`
,NVL(CONCAT(ROUND(ctr_push_open_uv/is_open_uv*100,2),'%'),0) as `ctr push拉起设备/push开启设备数`
,NVL(CONCAT(ROUND(ctr_push_open_pv/is_open_uv*100,2),'%'),0) as `ctr push拉起人次/push开启设备数`
,NVL(CONCAT(ROUND(ctr_push_open_2_uv/is_open_uv*100,2),'%'),0) as `ctr push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(ctr_push_open_2_pv/is_open_uv*100,2),'%'),0) as `ctr push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(ctr_push_open_2_pv/ctr_push_open_pv*100,2),'%'),0) as `ctr push二跳率=2跳人次/拉起人次`
,NVL(CONCAT(ROUND(non_ctr_push_open_uv/is_open_uv*100,2),'%'),0) as `非ctr push拉起设备/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_pv/is_open_uv*100,2),'%'),0) as `非ctr push拉起人次/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_uv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳设备/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/is_open_uv*100,2),'%'),0) as `非ctr push拉起有2跳人次/push开启设备数`
,NVL(CONCAT(ROUND(non_ctr_push_open_2_pv/non_ctr_push_open_pv*100,2),'%'),0) as `非ctr push二跳率=2跳人次/拉起人次`
from pm.tl_pm_push_click_d
where partition_day>='20200708' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>='20200626'
order by `日期`,`设备类型`,`活跃类型`
pm/daily_push/report/personal_push_received.sql 0 → 100644
View file @ e388a661
select day_id as `日期`
,device_os_type as `设备类型`
,active_type as `活跃类型`
,NVL(CONCAT(ROUND(sum(case when push_type = '101' then click_dev_num end)/sum(case when push_type = '101' then received_dev_num end)*100,2),'%'),0) as `ctr帖子点击率`
,NVL(CONCAT(ROUND(sum(case when push_type = '102' then click_dev_num end)/sum(case when push_type = '102' then received_dev_num end)*100,2),'%'),0) as `ctr日记点击率`
,NVL(CONCAT(ROUND(sum(case when push_type = '103' then click_dev_num end)/sum(case when push_type = '103' then received_dev_num end)*100,2),'%'),0) as `ctr回答点击率`
,NVL(CONCAT(ROUND(sum(case when push_type = '111' then click_dev_num end)/sum(case when push_type = '111' then received_dev_num end)*100,2),'%'),0) as `非ctr帖子点击率`
,NVL(CONCAT(ROUND(sum(case when push_type = '112' then click_dev_num end)/sum(case when push_type = '112' then received_dev_num end)*100,2),'%'),0) as `非ctr日记点击率`
,NVL(CONCAT(ROUND(sum(case when push_type = '113' then click_dev_num end)/sum(case when push_type = '113' then received_dev_num end)*100,2),'%'),0) as `非ctr回答点击率`
,NVL(ROUND(sum(case when push_type = '101' then received_dev_num end)/sum(case when push_type = '111' then received_dev_num end),2),0) as `ctr帖子接收设备数/非ctr帖子接收设备数`
,NVL(ROUND(sum(case when push_type = '102' then received_dev_num end)/sum(case when push_type = '112' then received_dev_num end),2),0) as `ctr日记接收设备数/非ctr日记接收设备数`
,NVL(ROUND(sum(case when push_type = '103' then received_dev_num end)/sum(case when push_type = '113' then received_dev_num end),2),0) as `ctr回答接收设备数/非ctr回答接收设备数`
,NVL(sum(case when push_type = '101' then received_dev_num end),0) as `ctr帖子接收设备数`
,NVL(sum(case when push_type = '102' then received_dev_num end),0) as `ctr日记接收设备数`
,NVL(sum(case when push_type = '103' then received_dev_num end),0) as `ctr回答接收设备数`
,NVL(sum(case when push_type = '111' then received_dev_num end),0) as `非ctr帖子接收设备数`
,NVL(sum(case when push_type = '112' then received_dev_num end),0) as `非ctr日记接收设备数`
,NVL(sum(case when push_type = '113' then received_dev_num end),0) as `非ctr回答接收设备数`
from pm.tl_pm_push_receive_d
where partition_day>='20200708' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>='20200626'
group by day_id
,device_os_type
,active_type
order by `日期`,`设备类型`,`活跃类型`
\ No newline at end of file
pm/daily_recommend_strategy/en-cn.properties 0 → 100644
View file @ e388a661
daily_recommend_strategy=首页推荐策略日报
\ No newline at end of file
tables/create_meigou_source-pv.sql → pm/daily_recommend_strategy/etl/create_daily_recommend_strategy.sql
View file @ e388a661
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*脚本名称:
--*功能: 首页推荐策略日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--*更新时间:
--***************************************************************
--设置全局变量&UDF
......@@ -14,20 +14,20 @@ SET mapreduce.job.queuename=data;
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_meigou_source_pv_d
CREATE TABLE IF NOT EXISTS pm.tl_pm_content_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
is_ai_channel string comment '{"chs_name":"是否AI渠道","description":"","etl":"","value":"","remark":""}',
welfare_pv int comment '{"chs_name":"美购详情页pv","description":"","etl":"","value":"","remark":""}',
to_welfare_pv int comment '{"chs_name":"从上级页面转化到美购详情页的pv总量","description":"","etl":"","value":"","remark":""}',
total_conversion_rate double comment '{"chs_name":"从上级页面转化到美购详情页的pv比美购详情页总pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_pv int comment '{"chs_name":"搜索结果美购页pv","description":"","etl":"","value":"","remark":""}',
search_result_welfare_to_welfare_pv int comment '{"chs_name":"从搜索结果美购页转化到美购详情页pv","description":"","etl":"","value":"","remark":""}',
result_wel_add_pv int comment '{"chs_name":"搜索结果美购页-转化点击加车pv","description":"","etl":"","value":"","remark":""}',
result_wel_buy_pv int comment '{"chs_name":"内容用户APP7留","description":"","etl":"","value":"","remark":""}',
result_wel_msg_pv int comment '{"chs_name":"内容用户APP30留","description":"","etl":"","value":"","remark":""}',
dau int comment '{"chs_name":"DAU","description":"","etl":"","value":"","remark":""}',
content_uv int comment '{"chs_name":"内容详情页uv","description":"","etl":"","value":"","remark":""}',
content_pv int comment '{"chs_name":"内容详情页PV","description":"","etl":"","value":"","remark":""}',
per_content_uv string comment '{"chs_name":"内容详情页uv/dau","description":"","etl":"","value":"","remark":""}',
per_content_pv string comment '{"chs_name":"内容详情页pv/dau","description":"","etl":"","value":"","remark":""}',
retention_1 string comment '{"chs_name":"内容用户APP次留","description":"","etl":"","value":"","remark":""}',
retention_7 string comment '{"chs_name":"内容用户APP7留","description":"","etl":"","value":"","remark":""}',
retention_30 string comment '{"chs_name":"内容用户APP30留","description":"","etl":"","value":"","remark":""}',
avg_app_duration double comment '{"chs_name":"内容用户单设备App时长(m)","description":"","etl":"","value":"","remark":""}',
avg_content_stay double comment '{"chs_name":"内容用户单设备内容时长(m)","description":"","etl":"","value":"","remark":""}',
avg_open_times double comment '{"chs_name":"内容用户单设备打开次数","description":"","etl":"","value":"","remark":""}',
......
sqls/daily_recommend_strategy_insert/daily_recommend_strategy_insert.sql → pm/daily_recommend_strategy/etl/daily_recommend_strategy.sql
View file @ e388a661
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY =#partition_day)
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT
t1.partition_date as day_id,
t1.device_os_type as device_os_type,
t1.active_type as active_type,
t2.card_content_type as card_content_type,
t2.recommend_type as recommend_type,
NVL(concat(round(sum(t3.session_pv)/sum(t2.session_pv)*100,2),'%'),0) as home_ctr,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t2.session_pv)*100,2),'%'),0) as second_rate,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t3.session_pv)*100,2),'%'),0) as first_rate,
NVL(sum(t3.session_pv),0) as card_click,
NVL(sum(t2.session_pv),0) as card_exposure,
NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)+NVL(sum(self_wel_pv),0)+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0) as total_second_click,
NVL(round(sum(page_stay)/count(distinct t4.cl_id)/60,2),0) as avg_page_stay,
NVL(sum(navbar_pv),0) as navbar_search,
NVL(sum(highlight_pv),0) as highlight_word,
......@@ -60,6 +64,7 @@ FROM
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
when transaction_type in ('samecity_ctr') then 'samecity_ctr'
END AS recommend_type,
card_id,
app_session_id
......@@ -69,7 +74,7 @@ FROM
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata','samecity_ctr')
AND card_content_type in ('qa','diary','user_post','answer')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
......@@ -78,7 +83,8 @@ FROM
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片' END,
when transaction_type in ('newdata') then '保量卡片'
when transaction_type in ('samecity_ctr') then 'samecity_ctr' END,
card_id,
app_session_id
)a
......@@ -103,6 +109,7 @@ FROM
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
when params['transaction_type'] in ('samecity_ctr') then 'samecity_ctr'
END AS recommend_type,
params['card_id'] as card_id,
app_session_id
......@@ -111,7 +118,7 @@ FROM
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata','samecity_ctr')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
GROUP BY partition_date,
cl_id,
......@@ -120,7 +127,8 @@ FROM
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片' END,
when params['transaction_type'] in ('newdata') then '保量卡片'
when params['transaction_type'] in ('samecity_ctr') then 'samecity_ctr' END,
params['card_id'],
app_session_id
)a
......@@ -213,10 +221,11 @@ FROM
FROM online.bl_hdfs_maidian_updates
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
or get_json_object(params['extra_param'], '$.jump_from')='msg_link'
or params['in_page_pos']='top' or params['in_page_pos']='bottom')
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND page_name IN ('diary_detail','topic_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
......@@ -309,8 +318,7 @@ FROM
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
WHERE partition_date=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)t1
JOIN
( --医生账号
......@@ -353,9 +361,7 @@ FROM
group by partition_date,device_id
)dev
on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
\ No newline at end of file
jobs/bak/demo_spark_param/step1_3.job → pm/daily_recommend_strategy/job/step1_3.job
View file @ e388a661
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_attr_tag_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/daily_recommend_strategy/job/step1_4.job 0 → 100644
View file @ e388a661
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/daily_recommend_strategy/step2.job → pm/daily_recommend_strategy/job/step2.job
View file @ e388a661
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5,step1_6,step1_7,step1_8,step1_9
command=/home/bi/bi-report/shells/daily_recommend_strategy/daily_recommend_strategy.sh
\ No newline at end of file
command=/home/bi/bi-report/lib/shell/hive daily_recommend_strategy
\ No newline at end of file
pm/daily_recommend_strategy/job/step4.job 0 → 100644
View file @ e388a661
#step4.job
type=command
dependencies=step3
command=curl -X GET http://localhost:8553/api/report/email/daily_recommend_strategy/liudi@igengmei.com,wangxin@igengmei.com,liweirui@igengmei.com,duanyingrong@igengmei.com,xuepengfei@igengmei.com,dengguangyu@igengmei.com,zhaowei@igengmei.com,songke@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
pm/daily_recommend_strategy/report/daily_recommend_strategy.sql 0 → 100644
View file @ e388a661
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_exposure*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND(card_click/card_exposure*100,2),'%'),0) as `首页卡片点击PV/首页卡片精准曝光PV`,
NVL(CONCAT(ROUND((navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)/card_click*100,2),'%'),0) as `来自首页推荐内容卡片的的有效二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
(navbar_search+highlight_word+self_welfare_card+recommend_welfare_card+recommend_content_card*0.2)as `有效二跳pv`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200627' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`;
\ No newline at end of file
pm/meigou_detail_page/en-cn.properties 0 → 100644
View file @ e388a661
meigou_detail_page=美购详情页来源与分发
meigou_detail_page_dispense_pv=美购详情页分发pv
meigou_detail_page_dispense_uv=美购详情页分发uv
meigou_detail_page_source_pv=美购详情页来源pv
meigou_detail_page_source_uv=美购详情页来源uv
\ No newline at end of file
pm/meigou_detail_page/job/step1_3.job 0 → 100644
View file @ e388a661
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
pm/meigou_detail_page/job/step2.job 0 → 100644
View file @ e388a661
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3
command=curl -X GET http://localhost:8553/api/report/email/meigou_detail_page/liudi@igengmei.com,wangxin@igengmei.com,zhaofei@igengmei.com/weiyimin@igengmei.com,yindanlei@igengmei.com
\ No newline at end of file
sqls/bak/meigou-detail-page/meigou-detail-page-dispense-pv.sql → pm/meigou_detail_page/report/meigou_detail_page_dispense_pv.sql
View file @ e388a661
--美购详情页的分发
SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
,`美购详情页浏览pv`
,(`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
......@@ -5,36 +8,36 @@ SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`) as `美购详情页分发pv加总`
,concat(round((`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
+`机构主页点击pv`+`机构的其他美购点击pv`+`推荐美购点击pv`+`收藏点击pv`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`)/`美购详情页浏览pv`*100,4),'%') as `美购详情页分发pv占比加总`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`)/`美购详情页浏览pv`*100,2),'%') as `美购详情页分发pv占比加总`
,`加购物车点击pv`
,concat(round(`加购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `加购物车点击pv占比`
,concat(round(`加购物车点击pv`/`美购详情页浏览pv`*100,2),'%') as `加购物车点击pv占比`
,`评价列表页点击pv`
,concat(round(`评价列表页点击pv`/`美购详情页浏览pv`*100,4),'%') as `评价列表页点击pv占比`
,concat(round(`评价列表页点击pv`/`美购详情页浏览pv`*100,2),'%') as `评价列表页点击pv占比`
,`立即支付点击pv`
,concat(round(`立即支付点击pv`/`美购详情页浏览pv`*100,4),'%') as `立即支付点击pv占比`
,concat(round(`立即支付点击pv`/`美购详情页浏览pv`*100,2),'%') as `立即支付点击pv占比`
,`私信及电话咨询pv`
,concat(round(`私信及电话咨询pv`/`美购详情页浏览pv`*100,4),'%') as `私信及电话咨询pv占比`
,concat(round(`私信及电话咨询pv`/`美购详情页浏览pv`*100,2),'%') as `私信及电话咨询pv占比`
,`选择美购项目点击pv`
,concat(round(`选择美购项目点击pv`/`美购详情页浏览pv`*100,4),'%') as `选择美购项目点击pv占比`
,concat(round(`选择美购项目点击pv`/`美购详情页浏览pv`*100,2),'%') as `选择美购项目点击pv占比`
,`机构主页点击pv`
,concat(round(`机构主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构主页点击pv占比`
,concat(round(`机构主页点击pv`/`美购详情页浏览pv`*100,2),'%') as `机构主页点击pv占比`
,`机构的其他美购点击pv`
,concat(round(`机构的其他美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构的其他美购点击pv占比`
,concat(round(`机构的其他美购点击pv`/`美购详情页浏览pv`*100,2),'%') as `机构的其他美购点击pv占比`
,`推荐美购点击pv`
,concat(round(`推荐美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `推荐美购点击pv占比`
,concat(round(`推荐美购点击pv`/`美购详情页浏览pv`*100,2),'%') as `推荐美购点击pv占比`
,`收藏点击pv`
,concat(round(`收藏点击pv`/`美购详情页浏览pv`*100,4),'%') as `收藏点击pv占比`
,concat(round(`收藏点击pv`/`美购详情页浏览pv`*100,2),'%') as `收藏点击pv占比`
,`右上角购物车点击pv`
,concat(round(`右上角购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `右上角购物车点击pv占比`
,concat(round(`右上角购物车点击pv`/`美购详情页浏览pv`*100,2),'%') as `右上角购物车点击pv占比`
,`医生主页点击pv`
,concat(round(`医生主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `医生主页点击pv占比`
,concat(round(`医生主页点击pv`/`美购详情页浏览pv`*100,2),'%') as `医生主页点击pv占比`
,`分享点击pv`
,concat(round(`分享点击pv`/`美购详情页浏览pv`*100,4),'%') as `分享点击pv占比`
,concat(round(`分享点击pv`/`美购详情页浏览pv`*100,2),'%') as `分享点击pv占比`
,`机构导航点击pv`
,concat(round(`机构导航点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构导航点击pv占比`
,concat(round(`机构导航点击pv`/`美购详情页浏览pv`*100,2),'%') as `机构导航点击pv占比`
FROM
(
(
SELECT t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
......@@ -64,32 +67,26 @@ FROM
(
SELECT partition_date,cl_id,'美购详情页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)pv
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on pv.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on pv.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
UNION ALL
SELECT partition_date,cl_id, '加购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_add'
......@@ -97,7 +94,7 @@ FROM
SELECT partition_date,cl_id, '右上角购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_navbar_cart'
......@@ -106,7 +103,7 @@ FROM
--点击查看更多评论
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_comment'
......@@ -115,7 +112,7 @@ FROM
--点击外显的评价标签
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='service_comment_click_tag'
......@@ -124,7 +121,7 @@ FROM
--点击外显的日记卡片
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_detail_comment_click_diary_card'
......@@ -133,7 +130,7 @@ FROM
SELECT partition_date,cl_id, '立即支付' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_buy'
......@@ -142,7 +139,7 @@ FROM
--点击选择更美项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -152,7 +149,7 @@ FROM
--7.22新版本--点击外露的美购项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -162,7 +159,7 @@ FROM
--7.22新版本--点击查看更多
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -172,7 +169,7 @@ FROM
--点击私信按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
......@@ -181,7 +178,7 @@ FROM
--7.20之后灰度上线的快捷咨询按钮和预约面诊按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -191,7 +188,7 @@ FROM
--点击机构下面的在线咨询和电话咨询
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea_contact'
......@@ -201,7 +198,7 @@ FROM
SELECT partition_date,cl_id, '机构主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
......@@ -211,7 +208,7 @@ FROM
SELECT partition_date,cl_id, '机构的其他美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_seller_service_item'
......@@ -220,7 +217,7 @@ FROM
SELECT partition_date,cl_id, '机构导航' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_address'
......@@ -229,7 +226,7 @@ FROM
SELECT partition_date,cl_id, '推荐美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date <regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_card'
......@@ -240,7 +237,7 @@ FROM
SELECT partition_date,cl_id, '收藏' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_favor'
......@@ -251,7 +248,7 @@ FROM
SELECT partition_date,cl_id, '医生主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
......@@ -261,42 +258,52 @@ FROM
SELECT partition_date,cl_id, '分享' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_click_share'
)click
JOIN
(
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
sqls/bak/demo_java/demo.sql → pm/meigou_detail_page/report/meigou_detail_page_dispense_uv.sql
View file @ e388a661
--美购详情页的分发
SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
,`美购详情页浏览pv`
,(`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
+`机构主页点击pv`+`机构的其他美购点击pv`+`推荐美购点击pv`+`收藏点击pv`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`) as `美购详情页分发pv加总`
,concat(round((`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
+`机构主页点击pv`+`机构的其他美购点击pv`+`推荐美购点击pv`+`收藏点击pv`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`)/`美购详情页浏览pv`*100,4),'%') as `美购详情页分发pv占比加总`
,`加购物车点击pv`
,concat(round(`加购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `加购物车点击pv占比`
,`评价列表页点击pv`
,concat(round(`评价列表页点击pv`/`美购详情页浏览pv`*100,4),'%') as `评价列表页点击pv占比`
,`立即支付点击pv`
,concat(round(`立即支付点击pv`/`美购详情页浏览pv`*100,4),'%') as `立即支付点击pv占比`
,`私信及电话咨询pv`
,concat(round(`私信及电话咨询pv`/`美购详情页浏览pv`*100,4),'%') as `私信及电话咨询pv占比`
,`选择美购项目点击pv`
,concat(round(`选择美购项目点击pv`/`美购详情页浏览pv`*100,4),'%') as `选择美购项目点击pv占比`
,`机构主页点击pv`
,concat(round(`机构主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构主页点击pv占比`
,`机构的其他美购点击pv`
,concat(round(`机构的其他美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构的其他美购点击pv占比`
,`推荐美购点击pv`
,concat(round(`推荐美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `推荐美购点击pv占比`
,`收藏点击pv`
,concat(round(`收藏点击pv`/`美购详情页浏览pv`*100,4),'%') as `收藏点击pv占比`
,`右上角购物车点击pv`
,concat(round(`右上角购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `右上角购物车点击pv占比`
,`医生主页点击pv`
,concat(round(`医生主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `医生主页点击pv占比`
,`分享点击pv`
,concat(round(`分享点击pv`/`美购详情页浏览pv`*100,4),'%') as `分享点击pv占比`
,`机构导航点击pv`
,concat(round(`机构导航点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构导航点击pv占比`
,`美购详情页浏览uv`
,(`加购物车点击uv`+`评价列表页点击uv`+`立即支付点击uv`+`私信及电话咨询uv` +`选择美购项目点击uv`
+`机构主页点击uv`+`机构的其他美购点击uv`+`推荐美购点击uv`+`收藏点击uv`
+`右上角购物车点击uv`+`医生主页点击uv`+`分享点击uv`+`机构导航点击uv`) as `美购详情页分发uv加总`
,concat(round((`加购物车点击uv`+`评价列表页点击uv`+`立即支付点击uv`+`私信及电话咨询uv` +`选择美购项目点击uv`
+`机构主页点击uv`+`机构的其他美购点击uv`+`推荐美购点击uv`+`收藏点击uv`
+`右上角购物车点击uv`+`医生主页点击uv`+`分享点击uv`+`机构导航点击uv`)/`美购详情页浏览uv`*100,2),'%') as `美购详情页分发uv占比加总`
,`加购物车点击uv`
,concat(round(`加购物车点击uv`/`美购详情页浏览uv`*100,2),'%') as `加购物车点击uv占比`
,`评价列表页点击uv`
,concat(round(`评价列表页点击uv`/`美购详情页浏览uv`*100,2),'%') as `评价列表页点击uv占比`
,`立即支付点击uv`
,concat(round(`立即支付点击uv`/`美购详情页浏览uv`*100,2),'%') as `立即支付点击uv占比`
,`私信及电话咨询uv`
,concat(round(`私信及电话咨询uv`/`美购详情页浏览uv`*100,2),'%') as `私信及电话咨询uv占比`
,`选择美购项目点击uv`
,concat(round(`选择美购项目点击uv`/`美购详情页浏览uv`*100,2),'%') as `选择美购项目点击uv占比`
,`机构主页点击uv`
,concat(round(`机构主页点击uv`/`美购详情页浏览uv`*100,2),'%') as `机构主页点击uv占比`
,`机构的其他美购点击uv`
,concat(round(`机构的其他美购点击uv`/`美购详情页浏览uv`*100,2),'%') as `机构的其他美购点击uv占比`
,`推荐美购点击uv`
,concat(round(`推荐美购点击uv`/`美购详情页浏览uv`*100,2),'%') as `推荐美购点击uv占比`
,`收藏点击uv`
,concat(round(`收藏点击uv`/`美购详情页浏览uv`*100,2),'%') as `收藏点击uv占比`
,`右上角购物车点击uv`
,concat(round(`右上角购物车点击uv`/`美购详情页浏览uv`*100,2),'%') as `右上角购物车点击uv占比`
,`医生主页点击uv`
,concat(round(`医生主页点击uv`/`美购详情页浏览uv`*100,2),'%') as `医生主页点击uv占比`
,`分享点击uv`
,concat(round(`分享点击uv`/`美购详情页浏览uv`*100,2),'%') as `分享点击uv占比`
,`机构导航点击uv`
,concat(round(`机构导航点击uv`/`美购详情页浏览uv`*100,2),'%') as `机构导航点击uv占比`
FROM
(
(
SELECT t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
,t2.channel as `渠道`
,count(case when type='美购详情页' then cl_id end ) as `美购详情页浏览pv`
,count(case when type='加购物车' then cl_id end ) as `加购物车点击pv`
,count(case when type='评价列表页' then cl_id end ) as `评价列表页点击pv`
,count(case when type='立即支付' then cl_id end ) as `立即支付点击pv`
,count(case when type='私信及电话咨询' then cl_id end ) as `私信及电话咨询pv`
,count(case when type='选择美购项目' then cl_id end ) as `选择美购项目点击pv`
,count(case when type='机构主页' then cl_id end ) as `机构主页点击pv`
,count(case when type='机构的其他美购' then cl_id end ) as `机构的其他美购点击pv`
,count(case when type='推荐美购' then cl_id end ) as `推荐美购点击pv`
,count(case when type='收藏' then cl_id end ) as `收藏点击pv`
,count(case when type='右上角购物车' then cl_id end ) as `右上角购物车点击pv`
,count(case when type='医生主页' then cl_id end ) as `医生主页点击pv`
,count(case when type='分享' then cl_id end ) as `分享点击pv`
,count(case when type='机构导航' then cl_id end ) as `机构导航点击pv`
,count(distinct case when type='美购详情页' then cl_id end ) as `美购详情页浏览uv`
,count(distinct case when type='加购物车' then cl_id end ) as `加购物车点击uv`
,count(distinct case when type='评价列表页' then cl_id end ) as `评价列表页点击uv`
,count(distinct case when type='立即支付' then cl_id end ) as `立即支付点击uv`
,count(distinct case when type='私信及电话咨询' then cl_id end ) as `私信及电话咨询uv`
,count(distinct case when type='选择美购项目' then cl_id end ) as `选择美购项目点击uv`
,count(distinct case when type='机构主页' then cl_id end ) as `机构主页点击uv`
,count(distinct case when type='机构的其他美购' then cl_id end ) as `机构的其他美购点击uv`
,count(distinct case when type='推荐美购' then cl_id end ) as `推荐美购点击uv`
,count(distinct case when type='收藏' then cl_id end ) as `收藏点击uv`
,count(distinct case when type='右上角购物车' then cl_id end ) as `右上角购物车点击uv`
,count(distinct case when type='医生主页' then cl_id end ) as `医生主页点击uv`
,count(distinct case when type='分享' then cl_id end ) as `分享点击uv`
,count(distinct case when type='机构导航' then cl_id end ) as `机构导航点击uv`
FROM
(
SELECT click.partition_date,click.cl_id,click.type,mas.active_type,mas.device_type,mas.channel
FROM
( --浏览需要去掉疑似机构刷量的设备
SELECT pv.partition_date as partition_date,pv.cl_id as cl_id,type
SELECT uv.partition_date as partition_date,uv.cl_id as cl_id,type
FROM
(
SELECT partition_date,cl_id,'美购详情页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)pv
)uv
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
( -- 去掉疑似机构刷量的UV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on pv.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on uv.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
UNION ALL
SELECT partition_date,cl_id, '加购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_add'
......@@ -97,7 +92,7 @@ FROM
SELECT partition_date,cl_id, '右上角购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_navbar_cart'
......@@ -106,7 +101,7 @@ FROM
--点击查看更多评论
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_comment'
......@@ -115,7 +110,7 @@ FROM
--点击外显的评价标签
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='service_comment_click_tag'
......@@ -124,7 +119,7 @@ FROM
--点击外显的日记卡片
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_detail_comment_click_diary_card'
......@@ -133,7 +128,7 @@ FROM
SELECT partition_date,cl_id, '立即支付' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_buy'
......@@ -142,7 +137,7 @@ FROM
--点击选择更美项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -152,7 +147,7 @@ FROM
--7.22新版本--点击外露的美购项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -162,7 +157,7 @@ FROM
--7.22新版本--点击查看更多
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -172,7 +167,7 @@ FROM
--点击私信按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
......@@ -181,7 +176,7 @@ FROM
--7.20之后灰度上线的快捷咨询按钮和预约面诊按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -191,7 +186,7 @@ FROM
--点击机构下面的在线咨询和电话咨询
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea_contact'
......@@ -201,7 +196,7 @@ FROM
SELECT partition_date,cl_id, '机构主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
......@@ -211,7 +206,7 @@ FROM
SELECT partition_date,cl_id, '机构的其他美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_seller_service_item'
......@@ -220,7 +215,7 @@ FROM
SELECT partition_date,cl_id, '机构导航' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_address'
......@@ -229,7 +224,7 @@ FROM
SELECT partition_date,cl_id, '推荐美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date <regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_card'
......@@ -240,7 +235,7 @@ FROM
SELECT partition_date,cl_id, '收藏' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_favor'
......@@ -251,7 +246,7 @@ FROM
SELECT partition_date,cl_id, '医生主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
......@@ -261,42 +256,52 @@ FROM
SELECT partition_date,cl_id, '分享' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_click_share'
)click
JOIN
(
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
sqls/bak/meigou-detail-page/meigou-detail-page-source-pv.sql → pm/meigou_detail_page/report/meigou_detail_page_source_pv.sql
View file @ e388a661
--美购详情页的来源
SELECT T1.partition_date as `日期`
,T1.device_type as `设备类型`
,T1.active_type as `活跃类型`
......@@ -8,170 +10,170 @@ SELECT T1.partition_date as `日期`
+T2.wel_pv+T2.order_detail_pv+T2.expert_detail_pv+T2.ai_report_pv+T2.diary_detail_pv+T2.zone_pv) as `转化美购pv总量`
,concat(round((T2.search_result_welfare_pv+T2.welfare_list_pv+T2.organization_detail_pv+T2.category_pv+T2.sign_pv
+T2.special_pv+T2.my_cart_pv+T2.search_result_more_pv+T2.welfare_home_pv+T2.conversation_detail_pv
+T2.wel_pv+T2.order_detail_pv+T2.expert_detail_pv+T2.ai_report_pv+T2.diary_detail_pv+T2.zone_pv)/T1.wel_pv*100,4),'%') as `转化美购pv占比加总`
+T2.wel_pv+T2.order_detail_pv+T2.expert_detail_pv+T2.ai_report_pv+T2.diary_detail_pv+T2.zone_pv)/T1.wel_pv*100,2),'%') as `转化美购pv占比加总`
,T1.search_result_welfare_pv as `搜索结果美购页pv`
,T2.search_result_welfare_pv as `搜索结果美购页-转化pv`
,T3.result_wel_add_pv as `搜索结果美购页-转化点击加车pv`
,T3.result_wel_buy_pv as `搜索结果美购页-转化点击立即支付pv`
,T3.result_wel_msg_pv as `搜索结果美购页-转化点击私信咨询pv`
,concat(round(T2.search_result_welfare_pv/T1.search_result_welfare_pv*100,4),'%') as `搜索结果美购页-转化pv率`
,concat(round(T2.search_result_welfare_pv/T1.wel_pv*100,4),'%') as `搜索结果美购页-转化美购pv占比`
,concat(round(T3.result_wel_add_pv/T2.search_result_welfare_pv*100,4),'%') as `搜索结果美购页-点击加车率`
,concat(round(T3.result_wel_buy_pv/T2.search_result_welfare_pv*100,4),'%') as `搜索结果美购页-点击立即支付率`
,concat(round(T3.result_wel_msg_pv/T2.search_result_welfare_pv*100,4),'%') as `搜索结果美购页-点击私信咨询率`
,T1.welfare_list_pv as `美购列表页pv`
,T2.welfare_list_pv as `美购列表页-转化pv`
,T3.wel_list_add_pv as `美购列表页-转化点击加车pv`
,T3.wel_list_buy_pv as `美购列表页-转化点击立即支付pv`
,T3.wel_list_msg_pv as `美购列表页-转化点击私信咨询pv`
,concat(round(T2.welfare_list_pv/T1.welfare_list_pv*100,4),'%') as `美购列表页-转化pv率`
,concat(round(T2.welfare_list_pv/T1.wel_pv*100,4),'%') as `美购列表页-转化美购pv占比`
,concat(round(T3.wel_list_add_pv/T2.welfare_list_pv*100,4),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_pv/T2.welfare_list_pv*100,4),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_pv/T2.welfare_list_pv*100,4),'%') as `美购列表页-点击私信咨询率`
,concat(round(T2.search_result_welfare_pv/T1.search_result_welfare_pv*100,2),'%') as `搜索结果美购页-转化pv率`
,concat(round(T2.search_result_welfare_pv/T1.wel_pv*100,2),'%') as `搜索结果美购页-转化美购pv占比`
,concat(round(T3.result_wel_add_pv/T2.search_result_welfare_pv*100,2),'%') as `搜索结果美购页-点击加车率`
,concat(round(T3.result_wel_buy_pv/T2.search_result_welfare_pv*100,2),'%') as `搜索结果美购页-点击立即支付率`
,concat(round(T3.result_wel_msg_pv/T2.search_result_welfare_pv*100,2),'%') as `搜索结果美购页-点击私信咨询率`
,T1.organization_detail_pv as `医院主页pv`
,T2.organization_detail_pv as `医院主页-转化pv`
,T3.org_add_pv as `医院主页-转化点击加车pv`
,T3.org_buy_pv as `医院主页-转化点击立即支付pv`
,T3.org_msg_pv as `医院主页-转化点击私信咨询pv`
,concat(round(T2.organization_detail_pv/T1.organization_detail_pv*100,4),'%') as `医院主页-转化pv率`
,concat(round(T2.organization_detail_pv/T1.wel_pv*100,4),'%') as `医院主页-转化美购pv占比`
,concat(round(T3.org_add_pv/T2.organization_detail_pv*100,4),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_pv/T2.organization_detail_pv*100,4),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_pv/T2.organization_detail_pv*100,4),'%') as `医院主页-点击私信咨询率`
,concat(round(T2.organization_detail_pv/T1.organization_detail_pv*100,2),'%') as `医院主页-转化pv率`
,concat(round(T2.organization_detail_pv/T1.wel_pv*100,2),'%') as `医院主页-转化美购pv占比`
,concat(round(T3.org_add_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_pv/T2.organization_detail_pv*100,2),'%') as `医院主页-点击私信咨询率`
,T1.category_pv as `品类聚合页pv`
,T2.category_pv as `品类聚合页-转化pv`
,T3.category_add_pv as `品类聚合页-转化点击加车pv`
,T3.category_buy_pv as `品类聚合页-转化点击立即支付pv`
,T3.category_msg_pv as `品类聚合页-转化点击私信咨询pv`
,concat(round(T2.category_pv/T1.category_pv*100,4),'%') as `品类聚合页-转化pv率`
,concat(round(T2.category_pv/T1.wel_pv*100,4),'%') as `品类聚合页-转化美购pv占比`
,concat(round(T3.category_add_pv/T2.category_pv*100,4),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_pv/T2.category_pv*100,4),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_pv/T2.category_pv*100,4),'%') as `品类聚合页-点击私信咨询率`
,T1.sign_pv as `签到页pv`
,T2.sign_pv as `签到页-转化pv`
,T3.sign_add_pv as `签到页-转化点击加车pv`
,T3.sign_buy_pv as `签到页-转化点击立即支付pv`
,T3.sign_msg_pv as `签到页-转化点击私信咨询pv`
,concat(round(T2.sign_pv/T1.sign_pv*100,4),'%') as `签到页-转化pv率`
,concat(round(T2.sign_pv/T1.wel_pv*100,4),'%') as `签到页-转化美购pv占比`
,concat(round(T3.sign_add_pv/T2.sign_pv*100,4),'%') as `签到页-点击加车率`
,concat(round(T3.sign_buy_pv/T2.sign_pv*100,4),'%') as `签到页-点击立即支付率`
,concat(round(T3.sign_msg_pv/T2.sign_pv*100,4),'%') as `签到页-点击私信咨询率`
,T1.special_pv as `专题和大促页pv`
,T2.special_pv as `专题和大促页-转化pv`
,T3.special_add_pv as `专题和大促页-转化点击加车pv`
,T3.special_buy_pv as `专题和大促页-转化点击立即支付pv`
,T3.special_msg_pv as `专题和大促页-转化点击私信咨询pv`
,concat(round(T2.special_pv/T1.special_pv*100,4),'%') as `专题和大促页-转化pv率`
,concat(round(T2.special_pv/T1.wel_pv*100,4),'%') as `专题和大促页-转化美购pv占比`
,concat(round(T3.special_add_pv/T2.special_pv*100,4),'%') as `专题和大促页-点击加车率`
,concat(round(T3.special_buy_pv/T2.special_pv*100,4),'%') as `专题和大促页-点击立即支付率`
,concat(round(T3.special_msg_pv/T2.special_pv*100,4),'%') as `专题和大促页-点击私信咨询率`
,concat(round(T2.category_pv/T1.category_pv*100,2),'%') as `品类聚合页-转化pv率`
,concat(round(T2.category_pv/T1.wel_pv*100,2),'%') as `品类聚合页-转化美购pv占比`
,concat(round(T3.category_add_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_pv/T2.category_pv*100,2),'%') as `品类聚合页-点击私信咨询率`
,T1.conversation_detail_pv as `私信详情页pv`
,T2.conversation_detail_pv as `私信详情页-转化pv`
,T3.conv_add_pv as `私信详情页-转化点击加车pv`
,T3.conv_buy_pv as `私信详情页-转化点击立即支付pv`
,T3.conv_msg_pv as `私信详情页-转化点击私信咨询pv`
,concat(round(T2.conversation_detail_pv/T1.conversation_detail_pv*100,2),'%') as `私信详情页-转化pv率`
,concat(round(T2.conversation_detail_pv/T1.wel_pv*100,2),'%') as `私信详情页-转化美购pv占比`
,concat(round(T3.conv_add_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_pv/T2.conversation_detail_pv*100,2),'%') as `私信详情页-点击私信咨询率`
,T1.welfare_list_pv as `美购列表页pv`
,T2.welfare_list_pv as `美购列表页-转化pv`
,T3.wel_list_add_pv as `美购列表页-转化点击加车pv`
,T3.wel_list_buy_pv as `美购列表页-转化点击立即支付pv`
,T3.wel_list_msg_pv as `美购列表页-转化点击私信咨询pv`
,concat(round(T2.welfare_list_pv/T1.welfare_list_pv*100,2),'%') as `美购列表页-转化pv率`
,concat(round(T2.welfare_list_pv/T1.wel_pv*100,2),'%') as `美购列表页-转化美购pv占比`
,concat(round(T3.wel_list_add_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_pv/T2.welfare_list_pv*100,2),'%') as `美购列表页-点击私信咨询率`
,T1.my_cart_pv as`购物车pv`
,T2.my_cart_pv as`购物车-转化pv`
,T3.cart_add_pv as `购物车-转化点击加车pv`
,T3.cart_buy_pv as `购物车-转化点击立即支付pv`
,T3.cart_msg_pv as `购物车-转化点击私信咨询pv`
,concat(round(T2.my_cart_pv/T1.my_cart_pv*100,4),'%') as `购物车-转化pv率`
,concat(round(T2.my_cart_pv/T1.wel_pv*100,4),'%') as `购物车-转化美购pv占比`
,concat(round(T3.cart_add_pv/T2.my_cart_pv*100,4),'%') as `购物车-点击加车率`
,concat(round(T3.cart_buy_pv/T2.my_cart_pv*100,4),'%') as `购物车-点击立即支付率`
,concat(round(T3.cart_msg_pv/T2.my_cart_pv*100,4),'%') as `购物车-点击私信咨询率`
,concat(round(T2.my_cart_pv/T1.my_cart_pv*100,2),'%') as `购物车-转化pv率`
,concat(round(T2.my_cart_pv/T1.wel_pv*100,2),'%') as `购物车-转化美购pv占比`
,concat(round(T3.cart_add_pv/T2.my_cart_pv*100,2),'%') as `购物车-点击加车率`
,concat(round(T3.cart_buy_pv/T2.my_cart_pv*100,2),'%') as `购物车-点击立即支付率`
,concat(round(T3.cart_msg_pv/T2.my_cart_pv*100,2),'%') as `购物车-点击私信咨询率`
,T1.search_result_more_pv as `搜索结果综合页pv`
,T2.search_result_more_pv as `搜索结果综合页-转化pv`
,T3.result_more_add_pv as `搜索结果综合页-转化点击加车pv`
,T3.result_more_buy_pv as `搜索结果综合页-转化点击立即支付pv`
,T3.result_more_msg_pv as `搜索结果综合页-转化点击私信咨询pv`
,concat(round(T2.search_result_more_pv/T1.search_result_more_pv*100,4),'%') as `搜索结果综合页-转化pv率`
,concat(round(T2.search_result_more_pv/T1.wel_pv*100,4),'%') as `搜索结果综合页-转化美购pv占比`
,concat(round(T3.result_more_add_pv/T2.search_result_more_pv*100,4),'%') as `搜索结果综合页-点击加车率`
,concat(round(T3.result_more_buy_pv/T2.search_result_more_pv*100,4),'%') as `搜索结果综合页-点击立即支付率`
,concat(round(T3.result_more_msg_pv/T2.search_result_more_pv*100,4),'%') as `搜索结果综合页-点击私信咨询率`
,T1.welfare_home_pv as `美购首页pv`
,T2.welfare_home_pv as `美购首页-转化pv`
,T3.wel_home_add_pv as `美购首页-转化点击加车pv`
,T3.wel_home_buy_pv as `美购首页-转化点击立即支付pv`
,T3.wel_home_msg_pv as `美购首页-转化点击私信咨询pv`
,concat(round(T2.welfare_home_pv/T1.welfare_home_pv*100,4),'%') as `美购首页-转化pv率`
,concat(round(T2.welfare_home_pv/T1.wel_pv*100,4),'%') as `美购首页-转化美购pv占比`
,concat(round(T3.wel_home_add_pv/T2.welfare_home_pv*100,4),'%') as `美购首页-点击加车率`
,concat(round(T3.wel_home_buy_pv/T2.welfare_home_pv*100,4),'%') as `美购首页-点击立即支付率`
,concat(round(T3.wel_home_msg_pv/T2.welfare_home_pv*100,4),'%') as `美购首页-点击私信咨询率`
,T1.conversation_detail_pv as `私信详情页pv`
,T2.conversation_detail_pv as `私信详情页-转化pv`
,T3.conv_add_pv as `私信详情页-转化点击加车pv`
,T3.conv_buy_pv as `私信详情页-转化点击立即支付pv`
,T3.conv_msg_pv as `私信详情页-转化点击私信咨询pv`
,concat(round(T2.conversation_detail_pv/T1.conversation_detail_pv*100,4),'%') as `私信详情页-转化pv率`
,concat(round(T2.conversation_detail_pv/T1.wel_pv*100,4),'%') as `私信详情页-转化美购pv占比`
,concat(round(T3.conv_add_pv/T2.conversation_detail_pv*100,4),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_pv/T2.conversation_detail_pv*100,4),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_pv/T2.conversation_detail_pv*100,4),'%') as `私信详情页-点击私信咨询率`
,T1.wel_pv as `美购详情页pv`
,T2.wel_pv as `其他美购详情页-转化pv`
,T3.wel_add_pv as `其他美购详情页-转化点击加车pv`
,T3.wel_buy_pv as `其他美购详情页-转化点击立即支付pv`
,T3.wel_msg_pv as `其他美购详情页-转化点击私信咨询pv`
,concat(round(T2.wel_pv/T1.wel_pv*100,4),'%') as `其他美购详情页-转化pv率`
,concat(round(T2.wel_pv/T1.wel_pv*100,4),'%') as `其他美购详情页-转化美购pv占比`
,concat(round(T3.wel_add_pv/T2.wel_pv*100,4),'%') as `其他美购详情页-点击加车率`
,concat(round(T3.wel_buy_pv/T2.wel_pv*100,4),'%') as `其他美购详情页-点击立即支付率`
,concat(round(T3.wel_msg_pv/T2.wel_pv*100,4),'%') as `其他美购详情页-点击私信咨询率`
,concat(round(T2.search_result_more_pv/T1.search_result_more_pv*100,2),'%') as `搜索结果综合页-转化pv率`
,concat(round(T2.search_result_more_pv/T1.wel_pv*100,2),'%') as `搜索结果综合页-转化美购pv占比`
,concat(round(T3.result_more_add_pv/T2.search_result_more_pv*100,2),'%') as `搜索结果综合页-点击加车率`
,concat(round(T3.result_more_buy_pv/T2.search_result_more_pv*100,2),'%') as `搜索结果综合页-点击立即支付率`
,concat(round(T3.result_more_msg_pv/T2.search_result_more_pv*100,2),'%') as `搜索结果综合页-点击私信咨询率`
,T1.order_detail_pv as `订单详情页pv`
,T2.order_detail_pv as `订单详情页-转化pv`
,T3.order_add_pv as `订单详情页-转化点击加车pv`
,T3.order_buy_pv as `订单详情页-转化点击立即支付pv`
,T3.order_msg_pv as `订单详情页-转化点击私信咨询pv`
,concat(round(T2.order_detail_pv/T1.order_detail_pv*100,4),'%') as `订单详情页-转化pv率`
,concat(round(T2.order_detail_pv/T1.wel_pv*100,4),'%') as `订单详情页-转化美购pv占比`
,concat(round(T3.order_add_pv/T2.order_detail_pv*100,4),'%') as `订单详情页-点击加车率`
,concat(round(T3.order_buy_pv/T2.order_detail_pv*100,4),'%') as `订单详情页-点击立即支付率`
,concat(round(T3.order_msg_pv/T2.order_detail_pv*100,4),'%') as `订单详情页-点击私信咨询率`
,concat(round(T2.order_detail_pv/T1.order_detail_pv*100,2),'%') as `订单详情页-转化pv率`
,concat(round(T2.order_detail_pv/T1.wel_pv*100,2),'%') as `订单详情页-转化美购pv占比`
,concat(round(T3.order_add_pv/T2.order_detail_pv*100,2),'%') as `订单详情页-点击加车率`
,concat(round(T3.order_buy_pv/T2.order_detail_pv*100,2),'%') as `订单详情页-点击立即支付率`
,concat(round(T3.order_msg_pv/T2.order_detail_pv*100,2),'%') as `订单详情页-点击私信咨询率`
,T1.special_pv as `专题和大促页pv`
,T2.special_pv as `专题和大促页-转化pv`
,T3.special_add_pv as `专题和大促页-转化点击加车pv`
,T3.special_buy_pv as `专题和大促页-转化点击立即支付pv`
,T3.special_msg_pv as `专题和大促页-转化点击私信咨询pv`
,concat(round(T2.special_pv/T1.special_pv*100,2),'%') as `专题和大促页-转化pv率`
,concat(round(T2.special_pv/T1.wel_pv*100,2),'%') as `专题和大促页-转化美购pv占比`
,concat(round(T3.special_add_pv/T2.special_pv*100,2),'%') as `专题和大促页-点击加车率`
,concat(round(T3.special_buy_pv/T2.special_pv*100,2),'%') as `专题和大促页-点击立即支付率`
,concat(round(T3.special_msg_pv/T2.special_pv*100,2),'%') as `专题和大促页-点击私信咨询率`
,T1.expert_detail_pv as `医生主页pv`
,T2.expert_detail_pv as `医生主页-转化pv`
,T3.exp_add_pv as `医生主页-转化点击加车pv`
,T3.exp_buy_pv as `医生主页-转化点击立即支付pv`
,T3.exp_msg_pv as `医生主页-转化点击私信咨询pv`
,concat(round(T2.expert_detail_pv/T1.expert_detail_pv*100,4),'%') as `医生主页-转化pv率`
,concat(round(T2.expert_detail_pv/T1.wel_pv*100,4),'%') as `医生主页-转化美购pv占比`
,concat(round(T3.exp_add_pv/T2.expert_detail_pv*100,4),'%') as `医生主页-点击加车率`
,concat(round(T3.exp_buy_pv/T2.expert_detail_pv*100,4),'%') as `医生主页-点击立即支付率`
,concat(round(T3.exp_msg_pv/T2.expert_detail_pv*100,4),'%') as `医生主页-点击私信咨询率`
,T1.ai_report_pv as `AI结果页pv`
,T2.ai_report_pv as `AI结果页-转化pv`
,T3.ai_add_pv as `AI结果页-转化点击加车pv`
,T3.ai_buy_pv as `AI结果页-转化点击立即支付pv`
,T3.ai_msg_pv as `AI结果页-转化点击私信咨询pv`
,concat(round(T2.ai_report_pv/T1.ai_report_pv*100,4),'%') as `AI结果页-转化pv率`
,concat(round(T2.ai_report_pv/T1.wel_pv*100,4),'%') as `AI结果页-转化美购pv占比`
,concat(round(T3.ai_add_pv/T2.ai_report_pv*100,4),'%') as `AI结果页-点击加车率`
,concat(round(T3.ai_buy_pv/T2.ai_report_pv*100,4),'%') as `AI结果页-点击立即支付率`
,concat(round(T3.ai_msg_pv/T2.ai_report_pv*100,4),'%') as `AI结果页-点击私信咨询率`
,concat(round(T2.expert_detail_pv/T1.expert_detail_pv*100,2),'%') as `医生主页-转化pv率`
,concat(round(T2.expert_detail_pv/T1.wel_pv*100,2),'%') as `医生主页-转化美购pv占比`
,concat(round(T3.exp_add_pv/T2.expert_detail_pv*100,2),'%') as `医生主页-点击加车率`
,concat(round(T3.exp_buy_pv/T2.expert_detail_pv*100,2),'%') as `医生主页-点击立即支付率`
,concat(round(T3.exp_msg_pv/T2.expert_detail_pv*100,2),'%') as `医生主页-点击私信咨询率`
,T1.wel_pv as `美购详情页pv`
,T2.wel_pv as `其他美购详情页-转化pv`
,T3.wel_add_pv as `其他美购详情页-转化点击加车pv`
,T3.wel_buy_pv as `其他美购详情页-转化点击立即支付pv`
,T3.wel_msg_pv as `其他美购详情页-转化点击私信咨询pv`
,concat(round(T2.wel_pv/T1.wel_pv*100,2),'%') as `其他美购详情页-转化pv率`
,concat(round(T2.wel_pv/T1.wel_pv*100,2),'%') as `其他美购详情页-转化美购pv占比`
,concat(round(T3.wel_add_pv/T2.wel_pv*100,2),'%') as `其他美购详情页-点击加车率`
,concat(round(T3.wel_buy_pv/T2.wel_pv*100,2),'%') as `其他美购详情页-点击立即支付率`
,concat(round(T3.wel_msg_pv/T2.wel_pv*100,2),'%') as `其他美购详情页-点击私信咨询率`
,T1.welfare_home_pv as `美购首页pv`
,T2.welfare_home_pv as `美购首页-转化pv`
,T3.wel_home_add_pv as `美购首页-转化点击加车pv`
,T3.wel_home_buy_pv as `美购首页-转化点击立即支付pv`
,T3.wel_home_msg_pv as `美购首页-转化点击私信咨询pv`
,concat(round(T2.welfare_home_pv/T1.welfare_home_pv*100,2),'%') as `美购首页-转化pv率`
,concat(round(T2.welfare_home_pv/T1.wel_pv*100,2),'%') as `美购首页-转化美购pv占比`
,concat(round(T3.wel_home_add_pv/T2.welfare_home_pv*100,2),'%') as `美购首页-点击加车率`
,concat(round(T3.wel_home_buy_pv/T2.welfare_home_pv*100,2),'%') as `美购首页-点击立即支付率`
,concat(round(T3.wel_home_msg_pv/T2.welfare_home_pv*100,2),'%') as `美购首页-点击私信咨询率`
,T1.diary_detail_pv as `日记本详情页pv`
,T2.diary_detail_pv as `日记本详情页-转化pv`
,T3.diary_add_pv as `日记本详情页-转化点击加车pv`
,T3.diary_buy_pv as `日记本详情页-转化点击立即支付pv`
,T3.diary_msg_pv as `日记本详情页-转化点击私信咨询pv`
,concat(round(T2.diary_detail_pv/T1.diary_detail_pv*100,4),'%') as `日记本详情页-转化pv率`
,concat(round(T2.diary_detail_pv/T1.wel_pv*100,4),'%') as `日记本详情页-转化美购pv占比`
,concat(round(T3.diary_add_pv/T2.diary_detail_pv*100,4),'%') as `日记本详情页-点击加车率`
,concat(round(T3.diary_buy_pv/T2.diary_detail_pv*100,4),'%') as `日记本详情页-点击立即支付率`
,concat(round(T3.diary_msg_pv/T2.diary_detail_pv*100,4),'%') as `日记本详情页-点击私信咨询率`
,concat(round(T2.diary_detail_pv/T1.diary_detail_pv*100,2),'%') as `日记本详情页-转化pv率`
,concat(round(T2.diary_detail_pv/T1.wel_pv*100,2),'%') as `日记本详情页-转化美购pv占比`
,concat(round(T3.diary_add_pv/T2.diary_detail_pv*100,2),'%') as `日记本详情页-点击加车率`
,concat(round(T3.diary_buy_pv/T2.diary_detail_pv*100,2),'%') as `日记本详情页-点击立即支付率`
,concat(round(T3.diary_msg_pv/T2.diary_detail_pv*100,2),'%') as `日记本详情页-点击私信咨询率`
,T1.sign_pv as `签到页pv`
,T2.sign_pv as `签到页-转化pv`
,T3.sign_add_pv as `签到页-转化点击加车pv`
,T3.sign_buy_pv as `签到页-转化点击立即支付pv`
,T3.sign_msg_pv as `签到页-转化点击私信咨询pv`
,concat(round(T2.sign_pv/T1.sign_pv*100,2),'%') as `签到页-转化pv率`
,concat(round(T2.sign_pv/T1.wel_pv*100,2),'%') as `签到页-转化美购pv占比`
,concat(round(T3.sign_add_pv/T2.sign_pv*100,2),'%') as `签到页-点击加车率`
,concat(round(T3.sign_buy_pv/T2.sign_pv*100,2),'%') as `签到页-点击立即支付率`
,concat(round(T3.sign_msg_pv/T2.sign_pv*100,2),'%') as `签到页-点击私信咨询率`
,T1.ai_report_pv as `AI结果页pv`
,T2.ai_report_pv as `AI结果页-转化pv`
,T3.ai_add_pv as `AI结果页-转化点击加车pv`
,T3.ai_buy_pv as `AI结果页-转化点击立即支付pv`
,T3.ai_msg_pv as `AI结果页-转化点击私信咨询pv`
,concat(round(T2.ai_report_pv/T1.ai_report_pv*100,2),'%') as `AI结果页-转化pv率`
,concat(round(T2.ai_report_pv/T1.wel_pv*100,2),'%') as `AI结果页-转化美购pv占比`
,concat(round(T3.ai_add_pv/T2.ai_report_pv*100,2),'%') as `AI结果页-点击加车率`
,concat(round(T3.ai_buy_pv/T2.ai_report_pv*100,2),'%') as `AI结果页-点击立即支付率`
,concat(round(T3.ai_msg_pv/T2.ai_report_pv*100,2),'%') as `AI结果页-点击私信咨询率`
,T1.zone_pv as `内容聚合页pv`
,T2.zone_pv as `内容聚合页-转化pv`
,T3.zone_add_pv as `内容聚合页-转化点击加车pv`
,T3.zone_buy_pv as `内容聚合页-转化点击立即支付pv`
,T3.zone_msg_pv as `内容聚合页-转化点击私信咨询pv`
,concat(round(T2.zone_pv/T1.zone_pv*100,4),'%') as `内容聚合页-转化pv率`
,concat(round(T2.zone_pv/T1.wel_pv*100,4),'%') as `内容聚合页-转化美购pv占比`
,concat(round(T3.zone_add_pv/T2.zone_pv*100,4),'%') as `内容聚合页-点击加车率`
,concat(round(T3.zone_buy_pv/T2.zone_pv*100,4),'%') as `内容聚合页-点击立即支付率`
,concat(round(T3.zone_msg_pv/T2.zone_pv*100,4),'%') as `内容聚合页-点击私信咨询率`
,concat(round(T2.zone_pv/T1.zone_pv*100,2),'%') as `内容聚合页-转化pv率`
,concat(round(T2.zone_pv/T1.wel_pv*100,2),'%') as `内容聚合页-转化美购pv占比`
,concat(round(T3.zone_add_pv/T2.zone_pv*100,2),'%') as `内容聚合页-点击加车率`
,concat(round(T3.zone_buy_pv/T2.zone_pv*100,2),'%') as `内容聚合页-点击立即支付率`
,concat(round(T3.zone_msg_pv/T2.zone_pv*100,2),'%') as `内容聚合页-点击私信咨询率`
FROM
(--能够走向美购详情页的各页面pv/uv
(--能够走向美购详情页的各页面pv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -189,7 +191,7 @@ FROM
,count(case when page_name = 'diary_detail' then t1.cl_id end) as diary_detail_pv
,count(case when page_name = 'order_detail' then t1.cl_id end) as order_detail_pv
,count(case when page_name ='new_sign' then t1.cl_id end) as sign_pv
,count(case when page_name in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') then t1.cl_id end) as special_pv --专题和大促
,count(case when page_name in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') then t1.cl_id end) as special_pv --专题和大促
,count(case when page_name = 'zone_v3' then t1.cl_id end) as zone_pv
,count(case when page_name in ('report_result','face_detect_result') then t1.cl_id end) as ai_report_pv --待之后增加模拟整形页面
FROM
......@@ -199,58 +201,63 @@ FROM
(
SELECT partition_date,page_name,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='page_view'
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
(
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
LEFT JOIN
(--不同referrer下的美购详情页pv/uv
LEFT JOIN
(--不同referrer下的美购详情页pv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -268,7 +275,7 @@ FROM
,count(case when referrer = 'diary_detail' then t1.cl_id end) as diary_detail_pv
,count(case when referrer = 'order_detail' then t1.cl_id end) as order_detail_pv
,count(case when referrer ='new_sign' then t1.cl_id end) as sign_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') then t1.cl_id end) as special_pv --专题和大促
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') then t1.cl_id end) as special_pv --专题和大促
,count(case when referrer = 'zone_v3' then t1.cl_id end) as zone_pv
,count(case when referrer in ('report_result','face_detect_result') then t1.cl_id end) as ai_report_pv --待之后增加模拟整形页面
FROM
......@@ -278,62 +285,67 @@ FROM
(
SELECT partition_date,referrer,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
(
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T2
ON T1.partition_date=T2.partition_date
ON T1.partition_date=T2.partition_date
and T1.device_type=T2.device_type
AND T1.active_type=T2.active_type
AND T1.channel=T2.channel
LEFT JOIN
(--点击加车/支付/私信的不同来源
LEFT JOIN
(--点击加车/支付/私信的不同来源
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -377,9 +389,9 @@ FROM
,count(case when referrer = 'new_sign' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as sign_add_pv
,count(case when referrer = 'new_sign' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as sign_buy_pv
,count(case when referrer = 'new_sign' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as sign_msg_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as special_add_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as special_buy_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as special_msg_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as special_add_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as special_buy_pv
,count(case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as special_msg_pv
,count(case when referrer = 'zone_v3' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as zone_add_pv
,count(case when referrer = 'zone_v3' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as zone_buy_pv
,count(case when referrer = 'zone_v3' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as zone_msg_pv
......@@ -394,7 +406,7 @@ FROM
( --点击加车/支付
SELECT partition_date,referrer,action,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action in ('welfare_multiattribute_click_add','welfare_multiattribute_click_buy')
......@@ -403,7 +415,7 @@ FROM
--点击私信按钮
SELECT partition_date,referrer,action,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
......@@ -419,7 +431,7 @@ FROM
( SELECT partition_date,params['referrer_link'] as referrer_link,cl_id,action
,json_split(params['referrer_link'],'string') as a
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -432,51 +444,57 @@ FROM
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T3
ON T1.partition_date=T3.partition_date
ON T1.partition_date=T3.partition_date
and T1.device_type=T3.device_type
AND T1.active_type=T3.active_type
AND T1.channel=T3.channel
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
sqls/bak/meigou-detail-page/meigou-detail-page-source-uv.sql → pm/meigou_detail_page/report/meigou_detail_page_source_uv.sql
View file @ e388a661
--美购详情页的来源
SELECT T1.partition_date as `日期`
,T1.device_type as `设备类型`
,T1.active_type as `活跃类型`
......@@ -8,164 +10,164 @@ SELECT T1.partition_date as `日期`
,T3.result_wel_add_uv as `搜索结果美购页-转化点击加车uv`
,T3.result_wel_buy_uv as `搜索结果美购页-转化点击立即支付uv`
,T3.result_wel_msg_uv as `搜索结果美购页-转化点击私信咨询uv`
,concat(round(T2.search_result_welfare_uv/T1.search_result_welfare_uv*100,4),'%') as `搜索结果美购页-转化uv率`
,concat(round(T2.search_result_welfare_uv/T1.wel_uv*100,4),'%') as `搜索结果美购页-转化美购uv占比`
,concat(round(T3.result_wel_add_uv/T2.search_result_welfare_uv*100,4),'%') as `搜索结果美购页-点击加车率`
,concat(round(T3.result_wel_buy_uv/T2.search_result_welfare_uv*100,4),'%') as `搜索结果美购页-点击立即支付率`
,concat(round(T3.result_wel_msg_uv/T2.search_result_welfare_uv*100,4),'%') as `搜索结果美购页-点击私信咨询率`
,T1.welfare_list_uv as `美购列表页uv`
,T2.welfare_list_uv as `美购列表页-转化uv`
,T3.wel_list_add_uv as `美购列表页-转化点击加车uv`
,T3.wel_list_buy_uv as `美购列表页-转化点击立即支付uv`
,T3.wel_list_msg_uv as `美购列表页-转化点击私信咨询uv`
,concat(round(T2.welfare_list_uv/T1.welfare_list_uv*100,4),'%') as `美购列表页-转化uv率`
,concat(round(T2.welfare_list_uv/T1.wel_uv*100,4),'%') as `美购列表页-转化美购uv占比`
,concat(round(T3.wel_list_add_uv/T2.welfare_list_uv*100,4),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_uv/T2.welfare_list_uv*100,4),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_uv/T2.welfare_list_uv*100,4),'%') as `美购列表页-点击私信咨询率`
,concat(round(T2.search_result_welfare_uv/T1.search_result_welfare_uv*100,2),'%') as `搜索结果美购页-转化uv率`
,concat(round(T2.search_result_welfare_uv/T1.wel_uv*100,2),'%') as `搜索结果美购页-转化美购uv占比`
,concat(round(T3.result_wel_add_uv/T2.search_result_welfare_uv*100,2),'%') as `搜索结果美购页-点击加车率`
,concat(round(T3.result_wel_buy_uv/T2.search_result_welfare_uv*100,2),'%') as `搜索结果美购页-点击立即支付率`
,concat(round(T3.result_wel_msg_uv/T2.search_result_welfare_uv*100,2),'%') as `搜索结果美购页-点击私信咨询率`
,T1.organization_detail_uv as `医院主页uv`
,T2.organization_detail_uv as `医院主页-转化uv`
,T3.org_add_uv as `医院主页-转化点击加车uv`
,T3.org_buy_uv as `医院主页-转化点击立即支付uv`
,T3.org_msg_uv as `医院主页-转化点击私信咨询uv`
,concat(round(T2.organization_detail_uv/T1.organization_detail_uv*100,4),'%') as `医院主页-转化uv率`
,concat(round(T2.organization_detail_uv/T1.wel_uv*100,4),'%') as `医院主页-转化美购uv占比`
,concat(round(T3.org_add_uv/T2.organization_detail_uv*100,4),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_uv/T2.organization_detail_uv*100,4),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_uv/T2.organization_detail_uv*100,4),'%') as `医院主页-点击私信咨询率`
,concat(round(T2.organization_detail_uv/T1.organization_detail_uv*100,2),'%') as `医院主页-转化uv率`
,concat(round(T2.organization_detail_uv/T1.wel_uv*100,2),'%') as `医院主页-转化美购uv占比`
,concat(round(T3.org_add_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击加车率`
,concat(round(T3.org_buy_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击立即支付率`
,concat(round(T3.org_msg_uv/T2.organization_detail_uv*100,2),'%') as `医院主页-点击私信咨询率`
,T1.category_uv as `品类聚合页uv`
,T2.category_uv as `品类聚合页-转化uv`
,T3.category_add_uv as `品类聚合页-转化点击加车uv`
,T3.category_buy_uv as `品类聚合页-转化点击立即支付uv`
,T3.category_msg_uv as `品类聚合页-转化点击私信咨询uv`
,concat(round(T2.category_uv/T1.category_uv*100,4),'%') as `品类聚合页-转化uv率`
,concat(round(T2.category_uv/T1.wel_uv*100,4),'%') as `品类聚合页-转化美购uv占比`
,concat(round(T3.category_add_uv/T2.category_uv*100,4),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_uv/T2.category_uv*100,4),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_uv/T2.category_uv*100,4),'%') as `品类聚合页-点击私信咨询率`
,T1.sign_uv as `签到页uv`
,T2.sign_uv as `签到页-转化uv`
,T3.sign_add_uv as `签到页-转化点击加车uv`
,T3.sign_buy_uv as `签到页-转化点击立即支付uv`
,T3.sign_msg_uv as `签到页-转化点击私信咨询uv`
,concat(round(T2.sign_uv/T1.sign_uv*100,4),'%') as `签到页-转化uv率`
,concat(round(T2.sign_uv/T1.wel_uv*100,4),'%') as `签到页-转化美购uv占比`
,concat(round(T3.sign_add_uv/T2.sign_uv*100,4),'%') as `签到页-点击加车率`
,concat(round(T3.sign_buy_uv/T2.sign_uv*100,4),'%') as `签到页-点击立即支付率`
,concat(round(T3.sign_msg_uv/T2.sign_uv*100,4),'%') as `签到页-点击私信咨询率`
,T1.special_uv as `专题和大促页uv`
,T2.special_uv as `专题和大促页-转化uv`
,T3.special_add_uv as `专题和大促页-转化点击加车uv`
,T3.special_buy_uv as `专题和大促页-转化点击立即支付uv`
,T3.special_msg_uv as `专题和大促页-转化点击私信咨询uv`
,concat(round(T2.special_uv/T1.special_uv*100,4),'%') as `专题和大促页-转化uv率`
,concat(round(T2.special_uv/T1.wel_uv*100,4),'%') as `专题和大促页-转化美购uv占比`
,concat(round(T3.special_add_uv/T2.special_uv*100,4),'%') as `专题和大促页-点击加车率`
,concat(round(T3.special_buy_uv/T2.special_uv*100,4),'%') as `专题和大促页-点击立即支付率`
,concat(round(T3.special_msg_uv/T2.special_uv*100,4),'%') as `专题和大促页-点击私信咨询率`
,concat(round(T2.category_uv/T1.category_uv*100,2),'%') as `品类聚合页-转化uv率`
,concat(round(T2.category_uv/T1.wel_uv*100,2),'%') as `品类聚合页-转化美购uv占比`
,concat(round(T3.category_add_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击加车率`
,concat(round(T3.category_buy_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击立即支付率`
,concat(round(T3.category_msg_uv/T2.category_uv*100,2),'%') as `品类聚合页-点击私信咨询率`
,T1.conversation_detail_uv as `私信详情页uv`
,T2.conversation_detail_uv as `私信详情页-转化uv`
,T3.conv_add_uv as `私信详情页-转化点击加车uv`
,T3.conv_buy_uv as `私信详情页-转化点击立即支付uv`
,T3.conv_msg_uv as `私信详情页-转化点击私信咨询uv`
,concat(round(T2.conversation_detail_uv/T1.conversation_detail_uv*100,2),'%') as `私信详情页-转化uv率`
,concat(round(T2.conversation_detail_uv/T1.wel_uv*100,2),'%') as `私信详情页-转化美购uv占比`
,concat(round(T3.conv_add_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_uv/T2.conversation_detail_uv*100,2),'%') as `私信详情页-点击私信咨询率`
,T1.welfare_list_uv as `美购列表页uv`
,T2.welfare_list_uv as `美购列表页-转化uv`
,T3.wel_list_add_uv as `美购列表页-转化点击加车uv`
,T3.wel_list_buy_uv as `美购列表页-转化点击立即支付uv`
,T3.wel_list_msg_uv as `美购列表页-转化点击私信咨询uv`
,concat(round(T2.welfare_list_uv/T1.welfare_list_uv*100,2),'%') as `美购列表页-转化uv率`
,concat(round(T2.welfare_list_uv/T1.wel_uv*100,2),'%') as `美购列表页-转化美购uv占比`
,concat(round(T3.wel_list_add_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击加车率`
,concat(round(T3.wel_list_buy_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击立即支付率`
,concat(round(T3.wel_list_msg_uv/T2.welfare_list_uv*100,2),'%') as `美购列表页-点击私信咨询率`
,T1.my_cart_uv as`购物车uv`
,T2.my_cart_uv as`购物车-转化uv`
,T3.cart_add_uv as `购物车-转化点击加车uv`
,T3.cart_buy_uv as `购物车-转化点击立即支付uv`
,T3.cart_msg_uv as `购物车-转化点击私信咨询uv`
,concat(round(T2.my_cart_uv/T1.my_cart_uv*100,4),'%') as `购物车-转化uv率`
,concat(round(T2.my_cart_uv/T1.wel_uv*100,4),'%') as `购物车-转化美购uv占比`
,concat(round(T3.cart_add_uv/T2.my_cart_uv*100,4),'%') as `购物车-点击加车率`
,concat(round(T3.cart_buy_uv/T2.my_cart_uv*100,4),'%') as `购物车-点击立即支付率`
,concat(round(T3.cart_msg_uv/T2.my_cart_uv*100,4),'%') as `购物车-点击私信咨询率`
,concat(round(T2.my_cart_uv/T1.my_cart_uv*100,2),'%') as `购物车-转化uv率`
,concat(round(T2.my_cart_uv/T1.wel_uv*100,2),'%') as `购物车-转化美购uv占比`
,concat(round(T3.cart_add_uv/T2.my_cart_uv*100,2),'%') as `购物车-点击加车率`
,concat(round(T3.cart_buy_uv/T2.my_cart_uv*100,2),'%') as `购物车-点击立即支付率`
,concat(round(T3.cart_msg_uv/T2.my_cart_uv*100,2),'%') as `购物车-点击私信咨询率`
,T1.search_result_more_uv as `搜索结果综合页uv`
,T2.search_result_more_uv as `搜索结果综合页-转化uv`
,T3.result_more_add_uv as `搜索结果综合页-转化点击加车uv`
,T3.result_more_buy_uv as `搜索结果综合页-转化点击立即支付uv`
,T3.result_more_msg_uv as `搜索结果综合页-转化点击私信咨询uv`
,concat(round(T2.search_result_more_uv/T1.search_result_more_uv*100,4),'%') as `搜索结果综合页-转化uv率`
,concat(round(T2.search_result_more_uv/T1.wel_uv*100,4),'%') as `搜索结果综合页-转化美购uv占比`
,concat(round(T3.result_more_add_uv/T2.search_result_more_uv*100,4),'%') as `搜索结果综合页-点击加车率`
,concat(round(T3.result_more_buy_uv/T2.search_result_more_uv*100,4),'%') as `搜索结果综合页-点击立即支付率`
,concat(round(T3.result_more_msg_uv/T2.search_result_more_uv*100,4),'%') as `搜索结果综合页-点击私信咨询率`
,T1.welfare_home_uv as `美购首页uv`
,T2.welfare_home_uv as `美购首页-转化uv`
,T3.wel_home_add_uv as `美购首页-转化点击加车uv`
,T3.wel_home_buy_uv as `美购首页-转化点击立即支付uv`
,T3.wel_home_msg_uv as `美购首页-转化点击私信咨询uv`
,concat(round(T2.welfare_home_uv/T1.welfare_home_uv*100,4),'%') as `美购首页-转化uv率`
,concat(round(T2.welfare_home_uv/T1.wel_uv*100,4),'%') as `美购首页-转化美购uv占比`
,concat(round(T3.wel_home_add_uv/T2.welfare_home_uv*100,4),'%') as `美购首页-点击加车率`
,concat(round(T3.wel_home_buy_uv/T2.welfare_home_uv*100,4),'%') as `美购首页-点击立即支付率`
,concat(round(T3.wel_home_msg_uv/T2.welfare_home_uv*100,4),'%') as `美购首页-点击私信咨询率`
,T1.conversation_detail_uv as `私信详情页uv`
,T2.conversation_detail_uv as `私信详情页-转化uv`
,T3.conv_add_uv as `私信详情页-转化点击加车uv`
,T3.conv_buy_uv as `私信详情页-转化点击立即支付uv`
,T3.conv_msg_uv as `私信详情页-转化点击私信咨询uv`
,concat(round(T2.conversation_detail_uv/T1.conversation_detail_uv*100,4),'%') as `私信详情页-转化uv率`
,concat(round(T2.conversation_detail_uv/T1.wel_uv*100,4),'%') as `私信详情页-转化美购uv占比`
,concat(round(T3.conv_add_uv/T2.conversation_detail_uv*100,4),'%') as `私信详情页-点击加车率`
,concat(round(T3.conv_buy_uv/T2.conversation_detail_uv*100,4),'%') as `私信详情页-点击立即支付率`
,concat(round(T3.conv_msg_uv/T2.conversation_detail_uv*100,4),'%') as `私信详情页-点击私信咨询率`
,T1.wel_uv as `美购详情页uv`
,T2.wel_uv as `其他美购详情页-转化uv`
,T3.wel_add_uv as `其他美购详情页-转化点击加车uv`
,T3.wel_buy_uv as `其他美购详情页-转化点击立即支付uv`
,T3.wel_msg_uv as `其他美购详情页-转化点击私信咨询uv`
,concat(round(T2.wel_uv/T1.wel_uv*100,4),'%') as `其他美购详情页-转化uv率`
,concat(round(T2.wel_uv/T1.wel_uv*100,4),'%') as `其他美购详情页-转化美购uv占比`
,concat(round(T3.wel_add_uv/T2.wel_uv*100,4),'%') as `其他美购详情页-点击加车率`
,concat(round(T3.wel_buy_uv/T2.wel_uv*100,4),'%') as `其他美购详情页-点击立即支付率`
,concat(round(T3.wel_msg_uv/T2.wel_uv*100,4),'%') as `其他美购详情页-点击私信咨询率`
,concat(round(T2.search_result_more_uv/T1.search_result_more_uv*100,2),'%') as `搜索结果综合页-转化uv率`
,concat(round(T2.search_result_more_uv/T1.wel_uv*100,2),'%') as `搜索结果综合页-转化美购uv占比`
,concat(round(T3.result_more_add_uv/T2.search_result_more_uv*100,2),'%') as `搜索结果综合页-点击加车率`
,concat(round(T3.result_more_buy_uv/T2.search_result_more_uv*100,2),'%') as `搜索结果综合页-点击立即支付率`
,concat(round(T3.result_more_msg_uv/T2.search_result_more_uv*100,2),'%') as `搜索结果综合页-点击私信咨询率`
,T1.order_detail_uv as `订单详情页uv`
,T2.order_detail_uv as `订单详情页-转化uv`
,T3.order_add_uv as `订单详情页-转化点击加车uv`
,T3.order_buy_uv as `订单详情页-转化点击立即支付uv`
,T3.order_msg_uv as `订单详情页-转化点击私信咨询uv`
,concat(round(T2.order_detail_uv/T1.order_detail_uv*100,4),'%') as `订单详情页-转化uv率`
,concat(round(T2.order_detail_uv/T1.wel_uv*100,4),'%') as `订单详情页-转化美购uv占比`
,concat(round(T3.order_add_uv/T2.order_detail_uv*100,4),'%') as `订单详情页-点击加车率`
,concat(round(T3.order_buy_uv/T2.order_detail_uv*100,4),'%') as `订单详情页-点击立即支付率`
,concat(round(T3.order_msg_uv/T2.order_detail_uv*100,4),'%') as `订单详情页-点击私信咨询率`
,concat(round(T2.order_detail_uv/T1.order_detail_uv*100,2),'%') as `订单详情页-转化uv率`
,concat(round(T2.order_detail_uv/T1.wel_uv*100,2),'%') as `订单详情页-转化美购uv占比`
,concat(round(T3.order_add_uv/T2.order_detail_uv*100,2),'%') as `订单详情页-点击加车率`
,concat(round(T3.order_buy_uv/T2.order_detail_uv*100,2),'%') as `订单详情页-点击立即支付率`
,concat(round(T3.order_msg_uv/T2.order_detail_uv*100,2),'%') as `订单详情页-点击私信咨询率`
,T1.special_uv as `专题和大促页uv`
,T2.special_uv as `专题和大促页-转化uv`
,T3.special_add_uv as `专题和大促页-转化点击加车uv`
,T3.special_buy_uv as `专题和大促页-转化点击立即支付uv`
,T3.special_msg_uv as `专题和大促页-转化点击私信咨询uv`
,concat(round(T2.special_uv/T1.special_uv*100,2),'%') as `专题和大促页-转化uv率`
,concat(round(T2.special_uv/T1.wel_uv*100,2),'%') as `专题和大促页-转化美购uv占比`
,concat(round(T3.special_add_uv/T2.special_uv*100,2),'%') as `专题和大促页-点击加车率`
,concat(round(T3.special_buy_uv/T2.special_uv*100,2),'%') as `专题和大促页-点击立即支付率`
,concat(round(T3.special_msg_uv/T2.special_uv*100,2),'%') as `专题和大促页-点击私信咨询率`
,T1.expert_detail_uv as `医生主页uv`
,T2.expert_detail_uv as `医生主页-转化uv`
,T3.exp_add_uv as `医生主页-转化点击加车uv`
,T3.exp_buy_uv as `医生主页-转化点击立即支付uv`
,T3.exp_msg_uv as `医生主页-转化点击私信咨询uv`
,concat(round(T2.expert_detail_uv/T1.expert_detail_uv*100,4),'%') as `医生主页-转化uv率`
,concat(round(T2.expert_detail_uv/T1.wel_uv*100,4),'%') as `医生主页-转化美购uv占比`
,concat(round(T3.exp_add_uv/T2.expert_detail_uv*100,4),'%') as `医生主页-点击加车率`
,concat(round(T3.exp_buy_uv/T2.expert_detail_uv*100,4),'%') as `医生主页-点击立即支付率`
,concat(round(T3.exp_msg_uv/T2.expert_detail_uv*100,4),'%') as `医生主页-点击私信咨询率`
,T1.ai_report_uv as `AI结果页uv`
,T2.ai_report_uv as `AI结果页-转化uv`
,T3.ai_add_uv as `AI结果页-转化点击加车uv`
,T3.ai_buy_uv as `AI结果页-转化点击立即支付uv`
,T3.ai_msg_uv as `AI结果页-转化点击私信咨询uv`
,concat(round(T2.ai_report_uv/T1.ai_report_uv*100,4),'%') as `AI结果页-转化uv率`
,concat(round(T2.ai_report_uv/T1.wel_uv*100,4),'%') as `AI结果页-转化美购uv占比`
,concat(round(T3.ai_add_uv/T2.ai_report_uv*100,4),'%') as `AI结果页-点击加车率`
,concat(round(T3.ai_buy_uv/T2.ai_report_uv*100,4),'%') as `AI结果页-点击立即支付率`
,concat(round(T3.ai_msg_uv/T2.ai_report_uv*100,4),'%') as `AI结果页-点击私信咨询率`
,concat(round(T2.expert_detail_uv/T1.expert_detail_uv*100,2),'%') as `医生主页-转化uv率`
,concat(round(T2.expert_detail_uv/T1.wel_uv*100,2),'%') as `医生主页-转化美购uv占比`
,concat(round(T3.exp_add_uv/T2.expert_detail_uv*100,2),'%') as `医生主页-点击加车率`
,concat(round(T3.exp_buy_uv/T2.expert_detail_uv*100,2),'%') as `医生主页-点击立即支付率`
,concat(round(T3.exp_msg_uv/T2.expert_detail_uv*100,2),'%') as `医生主页-点击私信咨询率`
,T1.wel_uv as `美购详情页uv`
,T2.wel_uv as `其他美购详情页-转化uv`
,T3.wel_add_uv as `其他美购详情页-转化点击加车uv`
,T3.wel_buy_uv as `其他美购详情页-转化点击立即支付uv`
,T3.wel_msg_uv as `其他美购详情页-转化点击私信咨询uv`
,concat(round(T2.wel_uv/T1.wel_uv*100,2),'%') as `其他美购详情页-转化uv率`
,concat(round(T2.wel_uv/T1.wel_uv*100,2),'%') as `其他美购详情页-转化美购uv占比`
,concat(round(T3.wel_add_uv/T2.wel_uv*100,2),'%') as `其他美购详情页-点击加车率`
,concat(round(T3.wel_buy_uv/T2.wel_uv*100,2),'%') as `其他美购详情页-点击立即支付率`
,concat(round(T3.wel_msg_uv/T2.wel_uv*100,2),'%') as `其他美购详情页-点击私信咨询率`
,T1.welfare_home_uv as `美购首页uv`
,T2.welfare_home_uv as `美购首页-转化uv`
,T3.wel_home_add_uv as `美购首页-转化点击加车uv`
,T3.wel_home_buy_uv as `美购首页-转化点击立即支付uv`
,T3.wel_home_msg_uv as `美购首页-转化点击私信咨询uv`
,concat(round(T2.welfare_home_uv/T1.welfare_home_uv*100,2),'%') as `美购首页-转化uv率`
,concat(round(T2.welfare_home_uv/T1.wel_uv*100,2),'%') as `美购首页-转化美购uv占比`
,concat(round(T3.wel_home_add_uv/T2.welfare_home_uv*100,2),'%') as `美购首页-点击加车率`
,concat(round(T3.wel_home_buy_uv/T2.welfare_home_uv*100,2),'%') as `美购首页-点击立即支付率`
,concat(round(T3.wel_home_msg_uv/T2.welfare_home_uv*100,2),'%') as `美购首页-点击私信咨询率`
,T1.diary_detail_uv as `日记本详情页uv`
,T2.diary_detail_uv as `日记本详情页-转化uv`
,T3.diary_add_uv as `日记本详情页-转化点击加车uv`
,T3.diary_buy_uv as `日记本详情页-转化点击立即支付uv`
,T3.diary_msg_uv as `日记本详情页-转化点击私信咨询uv`
,concat(round(T2.diary_detail_uv/T1.diary_detail_uv*100,4),'%') as `日记本详情页-转化uv率`
,concat(round(T2.diary_detail_uv/T1.wel_uv*100,4),'%') as `日记本详情页-转化美购uv占比`
,concat(round(T3.diary_add_uv/T2.diary_detail_uv*100,4),'%') as `日记本详情页-点击加车率`
,concat(round(T3.diary_buy_uv/T2.diary_detail_uv*100,4),'%') as `日记本详情页-点击立即支付率`
,concat(round(T3.diary_msg_uv/T2.diary_detail_uv*100,4),'%') as `日记本详情页-点击私信咨询率`
,concat(round(T2.diary_detail_uv/T1.diary_detail_uv*100,2),'%') as `日记本详情页-转化uv率`
,concat(round(T2.diary_detail_uv/T1.wel_uv*100,2),'%') as `日记本详情页-转化美购uv占比`
,concat(round(T3.diary_add_uv/T2.diary_detail_uv*100,2),'%') as `日记本详情页-点击加车率`
,concat(round(T3.diary_buy_uv/T2.diary_detail_uv*100,2),'%') as `日记本详情页-点击立即支付率`
,concat(round(T3.diary_msg_uv/T2.diary_detail_uv*100,2),'%') as `日记本详情页-点击私信咨询率`
,T1.sign_uv as `签到页uv`
,T2.sign_uv as `签到页-转化uv`
,T3.sign_add_uv as `签到页-转化点击加车uv`
,T3.sign_buy_uv as `签到页-转化点击立即支付uv`
,T3.sign_msg_uv as `签到页-转化点击私信咨询uv`
,concat(round(T2.sign_uv/T1.sign_uv*100,2),'%') as `签到页-转化uv率`
,concat(round(T2.sign_uv/T1.wel_uv*100,2),'%') as `签到页-转化美购uv占比`
,concat(round(T3.sign_add_uv/T2.sign_uv*100,2),'%') as `签到页-点击加车率`
,concat(round(T3.sign_buy_uv/T2.sign_uv*100,2),'%') as `签到页-点击立即支付率`
,concat(round(T3.sign_msg_uv/T2.sign_uv*100,2),'%') as `签到页-点击私信咨询率`
,T1.ai_report_uv as `AI结果页uv`
,T2.ai_report_uv as `AI结果页-转化uv`
,T3.ai_add_uv as `AI结果页-转化点击加车uv`
,T3.ai_buy_uv as `AI结果页-转化点击立即支付uv`
,T3.ai_msg_uv as `AI结果页-转化点击私信咨询uv`
,concat(round(T2.ai_report_uv/T1.ai_report_uv*100,2),'%') as `AI结果页-转化uv率`
,concat(round(T2.ai_report_uv/T1.wel_uv*100,2),'%') as `AI结果页-转化美购uv占比`
,concat(round(T3.ai_add_uv/T2.ai_report_uv*100,2),'%') as `AI结果页-点击加车率`
,concat(round(T3.ai_buy_uv/T2.ai_report_uv*100,2),'%') as `AI结果页-点击立即支付率`
,concat(round(T3.ai_msg_uv/T2.ai_report_uv*100,2),'%') as `AI结果页-点击私信咨询率`
,T1.zone_uv as `内容聚合页uv`
,T2.zone_uv as `内容聚合页-转化uv`
,T3.zone_add_uv as `内容聚合页-转化点击加车uv`
,T3.zone_buy_uv as `内容聚合页-转化点击立即支付uv`
,T3.zone_msg_uv as `内容聚合页-转化点击私信咨询uv`
,concat(round(T2.zone_uv/T1.zone_uv*100,4),'%') as `内容聚合页-转化uv率`
,concat(round(T2.zone_uv/T1.wel_uv*100,4),'%') as `内容聚合页-转化美购uv占比`
,concat(round(T3.zone_add_uv/T2.zone_uv*100,4),'%') as `内容聚合页-点击加车率`
,concat(round(T3.zone_buy_uv/T2.zone_uv*100,4),'%') as `内容聚合页-点击立即支付率`
,concat(round(T3.zone_msg_uv/T2.zone_uv*100,4),'%') as `内容聚合页-点击私信咨询率`
,concat(round(T2.zone_uv/T1.zone_uv*100,2),'%') as `内容聚合页-转化uv率`
,concat(round(T2.zone_uv/T1.wel_uv*100,2),'%') as `内容聚合页-转化美购uv占比`
,concat(round(T3.zone_add_uv/T2.zone_uv*100,2),'%') as `内容聚合页-点击加车率`
,concat(round(T3.zone_buy_uv/T2.zone_uv*100,2),'%') as `内容聚合页-点击立即支付率`
,concat(round(T3.zone_msg_uv/T2.zone_uv*100,2),'%') as `内容聚合页-点击私信咨询率`
FROM
(
(--能够走向美购详情页的各页面uv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -183,7 +185,7 @@ FROM
,count(distinct case when page_name = 'diary_detail' then t1.cl_id end) as diary_detail_uv
,count(distinct case when page_name = 'order_detail' then t1.cl_id end) as order_detail_uv
,count(distinct case when page_name ='new_sign' then t1.cl_id end) as sign_uv
,count(distinct case when page_name in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') then t1.cl_id end) as special_uv --专题和大促
,count(distinct case when page_name in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') then t1.cl_id end) as special_uv --专题和大促
,count(distinct case when page_name = 'zone_v3' then t1.cl_id end) as zone_uv
,count(distinct case when page_name in ('report_result','face_detect_result') then t1.cl_id end) as ai_report_uv --待之后增加模拟整形页面
FROM
......@@ -193,58 +195,63 @@ FROM
(
SELECT partition_date,page_name,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='page_view'
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
(
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
( -- 去掉疑似机构刷量的pV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
LEFT JOIN
(--不同referrer下的美购详情页uv/uv
)T1
LEFT JOIN
(--不同referrer下的美购详情页uv/uv
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -262,7 +269,7 @@ FROM
,count(distinct case when referrer = 'diary_detail' then t1.cl_id end) as diary_detail_uv
,count(distinct case when referrer = 'order_detail' then t1.cl_id end) as order_detail_uv
,count(distinct case when referrer ='new_sign' then t1.cl_id end) as sign_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') then t1.cl_id end) as special_uv --专题和大促
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') then t1.cl_id end) as special_uv --专题和大促
,count(distinct case when referrer = 'zone_v3' then t1.cl_id end) as zone_uv
,count(distinct case when referrer in ('report_result','face_detect_result') then t1.cl_id end) as ai_report_uv --待之后增加模拟整形页面
FROM
......@@ -272,62 +279,67 @@ FROM
(
SELECT partition_date,referrer,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
( -- 去掉疑似机构刷量的pV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T2
ON T1.partition_date=T2.partition_date
ON T1.partition_date=T2.partition_date
and T1.device_type=T2.device_type
AND T1.active_type=T2.active_type
AND T1.channel=T2.channel
LEFT JOIN
(--点击加车/支付/私信的不同来源
LEFT JOIN
(--点击加车/支付/私信的不同来源
SELECT t1.partition_date
,t1.device_type
,t1.active_type
......@@ -371,9 +383,9 @@ FROM
,count(distinct case when referrer = 'new_sign' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as sign_add_uv
,count(distinct case when referrer = 'new_sign' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as sign_buy_uv
,count(distinct case when referrer = 'new_sign' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as sign_msg_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as special_add_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as special_buy_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion') and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as special_msg_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as special_add_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as special_buy_uv
,count(distinct case when referrer in ('special_polymer','newt_welfare_special','welfare_special','activity_promotion_seckill','activity_promotion','custom_special') and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as special_msg_uv
,count(distinct case when referrer = 'zone_v3' and action = 'welfare_multiattribute_click_add' then t1.cl_id end) as zone_add_uv
,count(distinct case when referrer = 'zone_v3' and action = 'welfare_multiattribute_click_buy' then t1.cl_id end) as zone_buy_uv
,count(distinct case when referrer = 'zone_v3' and action in ('welfare_detail_click_message','on_click_button') then t1.cl_id end) as zone_msg_uv
......@@ -388,7 +400,7 @@ FROM
( --点击加车/支付
SELECT partition_date,referrer,action,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action in ('welfare_multiattribute_click_add','welfare_multiattribute_click_buy')
......@@ -397,7 +409,7 @@ FROM
--点击私信按钮
SELECT partition_date,referrer,action,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >=regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >=regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
......@@ -413,7 +425,7 @@ FROM
( SELECT partition_date,params['referrer_link'] as referrer_link,cl_id,action
,json_split(params['referrer_link'],'string') as a
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
WHERE partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
......@@ -426,51 +438,57 @@ FROM
)t1
JOIN
(
SELECT partition_date,device_id
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status
SELECT partition_date,device_os_type as device_type,m.device_id
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,array(CASE WHEN first_channel_source_type like '%xinyouxingkong%' or a.device_id is not NULL
or first_channel_source_type in ('promotion_aisi','promotion_koomobi_km00','promotion_koomobi_km01')
or first_channel_source_type like '%longyuzhixing%' or first_channel_source_type like '%jingmeng%'
or first_channel_source_type like '%mailuo%' THEN '可疑'
WHEN (partition_date>='20190601' and tmp.col2 = 'AI')
or (partition_date < '20200301' AND partition_date>='20190601' and first_channel_source_type like 'promotion_toutiao_jy%')
or (partition_date>='20200601' and ((first_channel_source_type like 'promotion_toutiao_jy%') or (first_channel_source_type like 'dyand%') or (first_channel_source_type like 'douyin%')))
THEN 'AI' ELSE '其他' END , '合计') as channel
FROM online.ml_device_day_active_status m
LEFT JOIN
(SELECT col1,col2 --col1:子渠道，col2：是否属于AI，col3:标识
FROM pm.tl_pm_ydl
WHERE col3='0204_danlei_channel')tmp
on first_channel_source_type=tmp.col1
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
(SELECT DISTINCT device_id
FROM al.al_pm_ct_dv_deviceappversionrollbackfrom20190101_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,2) ,'-',''))a
ON m.device_id = a.device_id
where partition_date >= regexp_replace(DATE_SUB(current_date,90) ,'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not like 'promotion\_jf\_%'
)mas
ON t1.cl_id=mas.device_id
AND t1.partition_date=mas.partition_date
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
)spam_pv
on t1.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
on t1.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T3
ON T1.partition_date=T3.partition_date
ON T1.partition_date=T3.partition_date
and T1.device_type=T3.device_type
AND T1.active_type=T3.active_type
AND T1.channel=T3.channel
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
pm/search_related/en-cn.properties 0 → 100644
View file @ e388a661
search_click_path=分搜索入口的搜索框点击数据
search_input_type=分搜索方式近7天数据
search_result_click_rate=分搜索词转化率数据
search_related=分搜索方式及搜索词日报
\ No newline at end of file
tables/create_search_click_path.sql → pm/search_related/etl/create_search_click_path.sql
View file @ e388a661
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*脚本名称:
--*功能: 分搜索入口搜索日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--*更新时间:
--***************************************************************
--设置全局变量&UDF
......
tables/create_search_input_type.sql → pm/search_related/etl/create_search_input_type.sql
View file @ e388a661
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*脚本名称:
--*功能: 分搜索方式日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--*更新时间:
--***************************************************************
--设置全局变量&UDF
......
tables/create_search_result_click_rate.sql → pm/search_related/etl/create_search_result_click_rate.sql
View file @ e388a661
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*脚本名称:
--*功能: 分搜索关键词转化日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--*更新时间:
--***************************************************************
--设置全局变量&UDF
......
sqls/search_related_insert/search_input_type_insert.sql → pm/search_related/etl/search_related.sql
View file @ e388a661
INSERT OVERWRITE TABLE pm.tl_pm_search_result_click_rate_d PARTITION (PARTITION_DAY = #partition_day)
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT day_id as day_id
,device_os_type as device_os_type
,active_type as active_type
,v.search_entrance as search_entrance
,sum(search_pv) as click_search_pv
,sum(search_uv) as click_search_uv
FROM
(
SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance
,search_pv,search_uv
FROM
(
SELECT t1.partition_date as day_id
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end as search_entrance
,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv
FROM
(
SELECT partition_date,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,device_id
FROM online.ml_device_day_active_status
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
join
(
SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action in ('do_search','on_click_navbar_search')
union all
SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date,cl_id,page_name,'热搜词' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='on_click_card'
and params['card_type']='highlight_word'
)t2
on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
group by t1.partition_date
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
)n
LATERAL VIEW explode(n.search_entrance) v AS search_entrance
group by day_id,device_os_type,active_type,v.search_entrance;
INSERT OVERWRITE TABLE pm.tl_pm_search_input_type_d PARTITION (PARTITION_DAY = ${partition_day})
SELECT partition_date as day_id
,t1.query
,t2.input_type
,sum(t1.all_search_pv) as search_pv
FROM
(
SELECT
partition_date
,query
,array(case when input_type in ('联想','聚合模块') then 'sug点击' else input_type end,'全部') as input_type
,count(click.cl_id) as all_search_pv
,count(distinct click.cl_id) as all_search_uv
FROM
(
SELECT cl_id
,partition_date
,case when params['input_type'] in ('热门','related_search') then '发现' else params['input_type'] end as input_type
,params['query'] as query
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action in ('do_search','search_result_click_search')
union all
SELECT cl_id
,partition_date
,'首页精选大家都在看' as input_type
,params['card_name'] as query
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT cl_id
,partition_date
,'美购首页热搜词' as input_type
,params['card_name'] as query
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select cl_id
,partition_date
,'详情页高亮词' as input_type
,params['card_name'] as query
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and int(split(app_version,'\\.')[1]) >= 27
and action='on_click_card'
and params['card_type']='highlight_word'
)click
JOIN
(
SELECT device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev_channel
on dev_channel.device_id = click.cl_id
GROUP BY partition_date,query,case when input_type in ('联想','聚合模块') then 'sug点击' else input_type end
)t1
LATERAL VIEW explode(t1.input_type) t2 AS input_type
GROUP BY t1.partition_date,t1.query,t2.input_type;
INSERT OVERWRITE TABLE pm.tl_pm_search_result_click_rate_d PARTITION (PARTITION_DAY = ${partition_day})
--搜索关键词日报,取近7天数据，每天取关键词搜索量前500
......@@ -119,7 +320,6 @@ from
where rank<=300
)t1
left join
--综合、美购、日记页面浏览pv
(
......@@ -430,11 +630,4 @@ left join
on t1.partition_date=t2.partition_date
and t1.query=t2.query
order by day_id desc,search_pv desc,query
order by day_id desc,search_pv desc,query;
\ No newline at end of file
jobs/bak/demo_java_gen/step1_1.job → pm/search_related/job/step1_1.job
View file @ e388a661
#step1_1.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_day_active_status
\ No newline at end of file
jobs/bak/demo_java/step1_2.job → pm/search_related/job/step1_2.job
View file @ e388a661
#step1_2.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_hospital_spam_pv_day
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_device_history_detail
\ No newline at end of file
pm/search_related/job/step1_3.job 0 → 100644
View file @ e388a661
#step1_3.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online bl_hdfs_maidian_updates
\ No newline at end of file
pm/search_related/job/step1_4.job 0 → 100644
View file @ e388a661
#step1_4.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive ml ml_d_ct_dv_devicespam_d
\ No newline at end of file
jobs/bak/demo_spark_param/step1_5.job → pm/search_related/job/step1_5.job
View file @ e388a661
#step1_5.job
type=command
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online tl_hdfs_api_tag_map_oldtag_view
\ No newline at end of file
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive online ml_community_precise_exposure_detail
\ No newline at end of file
pm/search_related/job/step2.job 0 → 100644
View file @ e388a661
#step2.job
type=command
dependencies=step1_1,step1_2,step1_3,step1_4,step1_5
command=/home/bi/bi-report/lib/shell/hive search_related
\ No newline at end of file
pm/search_related/job/step3_1.job 0 → 100644
View file @ e388a661
#step3_1.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_click_path_d
\ No newline at end of file
pm/search_related/job/step3_2.job 0 → 100644
View file @ e388a661
#step3_2.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_input_type_d
\ No newline at end of file
pm/search_related/job/step3_3.job 0 → 100644
View file @ e388a661
#step3_3.job
type=command
dependencies=step2
command=sh /home/bi/bi-report/lib/shell/waitsuccess.sh hive pm tl_pm_search_result_click_rate_d
\ No newline at end of file
pm/search_related/job/step4.job 0 → 100644
View file @ e388a661
#step4.job
type=command
dependencies=step3_1,step3_2,step3_3
command=curl -X GET http://localhost:8553/api/report/email/search_related/liudi@igengmei.com,wangxin@igengmei.com,duanyingrong@igengmei.com,dengguangyu@igengmei.com/zhaofei@igengmei.com,yindanlei@igengmei.com,weiyimin@igengmei.com
\ No newline at end of file
pm/search_related/report/search_input_type.sql 0 → 100644
View file @ e388a661
--近7天搜索方式的搜索量
SELECT regexp_replace(DATE_SUB(current_date,1) ,'-','') AS `日期`,
T4.QUERY AS `搜索词`,
T4.INPUT_TYPE AS `搜索类型`,
T4.SEARCH_PV AS `搜索量`
FROM
(SELECT T3.*
FROM (SELECT T2.*,
SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) SEARCH_PV_SUM,
ROW_NUMBER() OVER(PARTITION BY T2.QUERY ORDER BY SEARCH_PV DESC) SEARCH_PV_GROUP_SUM,
DENSE_RANK() OVER(ORDER BY SUM(SEARCH_PV_ALL) OVER(PARTITION BY T2.QUERY) DESC,T2.QUERY) SEARCH_PV_GROUP
FROM (SELECT T1.*,
CASE
WHEN INPUT_TYPE = '全部' THEN
0
ELSE
1
END INPUT_TYPE_RANK,
CASE
WHEN INPUT_TYPE = '全部' THEN
SEARCH_PV
ELSE
0
END SEARCH_PV_ALL
FROM (SELECT QUERY,INPUT_TYPE,SUM(SEARCH_PV) AS SEARCH_PV
FROM PM.TL_PM_SEARCH_INPUT_TYPE_D T
WHERE T.PARTITION_DAY >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.PARTITION_DAY<regexp_replace((current_date),'-','')
AND T.DAY_ID >= regexp_replace(DATE_SUB(current_date,7) ,'-','')
AND T.DAY_ID<regexp_replace((current_date),'-','')
GROUP BY T.QUERY,T.INPUT_TYPE) T1) T2)T3
WHERE SEARCH_PV_GROUP<=300
ORDER BY T3.SEARCH_PV_SUM DESC,
T3.QUERY,
T3.INPUT_TYPE_RANK,
T3.SEARCH_PV_GROUP_SUM)T4
\ No newline at end of file
sqls/search_related/search_result_click_rate.sql → pm/search_related/report/search_result_click_rate.sql
View file @ e388a661
......@@ -5,17 +5,17 @@ SELECT
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_pv*100/t2.page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_10_pv*100/t2.page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_diary_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_answer_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_topic_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_10_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_5_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_10_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_5_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_pv*100/page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_10_pv*100/page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_diary_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_answer_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_more_pv,0)>0 then concat(round(more_click_5_topic_pv*100/page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_10_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_wel_pv,0)>0 then concat(round(wel_click_5_pv*100/page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_10_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(page_diary_pv,0)>0 then concat(round(diary_click_5_pv*100/page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
......@@ -36,4 +36,6 @@ SELECT
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
and day_id>=regexp_replace(DATE_SUB(current_date,7) ,'-','')
and day_id<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
sbin/replace.sh
View file @ e388a661
......@@ -3,3 +3,6 @@
cd /home/bi/bi-report/lib/java
mv report-service-server-1.0.0-SNAPSHOT.jar report-service-server-1.0.0-SNAPSHOT.jar.`date +%Y%m%d%H%M%S`
cp /home/bi/bi-report/lib/java/bi-report-service/service-server/target/report-service-server-1.0.0-SNAPSHOT.jar .
cd /home/bi/bi-report/lib/java/bi-report-service
/opt/apache-maven-3.6.3/bin/mvn clean
\ No newline at end of file
shells/daily_recommend_strategy/daily_recommend_strategy.sh deleted 100755 → 0
View file @ 6e9fff6e
#!/bin/bash
# 定义变量方便修改
#hive="/opt/spark/spark-2.4.5-bin-hadoop2.7/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter003:10010 -n data"
hive="/opt/hive/hive-1.1.0-cdh5.16.1/bin/beeline -u jdbc:hive2://bj-gm-prod-cos-datacenter005:10000 -n data"
# 如果是输入的日期按照取输入日期；如果没输入日期取当前时间的前一天
if [ -n "$1" ] ;then
partition_day=$1
else
partition_day=`date --date="-1 day" +%Y%m%d`
fi
sql="
SET mapreduce.job.queuename=data;
SET mapreduce.map.memory.mb=8192;
SET mapreduce.map.java.opts=-Xmx8000m;
SET mapreduce.reduce.memory.mb=8192;
SET mapreduce.reduce.java.opts=-Xmx8000m;
set hive.auto.convert.join=true;
SET mapred.reduce.tasks=20;
SET role admin;
ADD JAR hdfs:///user/hive/share/lib/udf/hive-udf-1.0-SNAPSHOT.jar;
CREATE TEMPORARY FUNCTION convup AS 'com.gmei.hive.common.udf.UDFConvUpgrade';
INSERT OVERWRITE TABLE pm.tl_pm_recommend_strategy_d PARTITION (PARTITION_DAY =$partition_day)
SELECT
t1.partition_date as day_id,
t1.device_os_type as device_os_type,
t1.active_type as active_type,
t2.card_content_type as card_content_type,
t2.recommend_type as recommend_type,
NVL(concat(round(sum(t3.session_pv)/sum(t2.session_pv)*100,2),'%'),0) as home_ctr,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t2.session_pv)*100,2),'%'),0) as second_rate,
NVL(concat(round((NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)
+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0))/sum(t3.session_pv)*100,2),'%'),0) as first_rate,
NVL(sum(t3.session_pv),0) as card_click,
NVL(sum(t2.session_pv),0) as card_exposure,
NVL(sum(navbar_pv),0)+NVL(sum(highlight_pv),0)+NVL(sum(self_wel_pv),0)+NVL(sum(recom_wel_pv),0)+NVL(sum(recom_content_pv),0) as total_second_click,
NVL(round(sum(page_stay)/count(distinct t4.cl_id)/60,2),0) as avg_page_stay,
NVL(sum(navbar_pv),0) as navbar_search,
NVL(sum(highlight_pv),0) as highlight_word,
NVL(sum(self_wel_pv),0) as self_welfare_card,
NVL(sum(recom_wel_pv),0)-NVL(sum(self_wel_pv),0) as recommend_welfare_card,--需要排除关联的商品卡片点击
NVL(sum(recom_content_pv),0) as recommend_content_card,
NULL as recommend_special_card,
NULL as transfer_card,
NULL as video_consultation
FROM
(
SELECT partition_date
,device_os_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)t1
JOIN
(--精准曝光，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end as card_content_type,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片'
END AS recommend_type,
card_id,
app_session_id
from online.ml_community_precise_exposure_detail
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
AND is_exposure = '1' ----精准曝光
AND page_name ='home'
AND tab_name = '精选'
AND transaction_type in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND card_content_type in ('qa','diary','user_post','answer')
group by partition_date,
case when card_content_type in ('qa','answer') then 'qa' else card_content_type end,
cl_id,
CASE WHEN transaction_type in ('ctr') THEN 'ctr预估'
WHEN transaction_type in ('cvr') THEN 'cvr预估'
WHEN transaction_type in ('-1','smr') THEN 'smr'
when transaction_type in ('pgc','hotspot') then '热点卡片'
when transaction_type in ('newdata') then '保量卡片' END,
card_id,
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t2
on t1.device_id=t2.cl_id and t1.partition_date=t2.partition_date
LEFT JOIN
(--卡片，卡片id和session_id去重
SELECT partition_date,
card_content_type,
cl_id,
recommend_type,
card_id,
count(distinct app_session_id) as session_pv
FROM
(
SELECT partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end as card_content_type,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片'
END AS recommend_type,
params['card_id'] as card_id,
app_session_id
from online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
AND params['page_name'] ='home'
AND params['tab_name'] = '精选'
AND params['transaction_type'] in ('-1','ctr','smr','cvr','hotspot','pgc','newdata')
AND params['card_content_type'] in ('qa','diary','user_post','answer')
GROUP BY partition_date,
cl_id,
case when params['card_content_type'] in ('qa','answer') then 'qa' else params['card_content_type'] end,
CASE WHEN params['transaction_type'] in ('ctr') THEN 'ctr预估'
WHEN params['transaction_type'] in ('cvr') THEN 'cvr预估'
WHEN params['transaction_type'] in ('-1','smr') THEN 'smr'
when params['transaction_type'] in ('pgc','hotspot') then '热点卡片'
when params['transaction_type'] in ('newdata') then '保量卡片' END,
params['card_id'],
app_session_id
)a
group by partition_date,card_content_type,cl_id,recommend_type,card_id
)t3
on t2.partition_date=t3.partition_date
and t2.cl_id=t3.cl_id
and t2.card_id=t3.card_id
and t2.card_content_type=t3.card_content_type
and t2.recommend_type=t3.recommend_type
LEFT JOIN
(--页面浏览时长
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
sum(page_stay) as page_stay
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='page_view'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND referrer='home'
AND page_stay>=0 AND page_stay<1000
GROUP BY partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t4
on t4.partition_date=t3.partition_date
and t4.cl_id=t3.cl_id
and t4.business_id=t3.card_id
and t4.page_name=t3.card_content_type
LEFT JOIN
(--搜索框和点击行为
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as navbar_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action in ('on_click_navbar_search','do_search')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t5
on t5.partition_date=t3.partition_date
and t5.cl_id=t3.cl_id
and t5.business_id=t3.card_id
and t5.page_name=t3.card_content_type
LEFT JOIN
(--点击高亮词
SELECT partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as highlight_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_type']='highlight_word'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)t6
on t6.partition_date=t3.partition_date
and t6.cl_id=t3.cl_id
and t6.business_id=t3.card_id
and t6.page_name=t3.card_content_type
LEFT JOIN
(--关联的美购卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as self_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (get_json_object(params['extra_param'], '$.type')='交互栏'
or get_json_object(params['extra_param'], '$.jump_from')='msg_link')
AND action='on_click_card'
and params['card_content_type']='service'
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t7
on t7.partition_date=t3.partition_date
and t7.cl_id=t3.cl_id
and t7.business_id=t3.card_id
and t7.page_name=t3.card_content_type
LEFT JOIN
(--推荐的美购卡片(需要排除作者消费的美购)
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_wel_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND (action='on_click_card'and params['card_content_type']='service'
or action='on_click_button' and params['button_name']='unfold')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t8
on t8.partition_date=t3.partition_date
and t8.cl_id=t3.cl_id
and t8.business_id=t3.card_id
and t8.page_name=t3.card_content_type
LEFT JOIN
(--推荐的内容卡片
SELECT partition_date,cl_id,business_id,page_name,count(distinct app_session_id) as recom_content_pv
FROM
(
SELECT partition_date,cl_id,business_id,app_session_id,params['card_id'] as card_id,
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end as page_name,
count(1) as service_pv
FROM online.bl_hdfs_maidian_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND action='on_click_card'
and params['card_content_type'] in ('qa','diary','user_post','answer')
AND page_name IN ('diary_detail','topic_detail','post_detail','user_post_detail','doctor_post_detail','question_detail','answer_detail','question_answer_detail')
AND (referrer='home' or
(params['referrer_link'] like '%[%' and
json_split(params['referrer_link'])[size(json_split(params['referrer_link']))-1]='home'))
group by partition_date,cl_id,business_id,app_session_id,params['card_id'],
case when page_name in ('diary_detail','topic_detail') then 'diary'
when page_name in ('post_detail','user_post_detail','doctor_post_detail') then 'user_post'
when page_name in ('question_detail','answer_detail','question_answer_detail') then 'qa' else null end
)a
group by partition_date,cl_id,business_id,page_name
)t9
on t9.partition_date=t3.partition_date
and t9.cl_id=t3.cl_id
and t9.business_id=t3.card_id
and t9.page_name=t3.card_content_type
LEFT JOIN
(
select distinct device_id
from ml.ml_d_ct_dv_devicespam_d --去除机构刷单设备，即作弊设备（浏览和曝光事件去除）
WHERE partition_day=regexp_replace(DATE_SUB(current_date,1) ,'-','')
union all
select distinct device_id
from dim.dim_device_user_staff --去除内网用户
)spam_pv
on spam_pv.device_id=t2.cl_id
LEFT JOIN
(
SELECT partition_date,device_id
FROM
(--找出user_id当天活跃的第一个设备id
SELECT user_id,partition_date,
if(size(device_list) > 0, device_list [ 0 ], '') AS device_id
FROM online.ml_user_updates
WHERE partition_date>=regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND partition_date<regexp_replace((current_date),'-','')
)t1
JOIN
( --医生账号
SELECT distinct user_id
FROM online.tl_hdfs_doctor_view
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
--马甲账号/模特用户
UNION ALL
SELECT user_id
FROM ml.ml_c_ct_ui_user_dimen_d
WHERE partition_day = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND (is_puppet = 'true' or is_classifyuser = 'true')
UNION ALL
--公司内网覆盖用户
select distinct user_id
from dim.dim_device_user_staff
UNION ALL
--登陆过医生设备
SELECT distinct t1.user_id
FROM
(
SELECT user_id, v.device_id as device_id
FROM online.ml_user_history_detail
LATERAL VIEW EXPLODE(device_history_list) v AS device_id
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
) t1
JOIN
(
SELECT device_id
FROM online.ml_device_history_detail
WHERE partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND is_login_doctor = '1'
) t2
ON t1.device_id = t2.device_id
)t2
on t1.user_id=t2.user_id
group by partition_date,device_id
)dev
on t2.partition_date=dev.partition_date and t2.cl_id=dev.device_id
WHERE spam_pv.device_id IS NULL
and dev.device_id is null
GROUP BY t1.partition_date,t1.device_os_type,t1.active_type,t2.card_content_type,t2.recommend_type
order by day_id,device_os_type,active_type,card_content_type,recommend_type;
"
$hive -e "$sql"
\ No newline at end of file
sqls/bak/demo.sql deleted 100644 → 0
View file @ 6e9fff6e
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date between '20200401' and '20200408'
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
limit 10
\ No newline at end of file
sqls/bak/demo_hive/demo_hive.sql deleted 100644 → 0
View file @ 6e9fff6e
set mapreduce.job.queuename= data;
SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
,`美购详情页浏览pv`
,(`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
+`机构主页点击pv`+`机构的其他美购点击pv`+`推荐美购点击pv`+`收藏点击pv`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`) as `美购详情页分发pv加总`
,concat(round((`加购物车点击pv`+`评价列表页点击pv`+`立即支付点击pv`+`私信及电话咨询pv` +`选择美购项目点击pv`
+`机构主页点击pv`+`机构的其他美购点击pv`+`推荐美购点击pv`+`收藏点击pv`
+`右上角购物车点击pv`+`医生主页点击pv`+`分享点击pv`+`机构导航点击pv`)/`美购详情页浏览pv`*100,4),'%') as `美购详情页分发pv占比加总`
,`加购物车点击pv`
,concat(round(`加购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `加购物车点击pv占比`
,`评价列表页点击pv`
,concat(round(`评价列表页点击pv`/`美购详情页浏览pv`*100,4),'%') as `评价列表页点击pv占比`
,`立即支付点击pv`
,concat(round(`立即支付点击pv`/`美购详情页浏览pv`*100,4),'%') as `立即支付点击pv占比`
,`私信及电话咨询pv`
,concat(round(`私信及电话咨询pv`/`美购详情页浏览pv`*100,4),'%') as `私信及电话咨询pv占比`
,`选择美购项目点击pv`
,concat(round(`选择美购项目点击pv`/`美购详情页浏览pv`*100,4),'%') as `选择美购项目点击pv占比`
,`机构主页点击pv`
,concat(round(`机构主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构主页点击pv占比`
,`机构的其他美购点击pv`
,concat(round(`机构的其他美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构的其他美购点击pv占比`
,`推荐美购点击pv`
,concat(round(`推荐美购点击pv`/`美购详情页浏览pv`*100,4),'%') as `推荐美购点击pv占比`
,`收藏点击pv`
,concat(round(`收藏点击pv`/`美购详情页浏览pv`*100,4),'%') as `收藏点击pv占比`
,`右上角购物车点击pv`
,concat(round(`右上角购物车点击pv`/`美购详情页浏览pv`*100,4),'%') as `右上角购物车点击pv占比`
,`医生主页点击pv`
,concat(round(`医生主页点击pv`/`美购详情页浏览pv`*100,4),'%') as `医生主页点击pv占比`
,`分享点击pv`
,concat(round(`分享点击pv`/`美购详情页浏览pv`*100,4),'%') as `分享点击pv占比`
,`机构导航点击pv`
,concat(round(`机构导航点击pv`/`美购详情页浏览pv`*100,4),'%') as `机构导航点击pv占比`
FROM
(
SELECT t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
,t2.channel as `渠道`
,count(case when type='美购详情页' then cl_id end ) as `美购详情页浏览pv`
,count(case when type='加购物车' then cl_id end ) as `加购物车点击pv`
,count(case when type='评价列表页' then cl_id end ) as `评价列表页点击pv`
,count(case when type='立即支付' then cl_id end ) as `立即支付点击pv`
,count(case when type='私信及电话咨询' then cl_id end ) as `私信及电话咨询pv`
,count(case when type='选择美购项目' then cl_id end ) as `选择美购项目点击pv`
,count(case when type='机构主页' then cl_id end ) as `机构主页点击pv`
,count(case when type='机构的其他美购' then cl_id end ) as `机构的其他美购点击pv`
,count(case when type='推荐美购' then cl_id end ) as `推荐美购点击pv`
,count(case when type='收藏' then cl_id end ) as `收藏点击pv`
,count(case when type='右上角购物车' then cl_id end ) as `右上角购物车点击pv`
,count(case when type='医生主页' then cl_id end ) as `医生主页点击pv`
,count(case when type='分享' then cl_id end ) as `分享点击pv`
,count(case when type='机构导航' then cl_id end ) as `机构导航点击pv`
FROM
(
SELECT click.partition_date,click.cl_id,click.type,mas.active_type,mas.device_type,mas.channel
FROM
( --浏览需要去掉疑似机构刷量的设备
SELECT pv.partition_date as partition_date,pv.cl_id as cl_id,type
FROM
(
SELECT partition_date,cl_id,'美购详情页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)pv
LEFT JOIN
( -- 去掉疑似机构刷量的PV和UV
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
)spam_pv
on pv.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
UNION ALL
SELECT partition_date,cl_id, '加购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_add'
UNION ALL
SELECT partition_date,cl_id, '右上角购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_navbar_cart'
UNION ALL
--点击查看更多评论
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_comment'
UNION ALL
--点击外显的评价标签
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='service_comment_click_tag'
UNION ALL
--点击外显的日记卡片
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_detail_comment_click_diary_card'
UNION ALL
SELECT partition_date,cl_id, '立即支付' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_buy'
UNION ALL
--点击选择更美项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_choose'
UNION ALL
--7.22新版本--点击外露的美购项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='service_sku'
UNION ALL
--7.22新版本--点击查看更多
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_all'
UNION ALL
--点击私信按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
UNION ALL
--7.20之后灰度上线的快捷咨询按钮和预约面诊按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name'] in ('question_tag','appointment')
UNION ALL
--点击机构下面的在线咨询和电话咨询
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea_contact'
AND params['connect_type'] in ('onlineconsult','phone')
UNION ALL
SELECT partition_date,cl_id, '机构主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='organization'
UNION ALL
SELECT partition_date,cl_id, '机构的其他美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_seller_service_item'
UNION ALL
SELECT partition_date,cl_id, '机构导航' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_address'
UNION ALL
SELECT partition_date,cl_id, '推荐美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date <regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_card'
AND params['card_content_type']='service'
AND params['tab_name']='推荐'
UNION ALL
SELECT partition_date,cl_id, '收藏' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_favor'
AND params['favor_type']='service'
AND params['motion']='do'
UNION ALL
SELECT partition_date,cl_id, '医生主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='doctor'
UNION ALL
SELECT partition_date,cl_id, '分享' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= regexp_replace((current_date - interval '90' day),'-','')
AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_click_share'
)click
JOIN
(
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
sqls/bak/demo_java/demo_bak.sql deleted 100644 → 0
View file @ 6e9fff6e
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活'
WHEN active_type IN ('1','2') THEN '新增' END AS active_type
,device_id
FROM online.ml_device_day_active_status
WHERE partition_date between '20200401' and '20200408'
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
limit 10
\ No newline at end of file
sqls/bak/demo_param.sql deleted 100644 → 0
View file @ 6e9fff6e
select
t6.new_tag_name, --新标签name
t6.new_tag_type, --新标签type
t6.new_tag_id, --新标签id
t6.is_display, --是否在app展示
t6.is_online, --是否生效
collect_set(t6.jianyi_word) as jianyi_word, --近义词
collect_set(t6.like_new_tag) as tongyi_word, --同义词
collect_set(t6.fangshi_1f) as fangshi_1st, --一级方式_name
collect_set(t6.fangshi_2d) as fangshi_2nd, --二级方式_name
collect_set(t6.suqiu_1f) as suqiu_1st, --一级诉求_name
collect_set(t6.suqiu_2d) as suqiu_2nd, --二级诉求_name
collect_set(t6.buwei_if) as buwei_1st, --一级部位_name
collect_set(t6.buwei_2d) as buwei_2nd, --二级部位_name
collect_set(t6.old_tag_name) as old_tag_name, --老标签_name
collect_set(t6.old_tag_type) as old_tag_type, --老标签_type
collect_set(t6.old_tag_id) as old_tag_id, --老标签_id
collect_set(t6.fangshi_1f_id) as fangshi_1st_id, --一级方式_id
collect_set(t6.fangshi_2d_id) as fangshi_2nd_id, --二级方式_id
collect_set(t6.suqiu_1f_id) as suqiu_1st_id, --一级诉求_id
collect_set(t6.suqiu_2d_id) as suqiu_2nd_id, --二级诉求_id
collect_set(t6.buwei_if_id) as buwei_1st_id, --一级部位_id
collect_set(t6.buwei_2d_id) as buwei_2nd_id --二级部位_id
from
(
select
t1.name as new_tag_name, --新标签name
t1.tag_type as new_tag_type, --新标签type
t1.id as new_tag_id, --新标签id
t1.is_display as is_display, --是否在app展示
t1.is_online as is_online, --是否生效
t1.homoionym as jianyi_word, --近义词
t2.name as like_new_tag, --同义词
t4.aggregate_type, --对应类型的枚举值
(case when t4.aggregate_type='6' then t4.name else NULL end) as fangshi_1f, --一级方式_name
(case when t4.aggregate_type='2' then t4.name else NULL end) as fangshi_2d, --二级方式_name
(case when t4.aggregate_type='7' then t4.name else NULL end) as suqiu_1f, --一级诉求_name
(case when t4.aggregate_type='8' then t4.name else NULL end) as suqiu_2d, --二级诉求_name
(case when t4.aggregate_type='10' then t4.name else NULL end) as buwei_if, --一级部位_name
(case when t4.aggregate_type='3' then t4.name else NULL end) as buwei_2d, --二级部位_name
t7.old_tag_name as old_tag_name, --老标签_name
t7.old_tag_type as old_tag_type, --老标签_type
t5.old_tag_id as old_tag_id, --老标签_id
(case when t4.aggregate_type='6' then t4.id else NULL end) as fangshi_1f_id, --一级方式_id
(case when t4.aggregate_type='2' then t4.id else NULL end) as fangshi_2d_id, --二级方式_id
(case when t4.aggregate_type='7' then t4.id else NULL end) as suqiu_1f_id, --一级诉求_id
(case when t4.aggregate_type='8' then t4.id else NULL end) as suqiu_2d_id, --二级诉求_id
(case when t4.aggregate_type='10' then t4.id else NULL end) as buwei_if_id, --一级部位_id
(case when t4.aggregate_type='3' then t4.id else NULL end) as buwei_2d_id --二级部位_id
from
(select name,id,is_display,is_online,
case when tag_type in ('1') then '普通分类'
when tag_type in ('2') then '一级分类'
when tag_type in ('3') then '二级分类'
end as tag_type,
collect_list(t1.homoionym_detail) as homoionym
from online.tl_hdfs_api_tag_3_0_view --新标签id，近义词
lateral view explode(json_split(homoionym)) t1 as homoionym_detail --["\u8138\u578b\u77eb\u6b63"]转义 json_split
where partition_date='$V_PARYMD'
group by name,id,is_display,is_online,
case when tag_type in ('1') then '普通分类'
when tag_type in ('2') then '一级分类'
when tag_type in ('3') then '二级分类'
end
) t1
left join
(select name,tag_id from online.tl_hdfs_api_tag_aggregate_view where partition_date='$V_PARYMD') t2 --新标签id，同义词
on (t1.id=t2.tag_id)
left join
(select tag_attr_id as tag_ids,tag_id from online.tl_hdfs_api_tag_attr_tag_view where partition_date='$V_PARYMD') t3 --新标签id，属性ids
on (t1.id=t3.tag_id)
left join
(select name,id,aggregate_type from online.tl_hdfs_api_tag_attr_view where partition_date='$V_PARYMD') t4 --属性id，属性分类
on (t3.tag_ids=t4.id)
left join
(select old_tag_id,tag_id from online.tl_hdfs_api_tag_map_oldtag_view where partition_date='$V_PARYMD') t5
on (t1.id=t5.tag_id)
left join
(select id,name as old_tag_name,
case when tag_type in ('1') then '一级分类'
when tag_type in ('2') then '二级分类'
when tag_type in ('3') then '三级分类'
when tag_type in ('4') then '城市'
when tag_type in ('5') then '自由添加'
when tag_type in ('6') then '医生'
when tag_type in ('7') then '医院'
when tag_type in ('8') then '频道'
when tag_type in ('9') then '省份'
when tag_type in ('10') then '国家'
when tag_type in ('11') then '运营标签'
end as old_tag_type
from online.tl_hdfs_api_tag_view where partition_date='$V_PARYMD') t7
on (t7.id=t5.old_tag_id)
)t6
group by t6.new_tag_name,t6.new_tag_type,t6.new_tag_id,t6.is_display,t6.is_online
\ No newline at end of file
sqls/bak/demo_spark_param/demo_spark_param.sql deleted 100644 → 0
View file @ 6e9fff6e
select
t6.new_tag_name, --新标签name
t6.new_tag_type, --新标签type
t6.new_tag_id, --新标签id
t6.is_display, --是否在app展示
t6.is_online, --是否生效
collect_set(t6.jianyi_word) as jianyi_word, --近义词
collect_set(t6.like_new_tag) as tongyi_word, --同义词
collect_set(t6.fangshi_1f) as fangshi_1st, --一级方式_name
collect_set(t6.fangshi_2d) as fangshi_2nd, --二级方式_name
collect_set(t6.suqiu_1f) as suqiu_1st, --一级诉求_name
collect_set(t6.suqiu_2d) as suqiu_2nd, --二级诉求_name
collect_set(t6.buwei_if) as buwei_1st, --一级部位_name
collect_set(t6.buwei_2d) as buwei_2nd, --二级部位_name
collect_set(t6.old_tag_name) as old_tag_name, --老标签_name
collect_set(t6.old_tag_type) as old_tag_type, --老标签_type
collect_set(t6.old_tag_id) as old_tag_id, --老标签_id
collect_set(t6.fangshi_1f_id) as fangshi_1st_id, --一级方式_id
collect_set(t6.fangshi_2d_id) as fangshi_2nd_id, --二级方式_id
collect_set(t6.suqiu_1f_id) as suqiu_1st_id, --一级诉求_id
collect_set(t6.suqiu_2d_id) as suqiu_2nd_id, --二级诉求_id
collect_set(t6.buwei_if_id) as buwei_1st_id, --一级部位_id
collect_set(t6.buwei_2d_id) as buwei_2nd_id --二级部位_id
from
(
select
t1.name as new_tag_name, --新标签name
t1.tag_type as new_tag_type, --新标签type
t1.id as new_tag_id, --新标签id
t1.is_display as is_display, --是否在app展示
t1.is_online as is_online, --是否生效
t1.homoionym as jianyi_word, --近义词
t2.name as like_new_tag, --同义词
t4.aggregate_type, --对应类型的枚举值
(case when t4.aggregate_type='6' then t4.name else NULL end) as fangshi_1f, --一级方式_name
(case when t4.aggregate_type='2' then t4.name else NULL end) as fangshi_2d, --二级方式_name
(case when t4.aggregate_type='7' then t4.name else NULL end) as suqiu_1f, --一级诉求_name
(case when t4.aggregate_type='8' then t4.name else NULL end) as suqiu_2d, --二级诉求_name
(case when t4.aggregate_type='10' then t4.name else NULL end) as buwei_if, --一级部位_name
(case when t4.aggregate_type='3' then t4.name else NULL end) as buwei_2d, --二级部位_name
t7.old_tag_name as old_tag_name, --老标签_name
t7.old_tag_type as old_tag_type, --老标签_type
t5.old_tag_id as old_tag_id, --老标签_id
(case when t4.aggregate_type='6' then t4.id else NULL end) as fangshi_1f_id, --一级方式_id
(case when t4.aggregate_type='2' then t4.id else NULL end) as fangshi_2d_id, --二级方式_id
(case when t4.aggregate_type='7' then t4.id else NULL end) as suqiu_1f_id, --一级诉求_id
(case when t4.aggregate_type='8' then t4.id else NULL end) as suqiu_2d_id, --二级诉求_id
(case when t4.aggregate_type='10' then t4.id else NULL end) as buwei_if_id, --一级部位_id
(case when t4.aggregate_type='3' then t4.id else NULL end) as buwei_2d_id --二级部位_id
from
(select name,id,is_display,is_online,
case when tag_type in ('1') then '普通分类'
when tag_type in ('2') then '一级分类'
when tag_type in ('3') then '二级分类'
end as tag_type,
collect_list(t1.homoionym_detail) as homoionym
from online.tl_hdfs_api_tag_3_0_view --新标签id，近义词
lateral view explode(json_split(homoionym)) t1 as homoionym_detail --["\u8138\u578b\u77eb\u6b63"]转义 json_split
where partition_date='$V_PARYMD'
group by name,id,is_display,is_online,
case when tag_type in ('1') then '普通分类'
when tag_type in ('2') then '一级分类'
when tag_type in ('3') then '二级分类'
end
) t1
left join
(select name,tag_id from online.tl_hdfs_api_tag_aggregate_view where partition_date='$V_PARYMD') t2 --新标签id，同义词
on (t1.id=t2.tag_id)
left join
(select tag_attr_id as tag_ids,tag_id from online.tl_hdfs_api_tag_attr_tag_view where partition_date='$V_PARYMD') t3 --新标签id，属性ids
on (t1.id=t3.tag_id)
left join
(select name,id,aggregate_type from online.tl_hdfs_api_tag_attr_view where partition_date='$V_PARYMD') t4 --属性id，属性分类
on (t3.tag_ids=t4.id)
left join
(select old_tag_id,tag_id from online.tl_hdfs_api_tag_map_oldtag_view where partition_date='$V_PARYMD') t5
on (t1.id=t5.tag_id)
left join
(select id,name as old_tag_name,
case when tag_type in ('1') then '一级分类'
when tag_type in ('2') then '二级分类'
when tag_type in ('3') then '三级分类'
when tag_type in ('4') then '城市'
when tag_type in ('5') then '自由添加'
when tag_type in ('6') then '医生'
when tag_type in ('7') then '医院'
when tag_type in ('8') then '频道'
when tag_type in ('9') then '省份'
when tag_type in ('10') then '国家'
when tag_type in ('11') then '运营标签'
end as old_tag_type
from online.tl_hdfs_api_tag_view where partition_date='$V_PARYMD') t7
on (t7.id=t5.old_tag_id)
)t6
group by t6.new_tag_name,t6.new_tag_type,t6.new_tag_id,t6.is_display,t6.is_online
\ No newline at end of file
sqls/bak/meigou-detail-page/meigou-detail-page-dispense-uv.sql deleted 100644 → 0
View file @ 6e9fff6e
SELECT `日期`,`设备类型`,`活跃类型`,`渠道`
,`美购详情页浏览uv`
,`加购物车点击uv`
,concat(round(`加购物车点击uv`/`美购详情页浏览uv`*100,4),'%') as `加购物车点击uv占比`
,`评价列表页点击uv`
,concat(round(`评价列表页点击uv`/`美购详情页浏览uv`*100,4),'%') as `评价列表页点击uv占比`
,`立即支付点击uv`
,concat(round(`立即支付点击uv`/`美购详情页浏览uv`*100,4),'%') as `立即支付点击uv占比`
,`私信及电话咨询uv`
,concat(round(`私信及电话咨询uv`/`美购详情页浏览uv`*100,4),'%') as `私信及电话咨询uv占比`
,`选择美购项目点击uv`
,concat(round(`选择美购项目点击uv`/`美购详情页浏览uv`*100,4),'%') as `选择美购项目点击uv占比`
,`机构主页点击uv`
,concat(round(`机构主页点击uv`/`美购详情页浏览uv`*100,4),'%') as `机构主页点击uv占比`
,`机构的其他美购点击uv`
,concat(round(`机构的其他美购点击uv`/`美购详情页浏览uv`*100,4),'%') as `机构的其他美购点击uv占比`
,`推荐美购点击uv`
,concat(round(`推荐美购点击uv`/`美购详情页浏览uv`*100,4),'%') as `推荐美购点击uv占比`
,`收藏点击uv`
,concat(round(`收藏点击uv`/`美购详情页浏览uv`*100,4),'%') as `收藏点击uv占比`
,`右上角购物车点击uv`
,concat(round(`右上角购物车点击uv`/`美购详情页浏览uv`*100,4),'%') as `右上角购物车点击uv占比`
,`医生主页点击uv`
,concat(round(`医生主页点击uv`/`美购详情页浏览uv`*100,4),'%') as `医生主页点击uv占比`
,`分享点击uv`
,concat(round(`分享点击uv`/`美购详情页浏览uv`*100,4),'%') as `分享点击uv占比`
,`机构导航点击uv`
,concat(round(`机构导航点击uv`/`美购详情页浏览uv`*100,4),'%') as `机构导航点击uv占比`
FROM
(
SELECT t1.partition_date as `日期`
,t1.device_type as `设备类型`
,t1.active_type as `活跃类型`
,t2.channel as `渠道`
,count(distinct case when type='美购详情页' then cl_id end ) as `美购详情页浏览uv`
,count(distinct case when type='加购物车' then cl_id end ) as `加购物车点击uv`
,count(distinct case when type='评价列表页' then cl_id end ) as `评价列表页点击uv`
,count(distinct case when type='立即支付' then cl_id end ) as `立即支付点击uv`
,count(distinct case when type='私信及电话咨询' then cl_id end ) as `私信及电话咨询uv`
,count(distinct case when type='选择美购项目' then cl_id end ) as `选择美购项目点击uv`
,count(distinct case when type='机构主页' then cl_id end ) as `机构主页点击uv`
,count(distinct case when type='机构的其他美购' then cl_id end ) as `机构的其他美购点击uv`
,count(distinct case when type='推荐美购' then cl_id end ) as `推荐美购点击uv`
,count(distinct case when type='收藏' then cl_id end ) as `收藏点击uv`
,count(distinct case when type='右上角购物车' then cl_id end ) as `右上角购物车点击uv`
,count(distinct case when type='医生主页' then cl_id end ) as `医生主页点击uv`
,count(distinct case when type='分享' then cl_id end ) as `分享点击uv`
,count(distinct case when type='机构导航' then cl_id end ) as `机构导航点击uv`
FROM
(
SELECT click.partition_date,click.cl_id,click.type,mas.active_type,mas.device_type,mas.channel
FROM
( --浏览需要去掉疑似机构刷量的设备
SELECT pv.partition_date as partition_date,pv.cl_id as cl_id,type
FROM
(
SELECT partition_date,cl_id,'美购详情页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_view'
)pv
LEFT JOIN
(
SELECT cl_id
FROM online.ml_hospital_spam_pv_day
WHERE partition_date>='20180402' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
UNION ALL
SELECT cl_id
FROM online.ml_hospital_spam_pv_month
WHERE partition_date>='20171101' AND partition_date<regexp_replace((current_date),'-','')
AND pv_ratio>=0.95
)spam_pv
on pv.cl_id=spam_pv.cl_id
WHERE spam_pv.cl_id IS NULL
UNION ALL
SELECT partition_date,cl_id, '加购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_add'
UNION ALL
SELECT partition_date,cl_id, '右上角购物车' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_navbar_cart'
UNION ALL
--点击查看更多评论
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_comment'
UNION ALL
--点击外显的评价标签
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='service_comment_click_tag'
UNION ALL
--点击外显的日记卡片
SELECT partition_date,cl_id, '评价列表页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_detail_comment_click_diary_card'
UNION ALL
SELECT partition_date,cl_id, '立即支付' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='welfare_multiattribute_click_buy'
UNION ALL
--点击选择更美项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_choose'
UNION ALL
--7.22新版本--点击外露的美购项目
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='service_sku'
UNION ALL
--7.22新版本--点击查看更多
SELECT partition_date,cl_id, '选择美购项目' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name']='sku_all'
UNION ALL
--点击私信按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_message'
UNION ALL
--7.20之后灰度上线的快捷咨询按钮和预约面诊按钮
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_button'
AND params['button_name'] in ('question_tag','appointment')
UNION ALL
--点击机构下面的在线咨询和电话咨询
SELECT partition_date,cl_id, '私信及电话咨询' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea_contact'
AND params['connect_type'] in ('onlineconsult','phone')
UNION ALL
SELECT partition_date,cl_id, '机构主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='organization'
UNION ALL
SELECT partition_date,cl_id, '机构的其他美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_seller_service_item'
UNION ALL
SELECT partition_date,cl_id, '机构导航' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_address'
UNION ALL
SELECT partition_date,cl_id, '推荐美购' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date <regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_card'
AND params['card_content_type']='service'
AND params['tab_name']='推荐'
UNION ALL
SELECT partition_date,cl_id, '收藏' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='on_click_favor'
AND params['favor_type']='service'
AND params['motion']='do'
UNION ALL
SELECT partition_date,cl_id, '医生主页' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
--AND page_name = 'welfare_detail'
AND action='welfare_detail_click_curearea'
AND params['cure_type']='doctor'
UNION ALL
SELECT partition_date,cl_id, '分享' as type
FROM online.bl_hdfs_maidian_updates
WHERE partition_date >= '20191201' AND partition_date < regexp_replace((current_date),'-','')
AND page_name = 'welfare_detail'
AND action='page_click_share'
)click
JOIN
(
SELECT partition_date
,device_os_type AS device_type
,CASE WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type IN ('1','2') THEN '新增设备' END AS active_type
,array(CASE WHEN tmp.time = 'AI' THEN 'AI' ELSE '其他' END , '合计') as channel
,device_id
FROM online.ml_device_day_active_status
LEFT JOIN
(SELECT phone,time
FROM offline.tmp_zhx_20191227
WHERE flag='0204_danlei_channel')tmp
on first_channel_source_type=tmp.phone
WHERE partition_date>=regexp_replace((current_date - interval '90' day),'-','')
AND partition_date<regexp_replace((current_date),'-','')
AND active_type IN ('1','2','4')
AND first_channel_source_type not IN ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei','','unknown')
AND first_channel_source_type not LIKE 'promotion\_jf\_%'
)mas
ON click.cl_id=mas.device_id
AND click.partition_date=mas.partition_date
)t1
LATERAL VIEW explode(t1.channel) t2 AS channel
GROUP BY t1.partition_date,t1.device_type,t1.active_type,t2.channel
)T1
ORDER BY `日期` desc,`设备类型`,`活跃类型`,`渠道`
\ No newline at end of file
sqls/daily_recommend_strategy/daily_recommend_strategy.sql deleted 100644 → 0
View file @ 6e9fff6e
SELECT
day_id as `日期`,
device_os_type as `设备类型`,
active_type as `活跃类型`,
card_content_type as `卡片类型`,
recommend_type as `推荐类型`,
home_ctr as `首页卡片点击PV/首页卡片精准曝光PV`,
second_rate as `来自首页推荐内容卡片的的总商业二跳pv/首页卡片精准曝光PV`,
first_rate as `来自首页推荐内容卡片的的总商业二跳pv/首页卡片点击PV`,
card_click as `首页卡片点击PV`,
card_exposure as `首页卡片精准曝光PV`,
total_second_click as `来自I的总商业二跳pv（包含M列到T列）`,
avg_page_stay as `来自I的单PV平均浏览时长`,
navbar_search as `来自I的搜索框+搜索按钮点击PV`,
highlight_word as `来自I的文内搜索点击PV`,
self_welfare_card as `来自I的商品卡片点击PV`,
recommend_welfare_card as `来自I的推荐商品+查看全部商品点击pv`,
recommend_content_card as `来自I的推荐内容点击pv`,
'未配置' as `来自I的推荐专题点击pv`,
'未上线' as `来自I的转诊点击pv`,
'未上线' as `来自I的视频面诊点击pv`
FROM pm.tl_pm_recommend_strategy_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc ,`设备类型`,`活跃类型`,`卡片类型`,`推荐类型`
sqls/search_related/search_input_type.sql deleted 100644 → 0
View file @ 6e9fff6e
SELECT
day_id AS `日期`
,query AS `关键词`
,search_pv AS `搜索量`
,search_uv AS `搜索uv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_pv*100/t2.page_more_pv,2) ,'%') else 0 end, 0) as `搜索综合结果页内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_10_pv*100/t2.page_more_pv,2) ,'%') else 0 end,0) as `搜索综合结果页前10内容卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_diary_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5日记卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_answer_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5问答卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_more_pv,0)>0 then concat(round(t2.more_click_5_topic_pv*100/t2.page_more_pv,2),'%') else 0 end,0) as `搜索综合结果页前5帖子卡片点击PV/搜索综合结果页浏览pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_10_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前10卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_wel_pv,0)>0 then concat(round(t2.wel_click_5_pv*100/t2.page_wel_pv,2),'%') else 0 end,0) as `搜索美购结果页前5卡片点击PV/搜索美购结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_10_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前10卡片点击PV/搜索日记结果页pv`
,NVL(case when NVL(t2.page_diary_pv,0)>0 then concat(round(t2.diary_click_5_pv*100/t2.page_diary_pv,2),'%') else 0 end,0) as `搜索日记结果页前5卡片点击PV/搜索日记结果页pv`
,search_more_pv AS `综合tab下的搜索的点击pv`
,page_more_pv AS `搜索综合结果页浏览PV`
,more_click_pv AS `搜索综合结果页内容卡片点击PV`
,more_click_10_pv AS `搜索综合结果页前10内容卡片点击PV`
,more_click_5_diary_pv AS `搜索综合结果页前5日记卡片点击PV`
,more_click_5_answer_pv AS `搜索综合结果页前5问答卡片点击PV`
,more_click_5_topic_pv AS `搜索综合结果页前5帖子卡片点击PV`
,search_wel_pv AS `美购tab下的搜索的点击pv`
,page_wel_pv AS `搜索美购结果页浏览pv`
,wel_click_pv AS `搜索美购结果页卡片点击PV`
,wel_click_10_pv AS `搜索美购结果页前10卡片点击PV`
,wel_click_5_pv AS `搜索美购结果页前5卡片点击PV`
,search_diary_pv AS `日记tab下的搜索的点击pv`
,page_diary_pv AS `搜索日记结果页浏览pv`
,diary_click_pv AS `搜索日记结果页卡片点击PV`
,diary_click_10_pv AS `搜索日记结果页前10卡片点击PV`
,diary_click_5_pv AS `搜索日记结果页前5卡片点击PV`
FROM pm.tl_pm_search_result_click_rate_d
WHERE partition_day>='20200619' and partition_day<=regexp_replace(DATE_SUB(current_date,1) ,'-','')
order by `日期` desc,`搜索量` desc,`关键词`
sqls/search_related_insert/search_click_path_insert.sql deleted 100644 → 0
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_search_click_path_d PARTITION (PARTITION_DAY = #partition_day)
SELECT day_id as day_id
,device_os_type as device_os_type
,active_type as active_type
,v.search_entrance as search_entrance
,sum(search_pv) as click_search_pv
,sum(search_uv) as click_search_uv
FROM
(
SELECT day_id,device_os_type,active_type,array(search_entrance,'合计') as search_entrance
,search_pv,search_uv
FROM
(
SELECT t1.partition_date as day_id
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end as search_entrance
,count(cl_id) as search_pv
,count(distinct cl_id) as search_uv
FROM
(
SELECT partition_date,device_os_type
,case WHEN active_type = '4' THEN '老活跃设备'
WHEN active_type in ('1','2') then '新增设备' END as active_type
,device_id
FROM online.ml_device_day_active_status
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND active_type in ('1','2','4')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t1
join
(
SELECT partition_date,cl_id,page_name,action,params['referrer_link'] as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action in ('do_search','on_click_navbar_search')
union all
SELECT partition_date,cl_id,null as page_name,'大家都在看' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date,cl_id,null as page_name,'热搜词' as action, null as referrer_link
FROM online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date,cl_id,null as page_name,'详情页高亮词' as action,null as referrer_link
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(DATE_SUB(current_date,1) ,'-','')
and action='on_click_card'
and params['card_type']='highlight_word'
)t2
on t1.partition_date=t2.partition_date and t1.device_id=t2.cl_id
group by t1.partition_date
,t1.device_os_type
,t1.active_type
,case when action='on_click_navbar_search' and page_name='home' then '首页搜索框'
when action='on_click_navbar_search' and page_name='welfare_home' then '美购首页搜索框'
when action='on_click_navbar_search' and page_name='category' then '品类聚合页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["home","category"]' then '来自品类聚合的美购列表页搜索框'
when action='on_click_navbar_search' and page_name='welfare_list' and referrer_link ='["welfare_home"]' then '来自美购首页的美购列表页搜索框'
when action in ('do_search','on_click_navbar_search')
and page_name in ('diary_detail','topic_detail','post_detail','user_post_detail'
,'doctor_post_detail','question_detail','answer_detail','question_answer_detail') then '内容详情页搜索框'
when action ='详情页高亮词' then '详情页高亮词搜索'
when action='大家都在看' then '首页feed大家都在看'
when action='热搜词' then '美购首页热搜词' else null end
having search_entrance is not null
)t
)n
LATERAL VIEW explode(n.search_entrance) v AS search_entrance
group by day_id,device_os_type,active_type,v.search_entrance
sqls/search_related_insert/search_result_click_rate_insert.sql deleted 100644 → 0
View file @ 6e9fff6e
INSERT OVERWRITE TABLE pm.tl_pm_search_result_click_rate_d PARTITION (PARTITION_DAY = #partition_day)
--搜索关键词日报,取近7天数据，每天取关键词搜索量前500
select t1.partition_date as day_id
,t1.query as query
,NVL(t1.search_pv,0) as search_pv
,NVL(t1.search_uv,0) as search_uv
,NVL(t2.search_more_pv,0) as search_more_pv
,NVL(t2.page_more_pv,0) as page_more_pv
,NVL(t2.more_click_pv,0) as more_click_pv
,NVL(t2.more_click_10_pv,0) as more_click_10_pv
,NVL(t2.more_click_5_diary_pv,0) as more_click_5_diary_pv
,NVL(t2.more_click_5_answer_pv,0) as more_click_5_answer_pv
,NVL(t2.more_click_5_topic_pv,0) as more_click_5_topic_pv
,NVL(t2.search_wel_pv,0) as search_wel_pv
,NVL(t2.page_wel_pv,0) as page_wel_pv
,NVL(t2.wel_click_pv,0) as wel_click_pv
,NVL(t2.wel_click_10_pv,0) as wel_click_10_pv
,NVL(t2.wel_click_5_pv,0) as wel_click_5_pv
,NVL(t2.search_diary_pv,0) as search_diary_pv
,NVL(t2.page_diary_pv,0) as page_diary_pv
,NVL(t2.diary_click_pv,0) as diary_click_pv
,NVL(t2.diary_click_10_pv,0) as diary_click_10_pv
,NVL(t2.diary_click_5_pv,0) as diary_click_5_pv
,NVL(t2.search_more_uv,0) as search_more_uv
,NVL(t2.page_more_uv,0) as page_more_uv
,NVL(t2.more_click_uv,0) as more_click_uv
,NVL(t2.more_click_10_uv,0) as more_click_10_uv
,NVL(t2.more_click_5_diary_uv,0) as more_click_5_diary_uv
,NVL(t2.more_click_5_answer_uv,0) as more_click_5_answer_uv
,NVL(t2.more_click_5_topic_uv,0) as more_click_5_topic_uv
,NVL(t2.search_wel_uv,0) as search_wel_uv
,NVL(t2.page_wel_uv,0) as page_wel_uv
,NVL(t2.wel_click_uv,0) as wel_click_uv
,NVL(t2.wel_click_10_uv,0) as wel_click_10_uv
,NVL(t2.wel_click_5_uv,0) as wel_click_5_uv
,NVL(t2.search_diary_uv,0) as search_diary_uv
,NVL(t2.page_diary_uv,0) as page_diary_uv
,NVL(t2.diary_click_uv,0) as diary_click_uv
,NVL(t2.diary_click_10_uv,0) as diary_click_10_uv
,NVL(t2.diary_click_5_uv,0) as diary_click_5_uv
from
(--总搜索量
select partition_date
,query
,search_pv
,search_uv
from
(
select t1.partition_date
,query
,count(t1.cl_id) as search_pv
,count(distinct t1.cl_id) as search_uv
,row_number() over(partition by partition_date order by count(t1.cl_id) desc) as rank
from
(
select partition_date
,params['query'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('do_search','search_result_click_search')
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND params['in_page_pos']='猜你喜欢'
AND params['tab_name']='精选'
AND params['card_type']='search_word'
--AND page_name='home' android的page_name为空
union all
SELECT partition_date
,params['card_name'] as query
,cl_id
FROM online.bl_hdfs_maidian_updates
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
AND action = 'on_click_card'
AND page_name='welfare_home'
AND params['card_type'] ='search_word'
AND params['in_page_pos']='大家都在搜'
union all
select partition_date
,params['card_name'] as query
,cl_id
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and int(split(app_version,'\\.')[1]) >= 27
and action='on_click_card'
and params['card_type']='highlight_word'
)t1
join
(
SELECT device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)t2
on t1.cl_id=t2.device_id
group by t1.partition_date,query
)t
where rank<=300
)t1
left join
--综合、美购、日记页面浏览pv
(
select t1.partition_date,t1.query
,sum(case when tab_name='综合' then t1.pv end) as search_more_pv
,count(distinct case when tab_name='综合' then t1.cl_id end) as search_more_uv
,sum(case when tab_name='美购' then t1.pv end) as search_wel_pv
,count(distinct case when tab_name='美购' then t1.cl_id end) as search_wel_uv
,sum(case when tab_name='日记' then t1.pv end) as search_diary_pv
,count(distinct case when tab_name='日记' then t1.cl_id end) as search_diary_uv
,sum(case when t2.page_name='综合' then t2.pv end) as page_more_pv
,count(distinct case when t2.page_name='综合' then t2.cl_id end) as page_more_uv
,sum(case when t2.page_name='美购' then t2.pv end) as page_wel_pv
,count(distinct case when t2.page_name='美购' then t2.cl_id end) as page_wel_uv
,sum(case when t2.page_name='日记' then t2.pv end) as page_diary_pv
,count(distinct case when t2.page_name='日记' then t2.cl_id end) as page_diary_uv
,sum(case when t3.page_name='综合' then t3.pv end) as more_click_pv
,count(distinct case when t3.page_name='综合' then t3.cl_id end) as more_click_uv
,sum(case when t3.page_name='美购' then t3.pv end) as wel_click_pv
,count(distinct case when t3.page_name='美购' then t3.cl_id end) as wel_click_uv
,sum(case when t3.page_name='日记' then t3.pv end) as diary_click_pv
,count(distinct case when t3.page_name='日记' then t3.cl_id end) as diary_click_uv
,sum(case when t4.page_name='综合' then t4.pv end) as more_click_10_pv
,count(distinct case when t4.page_name='综合' then t4.cl_id end) as more_click_10_uv
,sum(case when t4.page_name='美购' then t4.pv end) as wel_click_10_pv
,count(distinct case when t4.page_name='美购' then t4.cl_id end) as wel_click_10_uv
,sum(case when t4.page_name='日记' then t4.pv end) as diary_click_10_pv
,count(distinct case when t4.page_name='日记' then t4.cl_id end) as diary_click_10_uv
,sum(case when t5.page_name='美购' then t5.pv end) as wel_click_5_pv
,count(distinct case when t5.page_name='美购' then t5.cl_id end) as wel_click_5_uv
,sum(case when t5.page_name='日记' then t5.pv end) as diary_click_5_pv
,count(distinct case when t5.page_name='日记' then t5.cl_id end) as diary_click_5_uv
,sum(case when t6.page_name='综合' then t6.pv end) as more_click_5_diary_pv
,count(distinct case when t6.page_name='综合' then t6.cl_id end) as more_click_5_diary_uv
,sum(case when t7.page_name='综合' then t7.pv end) as more_click_5_answer_pv
,count(distinct case when t7.page_name='综合' then t7.cl_id end) as more_click_5_answer_uv
,sum(case when t8.page_name='综合' then t8.pv end) as more_click_5_topic_pv
,count(distinct case when t8.page_name='综合' then t8.cl_id end) as more_click_5_topic_uv
from
(--各tab搜索
select partition_date
,params['query'] as query
,cl_id
,params['tab'] as tab_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and params['tab'] in ('综合','美购','日记')
and action in ('search_result_click_tab','search_result_click_search')
group by partition_date,params['query'],cl_id,params['tab']
)t1
left join
(--结果页浏览
select partition_date
,get_json_object(params['extra_param'],'$.query') as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
and action ='page_view'
group by partition_date
,get_json_object(params['extra_param'],'$.query')
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
and t1.cl_id=t2.cl_id
and t1.tab_name=t2.page_name
left join
(--内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t3
on t2.partition_date=t3.partition_date
and t2.query=t3.query
and t2.cl_id=t3.cl_id
and t2.page_name=t3.page_name
left join
(--前10内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and action in ('search_result_click_infomation_item','on_click_topic_card','on_click_diary_card','search_result_welfare_click_item')
and params['position'] in (0,1,2,3,4,5,6,7,8,9)
and page_name in ('search_result_more','search_result_welfare','search_result_diary')
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_more' then '综合'
when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t4
on t2.partition_date=t4.partition_date
and t2.query=t4.query
and t2.cl_id=t4.cl_id
and t2.page_name=t4.page_name
left join
(--日记、美购前5内容卡片点击
select partition_date
,params['query'] as query
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end as page_name
,count(1) as pv
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and ( action = 'on_click_diary_card' and page_name='search_result_diary'
or action ='search_result_welfare_click_item' and page_name='search_result_welfare')
and params['position'] in (0,1,2,3,4)
group by partition_date
,params['query']
,cl_id
,case when page_name='search_result_welfare' then '美购'
when page_name='search_result_diary' then '日记' end
)t5
on t2.partition_date=t5.partition_date
and t2.query=t5.query
and t2.cl_id=t5.cl_id
and t2.page_name=t5.page_name
left join
(--综搜页日记卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='diary'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item'and params['business_type']in('diary','0')
or action ='on_click_diary_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t6
on t2.partition_date=t6.partition_date
and t2.query=t6.query
and t2.cl_id=t6.cl_id
and t2.page_name=t6.page_name
left join
(--综搜页回答卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='answer'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action ='search_result_click_infomation_item'
and params['business_type']in ('answer','5')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t7
on t2.partition_date=t7.partition_date
and t2.query=t7.query
and t2.cl_id=t7.cl_id
and t2.page_name=t7.page_name
left join
(--综搜页帖子卡片前5
select c.partition_date,c.query,c.cl_id,'综合' as page_name,count(1) as pv
from
(
SELECT partition_date,query,position
FROM
(
select partition_date
,params['query'] as query
,cast(absolute_position as int) as position
,row_number() over(partition by params['query'], partition_date order by cast (absolute_position as int) asc) as rank
from online.ml_community_precise_exposure_detail
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and action in ('page_precise_exposure','home_choiceness_card_exposure') --7745版本action改为page_precise_exposure
and is_exposure = '1' ----精准曝光
and card_content_type='user_post'
group by partition_date,params['query'],cast(absolute_position as int)
)a
where rank<=5
)b
join
(
select partition_date
,params['query'] as query
,cl_id
,params['position'] as position
from online.bl_hdfs_maidian_updates
where partition_date = regexp_replace(date_sub(current_date,1),'-','')
and page_name='search_result_more'
and (action ='search_result_click_infomation_item' and params['business_type'] in ('post','user_post','doctor_post','11')
or action='on_click_topic_card')
)c
on b.partition_date=c.partition_date
and b.query=c.query
and b.position=c.position
group by c.partition_date,c.query,c.cl_id
)t8
on t2.partition_date=t8.partition_date
and t2.query=t8.query
and t2.cl_id=t8.cl_id
and t2.page_name=t8.page_name
join
(
select device_id
from online.ml_device_history_detail
WHERE partition_date = regexp_replace(date_sub(current_date,1),'-','')
and first_channel_source_type not in ('yqxiu1','yqxiu2','yqxiu3','yqxiu4','yqxiu5','mxyc1','mxyc2','mxyc3'
,'wanpu','jinshan','jx','maimai','zhuoyi','huatian','suopingjingling','mocha','mizhe','meika','lamabang'
,'js-az1','js-az2','js-az3','js-az4','js-az5','jfq-az1','jfq-az2','jfq-az3','jfq-az4','jfq-az5','toufang1'
,'toufang2','toufang3','toufang4','toufang5','toufang6','TF-toufang1','TF-toufang2','TF-toufang3','TF-toufang4'
,'TF-toufang5','tf-toufang1','tf-toufang2','tf-toufang3','tf-toufang4','tf-toufang5','benzhan','promotion_aso100'
,'promotion_qianka','promotion_xiaoyu','promotion_dianru','promotion_malioaso','promotion_malioaso-shequ'
,'promotion_shike','promotion_julang_jl03','promotion_zuimei')
AND first_channel_source_type not like 'promotion\_jf\_%'
)dev
on t1.cl_id=dev.device_id
left join
( -- 去掉疑似机构刷量的PV和UV
SELECT distinct device_id
FROM ml.ml_d_ct_dv_devicespam_d
WHERE partition_day=regexp_replace(date_sub(current_date,1),'-','')
)spam_pv
on t2.cl_id=spam_pv.device_id
WHERE spam_pv.device_id IS NULL
group by t1.partition_date,t1.query
)t2
on t1.partition_date=t2.partition_date
and t1.query=t2.query
order by day_id desc,search_pv desc,query
tables/create_recommend_daily_data.sql deleted 100644 → 0
View file @ 6e9fff6e
--***************************************************************
--*脚本名称: create_pm_c_op_co_content_dimen_d.sql
--*功能: 内容日报
--*业务名称: pm
--*输入数据:
--*作者: weiyimin@igengmei.com
--*更新时间: 2020-5-25 11:00
--***************************************************************
--设置全局变量&UDF
SET mapreduce.job.queuename=data;
--使用bl数据库
USE pm;
--创建BL层内部表
CREATE TABLE IF NOT EXISTS pm.tl_pm_recommend_strategy_d
(
day_id string comment '{"chs_name":"当天日期","description":"","etl":"","value":"","remark":""}',
device_os_type string comment '{"chs_name":"设备类型","description":"","etl":"","value":"","remark":""}',
active_type string comment '{"chs_name":"活跃类型","description":"","etl":"","value":"","remark":""}',
card_content_type string comment '{"chs_name":"卡片类型","description":"","etl":"","value":"","remark":""}',
recommend_type string comment '{"chs_name":"推荐类型","description":"","etl":"","value":"","remark":""}',
home_ctr string comment '{"chs_name":"首页卡片点击PV/首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
second_rate string comment '{"chs_name":"来自首页推荐内容卡片的的总商业二跳pv/首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
first_rate string comment '{"chs_name":"来自首页推荐内容卡片的的总商业二跳pv/首页卡片点击PV","description":"","etl":"","value":"","remark":""}',
card_click BIGINT comment '{"chs_name":"首页卡片点击PV","description":"","etl":"","value":"","remark":""}',
card_exposure BIGINT comment '{"chs_name":"首页卡片精准曝光PV","description":"","etl":"","value":"","remark":""}',
total_second_click BIGINT comment '{"chs_name":"来自首页卡片的总商业二跳pv（包含页面上搜索及卡片点击）","description":"","etl":"","value":"","remark":""}',
avg_page_stay double comment '{"chs_name":"来自首页卡片的单PV平均浏览时长","description":"","etl":"","value":"","remark":""}',
navbar_search BIGINT comment '{"chs_name":"来自首页卡片的搜索框+搜索按钮点击PV","description":"","etl":"","value":"","remark":""}',
highlight_word BIGINT comment '{"chs_name":"来自首页卡片的文内搜索点击PV","description":"","etl":"","value":"","remark":""}',
self_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的商品卡片点击PV","description":"","etl":"","value":"","remark":""}',
recommend_welfare_card BIGINT comment '{"chs_name":"来自首页卡片的推荐商品+查看全部商品点击pv","description":"","etl":"","value":"","remark":""}',
recommend_content_card BIGINT comment '{"chs_name":"来自首页卡片的推荐内容点击pv","description":"","etl":"","value":"","remark":""}',
recommend_special_card BIGINT comment '{"chs_name":"来自首页卡片的推荐专题点击pv","description":"","etl":"","value":"","remark":""}',
transfer_card BIGINT comment '{"chs_name":"来自首页卡片的转诊点击pv","description":"","etl":"","value":"","remark":""}',
video_consultation BIGINT comment '{"chs_name":"来自首页卡片的视频面诊点击pv","description":"","etl":"","value":"","remark":""}'
)comment '首页推荐策略日报'
PARTITIONED BY (PARTITION_DAY STRING comment '分区日期')
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
COLLECTION ITEMS TERMINATED BY '\002'
MAP KEYS TERMINATED BY '\003'
LINES TERMINATED BY '\n'
STORED AS TEXTFILE;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment