Skip to content

D
data-dqmonitor
Commit 404ad9fa authored by Pengfei Xue's avatar Pengfei Xue
Browse files
Options

add chain rate

parent 31f13811
Show whitespace changes
Inline Side-by-side
Showing with 217 additions and 36 deletions
.idea/misc.xml
View file @ 404ad9fa
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8.0_212" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/classes" />
</component>
</project>
\ No newline at end of file
.idea/workspace.xml
View file @ 404ad9fa
......@@ -2,7 +2,10 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="19be5465-2539-4713-8ab1-de44752080af" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/pvCheker.scala" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/pvCheker.scala" afterDir="false" />
<change afterPath="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/actionCheck.scala" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Main.scala" beforeDir="false" afterPath="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Main.scala" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/classes/" />
<ignored path="$PROJECT_DIR$/target/" />
......@@ -12,22 +15,79 @@
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="DatabaseView">
<option name="SHOW_INTERMEDIATE" value="true" />
<option name="GROUP_DATA_SOURCES" value="true" />
<option name="GROUP_SCHEMA" value="true" />
<option name="GROUP_CONTENTS" value="false" />
<option name="SORT_POSITIONED" value="false" />
<option name="SHOW_EMPTY_GROUPS" value="false" />
<option name="AUTO_SCROLL_FROM_SOURCE" value="false" />
<option name="HIDDEN_KINDS">
<set />
</option>
<expand />
<select />
</component>
<component name="DefaultGradleProjectSettings">
<option name="isMigrated" value="true" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Main.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="420">
<caret line="28" column="43" selection-start-line="28" selection-start-column="43" selection-end-line="28" selection-end-column="43" />
<folding>
<element signature="e#26#66#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/resources/data.txt">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/actionCheck.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2970">
<caret line="198" column="39" selection-start-line="198" selection-end-line="198" selection-end-column="40" />
<state relative-caret-position="606">
<caret line="50" column="2" selection-start-line="50" selection-start-column="2" selection-end-line="50" selection-end-column="2" />
<folding>
<element signature="e#26#59#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.2/spark-core_2.11-2.1.2-sources.jar!/org/apache/spark/rdd/PairRDDFunctions.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/pvCheker.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="55" column="37" selection-start-line="55" selection-start-column="37" selection-end-line="55" selection-end-column="37" />
<state relative-caret-position="220">
<caret line="47" selection-start-line="45" selection-start-column="2" selection-end-line="47" />
<folding>
<element signature="e#26#73#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="jar:///Library/Java/JavaVirtualMachines/jdk1.8.0_201.jdk/Contents/Home/src.zip!/java/util/Date.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="11766">
<caret line="929" column="19" selection-start-line="929" selection-start-column="19" selection-end-line="929" selection-end-column="19" />
<folding>
<element signature="e#39232#39233#0" expanded="true" />
<element signature="e#39292#39293#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Utils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="9" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
......@@ -42,6 +102,8 @@
<list>
<option value="$PROJECT_DIR$/pom.xml" />
<option value="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/pvCheker.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Main.scala" />
<option value="$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/actionCheck.scala" />
</list>
</option>
</component>
......@@ -68,11 +130,12 @@
<select />
</treeState>
</component>
<component name="ProjectFrameBounds">
<component name="ProjectFrameBounds" extendedState="6">
<option name="y" value="23" />
<option name="width" value="1440" />
<option name="height" value="797" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
......@@ -89,25 +152,27 @@
<path>
<item name="dq" type="b2602c69:ProjectViewProjectNode" />
<item name="data-dqmonitor" type="462c0819:PsiDirectoryNode" />
<item name=".settings" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dq" type="b2602c69:ProjectViewProjectNode" />
<item name="data-dqmonitor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dq" type="b2602c69:ProjectViewProjectNode" />
<item name="data-dqmonitor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="resources" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dq" type="b2602c69:ProjectViewProjectNode" />
<item name="data-dqmonitor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="resources" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dq" type="b2602c69:ProjectViewProjectNode" />
......@@ -115,6 +180,7 @@
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
<item name="dq" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
......@@ -126,6 +192,8 @@
<component name="PropertiesComponent">
<property name="ASKED_SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="settings.editor.selected.configurable" value="preferences.pluginManager" />
</component>
......@@ -151,20 +219,26 @@
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1561174147805</updated>
<workItem from="1561525770967" duration="3829000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="7006000" />
</component>
<component name="ToolWindowManager">
<frame x="0" y="23" width="1440" height="797" extended-state="0" />
<frame x="0" y="23" width="1440" height="828" extended-state="6" />
<editor active="true" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.22603719" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.49929875" visible="true" weight="0.24678111" />
<window_info id="Structure" order="1" sideWeight="0.50070125" side_tool="true" weight="0.24678111" />
<window_info id="Image Layers" order="2" />
<window_info id="Designer" order="3" />
<window_info id="UI Designer" order="4" />
<window_info id="Capture Tool" order="5" />
<window_info id="Favorites" order="6" side_tool="true" />
<window_info anchor="bottom" id="Docker" show_stripe_button="false" />
<window_info anchor="bottom" id="Database Changes" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="bottom" id="Run" order="2" />
......@@ -175,6 +249,8 @@
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info anchor="bottom" id="Event Log" order="8" side_tool="true" />
<window_info anchor="bottom" id="Version Control" order="9" />
<window_info anchor="bottom" id="Java Enterprise" order="10" />
<window_info anchor="right" id="Database" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.24964234" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
......@@ -183,8 +259,12 @@
<window_info anchor="right" id="Theme Preview" order="5" />
<window_info anchor="right" id="Capture Analysis" order="6" />
<window_info anchor="right" id="Palette&#9;" order="7" />
<window_info anchor="right" id="Bean Validation" order="8" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="antWorkspaceConfiguration">
<option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
<option name="FILTER_TARGETS" value="false" />
......@@ -197,41 +277,95 @@
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.2/spark-core_2.11-2.1.2-sources.jar!/org/apache/spark/rdd/RDD.scala">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.2/spark-core_2.11-2.1.2-sources.jar!/org/apache/spark/rdd/RDD.scala" />
<entry file="file://$PROJECT_DIR$/pom.xml">
<provider selected="true" editor-type-id="text-editor">
<state>
<caret selection-end-line="16" />
<state relative-caret-position="615">
<caret line="41" column="12" selection-start-line="41" selection-start-column="12" selection-end-line="41" selection-end-column="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/pom.xml">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.2/spark-core_2.11-2.1.2-sources.jar!/org/apache/spark/rdd/PairRDDFunctions.scala" />
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-mllib_2.11/2.1.2/spark-mllib_2.11-2.1.2-sources.jar!/org/apache/spark/mllib/linalg/Vectors.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="615">
<caret line="41" column="12" selection-start-line="41" selection-start-column="12" selection-end-line="41" selection-end-column="12" />
<state relative-caret-position="195">
<caret line="46" column="13" selection-start-line="46" selection-start-column="13" selection-end-line="46" selection-end-column="13" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/data.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="486">
<caret line="48" selection-start-line="48" selection-end-line="48" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.11/2.1.2/spark-sql_2.11-2.1.2-sources.jar!/org/apache/spark/sql/Dataset.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="590">
<caret line="1343" column="6" selection-start-line="1343" selection-start-column="6" selection-end-line="1343" selection-end-column="6" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-mllib_2.11/2.1.2/spark-mllib_2.11-2.1.2-sources.jar!/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="240">
<caret line="32" column="15" selection-start-line="32" selection-start-column="15" selection-end-line="32" selection-end-column="15" />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.11/2.1.2/spark-core_2.11-2.1.2-sources.jar!/org/apache/spark/rdd/PairRDDFunctions.scala">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Utils.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="55" column="37" selection-start-line="55" selection-start-column="37" selection-end-line="55" selection-end-column="37" />
<state relative-caret-position="135">
<caret line="9" selection-start-line="9" selection-end-line="9" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/Main.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="420">
<caret line="28" column="43" selection-start-line="28" selection-start-column="43" selection-end-line="28" selection-end-column="43" />
<folding>
<element signature="e#26#66#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar:///Library/Java/JavaVirtualMachines/jdk1.8.0_201.jdk/Contents/Home/src.zip!/java/util/Date.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="11766">
<caret line="929" column="19" selection-start-line="929" selection-start-column="19" selection-end-line="929" selection-end-column="19" />
<folding>
<element signature="e#39232#39233#0" expanded="true" />
<element signature="e#39292#39293#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar:///Library/Java/JavaVirtualMachines/jdk1.8.0_201.jdk/Contents/Home/src.zip!/java/util/Calendar.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-12801">
<caret line="885" column="20" selection-start-line="885" selection-start-column="20" selection-end-line="885" selection-end-column="20" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/pvCheker.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="509">
<caret line="43" column="2" selection-start-line="43" selection-start-column="2" selection-end-line="43" selection-end-column="2" />
<state relative-caret-position="220">
<caret line="47" selection-start-line="45" selection-start-column="2" selection-end-line="47" />
<folding>
<element signature="e#26#73#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/resources/data.txt">
<entry file="file://$PROJECT_DIR$/src/main/scala/com/gmei/data/dq/actionCheck.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2970">
<caret line="198" column="39" selection-start-line="198" selection-end-line="198" selection-end-column="40" />
<state relative-caret-position="606">
<caret line="50" column="2" selection-start-line="50" selection-start-column="2" selection-end-line="50" selection-end-column="2" />
<folding>
<element signature="e#26#59#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
......
src/main/scala/com/gmei/data/dq/Main.scala
View file @ 404ad9fa
......@@ -30,7 +30,8 @@ object Main {
}
// check pv
pvReferCheker.check(spark, partition_date)
// pvReferCheker.check(spark, partition_date)
actionCheck.check(spark, partition_date)
spark.stop()
}
......
src/main/scala/com/gmei/data/dq/actionCheck.scala 0 → 100644
View file @ 404ad9fa
package com.gmei.data.dq
import java.text.SimpleDateFormat
import java.util.Date
import org.apache.spark.sql.SparkSession
object actionCheck {
import java.util.Calendar
private def getYesterday(date: Date) = {
val cal = Calendar.getInstance
cal.setTime(date)
cal.add(Calendar.DATE, -1)
cal.getTime
}
def check(sc: SparkSession, partition_date: String) = {
import sc.implicits._
val dateFormat = new SimpleDateFormat("yyyyMMdd")
val time_date = dateFormat.parse(partition_date)
val yesterday = getYesterday(time_date)
val df = sc.sql(
s"""
|select
| a.partition_date as today,
| b.partition_date as yesterday,
| a.cl_type,
| a.app_version,
| a.c as todayCount,
| b.c as yesterdayCount,
| case when b.c = 0 then 1.0 else (a.c - b.c) * 1.0 / b.c end as chainRate
|(
| select partition_date, cl_type, app_version, count(1) as c
| from online.bl_hdfs_maidian_updates
| where partition_date = '${partition_date}'
| group by partition_date, cl_type, app_version
|) a left join (
| select
| select partition_date, cl_type, app_version, count(1) as c
| from online.bl_hdfs_maidian_updates
| where partition_date = '${yesterday}'
| group by partition_date, cl_type, app_version
|) b on a.cl_type = b.cl_type and a.app_version = b.app_version
""".stripMargin)
df.show
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment