Merge branch 'master' of http://git.wanmeizhensuo.com/ML/ffm-baseline

eda/feededa/src/main/scala/com/gmei/EsmmData.scala

@@ -548,25 +548,39 @@ object GetDevicePortrait {
 
       val device_search_tag = sc.sql(
         s"""
-           |select a.cl_id as device_id,COALESCE(a.params['diary_id'], a.params['business_id'], 0) as cid_id,split(c.level1_ids,',')[0] as level1_id
-           |from online.tl_hdfs_maidian_view as a
-           |left join
-           |	(select cl_id as device_id,max(partition_date) as stat_date
-           |	from online.tl_hdfs_maidian_view
-           |	where action = 'on_click_diary_card'
-           |	and (params['page_name']='search_result_diary' or params['page_name']='search_result_more')
-           |	and partition_date > '20180601' group by cl_id) b
-           |on a.cl_id = b.device_id and a.partition_date=b.stat_date
-           |left join diary_feat c
-           |on COALESCE(a.params['diary_id'], a.params['business_id'], 0) = c.diary_id
-           |where a.partition_date > '20180601'
-           |and COALESCE(a.params['diary_id'], a.params['business_id'], 0) != 0
-           |and c.level1_ids != ""
+           |select c.device_id,c.stat_date,c.level1_id,count(c.level1_id) as level1_count
+           |from (select
+           |		a.cl_id as device_id,a.partition_date as stat_date,
+           |		COALESCE(a.params['diary_id'], a.params['business_id'], 0) as cid_id,
+           |		b.level1_ids as level1_id
+           |	from online.tl_hdfs_maidian_view a
+           |	left join diary_feat b
+           |	on COALESCE(a.params['diary_id'], a.params['business_id'], 0) = b.diary_id
+           |	where
+           |    b.level1_ids is not null and
+           |		a.partition_date = '20181224'
+           |		and (a.action = 'on_click_diary_card' or (a.action="full_stack_click_video_card_full_screen_play" and a.params["card_type"]="diary"))) c
+           |group by c.device_id,c.level1_id,c.stat_date
          """.stripMargin
       )
       device_search_tag.show()
       println(device_search_tag.count())
 
+      device_search_tag.createOrReplaceTempView("tag_count")
+
+      val max_count_tag = sc.sql(
+      s"""
+         |select a.device_id,a.stat_date,a.level1_id,a.level1_count
+         |from tag_count a
+         |inner join
+         |(select device_id,max(level1_count) as max_count from tag_count group by device_id) b
+         |on a.level1_count = b.max_count and a.device_id = b.device_id
+       """.stripMargin
+    )
+      max_count_tag.show()
+      println(max_count_tag.count())
+
+
 //      GmeiConfig.writeToJDBCTable(diary_tag,"diary_feat",SaveMode.Overwrite)
 
 

tensnsorflow/applist.py

@@ -34,7 +34,7 @@ def sort_app():
     df = df.rename(columns={0: "device_id", 1: "app_list"})
     df = df.loc[df["app_list"].apply(is_json)]
 
-    category = {"competitor":{"新氧美容"},
+    category = {"competitor":{"新氧美容","悦美","美呗整形","悦美微整形","如丽美容","医美咖","整形去哪儿","美黛拉","整形思密达","美芽"},
                 "dianshang":{"京东","淘宝","唯品会","天猫","苏宁易购","国美","当当","亚马逊","网易严选","小米有品"},
                 "kuajing_dianshang": {"小红书", "网易考拉", "洋码头", "达令全球好货", "海狐海淘",
                                       "HIG0", "豌豆公主", "尚品网", "丰趣海淘", "比呀比海外购"},
@@ -72,7 +72,7 @@ def sort_app():
     for i in category.keys():
         df[i] = df["app_list"].apply(lambda x: 1 if len(x & category[i]) > 0 else 0)
         print(i)
-        print(df[i].unique())
+        print(df[i].value_counts())
     df = df.drop("app_list",axis=1)
 
     yconnect = create_engine('mysql+pymysql://root:3SYz54LS9#^9sBvC@10.66.157.22:4000/jerry_test?charset=utf8')