Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

eda/feededa/src/main/scala/com/gmei/temp_analysis.scala

@@ -615,19 +615,100 @@ object smart_rank_count {
       final_id.createOrReplaceTempView("final_id")
 
 
-            val meigou_price = sc.sql(
-              s"""
-                 |select cl_id,city_id,params['business_id'] as meigou_id
-                 |from online.tl_hdfs_maidian_view
-                 |where action = "page_view"
-                 |and params['page_name']="welfare_detail"
-                 |and partition_date ='20181201'
-                 |LIMIT 10
-             """.stripMargin
-            )
-            meigou_price.show(80)
-
-      //      GmeiConfig.writeToJDBCTable(meigou_price, "meigou_price", SaveMode.Overwrite)
+//      val user_city_meigou_view = sc.sql(
+//        s"""
+//           |select cl_id as device_id,city_id as device_city,params['business_id'] as meigou_id
+//           |from online.tl_hdfs_maidian_view
+//           |where action = "page_view"
+//           |and params['page_name']="welfare_detail"
+//           |and partition_date >='20181201'
+//           |and city_id is not null
+//       """.stripMargin
+//      )
+//      user_city_meigou_view.createOrReplaceTempView("user_city_meigou_view")
+//
+//      val meigou_city = sc.sql(
+//        s"""
+//           |select b.id as meigou_id,d.city_id as meigou_city
+//           |from online.tl_meigou_service_view b
+//           |left join online.tl_hdfs_doctor_view c on b.doctor_id=c.id
+//           |left join online.tl_hdfs_hospital_view d on c.hospital_id=d.id
+//           |where b.partition_date='20181227'
+//           |and c.partition_date='20181227'
+//           |and d.partition_date='20181227'
+//       """.stripMargin
+//      )
+//      meigou_city.createOrReplaceTempView("meigou_city")
+//
+//
+//      val meigou_pv_tongcheng = sc.sql(
+//        s"""
+//           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
+//           |from user_city_meigou_view a
+//           |left join meigou_city b
+//           |on a.meigou_id=b.meigou_id
+//       """.stripMargin
+//      )
+//      meigou_pv_tongcheng.createOrReplaceTempView("meigou_pv_tongcheng")
+//
+//      val meigou_pv_count = sc.sql(
+//        s"""
+//           |select meigou_city,count(device_id) as meigou_pv,count(distinct(device_id)) as meigou_device_num
+//           |from meigou_pv_tongcheng
+//           |where device_city=meigou_city
+//           |group by meigou_city
+//       """.stripMargin
+//      )
+//      meigou_pv_count.show()
+//
+//
+////开始计算咨询
+//      val zixun_meigou_view = sc.sql(
+//        s"""
+//           |select cl_id as device_id,city_id as device_city,params['service_id'] as meigou_id
+//           |from online.tl_hdfs_maidian_view
+//           |where partition_date >= '20181201'
+//           |and action = 'welfare_detail_click_message'
+//       """.stripMargin
+//      )
+//      zixun_meigou_view.createOrReplaceTempView("zixun_meigou_view")
+//
+//      val zixun_meigou_tongcheng = sc.sql(
+//        s"""
+//           |select a.device_id,a.device_city,a.meigou_id,b.meigou_city
+//           |from zixun_meigou_view a
+//           |left join meigou_city b
+//           |on a.meigou_id=b.meigou_id
+//       """.stripMargin
+//      )
+//      zixun_meigou_tongcheng.createOrReplaceTempView("zixun_meigou_tongcheng")
+//
+//      val zixun_pv_count = sc.sql(
+//        s"""
+//           |select meigou_city,count(device_id) as meigou_zixun,count(distinct(device_id)) as meigou_zixun_device_num
+//           |from zixun_meigou_tongcheng
+//           |where device_city=meigou_city
+//           |group by meigou_city
+//       """.stripMargin
+//      )
+//      zixun_pv_count.show()
+
+
+
+      //开始计算每个地区每月新增设备
+
+      val device_new_count = sc.sql(
+        s"""
+           |select first_city,count(distinct(device_id))
+           |from online.ml_device_day_active_status
+           |where active_type != '4'
+           |and partition_date >='20181201'
+           |group by first_city
+       """.stripMargin
+      )
+      device_new_count.show()
+
+
 
     }
 

tensnsorflow/applist.py

@@ -29,11 +29,18 @@ def json_format(x):
 
 def sort_app():
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
-    sql = "select device_id,app_list from device_id_applist"
+    sql = "select device_id,app_list,stat_date from device_id_applist"
     df = con_sql(db, sql).dropna()
-    df = df.rename(columns={0: "device_id", 1: "app_list"})
+    df = df.rename(columns={0: "device_id", 1: "app_list",2:"stat_date"})
+    print(df.shape)
+    df = df.sort_values(by="stat_date",ascending=False)
+    print(df.head())
+    df = df.drop("stat_date",axis=1)
+    df = df.drop_duplicates("device_id")
+    print(df.shape)
     df = df.loc[df["app_list"].apply(is_json)]
 
+
     category = {"competitor":{"新氧美容","悦美","美呗整形","悦美微整形","如丽美容","医美咖","整形去哪儿","美黛拉","整形思密达","美芽"},
                 "dianshang":{"京东","淘宝","唯品会","天猫","苏宁易购","国美","当当","亚马逊","网易严选","小米有品"},
                 "kuajing_dianshang": {"小红书", "网易考拉", "洋码头", "达令全球好货", "海狐海淘",
@@ -100,3 +107,6 @@ def sort_app():
 
 if __name__ == "__main__":
     sort_app()
+
+
+

tensnsorflow/ffm.py

@@ -147,10 +147,16 @@ def get_data():
     print(start)
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_test')
     sql = "select e.y,e.z,e.stat_date,e.ucity_id,e.clevel1_id,e.ccity_name," \
-          "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id " \
+          "u.device_type,u.manufacturer,u.channel,c.top,cid_time.time,e.device_id," \
+          "a.competitor,a.dianshang,a.kuajing_dianshang,a.zhibo,a.youxizhibo,a.short_video,a.meitu,a.tiyu," \
+          "a.read,a.finance,a.fashion_clothes,a.muying,a.fresh,a.bijia,a.travel,a.airplane," \
+          "a.love,a.stock,a.car,a.child,a.homework,a.work,a.job " \
           "from esmm_train_data e left join user_feature_clean u on e.device_id = u.device_id " \
-          "left join cid_type_top c on e.device_id = c.device_id left join cid_time on e.cid_id = cid_time.cid_id " \
+          "left join cid_type_top c on e.device_id = c.device_id " \
+          "left join cid_time on e.cid_id = cid_time.cid_id " \
+          "left join app_list_sort a on e.device_id = a.device_id " \
           "where e.stat_date >= '{}'".format(start)
+
     df = con_sql(db, sql)
     print(df.shape)
     df = df.rename(columns={0: "y", 1: "z", 2: "stat_date", 3: "ucity_id",4: "clevel1_id", 5: "ccity_name",
@@ -158,24 +164,29 @@ def get_data():
     print("esmm data ok")
     print(df.head(2))
 
+    for i in range(12,35):
+        df[i] = df[i].astype("str")
+        df[i] = df[i].fillna(0)
     df["clevel1_id"] = df["clevel1_id"].astype("str")
     df["y"] = df["y"].astype("str")
     df["z"] = df["z"].astype("str")
     df["top"] = df["top"].astype("str")
     df["y"] = df["stat_date"].str.cat([df["device_id"].values.tolist(),df["y"].values.tolist(),df["z"].values.tolist()], sep=",")
-    df = df.drop(["z","stat_date","device_id"], axis=1).fillna(0.0)
-    print(df.head(2))
+    df = df.drop(["z","stat_date","device_id"], axis=1)
     features = 0
-    for i in ["ucity_id","clevel1_id","ccity_name","device_type","manufacturer","channel"]:
+    for i in ["ucity_id","clevel1_id","ccity_name","device_type","manufacturer","channel","top"]:
         features = features + len(df[i].unique())
+        df[i] = df[i].fillna(0)
+    df["time"] = df["time"].fillna(df["time"].mode()[0])
+    print(df.count())
+    print(df.head(2))
     print("fields:{}".format(df.shape[1]-1))
-    print("features:{}".format(features))
+    print("features:{}".format(features+46))
     ccity_name = list(set(df["ccity_name"].values.tolist()))
     ucity_id = list(set(df["ucity_id"].values.tolist()))
     manufacturer = list(set(df["manufacturer"].values.tolist()))
     channel = list(set(df["channel"].values.tolist()))
 
-
     return df,validate_date,ucity_id,ccity_name,manufacturer,channel
 
 
@@ -291,7 +302,7 @@ if __name__ == "__main__":
     a = time.time()
     temp, validate_date, ucity_id,ccity_name,manufacturer,channel = get_data()
     model = transform(temp, validate_date)
-    get_predict_set(ucity_id,model,ccity_name,manufacturer,channel)
+    # get_predict_set(ucity_id,model,ccity_name,manufacturer,channel)
     b = time.time()
     print("cost(分钟)")
     print((b-a)/60)