Merge branch 'master' of git.wanmeizhensuo.com:ML/ffm-baseline

change pre data from merge 3 recall

eda/feededa/src/main/scala/com/gmei/testt.scala

@@ -65,18 +65,15 @@ object testt {
 //      )
 ////      agency_id.show()
 //      agency_id.createOrReplaceTempView("agency_id")
-      val a = Array("personal_store_diary_list","question_detail","received_votes","search_result","search_result_diary","search_result_more"
-,"search_result_welfare","service_all_cases_service_comment","spell_special_list","topic_detail","welfare_detail","welfare_list","welfare_special","wiki_detail"
-,"zone_detail","zone_v3")
+      val a = Array("answer_detail","article_detail","diary_detail","home_page","question_detail","search_content","search_diary")
       for (i <- 0 until a.length){
         val diary_pv = sc.sql(
           s"""
              |select partition_date,count(cl_id)
-             |from online.tl_hdfs_maidian_view
-             |where action="page_view"
-             |and params["page_name"]="diary_detail"
-             |and params["referrer"]='${a(i)}'
+             |from online.ml_community_exposure_detail
+             |where business_type ='diary'
              |and partition_date >='20181120'
+             |and event='${a(i)}'
              |GROUP BY partition_date
              |order by partition_date
            """.stripMargin
@@ -86,6 +83,22 @@ object testt {
       }
 
 
+//      val diary_pv = sc.sql(
+//        s"""
+//           |select partition_date,count(cl_id)
+//           |from online.tl_hdfs_maidian_view
+//           |where action ='on_click_diary_card'
+//           |and params["page_name"]="home"
+//           |and params["tab_name"]="精选"
+//           |and partition_date >='20181120'
+//           |GROUP BY partition_date
+//           |order by partition_date
+//           """.stripMargin
+//      )
+//      diary_pv.show()
+
+
+
 
 //      GmeiConfig.writeToJDBCTable(view_count, "avg", SaveMode.Overwrite)
 

tensnsorflow/ffm.py

@@ -21,7 +21,7 @@ def con_sql(db,sql):
 
 def get_data():
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='jerry_prod')
-    sql = "select * from esmm_data where stat_date >= '2018-11-20'"
+    sql = "select * from esmm_data where stat_date >= '2018-11-20' limit 6"
     esmm = con_sql(db,sql)
     esmm = esmm.rename(columns={0:"stat_date",1: "device_id",2:"ucity_id",3:"cid_id",4:"diary_service_id",5:"y",
                                 6:"z",7:"clevel1_id",8:"slevel1_id"})
@@ -29,13 +29,13 @@ def get_data():
     print(esmm.head())
     print(esmm.shape)
     db = pymysql.connect(host='10.66.157.22', port=4000, user='root', passwd='3SYz54LS9#^9sBvC', db='eagle')
-    sql = "select * from home_tab_click"
+    sql = "select * from home_tab_click limit 6"
     temp = con_sql(db,sql)
     temp = temp.rename(columns={0: "device_id"})
     print("click data ok")
     # print(temp.head())
     df = pd.merge(esmm,temp,on = "device_id",how='left').fillna(0)
-    print("合并后：")
+    # print("合并后：")
     print(df.shape)
 
     df["diary_service_id"] = df["diary_service_id"].astype("str")
@@ -135,9 +135,9 @@ class multiFFMFormatPandas:
             col_type = t[col]
             name = '{}_{}'.format(col, val)
             if col_type.kind == 'O':
-                ffm.append('{}:{}:1'.format(self.field_index_[col]+1, self.feature_index_[name]+1))
+                ffm.append('{}:{}:1'.format(self.field_index_[col]+1, self.feature_index_[name]))
             elif col_type.kind == 'i':
-                ffm.append('{}:{}:{}'.format(self.field_index_[col]+1, self.feature_index_[col]+1, val))
+                ffm.append('{}:{}:{}'.format(self.field_index_[col]+1, self.feature_index_[col], val))
         result = ' '.join(ffm)
         if self.y is not None:
             result = str(row.loc[row.index == self.y][0]) + "," + result

tensnsorflow/test.py

+import time
+for i in range(1,100):
+    print(i)
+    time.sleep(5)
\ No newline at end of file