update

eda/smart_rank/dist_update_user_portrait.py

@@ -139,9 +139,9 @@ if __name__ == '__main__':
         spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
         spark.sparkContext.setLogLevel("WARN")
 
-        device_ids_lst_rdd = spark.sparkContext.parallelize(device_ids_lst)
+        device_ids_lst_rdd = spark.sparkContext.parallelize(device_ids_lst, numSlices=100)
         gm_kv_cli = redis.Redis(host="172.16.40.135", port=5379, db=6, socket_timeout=2000)
-        result = device_ids_lst_rdd.repartition(40).map(lambda x: get_user_tag_score(x, all_log_df))
+        result = device_ids_lst_rdd.repartition(100).map(lambda x: get_user_tag_score(x, all_log_df))
         result.take(10).foreach(print)
     except Exception as e:
         print(e)
\ No newline at end of file