from common.log_utils import logFactory from common.database_utils import database_util from common import constant import pandas as pd from common.process_util import ProcessUtil from tqdm import tqdm from random import shuffle from sklearn.model_selection import train_test_split import lightgbm as lgb from sklearn import metrics import matplotlib.pyplot as plt import seaborn as sns from sklearn.cluster import DBSCAN logger = logFactory("local train").log drop_columns = ['uuid', 'row_id', 'mark'] def train(total_pos, total_neg, model_name): lgb_model = None train_params = { 'task': 'train', 'objective': 'binary', 'boosting_type': 'gbdt', 'learning_rate': 0.01, 'num_leaves': 40, 'tree_learner': 'serial', 'metric': {'binary_logloss', 'auc', 'average_precision'}, # l1:mae, l2:mse 'max_bin': 80, # 较小的max_bin会导致更快的速度，较大的值会提高准确性 'max_depth': 6, # 'min_child_samples': 5, # "min_data_in_leaf": 10, "bagging_fraction": 0.9, # 样本采样比例，同 XGBoost ，调小可以防止过拟合，加快运算速度 "feature_fraction": 0.9, # 样本采样比例，同 XGBoost ，调小可以防止过拟合，加快运算速度 "n_jobs": 8, "boost_from_average": False, 'seed': 2022, "lambda_l1": 1e-5, "lambda_l2": 1e-5, } for i, data_tuple in enumerate(tqdm(train_tuple)): step_num = str(i) logger.info(f"开始第{step_num}轮") pos_ids = data_tuple[0] neg_ids = data_tuple[1] train_data_frame_pos = total_pos[total_pos.row_id.isin(pos_ids)] train_data_frame_neg = total_neg[total_neg.row_id.isin(neg_ids)] train_data_frame_pos['mark'] = 0 train_data_frame_neg['mark'] = 1 total_train_data = pd.concat([train_data_frame_pos, train_data_frame_neg], axis=0) # 划分训练集和测试集 data_train, data_test = train_test_split(total_train_data, train_size=0.8) train_y_data = data_train[['mark']] train_x_data = data_train.drop(columns=drop_columns) # train_x_data = data_train[['diff_EVENT_FLUX_V','diff_EVENT_CONSUM_V','diff_EVENT_VIDEO_FLUX_V','diff_kuaishou_use']] test_y_data = data_test[['mark']] test_x_data = data_test.drop(columns=drop_columns) # test_x_data = data_test[['diff_EVENT_FLUX_V','diff_EVENT_CONSUM_V','diff_EVENT_VIDEO_FLUX_V','diff_kuaishou_use']] # 创建lgb的数据集 lgb_train = lgb.Dataset(train_x_data, train_y_data.values, silent=True) lgb_eval = lgb.Dataset(test_x_data, test_y_data.values, reference=lgb_train, silent=True) lgb_model = lgb.train(params=train_params, train_set=lgb_train, num_boost_round=6000, valid_sets=lgb_eval, init_model=lgb_model, feature_name=train_x_data.columns.tolist(), early_stopping_rounds=20, verbose_eval=False, keep_training_booster=True) # 输出模型评估分数 score_train = str(dict([(s[1], s[2]) for s in lgb_model.eval_train()])) score_valid = str(dict([(s[1], s[2]) for s in lgb_model.eval_valid()])) logger.info(f"第{step_num}轮的结果如下:") logger.info(f"在训练集上:{score_train}") logger.info(f"在测试集上:{score_valid}") if i == len(train_tuple) - 1: test_predict = lgb_model.predict(test_x_data) test_result = [] for x in test_predict: if x < 0.5: test_result.append(0) else: test_result.append(1) result = metrics.classification_report(test_y_data.values, test_result) logger.info(result) ProcessUtil.draw_roc_auc(test_y_data.values.reshape([-1, ]), test_predict) sns.set(rc={'figure.figsize': (50, 50)}) sns.barplot(y=train_x_data.columns, x=lgb_model.feature_importance()) plt.show() lgb_model.save_model(model_name) def test(model_path, test_pos_path, test_neg_path): logger.info("准备开始加载验证数据") test_pos = pd.read_pickle(test_pos_path) test_neg = pd.read_pickle(test_neg_path) test_pos['mark'] = 0 test_neg['mark'] = 1 test_all_data = pd.concat([test_pos, test_neg], axis=0) test_x_data = test_all_data test_y_data = test_all_data[['mark']] test_x_data.drop(columns=['uuid', 'mark'], inplace=True) # test_x_data=test_x_data[['diff_EVENT_FLUX_V','diff_EVENT_CONSUM_V','diff_EVENT_VIDEO_FLUX_V','diff_kuaishou_use']] lgb_model = lgb.Booster(model_file=model_path) test_predict = lgb_model.predict(test_x_data, num_iteration=lgb_model.best_iteration) test_result = [] for x in test_predict: if x > 0.2: test_result.append(1) else: test_result.append(0) result = metrics.classification_report(test_y_data.values, test_result) logger.info(result) ProcessUtil.draw_confusion_matrix(test_y_data.values, test_result) ProcessUtil.draw_roc_auc(test_y_data.values.reshape([-1, ]), test_predict) pass def dbscan_data(df_list): df_all = pd.concat(df_list, axis=0) dbscan = DBSCAN(eps=0.8, min_samples=1) dbscan.fit(df_all[ [ 'EVENT_FLUX_C_0', 'EVENT_FLUX_C_1', 'EVENT_FLUX_C_2', 'EVENT_FLUX_C_3', 'EVENT_FLUX_C_4', 'EVENT_FLUX_C_5', 'EVENT_FLUX_C_6', 'EVENT_FLUX_C_7', 'EVENT_FLUX_C_8', 'EVENT_CONSUM_C_0', 'EVENT_CONSUM_C_1', 'EVENT_CONSUM_C_2', 'EVENT_CONSUM_C_3', 'EVENT_CONSUM_C_4', 'EVENT_CONSUM_C_5', 'EVENT_CONSUM_C_6', 'EVENT_CONSUM_C_7', 'EVENT_VIDEO_FLUX_C_0', 'EVENT_VIDEO_FLUX_C_1', 'EVENT_VIDEO_FLUX_C_2', 'EVENT_VIDEO_FLUX_C_3', 'EVENT_VIDEO_FLUX_C_4', 'EVENT_VIDEO_FLUX_C_5', 'EVENT_VIDEO_FLUX_C_6', 'EVENT_VIDEO_FLUX_C_7', 'app_use_kuaishou_0', 'app_use_kuaishou_1', 'app_use_kuaishou_2', 'app_use_kuaishou_3', 'app_use_kuaishou_4', 'app_use_kuaishou_5', 'app_use_kuaishou_6', 'app_use_kuaishou_7', 'EVENT_ORDER_MONTH_C_0', 'EVENT_ORDER_MONTH_C_1', 'EVENT_ORDER_MONTH_C_2', 'EVENT_ORDER_MONTH_C_3', 'EVENT_ORDER_MONTH_C_4', 'EVENT_ORDER_MONTH_C_5', 'EVENT_ORDER_MONTH_C_6']]) label_pred = dbscan.labels_ print(label_pred) if __name__ == "__main__": batch_num_pos = 10 batch_num_neg = 100 total_pos = pd.read_pickle("./data/pkl/train_neg_all.pkl") total_neg = pd.read_pickle("./data/pkl/train_pos_all.pkl") positive_ids = range(0, total_pos.shape[0]) negative_ids = range(0, total_neg.shape[0]) pos_row_ids = ProcessUtil.partition_preserve_order(positive_ids, batch_num_pos) neg_row_ids = ProcessUtil.partition_preserve_order(negative_ids, batch_num_neg) train_tuple = ProcessUtil.gen_train_tuple(pos_row_ids, neg_row_ids, batch_num_pos) # 增加自增列 total_pos['row_id'] = range(0, len(total_pos)) total_neg['row_id'] = range(0, len(total_neg)) train(total_pos, total_neg, "./data/model/temp_model030901.model") # test("./data/model/temp_model030802.model", './data/pkl/valid_neg_all.pkl','./data/pkl/valid_pos_all.pkl') # dbscan_data([pd.read_pickle('./data/pkl/process_dbscan.pkl')])