unsubscribe_predict/gen_target_month_data.py

import pandas as pd
from common.database_utils import database_util
from common.log_utils import logFactory
from common import constant
from tqdm import tqdm


click_client = database_util.get_client()
logger = logFactory("preprocess data").log

def write_df_to_pickle(data, filename):
    logger.info(f"开始写入pickle")
    data.to_pickle(f"./data/pkl/{filename}")
    logger.info(f"写入pickle完成，文件名{filename}，文件大小{data.shape}", )

def get_df_by_id_month(uuid_list, month):
    logger.info(f"开始读取{month}数据")
    sql1 = f"select *, length(EVENT_APP_USE.C) as I_appuse from Z_USER_TAG_FLAT_out_{month} where uuid in {uuid_list} order by I_appuse desc limit 1 by uuid,EVENT_SPNAME_C"
    all_data1 = click_client.execute(sql1)


    data_frame1 = pd.DataFrame(all_data1, columns=constant.origin_column_names)
    logger.info(f"data1的shape{data_frame1.shape}")

    return data_frame1





if __name__ == '__main__':
    path = "./data/csv/8910/tuidinguuids.csv"

    uuids = list(set(pd.read_csv(path).values.reshape(-1).tolist()))
    df = get_df_by_id_month(uuids,'202110')
    write_df_to_pickle(df, f"dbscan.pkl")