user_preference_predict/data_join.py

import pandas as pd
from common.log_utils import logFactory
from common.database_utils import database_util
from tqdm import tqdm

click_client = database_util.get_client()

logger = logFactory("data gen").log

if __name__ == "__main__":
    index = 0
    fetch_size = 1000000

    # part1
    # for i in tqdm(range(0, 15)):
    #     t1_start = i * fetch_size
    #     for j in tqdm(range(0, 74)):
    #         try:
    #             t2_start = j * fetch_size
    #             sql = f"insert into l_neg_origin_06 " \
    #                   f"select t2.*,length(t2.EVENT_APP_USE.C) as I_appuse,{i} as PART from (select * from l_neg_uuid_06 limit {t1_start},{fetch_size}) t1  inner join (select * from Z_USER_TAG_FLAT_out_202106 limit {t2_start},{fetch_size}) t2 on t1.uuid==t2.uuid  order by I_appuse desc limit 1 by uuid,EVENT_SPNAME_C"
    #             # logger.info(sql)
    #             click_client.execute(sql)
    #         except Exception as e:
    #             logger.info(f"第{i}大循环的第{j}小循环出错")
    #             logger.info(str(e))
    #             continue


    # part2
    # for i in tqdm(range(15, 30)):
    #     t1_start = i * fetch_size
    #     for j in tqdm(range(0, 74)):
    #         try:
    #             t2_start = j * fetch_size
    #             sql = f"insert into l_neg_origin_06 " \
    #                   f"select t2.*,length(t2.EVENT_APP_USE.C) as I_appuse,{i} as PART from (select * from l_neg_uuid_06 limit {t1_start},{fetch_size}) t1  inner join (select * from Z_USER_TAG_FLAT_out_202106 limit {t2_start},{fetch_size}) t2 on t1.uuid==t2.uuid  order by I_appuse desc limit 1 by uuid,EVENT_SPNAME_C"
    #             # logger.info(sql)
    #             click_client.execute(sql)
    #         except Exception as e:
    #             logger.info(f"第{i}大循环的第{j}小循环出错")
    #             logger.info(str(e))
    #             continue
    #
    # # part3
    for i in tqdm(range(30, 47)):
        t1_start = i * fetch_size
        for j in tqdm(range(0, 74)):
            try:
                t2_start = j * fetch_size
                sql = f"insert into l_neg_origin_06 " \
                      f"select t2.*,length(t2.EVENT_APP_USE.C) as I_appuse,{i} as PART from (select * from l_neg_uuid_06 limit {t1_start},{fetch_size}) t1  inner join (select * from Z_USER_TAG_FLAT_out_202106 limit {t2_start},{fetch_size}) t2 on t1.uuid==t2.uuid  order by I_appuse desc limit 1 by uuid,EVENT_SPNAME_C"
                # logger.info(sql)
                click_client.execute(sql)
            except Exception as e:
                logger.info(f"第{i}大循环的第{j}小循环出错")
                logger.info(str(e))
                continue

    pass