/**
 * 
 */
package com.platomix.userprofile.handler;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

import com.alibaba.fastjson.JSONObject;
import com.platomix.userprofile.enums.DsEnum;
import com.platomix.userprofile.service.DynamicExecuteService;
import com.platomix.userprofile.service.UserprofileTagService;
import com.platomix.userprofile.utils.StrBuilder;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

/**
 * @author wookvn
 *
 *  根据临时表创建大宽表
 */
@Slf4j
@Service
public class FlatTagHandler {
	final static String SHELL_DIC = "shell/";

	@Value("${dflow.flat-temp-table-name}")
	private String tempTableName;
	@Value("${dflow.flat-clu-table-name}")
	private String cluTableName;
	@Value("${dflow.flat-all-view-name}")
	private String flatAllViewName;
	
	final static String CLUSTER_NAME = "cluster_3s_1r";

	final static String[] INVALID_FIELD_NAMES = new String[] { "USERID", "MONTH", "DEVICE_NUMBER", "cTime" };

	@Value("${dflow.use-debug}")
	private Boolean useDebug;

	@Value("${dbcluster.clickhouse}")
	private String[] clickhouseDbCluster;

	@Value("${spring.datasource.dynamic.datasource.clickhouse.username}")
	private String clickhouseUserName;
	@Value("${spring.datasource.dynamic.datasource.clickhouse.password}")
	private String clickhousePassWord;

	@Autowired
	private DynamicExecuteService dynamicExecuteService;

	@Autowired
	private UserprofileTagService userprofileTagService;

	/**
	 * 
	 * @param month
	 */
	public void tempDataToFlatView(String month) {
		//校验并处理列变动
		_checkAndDealColumn();
		
		//删除月份分区数据
		String dropViewDataSql = String.format("alter table ads.%s on cluster %s drop partition '%s'", cluTableName, CLUSTER_NAME, month);
		log.info("\n" + dropViewDataSql);
		dynamicExecuteService.execute(DsEnum.clickhouse, dropViewDataSql, "删除分区[month="+month+"]数据完成");
		
		//再次查询列变动后的字段名
		List<JSONObject> checkTableDiffList = _getDiffColumnList();
		List<String> columnList = new ArrayList<>();
		for(JSONObject columnObj : checkTableDiffList) {
			String tempName = columnObj.getString("temp_name"),
				   cluName  = columnObj.getString("clu_name");
			//仅当2个表都存在该列时，入库该字段
			if(!StringUtils.isEmpty(tempName) && !StringUtils.isEmpty(cluName)) {
				columnList.add("`" + cluName + "`");
			}
		}

		//生成shell文件 在导入
		shellToCluster(month,columnList);
		
//		StrBuilder sqlBuilder = new StrBuilder();
//		sqlBuilder.appendLn("insert into " + FLAT_VIEW_NAME + "(")
//				  .appendLn(String.join(",", columnList))
//				  .appendLn(")")
//				  .appendLn("select")
//				  .appendLn(String.join(",", columnList))
//				  .appendLn("\nfrom ads." + TEMP_TABLE_NAME)
//				  .appendLn("where THEMONTH_V = '"+month+"'");
//		log.info("\n" + sqlBuilder.toString());
//		dynamicExecuteService.execute(DsEnum.clickhouse, sqlBuilder.toString(), "大宽表数据完成");
	}

	/**
	 * 本地临时表数据导入Flat视图表
	 */
	@Deprecated
	public void tmpDataToFlat(String month) {
		List<JSONObject> sourceColumnList = dynamicExecuteService.list(DsEnum.clickhouse, "select name from system.columns where `table`='" + tempTableName + "'");
		List<String> sourceColumnNameList = sourceColumnList.stream().map(item -> item.getString("name")).collect(Collectors.toList());

		List<JSONObject> targetColumnList = dynamicExecuteService.list(DsEnum.clickhouse, "select name from system.columns where `table`='" + cluTableName + "'");
		Map<String, String> targetColumnMap = new HashMap<String, String>();
		for (JSONObject jsonObject : targetColumnList) {
			targetColumnMap.put(jsonObject.getString("name"), jsonObject.getString("type"));
		}

		StrBuilder sqlBuilder = new StrBuilder();
		sqlBuilder.appendLn("insert into " + flatAllViewName + "(");
		targetColumnMap.keySet().stream().filter(f -> sourceColumnNameList.contains(f)).map(item -> "`" + item + "`")
				.forEach(sqlBuilder::appendLn);
		sqlBuilder.appendLn(")").appendLn("with");
		// 省份
		String provinceSql = userprofileTagService.genSelectSql("省份", "b.PROVINCE", null, "TMP_PROVINCE");
		sqlBuilder.append(provinceSql).appendLn(",");
		// 城市
		String citySql = userprofileTagService.genSelectSql("城市", "b.AREA", null, "TMP_AREA");
		sqlBuilder.appendLn(citySql);

		sqlBuilder.appendLn("select THEMONTH_V,USERID_V,");
		for (Entry<String, String> entry : targetColumnMap.entrySet()) {
			String name = entry.getKey(), type = entry.getValue();
			// 包含一些特殊字段/或者源表中不存在该字段(大宽表比临时表多的字段)，不处理
			if(Stream.of(INVALID_FIELD_NAMES).anyMatch(f -> name.contains(f))
					|| !sourceColumnNameList.contains(name)) {
				continue;
			}
			if(	// 投诉标签处理
				_handleComplain(sqlBuilder, name) ||
				// 省份标签处理
				_handleProvince(sqlBuilder, name) ||
				// 城市标签处理
				_handleCity(sqlBuilder, name) ||
				//其它标签处理
				_handleOthers(sqlBuilder, name, type)) {
			}
		}
		sqlBuilder.deleteLastChar().deleteLastChar();
		sqlBuilder.appendLn("\nfrom ads." + tempTableName + " a left join ETL_VW_TB_CALLER_AREACODE b on substring(a.USERID_V , 1, 7) = b.CALLERHEAD");
		log.info("\n" + sqlBuilder.toString());
		//dynamicExecuteService.execute(DsEnum.clickhouse, sqlBuilder.toString(), "临时表导入视图生成");
	}

	/**
	 * 手动生成shell文件，插入到不同服务器【只能在Linux下运行 需要clickhouse docker 环境】
	 * @param month 插入月份
	 * @param columnList 字段集合
	 */
	private void shellToCluster(String month,List<String> columnList){
		String firstDb=clickhouseDbCluster[0];
		//System.out.println(firstDb);
		//按手机号 查找分段
		Set<String> setps=new HashSet<>();
		StrBuilder sqlBuilder=new StrBuilder();
		sqlBuilder.appendLn(String.format("with intDiv((select COUNT() from ads.%s),%d) as step",tempTableName,clickhouseDbCluster.length))
				.appendLn("select USERID_V,(rowNumberInAllBlocks()+1) rownum from (")
				.appendLn(String.format("select USERID_V from ads.%s order by USERID_V ",tempTableName)+")")
				.append("where rownum in (");
		IntStream.range(1, clickhouseDbCluster.length).forEachOrdered(idx -> {setps.add(String.format("step*%d",idx));});
		sqlBuilder.append(String.join(",", setps));
		sqlBuilder.append(")");
		//动态拼接SQL
		List<String> dataList = dynamicExecuteService.list(DsEnum.clickhouse, sqlBuilder.toString()).stream().map(item ->item.getString("USERID_V")).collect(Collectors.toList());

		try {
			FileUtils.forceMkdir(new File(SHELL_DIC));
			for(int i = 0; i < clickhouseDbCluster.length; i++) {
				String leftUserId=i==0?"0": dataList.get(i-1);
				String rightUserId=i==clickhouseDbCluster.length-1?"999": dataList.get(i);
				String userIdSql= leftUserId == null || rightUserId == null ? "" : String.format(" and USERID_V>=toString(%s) and USERID_V<toString(%s)",  leftUserId, rightUserId);

				StrBuilder builder=new StrBuilder();
				//如果是向本机插入
				if(i==0){
					builder.appendLn("insert into " + cluTableName + "(")
							.appendLn(String.join(",", columnList))
							.appendLn(")")
							.appendLn("select")
							.appendLn(String.join(",", columnList))
							.appendLn("\nfrom ads." + tempTableName)
							.appendLn("where THEMONTH_V = '"+month+"' "+userIdSql);
					_shellToCluster(new ShellCommond(clickhouseDbCluster[i],i==0,builder.toString()));
				}else{
					builder.append("echo 'select ")
							.append(String.join(",", columnList))
							.append(String.format(" from ads.%s where THEMONTH_V = toString(%s) %s' | ",tempTableName,month,userIdSql))
							.append(String.format(" curl %s:8123?database=ads -u%s:%s -d @- | ",firstDb,clickhouseUserName,clickhousePassWord))
							.append(String.format(" docker run -i --rm yandex/clickhouse-client --host %s --port 9007 --user %s --password %s --input_format_allow_errors_num=100",clickhouseDbCluster[i],clickhouseUserName,clickhousePassWord))
							.append(String.format(" --query='INSERT INTO ads.%s( ",cluTableName))
							.append(String.join(",", columnList))
							.appendLn(") FORMAT TabSeparated'");
					_shellToCluster(new ShellCommond(clickhouseDbCluster[i],i==0,builder.toString()));
				}
			}
		}catch (IOException iOException){
			log.info(iOException.getMessage());
		}
	}

	/**
	 * 异步从主服务器向其他db分发，先控制在 3个并发
	 * @param shellData
	 * @throws IOException
	 */
	@Async("shellThreadPool")
	public void _shellToCluster(ShellCommond shellData) throws IOException {
		//如果是插入本地
		if(shellData.getIsLocalIp()){
			dynamicExecuteService.execute(DsEnum.clickhouse, shellData.getShellCmd(), "插入temp宽表 到本机clu表");
		}
		else {
			String filePath = getShellFilePath(shellData);
			File shellFile = new File(filePath);
			FileUtils.writeStringToFile(shellFile, shellData.getShellCmd(), "UTF-8");
			Runtime.getRuntime().exec("chmod 777 " + filePath);

			String commond="sh " + shellFile.getAbsolutePath();
			log.info(commond);
			Runtime.getRuntime().exec(commond);
		}
		//log.info("\n" + shellData.getShellCmd());
	}
	/**
	 * 创建Flat表 [初始化执行一次]
	 */
	public void initCreateFlatTable() {
		List<JSONObject> columnList = dynamicExecuteService.list(DsEnum.clickhouse, "select name,`type` from system.columns where `table`='" + tempTableName + "'");
		StrBuilder sqlBuilder = new StrBuilder();
		sqlBuilder.appendLn("create table ads." + cluTableName + " on cluster " + CLUSTER_NAME + " (")
				  .appendLn("`THEMONTH_V` String, ").appendLn("`USERID_V` String,");
		for (JSONObject column : columnList) {
			String name = column.getString("name"), type = column.getString("type");
			if (!Stream.of(INVALID_FIELD_NAMES).anyMatch(f -> name.contains(f))) {
				sqlBuilder.append("`").append(name).append("` ").append(type).appendLn(",");
			}
		}
		// 删除一个空格和一个逗号
		sqlBuilder.deleteLastChar().deleteLastChar();
		sqlBuilder.appendLn(") ENGINE = MergeTree PARTITION BY THEMONTH_V ORDER BY USERID_V");
		log.info("\n" + sqlBuilder.toString());
		dynamicExecuteService.execute(DsEnum.clickhouse, sqlBuilder.toString(), "创建ads.Z_USER_TAG_FLAT_clu表完成");
	}

	/**
	 * 校验字段列是否有改变
	 */
	private void _checkAndDealColumn() {
		List<JSONObject> checkTableDiffList = _getDiffColumnList();
		List<String> addColumns = new ArrayList<>();
//		List<String> delColumns = new ArrayList<>();
		for(JSONObject dataObj : checkTableDiffList) {
			String tempName = dataObj.getString("temp_name"),
				   cluName  = dataObj.getString("clu_name"),
				   type     = dataObj.getString("tp");
			if(StringUtils.isEmpty(cluName)) { //需要添加的列
				addColumns.add("add column `" + tempName + "` " + type);
			}
//			if(StringUtils.isEmpty(tempName)) {//需要删除的列
//				delColumns.add("drop column `"+cluName+"`");
//			}
		}
		
		try {
			//是否需要增删字段； 是->先drop视图
			if(!addColumns.isEmpty()) { //|| !delColumns.isEmpty()
				//1.drop视图
				String dropViewSql = String.format("drop table %s ON cluster %s", flatAllViewName, CLUSTER_NAME);
				log.info("\n" + dropViewSql);
				dynamicExecuteService.execute(DsEnum.clickhouse, dropViewSql, "删除视图完成");
				log.info("删除视图完成，开始等待8s处理增删列...");
				TimeUnit.SECONDS.sleep(8);
				
				//2.处理增删列
				_dealColumn(addColumns);
//				_dealColumn(delColumns);
				
				//3.创建视图
				String createViewSql = String.format("CREATE TABLE %s ON cluster %s AS ads.%s ENGINE = Distributed(%s, ads, %s, rand())", 
						flatAllViewName, CLUSTER_NAME, cluTableName, CLUSTER_NAME, cluTableName);
				log.info("\n" + createViewSql);
				dynamicExecuteService.execute(DsEnum.clickhouse, createViewSql, "创建视图完成");
				log.info("创建视图完成，开始等待8s进入下一操作...");
				TimeUnit.SECONDS.sleep(8);
			}
		} catch (Exception e) {
			log.error(e.getMessage(), e);
		}
	}
	
	private List<JSONObject> _getDiffColumnList(){
		StrBuilder checkTableDiffSql = new StrBuilder()
				.appendLn("select temp.name as temp_name,temp.type as tp,clu.name as clu_name from")
				.appendLn("(select name,`type` from system.columns where `table`='" + tempTableName +(useDebug==true?"_dev":"") +"')temp")
				.appendLn("full outer join")
				.appendLn("(select name,`type` from system.columns where `table`='" + cluTableName +(useDebug==true?"_dev":"")+ "')clu")
				.appendLn("on temp.name=clu.name");
		return dynamicExecuteService.list(DsEnum.clickhouse, checkTableDiffSql.toString());
	}
	
	/**
	 * 处理字段列
	 * @param columns
	 */
	private void _dealColumn(List<String> columns) {
		if(!columns.isEmpty()) {
			String addColumnSql = String.format("alter table ads.%s ON cluster %s %s", cluTableName, CLUSTER_NAME, String.join(",", columns));
			log.info("\n" + addColumnSql);
			dynamicExecuteService.execute(DsEnum.clickhouse, addColumnSql, "更新字段完成");
		}
	}
	
	/*
	 * 其它标签处理
	 */
	private boolean _handleOthers(StrBuilder sqlBuilder, String name, String type) {
		if(name.endsWith("_V") || name.endsWith(".V") || type.contains("Array")) {
			sqlBuilder.appendLn("`"+name+"`,");
		}else {
			String nullValue = name.endsWith("_C") || name.endsWith(".C") ? "未知" : "AAA";
			if(type.contains("Nullable")) {
				sqlBuilder.appendLn("ifNull("+name+",'"+nullValue+"') as `"+name+"`,");
			}else {
				sqlBuilder.appendLn("if(" + name + "='','"+nullValue+"'," + name + ") as `" + name + "`,");
			}
		}
		return true;
	}

	/**
	 * 投诉标签为空处理为0
	 * 
	 * @param sqlBuilder
	 * @param name
	 */
	private boolean _handleComplain(StrBuilder sqlBuilder, String name) {
		if (name.contains("TAG_COMPLAIN")) {
			if (name.contains("_C")) {
				sqlBuilder.appendLn("ifNull(" + name + ",'0次') as `" + name + "`,");
			} else if (name.contains("_V")) {
				sqlBuilder.appendLn("`" + name + "`,");
			} else {
				sqlBuilder.appendLn("ifNull(" + name + ",'B112_22_001') as `" + name + "`,");
			}
			return true;
		}
		return false;
	}

	/**
	 * 省份标签处理
	 * 
	 * @param sqlBuilder
	 * @param name
	 */
	private boolean _handleProvince(StrBuilder sqlBuilder, String name) {
		if (name.contains("TAG_PROVINCE")) {
			if (name.endsWith("_C")) {
				sqlBuilder.appendLn("if(" + name + "='',TMP_PROVINCE_C," + name + ") as `" + name + "`,");
			} else if (name.endsWith("_V")) {
				sqlBuilder.appendLn("`TAG_PROVINCE_C` as `" + name + "`,");
			} else {
				sqlBuilder.appendLn("if(" + name + "='',TMP_PROVINCE, " + name + ") as `" + name + "`,");
			}
			return true;
		}
		return false;
	}

	/**
	 * 城市标签处理
	 * 
	 * @param sqlBuilder
	 * @param name
	 */
	private boolean _handleCity(StrBuilder sqlBuilder, String name) {
		if (name.contains("TAG_AREA")) {
			if (name.endsWith("_C")) {
				sqlBuilder.appendLn("if(" + name + "='',TMP_AREA_C," + name + ") as `" + name + "`,");
			} else if (name.endsWith("_V")) {
				sqlBuilder.appendLn("`TMP_AREA_C` as `" + name + "`,");
			} else {
				sqlBuilder.appendLn("if(" + name + "='',TMP_AREA, " + name + ") as `" + name + "`,");
			}
			return true;
		}
		return false;
	}


	/**
	 * 获取shell文件路径
	 * @param shellData
	 * @return
	 */
	private String getShellFilePath(ShellCommond shellData)
	{
		return 	SHELL_DIC + "to"+shellData.getIp().replace('.','_')+".sh";
	}

	@Data
	public class ShellCommond {
		private String ip;
		private Boolean isLocalIp=false;
		private String shellCmd;
		public ShellCommond(String ip, Boolean isLocalIp,String shellCmd) {
			this.ip=ip;
			this.isLocalIp=isLocalIp;
			this.shellCmd=shellCmd;
		}
	}
}
