소스 검색

生成long表时会出现先删除的情况,分段通过缓存表提速

huaerzx 2 년 전
부모
커밋
faf591005b

+ 2 - 2
src/main/java/com/platomix/userprofile/handler/support/XxhTagManager.java

@@ -33,7 +33,7 @@ public class XxhTagManager {
 	@Autowired
 	private MobileRangeService mobileRangeService;
 
-	final static long _STEP=22000000L;
+	final static long _STEP=17000000L;
 	/**
 	 * 从etl表向 xxh表转化 转化时 重复消除重复
 	 * @param month 月份
@@ -72,7 +72,7 @@ public class XxhTagManager {
 	 * @param month 月份
 	 */
 	public void xxhFromEtlSp(String month) {
-		SortedSet<String> mobileList = mobileRangeService.getList(_STEP);
+		SortedSet<String> mobileList = mobileRangeService.getList(MobileRangeService.DEFAULT_STEP);
 		String mobileLeft = mobileList.first();
 		for(Iterator<String> it = mobileList.iterator(); it.hasNext(); ) {
 			String mobileRight = it.next();

+ 3 - 3
src/main/java/com/platomix/userprofile/service/MobileRangeService.java

@@ -39,14 +39,14 @@ public class MobileRangeService {
 	 * @param step
 	 * @return
 	 */
-	@Cacheable(cacheNames = "userProfileCache", key = "'mobile_range_'+#p0")
+	//@Cacheable(cacheNames = "userProfileCache", key = "'mobile_range_'+#p0")
 	public SortedSet<String> getList(long step){
 		//todo
 		log.info("计算用户手机号范围开始.....");
 		long startTime = System.currentTimeMillis();
 		StrBuilder sqlBuilder = new StrBuilder();
-		sqlBuilder.appendLn("SELECT rowNumberInAllBlocks() % " + step + " as rNum, USERID")
-				  .appendLn("from (select DEVICE_NUMBER as USERID from ads.XXH_ALL_MONTH_LONG order by USERID) where rNum=0");
+		sqlBuilder.appendLn("SELECT rowNumberInAllBlocks() % " + (long) Math.floor(step/1000) + " as rNum, USERID")
+				  .appendLn("from (select USERID from ads.CFG_ZT_USERS order by USERID) where rNum=0");
 		List<JSONObject> dataList = dynamicExecuteService.list(DsEnum.clickhouse, sqlBuilder.toString());
 		log.info("计算用户手机号范围结束.....|cost:{}s", (System.currentTimeMillis()-startTime)/1000);
 		SortedSet<String> resultSet = new TreeSet<String>();