platomix-tool/platomix-docker/docker-flo/logstash/bi-watch.conf

input{
    beats {
        host => "0.0.0.0"
        port => 13838
    }
}

filter{
   # mutate{
   #     add_field => {"pq1" =>"value1"}
   # }
    mutate{
        gsub => ["message", "[\\\\]", ""]
        remove_field => ["container","@timestamp","log","@version","version","agent","offset","input","beat","tags","host","ecs","_flush_time"]
    #   remove_field => "container"
    }
    
    json {
        source => "message" # 解析成json
    }
     mutate{
        remove_field => "message"
        remove_field => "lib"
	    remove_field => "[properties][$latest_search_keyword]"
        remove_field => "[properties][$url_path]"
        remove_field => "[properties][$lib_version]"
     }
     mutate{
        rename => {
            "[properties][$lib]"=>"lib"
            "[properties][$screen_width]"=>"screen_width"
            "[properties][$screen_height]"=>"screen_height"
            "[properties][$url]"=>"url"
            "[properties][$title]"=>"title"
            "[properties][$element_target_url]"=>"element_target_url"
            "[properties][$is_first_day]"=>"is_first_day"
            "[properties][$element_path]"=>"element_path"
            "[properties][$element_class_name]"=>"element_class_name"
            "[properties][$element_content]"=>"element_content"
            "[properties][$element_type]"=>"element_type"
            "[properties][$latest_referrer]"=>"latest_referrer"
            "[properties][aid]"=>"aid"
            "[properties][taskid]"=>"taskid"
            "[properties][attr1]"=>"attr1"
            "[properties][attr2]"=>"attr2"
            "[properties][attr3]"=>"attr3"
            "[properties][attr4]"=>"attr4"
            "[properties][attr5]"=>"attr5"
            "[properties][attr6]"=>"attr6"
            "[properties][attr7]"=>"attr7"
            "[properties][attr8]"=>"attr8"
            "[properties][attr9]"=>"attr9"
            "[properties][attr10]"=>"attr10"
            "[properties][attr11]"=>"attr11"
            "[properties][attr12]"=>"attr12"
            "[properties][attr13]"=>"attr13"
            "[properties][attr14]"=>"attr14"
            "[properties][attr15]"=>"attr15"
            "[properties][attr16]"=>"attr16"
            "[properties][attr17]"=>"attr17"
            "[properties][attr18]"=>"attr18"
            "[properties][attr19]"=>"attr19"
            "[properties][attr20]"=>"attr20"
            "[properties][platform_type]"=>"platform_type"
            "[properties][source]"=>"source"
            "[properties][plan]"=>"plan"
            "[properties][unit]"=>"unit"
            "[properties][keyword]"=>"keyword"
            "[properties][e_creative]"=>"e_creative"
            "[properties][e_keywordid]"=>"e_keywordid"
            "[properties][account]"=>"account"
            "[properties][referer]"=>"referer"
        }

        convert => {
            "attr1" => "string"
            "attr2" => "string"
            "attr3" => "string"
            "attr4" => "string"
            "attr5" => "string"
            "attr6" => "string"
            "attr7" => "string"
            "attr8" => "string"
            "attr9" => "string"
            "attr10" => "string"
            "attr11" => "string"
            "attr12" => "string"
            "attr13" => "string"
            "attr14" => "string"
            "attr15" => "string"
            "attr16" => "string"
            "attr17" => "string"
            "attr18" => "string"
            "attr19" => "string"
            "attr20" => "string"
            "source" => "string"
            "plan" => "string"
            "unit" => "string"
            "keyword" => "string"
            "e_creative" => "string"
            "e_keywordid" => "string"
            "account" => "string"
		    "referer" => "string"
        }
     }

     date {
        match => ["time", "UNIX_MS"]
        target =>"logdatetime"
        locale =>"cn"
        tag_on_failure =>"1970-01-01 00:00:00"
    }
    # time时区必须加个00:00 不然插入时会+8小时
	#event.set('timezone', Time.at(tstamp).zone)
    ruby {
    code => "tstamp = event.get('logdatetime').to_i
            event.set('d_timestamp', Time.at(tstamp).localtime('+00:00').strftime('%Y-%m-%d %H:%M:%S'))
            event.set('d_date', Time.at(tstamp).strftime('%Y-%m-%d'))"
    }
}

output{
    stdout {
        codec => rubydebug
    }

  	# base64.encode(userName:passWord) => ZGVmYXVsdDpMaWFudG9uZ2JpWWlkb25naGFvNjY=
  	clickhouse {
        headers => ["Authorization", "Basic ZGVmYXVsdDpQbGF0b21peEBjazEyMw=="]
        http_hosts => "http://clickhouse:8123"
        table => "ads.user_event"
        request_tolerance => 1 #失败（响应码不是200）的重试次数
        #flush_size => 1000 #每次批量发送的数据大小，默认值为50
        #pool_max => 1000
        mutations => {    
            proj => proj
            ptype => ptype
            ip => ip
            distinct_id => distinct_id
            anonymous_id => anonymous_id
            login_id => login_id
            d_time => time
            d_timestamp=> d_timestamp
            d_date => d_date
            d_type => type
            event => event
            screen_width => screen_width
            screen_height => screen_height
            lib => lib
            _track_id => _track_id
            url => url
            title => title
            element_target_url => element_target_url
            is_first_day => is_first_day
            element_path => element_path
            element_class_name => element_class_name
            element_content => element_content
            element_type => element_type
            latest_referrer => latest_referrer
            aid=> aid
            taskid=> taskid
            attr1=> attr1
            attr2=> attr2
            attr3=> attr3
            attr4=> attr4
            attr5=> attr5
            attr6=> attr6
            attr7=> attr7
            attr8=> attr8
            attr9=> attr9
            attr10=> attr10
            attr11=> attr11
            attr12=> attr12
            attr13=> attr13
            attr14=> attr14
            attr15=> attr15
            attr16=> attr16
            attr17=> attr17
            attr18=> attr18
            attr19=> attr19
            attr20=> attr20
            platform_type=> platform_type
            source=> source
            plan=> plan
            unit=> unit
            keyword=> keyword
            e_creative=> e_creative
            e_keywordid=> e_keywordid
            account=> account  
            referer=> referer
   	    }
    }
}