博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
ELK-Python(一)
阅读量:7226 次
发布时间:2019-06-29

本文共 2903 字,大约阅读时间需要 9 分钟。

不具有通用性,留作纪念。

[root@GXB-CTRLCENTER python]# cat insert_active_user.py #!/usr/bin/env python# -*- coding:utf-8 -*-from datetime import *from with_conn_to_db import conn_to_mysqlimport urllib2,jsonimport time###define yestoday 0-24 hours delta part##########today = date.today()yestoday = today - timedelta(days=1)#print today,yestodaya = str(yestoday) + ' ' + '00:00:00'b = str(today) + ' ' + '00:00:00'timeArray1 = time.strptime(a, "%Y-%m-%d %H:%M:%S")timeArray2 = time.strptime(b, "%Y-%m-%d %H:%M:%S")start_time = int(time.mktime(timeArray1)) * 1000end_time = int(time.mktime(timeArray2)) * 1000#####define es index and search part########server = 'http://elk.xkops.com:9200/'#stat_index = 'client-visit-*'index='client-*'#start_time = 1459146210879#stop_time = 1459147110879url = server + index + "/_search?pretty=true"query_date={  "query": {    "filtered": {      "query": {        "query_string": {          "query": "*",          "analyze_wildcard": True        }      },      "filter": {        "bool": {          "must": [            {              "range": {                "@timestamp": {                  "gte": start_time,                  "lte": end_time,                  "format": "epoch_millis"                }              }            }          ],          "must_not": []        }      }    }  },  "size": 0,  "aggs": {    "2": {      "terms": {        "field": "visit_tenant_id",        "size": 10000000,        "order": {          "_count": "desc"        }      },      "aggs": {        "3": {          "terms": {            "field": "user_id",            "size": 0,            "order": {              "_count": "desc"            }          },          "aggs": {            "4": {              "terms": {                "field": "ip_address",                "size": 1,                "order": {                  "_count": "desc"                }              }            }          }        }      }    }  }}query_date = json.dumps(query_date)req = urllib2.Request(url,query_date)response = urllib2.urlopen(req)page = response.read()#print pageresult = json.loads(page)###避免当天多次插入,插入前先删除#######sql = "delete from active_user where create_time = '%s'" % (yestoday)with conn_to_mysql('logstash') as db:    db.execute(sql)for s in result['aggregations']['2']['buckets']:    #print s    tenant_id =  s['key']    if len(s['3']['buckets']) != 0:        for a in  range(len(s['3']['buckets'])):            user_id = s['3']['buckets'][a]['key']            ip_address = s['3']['buckets'][a]['4']['buckets'][0]['key']            #print tenant_id,user_id,ip_address            sql = "insert into active_user(tenant_id,create_time,user_id,ip_addr) values('%s','%s','%s','%s')" %(tenant_id,yestoday,user_id,ip_address)            #print sql            with conn_to_mysql('logstash') as db:                db.execute(sql)    else:        continue

 

转载于:https://www.cnblogs.com/xkops/p/5602057.html

你可能感兴趣的文章
关于冰岛足球的段子
查看>>
在 Windows 中安装 Laravel 5.1.X
查看>>
TeamViewer 9发布-在Linux下安装运行
查看>>
Centos7 Gitea安装教程 - 一款易搭建,运行快的Git服务器
查看>>
CentOS minimal 网络配置
查看>>
Nginx架构
查看>>
为什么结构体中的数组不能用const int变量指定大小?
查看>>
模板特化疑问
查看>>
ruby多线程理解
查看>>
享用甜点时要不要来杯咖啡?Java 与树莓派 (Raspberry PI)
查看>>
精通Oracle 10g Pl/SQL编程
查看>>
邱怡轩:R中大规模矩阵的SVD与矩阵补全
查看>>
C++ Primer习题集(第5版)
查看>>
centos7 mysql 5.7 yum安装
查看>>
JSOUP简单应用
查看>>
Mysql,SqlServer,Oracle主键自动增长的设置
查看>>
开源 java CMS - FreeCMS2.3会员登录
查看>>
malloc(0)的返回值
查看>>
析构方法、克隆对象
查看>>
Python字符编码详解
查看>>