async test

author: Jinwei Zhao <[email protected]> 2017-01-31 08:45:36 +0800
committer: Jinwei Zhao <[email protected]> 2017-01-31 08:45:36 +0800
commit: 3e3bd71f48821d3cdb34536be4dcecdcde9b49ab (patch)
tree: 6f1c42eeb3d809d26962893e3c7510c1daff50d6
parent: 66d411c97ba82bfe3b072b57d2b7f8a6554723d6 (diff)
download: COMM2TG-3e3bd71f48821d3cdb34536be4dcecdcde9b49ab.tar.gz
2 files changed, 326 insertions, 1 deletions
diff --git a/async_bot.py b/async_bot.py
new file mode 100755
index 0000000..3e6d9b0
--- /dev/null
+++ b/async_bot.py
@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+import os
+import re
+import time
+import json
+import logging
+import urllib
+from selenium import webdriver
+import telegram
+from telegram.error import NetworkError
+from pymongo import MongoClient
+import ingrex
+Debug = True
+bot = None
+BOT_TOKEN = ''
+CHANNEL_NAME = ''
+Email = ''
+Passwd = ''
+PhantomjsPath = ''
+DBName = ''
+DBUser = ''
+DBPass = ''
+DBHost = ''
+BlockList = ''
+LOG_FILENAME = 'voh.log'
+logging.basicConfig(level=logging.DEBUG,
+                    filename=LOG_FILENAME,
+                    filemode='w')
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+# set a format which is simpler for console use
+formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
+# tell the handler to use this format
+console.setFormatter(formatter)
+# add the handler to the root logger
+logging.getLogger('').addHandler(console)
+class CookieException(Exception):
+    ''' CookieError '''
+def getTime():
+    return time.strftime('%x %X %Z')
+def readConfig():
+    global Email
+    global Passwd
+    global BOT_TOKEN
+    global CHANNEL_NAME
+    global PhantomjsPath
+    global DBName
+    global DBUser
+    global DBPass
+    global DBHost
+    global BlockList
+    configfile = open("./config.json")
+    config = json.load(configfile)
+    Email = config["Email"]
+    Passwd = config["Passwd"]
+    BOT_TOKEN = config["BOT_TOKEN"]
+    CHANNEL_NAME = config["CHANNEL_NAME"]
+    PhantomjsPath = config["PhantomjsPath"]
+    DBName = config["DBName"]
+    DBUser = config["DBUser"]
+    DBPass = config["DBPass"]
+    DBHost = config["DBHost"]
+    BlockList = config["BlockList"]
+    os.environ['TZ'] = 'Asia/Shanghai'
+    time.tzset()
+def fetchCookie():
+    global Debug
+    global Email
+    global Passwd
+    global PhantomjsPath
+    logger = logging.getLogger('fetchCookie')
+    logger.info(getTime() + ': Fetching Cookie...')
+    driver = webdriver.PhantomJS(PhantomjsPath)
+    driver.get('https://www.ingress.com/intel')
+    # get login page
+    link = driver.find_elements_by_tag_name('a')[0].get_attribute('href')
+    driver.get(link)
+    if Debug:
+        driver.get_screenshot_as_file('1.png')
+    # simulate manual login
+    driver.set_page_load_timeout(10)
+    driver.set_script_timeout(20)
+    driver.find_element_by_id('Email').send_keys(Email)
+    if Debug:
+        driver.get_screenshot_as_file('2.png')
+    driver.find_element_by_css_selector('#next').click()
+    time.sleep(3)
+    driver.find_element_by_id('Passwd').send_keys(Passwd)
+    if Debug:
+        driver.get_screenshot_as_file('3.png')
+    driver.find_element_by_css_selector('#signIn').click()
+    time.sleep(3)
+    if Debug:
+        driver.get_screenshot_as_file('3.png')
+    # get cookies
+    temp = driver.get_cookies()
+    csrftoken = ''
+    SACSID = ''
+    for a in temp:
+        if (a['name'] == 'csrftoken'):
+            csrftoken = a['value']
+        if (a['name'] == 'SACSID'):
+            SACSID = a['value']
+    if csrftoken == '' or SACSID == '':
+        logger.error(getTime() + ': Fetch Cookie Failed')
+        raise CookieException
+    with open('cookie', 'w') as file:
+        cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17'
+        file.write(cookie)
+    driver.quit()
+    logger.info(getTime() + ': Fetching Cookie Succeed')
+    return True
+def sendMessge(bot, msg):
+    "sendMsg"
+    logger = logging.getLogger('sendMessage')
+    while True:
+        try:
+            url = 'https://api.telegram.org/bot'
+            url += BOT_TOKEN
+            url += '/sendMessage?chat_id='
+            url += CHANNEL_NAME
+            url += '&text='
+            url += urllib.parse.quote(msg)
+            req = urllib.request.Request(url, headers={'Content-Type': 'application/x-www-form-urlencoded'})
+            resp = urllib.request.urlopen(req)
+            data = resp.read()
+            logger.info(data)
+            logger.info(getTime() + ": sendMsg " + msg)
+            break
+        except NetworkError:
+            time.sleep(1)
+def sendMonitor(bot, msg):
+    logger = logging.getLogger('sendMonitor')
+    while True:
+        try:
+            bot.sendMessage(chat_id="@voamonitor", text=msg)
+            logger.info(getTime() + ": sendMsg " + msg)
+            break
+        except NetworkError:
+            time.sleep(1)
+def formatMessage(raw):
+    pattern = re.compile(BlockList)
+    match = pattern.search(str(raw))
+    if match:
+        return "Blocked"
+    msg = ''
+    plext = raw[2]['plext']
+    markup = plext['markup']
+    for mark in markup:
+        if mark[0] == 'SECURE':
+            msg += mark[1]['plain']
+        elif mark[0] == 'SENDER':
+            player = mark[1]['plain']
+            team = mark[1]['team']
+            pattern = re.compile(':')
+            match = pattern.search(player)
+            if match:
+                if team == 'RESISTANCE':
+                    player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:]
+                elif team == 'ENLIGHTENED':
+                    player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:]
+            msg += player
+        elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER':
+            player = mark[1]['plain']
+            team = mark[1]['team']
+            msg += player
+            if team == 'RESISTANCE':
+                msg += ' 🐳'
+            elif team == 'ENLIGHTENED':
+                msg += ' 🐸'
+        elif mark[0] == 'TEXT':
+            msg += mark[1]['plain']
+    pattern = re.compile('\[secure\]')
+    match = pattern.search(msg)
+    if match:
+        if msg.find(':') != -1:
+            msg = msg[:9] + '@' + msg[9:]
+        else:
+            msg = msg[:10] + '@' + msg[10:]
+    else:
+        msg = '@' + msg
+    return msg
+def FindRecord(id):
+    uri = 'mongodb://' + DBHost
+    Conn = MongoClient(uri)
+    Conn.api.authenticate(DBUser, DBPass, DBName)
+    database = Conn[DBName]
+    mycollection = database.entries
+    res = mycollection.find({"id": id})
+    if res.count() == 0:
+        return False
+    else:
+        return True
+def insertDB(time, id, msg):
+    uri = 'mongodb://' + DBHost
+    Conn = MongoClient(uri)
+    Conn.api.authenticate(DBUser, DBPass, DBName)
+    database = Conn[DBName]
+    mycollection = database.entries
+    post = {"id": id, "time": time, "msg": msg}
+    mycollection.insert(post)
+    Conn.close()
+def main():
+    logger = logging.getLogger('main')
+    field = {
+        'minLngE6': 119618783,
+        'minLatE6': 29912919,
+        'maxLngE6': 121018722,
+        'maxLatE6': 30573739,
+    }
+    mints = -1
+    maxts=-1
+    reverse=False
+    tab='all'
+    while True:
+        try:
+            if fetchCookie():
+                break
+        except CookieException:
+            time.sleep(3)
+    count = 0
+    while True:
+        count += 1
+        with open('cookie') as cookies:
+            cookies = cookies.read().strip()
+        logger.info(getTime() + ": {} Fetching from Intel...".format(str(count)))
+        while True:
+            try:
+                intel = ingrex.Intel(cookies, field)
+                result = intel.fetch_msg(mints, maxts, reverse, tab)
+                if result:
+                    mints = result[0][1] + 1
+                    break
+            except CookieException:
+                while True:
+                    try:
+                        if fetchCookie():
+                            break
+                    except CookieException:
+                        time.sleep(3)
+            except Exception:
+                pass
+        for item in result[::-1]:
+            message = ingrex.Message(item)
+            if message.ptype == 'PLAYER_GENERATED':
+                # logger.info(getTime() + str(item))
+                msg = formatMessage(item)
+                if msg == 'Blocked':
+                    logger.info(getTime() + " " + message.text)
+                else:
+                    msg = message.time + " " + msg
+                    # logger.info(getTime() + " " + msg)
+                    if FindRecord(message.guid) is False:
+                        insertDB(message.time, message.guid, msg)
+                        # sendMonitor(bot, msg)
+                        sendMessge(bot, msg)
+        time.sleep(10)
+if __name__ == '__main__':
+    readConfig()
+    bot = telegram.Bot(BOT_TOKEN)
+    while True:
+        try:
+            main()
+        except Exception:
+            sendMonitor(bot, 'Main Error')
diff --git a/bot.py b/bot.py
index dc50264..3e6d9b0 100755
--- a/bot.py
+++ b/bot.py
@@ -34,7 +34,7 @@ logging.basicConfig(level=logging.DEBUG,
 console = logging.StreamHandler()
 console.setLevel(logging.INFO)
 # set a format which is simpler for console use
-formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
+formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
 # tell the handler to use this format
 console.setFormatter(formatter)
 # add the handler to the root logger
author	Jinwei Zhao <[email protected]>	2017-01-31 08:45:36 +0800
committer	Jinwei Zhao <[email protected]>	2017-01-31 08:45:36 +0800
commit	3e3bd71f48821d3cdb34536be4dcecdcde9b49ab (patch)
tree	6f1c42eeb3d809d26962893e3c7510c1daff50d6
parent	66d411c97ba82bfe3b072b57d2b7f8a6554723d6 (diff)
download	COMM2TG-3e3bd71f48821d3cdb34536be4dcecdcde9b49ab.tar.gz

diff --git a/async_bot.py b/async_bot.py new file mode 100755 index 0000000..3e6d9b0 --- /dev/null +++ b/async_bot.py
@@ -0,0 +1,325 @@
		1	# -- coding: utf-8 --
		2
		3	import os
		4	import re
		5	import time
		6	import json
		7	import logging
		8	import urllib
		9	from selenium import webdriver
		10
		11	import telegram
		12	from telegram.error import NetworkError
		13	from pymongo import MongoClient
		14
		15	import ingrex
		16
		17	Debug = True
		18	bot = None
		19	BOT_TOKEN = ''
		20	CHANNEL_NAME = ''
		21	Email = ''
		22	Passwd = ''
		23	PhantomjsPath = ''
		24	DBName = ''
		25	DBUser = ''
		26	DBPass = ''
		27	DBHost = ''
		28	BlockList = ''
		29
		30	LOG_FILENAME = 'voh.log'
		31	logging.basicConfig(level=logging.DEBUG,
		32	filename=LOG_FILENAME,
		33	filemode='w')
		34	console = logging.StreamHandler()
		35	console.setLevel(logging.INFO)
		36	# set a format which is simpler for console use
		37	formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
		38	# tell the handler to use this format
		39	console.setFormatter(formatter)
		40	# add the handler to the root logger
		41	logging.getLogger('').addHandler(console)
		42
		43
		44	class CookieException(Exception):
		45	''' CookieError '''
		46
		47
		48	def getTime():
		49	return time.strftime('%x %X %Z')
		50
		51
		52	def readConfig():
		53	global Email
		54	global Passwd
		55	global BOT_TOKEN
		56	global CHANNEL_NAME
		57	global PhantomjsPath
		58	global DBName
		59	global DBUser
		60	global DBPass
		61	global DBHost
		62	global BlockList
		63
		64	configfile = open("./config.json")
		65	config = json.load(configfile)
		66	Email = config["Email"]
		67	Passwd = config["Passwd"]
		68	BOT_TOKEN = config["BOT_TOKEN"]
		69	CHANNEL_NAME = config["CHANNEL_NAME"]
		70	PhantomjsPath = config["PhantomjsPath"]
		71	DBName = config["DBName"]
		72	DBUser = config["DBUser"]
		73	DBPass = config["DBPass"]
		74	DBHost = config["DBHost"]
		75	BlockList = config["BlockList"]
		76
		77	os.environ['TZ'] = 'Asia/Shanghai'
		78	time.tzset()
		79
		80
		81	def fetchCookie():
		82	global Debug
		83	global Email
		84	global Passwd
		85	global PhantomjsPath
		86
		87	logger = logging.getLogger('fetchCookie')
		88	logger.info(getTime() + ': Fetching Cookie...')
		89
		90	driver = webdriver.PhantomJS(PhantomjsPath)
		91	driver.get('https://www.ingress.com/intel')
		92
		93	# get login page
		94	link = driver.find_elements_by_tag_name('a')[0].get_attribute('href')
		95	driver.get(link)
		96	if Debug:
		97	driver.get_screenshot_as_file('1.png')
		98	# simulate manual login
		99	driver.set_page_load_timeout(10)
		100	driver.set_script_timeout(20)
		101	driver.find_element_by_id('Email').send_keys(Email)
		102	if Debug:
		103	driver.get_screenshot_as_file('2.png')
		104	driver.find_element_by_css_selector('#next').click()
		105	time.sleep(3)
		106	driver.find_element_by_id('Passwd').send_keys(Passwd)
		107	if Debug:
		108	driver.get_screenshot_as_file('3.png')
		109	driver.find_element_by_css_selector('#signIn').click()
		110	time.sleep(3)
		111	if Debug:
		112	driver.get_screenshot_as_file('3.png')
		113	# get cookies
		114	temp = driver.get_cookies()
		115
		116	csrftoken = ''
		117	SACSID = ''
		118	for a in temp:
		119	if (a['name'] == 'csrftoken'):
		120	csrftoken = a['value']
		121	if (a['name'] == 'SACSID'):
		122	SACSID = a['value']
		123
		124	if csrftoken == '' or SACSID == '':
		125	logger.error(getTime() + ': Fetch Cookie Failed')
		126	raise CookieException
		127
		128	with open('cookie', 'w') as file:
		129	cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17'
		130	file.write(cookie)
		131
		132	driver.quit()
		133	logger.info(getTime() + ': Fetching Cookie Succeed')
		134	return True
		135
		136
		137	def sendMessge(bot, msg):
		138	"sendMsg"
		139
		140	logger = logging.getLogger('sendMessage')
		141	while True:
		142	try:
		143	url = 'https://api.telegram.org/bot'
		144	url += BOT_TOKEN
		145	url += '/sendMessage?chat_id='
		146	url += CHANNEL_NAME
		147	url += '&text='
		148	url += urllib.parse.quote(msg)
		149
		150	req = urllib.request.Request(url, headers={'Content-Type': 'application/x-www-form-urlencoded'})
		151	resp = urllib.request.urlopen(req)
		152	data = resp.read()
		153	logger.info(data)
		154	logger.info(getTime() + ": sendMsg " + msg)
		155	break
		156	except NetworkError:
		157	time.sleep(1)
		158
		159
		160	def sendMonitor(bot, msg):
		161	logger = logging.getLogger('sendMonitor')
		162	while True:
		163	try:
		164	bot.sendMessage(chat_id="@voamonitor", text=msg)
		165	logger.info(getTime() + ": sendMsg " + msg)
		166	break
		167	except NetworkError:
		168	time.sleep(1)
		169
		170
		171	def formatMessage(raw):
		172	pattern = re.compile(BlockList)
		173	match = pattern.search(str(raw))
		174	if match:
		175	return "Blocked"
		176
		177	msg = ''
		178	plext = raw[2]['plext']
		179	markup = plext['markup']
		180
		181	for mark in markup:
		182	if mark[0] == 'SECURE':
		183	msg += mark[1]['plain']
		184	elif mark[0] == 'SENDER':
		185	player = mark[1]['plain']
		186	team = mark[1]['team']
		187
		188	pattern = re.compile(':')
		189	match = pattern.search(player)
		190	if match:
		191	if team == 'RESISTANCE':
		192	player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:]
		193	elif team == 'ENLIGHTENED':
		194	player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:]
		195	msg += player
		196
		197	elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER':
		198	player = mark[1]['plain']
		199	team = mark[1]['team']
		200
		201	msg += player
		202	if team == 'RESISTANCE':
		203	msg += ' 🐳'
		204	elif team == 'ENLIGHTENED':
		205	msg += ' 🐸'
		206
		207	elif mark[0] == 'TEXT':
		208	msg += mark[1]['plain']
		209
		210	pattern = re.compile('\[secure\]')
		211	match = pattern.search(msg)
		212	if match:
		213	if msg.find(':') != -1:
		214	msg = msg[:9] + '@' + msg[9:]
		215	else:
		216	msg = msg[:10] + '@' + msg[10:]
		217	else:
		218	msg = '@' + msg
		219
		220	return msg
		221
		222
		223	def FindRecord(id):
		224	uri = 'mongodb://' + DBHost
		225	Conn = MongoClient(uri)
		226	Conn.api.authenticate(DBUser, DBPass, DBName)
		227	database = Conn[DBName]
		228	mycollection = database.entries
		229	res = mycollection.find({"id": id})
		230	if res.count() == 0:
		231	return False
		232	else:
		233	return True
		234
		235
		236	def insertDB(time, id, msg):
		237	uri = 'mongodb://' + DBHost
		238	Conn = MongoClient(uri)
		239	Conn.api.authenticate(DBUser, DBPass, DBName)
		240	database = Conn[DBName]
		241	mycollection = database.entries
		242	post = {"id": id, "time": time, "msg": msg}
		243	mycollection.insert(post)
		244	Conn.close()
		245
		246
		247	def main():
		248	logger = logging.getLogger('main')
		249
		250	field = {
		251	'minLngE6': 119618783,
		252	'minLatE6': 29912919,
		253	'maxLngE6': 121018722,
		254	'maxLatE6': 30573739,
		255	}
		256
		257	mints = -1
		258	maxts=-1
		259	reverse=False
		260	tab='all'
		261
		262	while True:
		263	try:
		264	if fetchCookie():
		265	break
		266	except CookieException:
		267	time.sleep(3)
		268
		269	count = 0
		270	while True:
		271	count += 1
		272
		273	with open('cookie') as cookies:
		274	cookies = cookies.read().strip()
		275
		276	logger.info(getTime() + ": {} Fetching from Intel...".format(str(count)))
		277
		278	while True:
		279	try:
		280
		281	intel = ingrex.Intel(cookies, field)
		282	result = intel.fetch_msg(mints, maxts, reverse, tab)
		283
		284	if result:
		285	mints = result[0][1] + 1
		286	break
		287
		288	except CookieException:
		289	while True:
		290	try:
		291	if fetchCookie():
		292	break
		293	except CookieException:
		294	time.sleep(3)
		295	except Exception:
		296	pass
		297
		298	for item in result[::-1]:
		299	message = ingrex.Message(item)
		300
		301	if message.ptype == 'PLAYER_GENERATED':
		302	# logger.info(getTime() + str(item))
		303
		304	msg = formatMessage(item)
		305	if msg == 'Blocked':
		306	logger.info(getTime() + " " + message.text)
		307	else:
		308	msg = message.time + " " + msg
		309	# logger.info(getTime() + " " + msg)
		310	if FindRecord(message.guid) is False:
		311	insertDB(message.time, message.guid, msg)
		312	# sendMonitor(bot, msg)
		313	sendMessge(bot, msg)
		314
		315	time.sleep(10)
		316
		317	if __name__ == '__main__':
		318	readConfig()
		319	bot = telegram.Bot(BOT_TOKEN)
		320
		321	while True:
		322	try:
		323	main()
		324	except Exception:
		325	sendMonitor(bot, 'Main Error')


diff --git a/bot.py b/bot.py index dc50264..3e6d9b0 100755 --- a/bot.py +++ b/bot.py
@@ -34,7 +34,7 @@ logging.basicConfig(level=logging.DEBUG,
34	console = logging.StreamHandler()	34	console = logging.StreamHandler()
35	console.setLevel(logging.INFO)	35	console.setLevel(logging.INFO)
36	# set a format which is simpler for console use	36	# set a format which is simpler for console use
37	formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')	37	formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
38	# tell the handler to use this format	38	# tell the handler to use this format
39	console.setFormatter(formatter)	39	console.setFormatter(formatter)
40	# add the handler to the root logger	40	# add the handler to the root logger