From 4d2a94988cf0e51b51bfc4d6f5e73aa6351402c6 Mon Sep 17 00:00:00 2001 From: Jinwei Zhao Date: Wed, 1 Feb 2017 21:34:30 +0800 Subject: refactor --- async_bot.py | 263 ---------------------------------------------------- bot.py | 294 +++++++++++++++++++++++------------------------------------ searchbot.py | 41 --------- 3 files changed, 113 insertions(+), 485 deletions(-) delete mode 100755 async_bot.py delete mode 100644 searchbot.py diff --git a/async_bot.py b/async_bot.py deleted file mode 100755 index 5c60503..0000000 --- a/async_bot.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re -import time -import json -import logging -import inspect -import urllib -import sys -from selenium import webdriver - -import telegram -from pymongo import MongoClient - -import ingrex - -import aiohttp -import asyncio -import async_timeout - -bot = None -BOT_TOKEN = '' -CHANNEL_NAME = '' -Email = '' -Passwd = '' -PhantomJSPath = '' -DBName = '' -DBUser = '' -DBPass = '' -DBHost = '' -BlockList = '' -LOG_FILENAME = 'voh.log' -TIME_ZONE='Asia/Shanghai' -minLngE6 = 0 -minLatE6 = 0 -maxLngE6 = 0 -maxLatE6 = 0 - - -class CookieException(Exception): - """Intel Error""" - pass - - -def get_time(): - return time.strftime('%x %X %Z') - - -def read_config(): - global Email - global Passwd - global BOT_TOKEN - global CHANNEL_NAME - global PhantomJSPath - global DBName - global DBUser - global DBPass - global DBHost - global BlockList - global LOG_FILENAME - global minLngE6 - global minLatE6 - global maxLngE6 - global maxLatE6 - - configfile = open("./config.json") - config = json.load(configfile) - Email = config["Email"] - Passwd = config["Passwd"] - BOT_TOKEN = config["BOT_TOKEN"] - CHANNEL_NAME = config["CHANNEL_NAME"] - PhantomJSPath = config["PhantomJSPath"] - DBName = config["DBName"] - DBUser = config["DBUser"] - DBPass = config["DBPass"] - DBHost = config["DBHost"] - BlockList = config["BlockList"] - minLngE6 = config["minLngE6"] - minLatE6 = config["minLatE6"] - maxLngE6 = config["maxLngE6"] - maxLatE6 = config["maxLatE6"] - - os.environ['TZ'] = TIME_ZONE - time.tzset() - - logging.basicConfig(level=logging.DEBUG, - filename=LOG_FILENAME, - filemode='w') - console = logging.StreamHandler() - console.setLevel(logging.INFO) - formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s') - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - - -def fetch_cookie(): - logger = logging.getLogger('fetch_cookie') - logger.info(get_time() + ': Fetching Cookie...') - - driver = webdriver.PhantomJS(PhantomJSPath) - driver.get('https://www.ingress.com/intel') - - # get login page - link = driver.find_elements_by_tag_name('a')[0].get_attribute('href') - driver.get(link) - driver.get_screenshot_as_file('1.png') - - # simulate manual login - driver.set_page_load_timeout(10) - driver.set_script_timeout(20) - driver.find_element_by_id('Email').send_keys(Email) - driver.get_screenshot_as_file('2.png') - driver.find_element_by_css_selector('#next').click() - time.sleep(3) - driver.find_element_by_id('Passwd').send_keys(Passwd) - driver.get_screenshot_as_file('3.png') - driver.find_element_by_css_selector('#signIn').click() - time.sleep(3) - driver.get_screenshot_as_file('4.png') - - # get cookies - cookies = driver.get_cookies() - - csrftoken = '' - SACSID = '' - for key in cookies: - if key['name'] == 'csrftoken': - csrftoken = key['value'] - if key['name'] == 'SACSID': - SACSID = key['value'] - - if csrftoken == '' or SACSID == '': - raise CookieException - - with open('cookie', 'w') as file: - cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17' - file.write(cookie) - - logger.info(get_time() + ': Fetching Cookie Succeed') - driver.quit() - return True - - -def send_message(bot, message, monitor=False): - logger = logging.getLogger('send_message') - while True: - try: - if monitor is True: - bot.sendMessage(chat_id="@voamonitor", text=message) - else: - print(type(message)) - bot.sendMessage(chat_id=CHANNEL_NAME, text=message) - logger.info(get_time() + ": sendMsg " + message) - break - except telegram.TelegramError: - logger.error(get_time() + ": Send Message to Channel Failed") - time.sleep(1) - except Exception: - logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " " + str(inspect.currentframe().f_lineno)) - time.sleep(1) - - -def find_message_record(id): - uri = 'mongodb://' + DBHost - conn = MongoClient(uri) - conn.api.authenticate(DBUser, DBPass, DBName) - database = conn[DBName] - collection = database.entries - count = collection.find({"id": id}).count() - conn.close() - if count == 0: - return False - else: - return True - - -def insert_message_to_database(time, id, msg): - uri = 'mongodb://' + DBHost - conn = MongoClient(uri) - conn.api.authenticate(DBUser, DBPass, DBName) - database = conn[DBName] - collection = database.entries - post = {"id": id, "time": time, "msg": msg} - collection.insert(post) - conn.close() - - -def main(): - logger = logging.getLogger(__name__) - - # Lat & Lng of fetch region - field = { - 'minLngE6': minLngE6, - 'minLatE6': minLatE6, - 'maxLngE6': maxLngE6, - 'maxLatE6': maxLatE6, - } - - mints = -1 - maxts = -1 - reverse = False - tab = 'all' - - # fetch cookie - while True: - try: - if fetch_cookie(): - break - except CookieException: - logger.error(get_time() + ': Fetch Cookie Failed') - time.sleep(3) - except: - logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " " + str(inspect.currentframe().f_lineno)) - time.sleep(3) - - - # fetch message - count = 0 - while True: - count += 1 - logger.info(get_time() + ": {} Fetching from Intel...".format(str(count))) - - with open('cookie') as cookies: - cookies = cookies.read().strip() - - # fetch message per time - while True: - try: - intel = ingrex.Intel(cookies, field) - result = intel.fetch_msg(mints, maxts, reverse, tab) - if result: - mints = result[0][1] + 1 - break - except: - logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " " + str(inspect.currentframe().f_lineno)) - time.sleep(3) - - for item in result[::-1]: - # Check spam message - pattern = re.compile(BlockList) - match = pattern.search(str(item)) - if match: - continue - - message = ingrex.Message(item) - if message.ptype == 'PLAYER_GENERATED': - if find_message_record(message.guid) is False: - insert_message_to_database(message.time, message.guid, message.msg) - send_message(bot, message.msg, False) - - time.sleep(10) - -if __name__ == '__main__': - read_config() - bot = telegram.Bot(BOT_TOKEN) - - while True: - try: - main() - except Exception: - send_message(bot, 'Main Unexpected error' + str(sys.exc_info()[0]) + " " + str(inspect.currentframe().f_lineno), True) - time.sleep(3) \ No newline at end of file diff --git a/bot.py b/bot.py index 80ee4ed..10b1107 100755 --- a/bot.py +++ b/bot.py @@ -2,19 +2,17 @@ import os import re +import sys import time import json +import ingrex import logging -import urllib -from selenium import webdriver - +import inspect import telegram -from telegram.error import NetworkError -from pymongo import MongoClient -import ingrex +from selenium import webdriver +from pymongo import MongoClient -Debug = True bot = None BOT_TOKEN = '' CHANNEL_NAME = '' @@ -26,30 +24,24 @@ DBUser = '' DBPass = '' DBHost = '' BlockList = '' - LOG_FILENAME = 'voh.log' -logging.basicConfig(level=logging.DEBUG, - filename=LOG_FILENAME, - filemode='w') -console = logging.StreamHandler() -console.setLevel(logging.INFO) -# set a format which is simpler for console use -formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s') -# tell the handler to use this format -console.setFormatter(formatter) -# add the handler to the root logger -logging.getLogger('').addHandler(console) +TIME_ZONE='Asia/Shanghai' +minLngE6 = 0 +minLatE6 = 0 +maxLngE6 = 0 +maxLatE6 = 0 class CookieException(Exception): - ''' CookieError ''' + """Intel Error""" + pass -def getTime(): +def get_time(): return time.strftime('%x %X %Z') -def readConfig(): +def read_config(): global Email global Passwd global BOT_TOKEN @@ -60,6 +52,11 @@ def readConfig(): global DBPass global DBHost global BlockList + global LOG_FILENAME + global minLngE6 + global minLatE6 + global maxLngE6 + global maxLatE6 configfile = open("./config.json") config = json.load(configfile) @@ -67,259 +64,194 @@ def readConfig(): Passwd = config["Passwd"] BOT_TOKEN = config["BOT_TOKEN"] CHANNEL_NAME = config["CHANNEL_NAME"] - PhantomjsPath = config["PhantomjsPath"] + PhantomJSPath = config["PhantomJSPath"] DBName = config["DBName"] DBUser = config["DBUser"] DBPass = config["DBPass"] DBHost = config["DBHost"] BlockList = config["BlockList"] + minLngE6 = config["minLngE6"] + minLatE6 = config["minLatE6"] + maxLngE6 = config["maxLngE6"] + maxLatE6 = config["maxLatE6"] - os.environ['TZ'] = 'Asia/Shanghai' + os.environ['TZ'] = TIME_ZONE time.tzset() + logging.basicConfig(level=logging.DEBUG, + filename=LOG_FILENAME, + filemode='w') + console = logging.StreamHandler() + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s') + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) -def fetchCookie(): - global Debug - global Email - global Passwd - global PhantomJSPath - logger = logging.getLogger('fetchCookie') - logger.info(getTime() + ': Fetching Cookie...') +def fetch_cookie(): + logger = logging.getLogger('fetch_cookie') + logger.info(get_time() + ': Fetching Cookie...') - driver = webdriver.PhantomJS(PhantomjsPath) + driver = webdriver.PhantomJS(PhantomJSPath) driver.get('https://www.ingress.com/intel') # get login page link = driver.find_elements_by_tag_name('a')[0].get_attribute('href') driver.get(link) - if Debug: - driver.get_screenshot_as_file('1.png') + driver.get_screenshot_as_file('1.png') + # simulate manual login driver.set_page_load_timeout(10) driver.set_script_timeout(20) driver.find_element_by_id('Email').send_keys(Email) - if Debug: - driver.get_screenshot_as_file('2.png') + driver.get_screenshot_as_file('2.png') driver.find_element_by_css_selector('#next').click() time.sleep(3) driver.find_element_by_id('Passwd').send_keys(Passwd) - if Debug: - driver.get_screenshot_as_file('3.png') + driver.get_screenshot_as_file('3.png') driver.find_element_by_css_selector('#signIn').click() time.sleep(3) - if Debug: - driver.get_screenshot_as_file('3.png') + driver.get_screenshot_as_file('4.png') + # get cookies - temp = driver.get_cookies() + cookies = driver.get_cookies() csrftoken = '' SACSID = '' - for a in temp: - if (a['name'] == 'csrftoken'): - csrftoken = a['value'] - if (a['name'] == 'SACSID'): - SACSID = a['value'] + for key in cookies: + if key['name'] == 'csrftoken': + csrftoken = key['value'] + if key['name'] == 'SACSID': + SACSID = key['value'] if csrftoken == '' or SACSID == '': - logger.error(getTime() + ': Fetch Cookie Failed') raise CookieException with open('cookie', 'w') as file: cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17' file.write(cookie) + logger.info(get_time() + ': Fetching Cookie Succeed') driver.quit() - logger.info(getTime() + ': Fetching Cookie Succeed') return True -def sendMessge(bot, msg): - "sendMsg" - - logger = logging.getLogger('sendMessage') +def send_message(bot, message, monitor=False): + logger = logging.getLogger('send_message') while True: try: - url = 'https://api.telegram.org/bot' - url += BOT_TOKEN - url += '/sendMessage?chat_id=' - url += CHANNEL_NAME - url += '&text=' - url += urllib.parse.quote(msg) - - req = urllib.request.Request(url, headers={'Content-Type': 'application/x-www-form-urlencoded'}) - resp = urllib.request.urlopen(req) - data = resp.read() - logger.info(data) - logger.info(getTime() + ": sendMsg " + msg) + if monitor is True: + bot.sendMessage(chat_id="@voamonitor", text=message) + else: + print(type(message)) + bot.sendMessage(chat_id=CHANNEL_NAME, text=message) + logger.info(get_time() + ": sendMsg " + message) break - except NetworkError: + except telegram.TelegramError: + logger.error(get_time() + ": Send Message to Channel Failed") time.sleep(1) - - -def sendMonitor(bot, msg): - logger = logging.getLogger('sendMonitor') - while True: - try: - bot.sendMessage(chat_id="@voamonitor", text=msg) - logger.info(getTime() + ": sendMsg " + msg) - break - except NetworkError: + except Exception: + logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " Line: " + str(inspect.currentframe().f_lineno)) time.sleep(1) -def formatMessage(raw): - pattern = re.compile(BlockList) - match = pattern.search(str(raw)) - if match: - return "Blocked" - - msg = '' - plext = raw[2]['plext'] - markup = plext['markup'] - - for mark in markup: - if mark[0] == 'SECURE': - msg += mark[1]['plain'] - elif mark[0] == 'SENDER': - player = mark[1]['plain'] - team = mark[1]['team'] - - pattern = re.compile(':') - match = pattern.search(player) - if match: - if team == 'RESISTANCE': - player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:] - elif team == 'ENLIGHTENED': - player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:] - msg += player - - elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER': - player = mark[1]['plain'] - team = mark[1]['team'] - - msg += player - if team == 'RESISTANCE': - msg += ' 🐳' - elif team == 'ENLIGHTENED': - msg += ' 🐸' - - elif mark[0] == 'TEXT': - msg += mark[1]['plain'] - - pattern = re.compile('\[secure\]') - match = pattern.search(msg) - if match: - if msg.find(':') != -1: - msg = msg[:9] + '@' + msg[9:] - else: - msg = msg[:10] + '@' + msg[10:] - else: - msg = '@' + msg - - return msg - - -def FindRecord(id): +def find_message_record(id): uri = 'mongodb://' + DBHost - Conn = MongoClient(uri) - Conn.api.authenticate(DBUser, DBPass, DBName) - database = Conn[DBName] - mycollection = database.entries - res = mycollection.find({"id": id}) - if res.count() == 0: + conn = MongoClient(uri) + conn.api.authenticate(DBUser, DBPass, DBName) + database = conn[DBName] + collection = database.entries + count = collection.find({"id": id}).count() + conn.close() + if count == 0: return False else: return True -def insertDB(time, id, msg): +def insert_message_to_database(time, id, msg): uri = 'mongodb://' + DBHost - Conn = MongoClient(uri) - Conn.api.authenticate(DBUser, DBPass, DBName) - database = Conn[DBName] - mycollection = database.entries + conn = MongoClient(uri) + conn.api.authenticate(DBUser, DBPass, DBName) + database = conn[DBName] + collection = database.entries post = {"id": id, "time": time, "msg": msg} - mycollection.insert(post) - Conn.close() + collection.insert(post) + conn.close() def main(): - logger = logging.getLogger('main') + logger = logging.getLogger(__name__) + # Lat & Lng of fetch region field = { - 'minLngE6': 119618783, - 'minLatE6': 29912919, - 'maxLngE6': 121018722, - 'maxLatE6': 30573739, + 'minLngE6': minLngE6, + 'minLatE6': minLatE6, + 'maxLngE6': maxLngE6, + 'maxLatE6': maxLatE6, } mints = -1 - maxts=-1 - reverse=False - tab='all' + maxts = -1 + reverse = False + tab = 'all' + # fetch cookie while True: try: - if fetchCookie(): + if fetch_cookie(): break except CookieException: + logger.error(get_time() + ': Fetch Cookie Failed') + time.sleep(3) + except: + logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " Line: " + str(inspect.currentframe().f_lineno)) time.sleep(3) + + # fetch message count = 0 while True: count += 1 + logger.info(get_time() + ": {} Fetching from Intel...".format(str(count))) with open('cookie') as cookies: cookies = cookies.read().strip() - logger.info(getTime() + ": {} Fetching from Intel...".format(str(count))) - + # fetch message per time while True: try: - intel = ingrex.Intel(cookies, field) result = intel.fetch_msg(mints, maxts, reverse, tab) - if result: mints = result[0][1] + 1 - break - - except CookieException: - while True: - try: - if fetchCookie(): - break - except CookieException: - time.sleep(3) - except Exception: - pass + break + except: + logger.error(get_time() + ": Unexpected error: " + str(sys.exc_info()[0]) + " Line: " + str(inspect.currentframe().f_lineno)) + time.sleep(3) for item in result[::-1]: - message = ingrex.Message(item) + # Check spam message + pattern = re.compile(BlockList) + match = pattern.search(str(item)) + if match: + continue + message = ingrex.Message(item) if message.ptype == 'PLAYER_GENERATED': - # logger.info(getTime() + str(item)) - - msg = formatMessage(item) - if msg == 'Blocked': - logger.info(getTime() + " " + message.text) - else: - msg = message.time + " " + msg - # logger.info(getTime() + " " + msg) - if FindRecord(message.guid) is False: - insertDB(message.time, message.guid, msg) - # sendMonitor(bot, msg) - sendMessge(bot, msg) + if find_message_record(message.guid) is False: + insert_message_to_database(message.time, message.guid, message.msg) + send_message(bot, message.msg, False) time.sleep(10) if __name__ == '__main__': - readConfig() + read_config() bot = telegram.Bot(BOT_TOKEN) while True: try: main() - except Exception: - sendMonitor(bot, 'Main Error') + except Exception as e: + send_message(bot, 'Main Unexpected error' + str(sys.exc_info()[0]) + " Line: " + str(inspect.currentframe().f_lineno), True) + time.sleep(3) \ No newline at end of file diff --git a/searchbot.py b/searchbot.py deleted file mode 100644 index ee504d5..0000000 --- a/searchbot.py +++ /dev/null @@ -1,41 +0,0 @@ -from pymongo import MongoClient -from telegram.ext import Updater, CommandHandler -import logging -# Enable logging -logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - level=logging.DEBUG) -logger = logging.getLogger(__name__) - - -def search(bot, update, args): - agentid = args[0] - print(agentid) - conn = MongoClient('Mongo Address') - db = conn['COMM_Hangzhou'] - collection = db.entries - res = collection.find({"$text": {"$search": agentid}}) - for i in res: - print(i) - bot.sendMessage(update.message.chat_id, text=str(i)) - - -def start(bot, update): - bot.sendMessage(update.message.chat_id, text="Hi! Use /search agentid to start, no need for @") - - -def error(bot, update, error): - logger.warn('Update "%s" caused error "%s"' % (update, error)) - - -def main(): - updater = Updater('Telegram Bot ID') - dp = updater.dispatcher - dp.add_handler(CommandHandler("start", start)) - dp.add_handler(CommandHandler("help", start)) - dp.add_handler(CommandHandler("search", search, pass_args=True)) - dp.add_error_handler(error) - updater.start_polling() - updater.idle() - -if __name__ == "__main__": - main() -- cgit v1.2.3