From e0c3ae70a20fedc3bad638ef6676363f8eb3efcb Mon Sep 17 00:00:00 2001 From: Jinwei Zhao Date: Tue, 31 Jan 2017 18:32:56 +0800 Subject: refactor --- async_bot.py | 258 +++++++++++++++++++--------------------------------- bot.py | 6 +- config.json.example | 2 +- ingrex/praser.py | 62 ++++++++++--- 4 files changed, 149 insertions(+), 179 deletions(-) diff --git a/async_bot.py b/async_bot.py index 3e6d9b0..3a5aadc 100755 --- a/async_bot.py +++ b/async_bot.py @@ -6,60 +6,53 @@ import time import json import logging import urllib +import sys from selenium import webdriver import telegram -from telegram.error import NetworkError from pymongo import MongoClient import ingrex -Debug = True +import aiohttp +import asyncio +import async_timeout + bot = None BOT_TOKEN = '' CHANNEL_NAME = '' Email = '' Passwd = '' -PhantomjsPath = '' +PhantomJSPath = '' DBName = '' DBUser = '' DBPass = '' DBHost = '' BlockList = '' - LOG_FILENAME = 'voh.log' -logging.basicConfig(level=logging.DEBUG, - filename=LOG_FILENAME, - filemode='w') -console = logging.StreamHandler() -console.setLevel(logging.INFO) -# set a format which is simpler for console use -formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s') -# tell the handler to use this format -console.setFormatter(formatter) -# add the handler to the root logger -logging.getLogger('').addHandler(console) +TIME_ZONE='Asia/Shanghai' class CookieException(Exception): - ''' CookieError ''' + pass -def getTime(): +def get_time(): return time.strftime('%x %X %Z') -def readConfig(): +def read_config(): global Email global Passwd global BOT_TOKEN global CHANNEL_NAME - global PhantomjsPath + global PhantomJSPath global DBName global DBUser global DBPass global DBHost global BlockList + global LOG_FILENAME configfile = open("./config.json") config = json.load(configfile) @@ -67,186 +60,121 @@ def readConfig(): Passwd = config["Passwd"] BOT_TOKEN = config["BOT_TOKEN"] CHANNEL_NAME = config["CHANNEL_NAME"] - PhantomjsPath = config["PhantomjsPath"] + PhantomJSPath = config["PhantomJSPath"] DBName = config["DBName"] DBUser = config["DBUser"] DBPass = config["DBPass"] DBHost = config["DBHost"] BlockList = config["BlockList"] - os.environ['TZ'] = 'Asia/Shanghai' + os.environ['TZ'] = TIME_ZONE time.tzset() + logging.basicConfig(level=logging.DEBUG, + filename=LOG_FILENAME, + filemode='w') + console = logging.StreamHandler() + console.setLevel(logging.INFO) + formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s') + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) -def fetchCookie(): - global Debug - global Email - global Passwd - global PhantomjsPath - logger = logging.getLogger('fetchCookie') - logger.info(getTime() + ': Fetching Cookie...') +def fetch_cookie(): + logger = logging.getLogger('fetch_cookie') + logger.info(get_time() + ': Fetching Cookie...') - driver = webdriver.PhantomJS(PhantomjsPath) + driver = webdriver.PhantomJS(PhantomJSPath) driver.get('https://www.ingress.com/intel') # get login page link = driver.find_elements_by_tag_name('a')[0].get_attribute('href') driver.get(link) - if Debug: - driver.get_screenshot_as_file('1.png') + driver.get_screenshot_as_file('1.png') + # simulate manual login driver.set_page_load_timeout(10) driver.set_script_timeout(20) driver.find_element_by_id('Email').send_keys(Email) - if Debug: - driver.get_screenshot_as_file('2.png') + driver.get_screenshot_as_file('2.png') driver.find_element_by_css_selector('#next').click() time.sleep(3) driver.find_element_by_id('Passwd').send_keys(Passwd) - if Debug: - driver.get_screenshot_as_file('3.png') + driver.get_screenshot_as_file('3.png') driver.find_element_by_css_selector('#signIn').click() time.sleep(3) - if Debug: - driver.get_screenshot_as_file('3.png') + driver.get_screenshot_as_file('4.png') + # get cookies - temp = driver.get_cookies() + cookies = driver.get_cookies() csrftoken = '' SACSID = '' - for a in temp: - if (a['name'] == 'csrftoken'): - csrftoken = a['value'] - if (a['name'] == 'SACSID'): - SACSID = a['value'] + for key in cookies: + if key['name'] == 'csrftoken': + csrftoken = key['value'] + if key['name'] == 'SACSID': + SACSID = key['value'] if csrftoken == '' or SACSID == '': - logger.error(getTime() + ': Fetch Cookie Failed') raise CookieException with open('cookie', 'w') as file: cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17' file.write(cookie) + logger.info(get_time() + ': Fetching Cookie Succeed') driver.quit() - logger.info(getTime() + ': Fetching Cookie Succeed') return True -def sendMessge(bot, msg): - "sendMsg" - - logger = logging.getLogger('sendMessage') +def send_message(bot, message, monitor=False): + logger = logging.getLogger('send_message') while True: try: - url = 'https://api.telegram.org/bot' - url += BOT_TOKEN - url += '/sendMessage?chat_id=' - url += CHANNEL_NAME - url += '&text=' - url += urllib.parse.quote(msg) - - req = urllib.request.Request(url, headers={'Content-Type': 'application/x-www-form-urlencoded'}) - resp = urllib.request.urlopen(req) - data = resp.read() - logger.info(data) - logger.info(getTime() + ": sendMsg " + msg) + if monitor is True: + bot.sendMessage(chat_id="@voamonitor", text=message) + else: + bot.sendMessage(chat_id=CHANNEL_NAME, text=urllib.parse.quote(message)) + logger.info(get_time() + ": sendMsg " + message) break - except NetworkError: + except telegram.TelegramError: + logger.error(get_time() + ": Send Message to Channel Failed") time.sleep(1) - - -def sendMonitor(bot, msg): - logger = logging.getLogger('sendMonitor') - while True: - try: - bot.sendMessage(chat_id="@voamonitor", text=msg) - logger.info(getTime() + ": sendMsg " + msg) - break - except NetworkError: + except Exception: + logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0]) time.sleep(1) -def formatMessage(raw): - pattern = re.compile(BlockList) - match = pattern.search(str(raw)) - if match: - return "Blocked" - - msg = '' - plext = raw[2]['plext'] - markup = plext['markup'] - - for mark in markup: - if mark[0] == 'SECURE': - msg += mark[1]['plain'] - elif mark[0] == 'SENDER': - player = mark[1]['plain'] - team = mark[1]['team'] - - pattern = re.compile(':') - match = pattern.search(player) - if match: - if team == 'RESISTANCE': - player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:] - elif team == 'ENLIGHTENED': - player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:] - msg += player - - elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER': - player = mark[1]['plain'] - team = mark[1]['team'] - - msg += player - if team == 'RESISTANCE': - msg += ' 🐳' - elif team == 'ENLIGHTENED': - msg += ' 🐸' - - elif mark[0] == 'TEXT': - msg += mark[1]['plain'] - - pattern = re.compile('\[secure\]') - match = pattern.search(msg) - if match: - if msg.find(':') != -1: - msg = msg[:9] + '@' + msg[9:] - else: - msg = msg[:10] + '@' + msg[10:] - else: - msg = '@' + msg - - return msg - - -def FindRecord(id): +def find_message_record(id): uri = 'mongodb://' + DBHost - Conn = MongoClient(uri) - Conn.api.authenticate(DBUser, DBPass, DBName) - database = Conn[DBName] - mycollection = database.entries - res = mycollection.find({"id": id}) - if res.count() == 0: + conn = MongoClient(uri) + conn.api.authenticate(DBUser, DBPass, DBName) + database = conn[DBName] + collection = database.entries + count = collection.find({"id": id}).count() + conn.close() + if count == 0: return False else: return True -def insertDB(time, id, msg): +def insert_message_to_database(time, id, msg): uri = 'mongodb://' + DBHost - Conn = MongoClient(uri) - Conn.api.authenticate(DBUser, DBPass, DBName) - database = Conn[DBName] - mycollection = database.entries + conn = MongoClient(uri) + conn.api.authenticate(DBUser, DBPass, DBName) + database = conn[DBName] + collection = database.entries post = {"id": id, "time": time, "msg": msg} - mycollection.insert(post) - Conn.close() + collection.insert(post) + conn.close() def main(): logger = logging.getLogger('main') + # Lat Lng of Hangzhou field = { 'minLngE6': 119618783, 'minLatE6': 29912919, @@ -255,71 +183,73 @@ def main(): } mints = -1 - maxts=-1 - reverse=False - tab='all' + maxts = -1 + reverse = False + tab = 'all' + # fetch cookie while True: try: - if fetchCookie(): + if fetch_cookie(): break except CookieException: + logger.error(get_time() + ': Fetch Cookie Failed') + time.sleep(3) + except Exception: + logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0]) time.sleep(3) + # fetch message count = 0 while True: count += 1 with open('cookie') as cookies: cookies = cookies.read().strip() + logger.info(get_time() + ": {} Fetching from Intel...".format(str(count))) - logger.info(getTime() + ": {} Fetching from Intel...".format(str(count))) - + # fetch message per time while True: try: - intel = ingrex.Intel(cookies, field) result = intel.fetch_msg(mints, maxts, reverse, tab) - if result: mints = result[0][1] + 1 break - except CookieException: while True: try: - if fetchCookie(): + if fetch_cookie(): break except CookieException: time.sleep(3) except Exception: - pass + logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0]) + time.sleep(3) for item in result[::-1]: - message = ingrex.Message(item) + # Check spam message + pattern = re.compile(BlockList) + match = pattern.search(str(item)) + if match: + continue + message = ingrex.Message(item) if message.ptype == 'PLAYER_GENERATED': - # logger.info(getTime() + str(item)) - - msg = formatMessage(item) - if msg == 'Blocked': - logger.info(getTime() + " " + message.text) - else: - msg = message.time + " " + msg - # logger.info(getTime() + " " + msg) - if FindRecord(message.guid) is False: - insertDB(message.time, message.guid, msg) - # sendMonitor(bot, msg) - sendMessge(bot, msg) + logger.info(get_time() + " " + message) + if find_message_record(message.guid) is False: + insert_message_to_database(message.time, message.guid, message) + send_message(bot, message, False) time.sleep(10) if __name__ == '__main__': - readConfig() + read_config() bot = telegram.Bot(BOT_TOKEN) while True: try: main() except Exception: - sendMonitor(bot, 'Main Error') + send_message(bot, 'Main Unexpected error' + sys.exc_info()[0], True) + time.sleep(3) \ No newline at end of file diff --git a/bot.py b/bot.py index 3e6d9b0..80ee4ed 100755 --- a/bot.py +++ b/bot.py @@ -20,7 +20,7 @@ BOT_TOKEN = '' CHANNEL_NAME = '' Email = '' Passwd = '' -PhantomjsPath = '' +PhantomJSPath = '' DBName = '' DBUser = '' DBPass = '' @@ -54,7 +54,7 @@ def readConfig(): global Passwd global BOT_TOKEN global CHANNEL_NAME - global PhantomjsPath + global PhantomJSPath global DBName global DBUser global DBPass @@ -82,7 +82,7 @@ def fetchCookie(): global Debug global Email global Passwd - global PhantomjsPath + global PhantomJSPath logger = logging.getLogger('fetchCookie') logger.info(getTime() + ': Fetching Cookie...') diff --git a/config.json.example b/config.json.example index 7e61d75..ea29c95 100755 --- a/config.json.example +++ b/config.json.example @@ -3,7 +3,7 @@ "Passwd": "Your Ingress Account Password", "BOT_TOKEN": "Your Telegram Bot Token", "CHANNEL_NAME": "Your Channel ID", - "PhantomjsPath": "/usr/local/phantomjs/bin/phantomjs", + "PhantomJSPath": "/usr/local/phantomjs/bin/phantomjs", "DBName": "", "DBUser": "", "DBPass": "", diff --git a/ingrex/praser.py b/ingrex/praser.py index a05ec1e..1d2c03d 100755 --- a/ingrex/praser.py +++ b/ingrex/praser.py @@ -1,26 +1,66 @@ -"Ingrex praser deal with message" from datetime import datetime, timedelta -import platform +import re import os import time -osname = platform.system() -if osname == "Linux": - os.environ['TZ'] = 'Asia/Shanghai' - time.tzset() - class Message(object): - "Message object" def __init__(self, raw_msg): + os.environ['TZ'] = 'Asia/Shanghai' + time.tzset() + + seconds, millis = divmod(raw_msg[1], 1000) + timestamp = datetime.fromtimestamp(seconds) + timedelta(milliseconds=millis) + self.time = timestamp.strftime('%Y/%m/%d %H:%M:%S:%f')[:-3] + self.raw = raw_msg self.guid = raw_msg[0] self.timestamp = raw_msg[1] - seconds, millis = divmod(raw_msg[1], 1000) - time = datetime.fromtimestamp(seconds) + timedelta(milliseconds=millis) - self.time = time.strftime('%Y/%m/%d %H:%M:%S:%f')[:-3] self.text = raw_msg[2]['plext']['text'] self.ptype = raw_msg[2]['plext']['plextType'] self.team = raw_msg[2]['plext']['team'] self.type = raw_msg[2]['plext']['markup'][1][1]['plain'] + self.msg = '' + self.markup = raw_msg[2]['plext']['markup'] + + for mark in self.markup: + if mark[0] == 'SECURE': + self.msg += mark[1]['plain'] + elif mark[0] == 'SENDER': + player = mark[1]['plain'] + team = mark[1]['team'] + + pattern = re.compile(':') + match = pattern.search(player) + if match: + if team == 'RESISTANCE': + player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:] + elif team == 'ENLIGHTENED': + player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:] + self.msg += player + + elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER': + player = mark[1]['plain'] + team = mark[1]['team'] + + self.msg += player + if team == 'RESISTANCE': + self.msg += ' 🐳' + elif team == 'ENLIGHTENED': + self.msg += ' 🐸' + + elif mark[0] == 'TEXT': + self.msg += mark[1]['plain'] + + pattern = re.compile('\[secure\]') + match = pattern.search(self.msg) + if match: + if self.msg.find(':') != -1: + self.msg = self.msg[:9] + '@' + self.msg[9:] + else: + self.msg = self.msg[:10] + '@' + self.msg[10:] + else: + self.msg = '@' + self.msg + + self.msg = self.time + self.msg -- cgit v1.2.3