aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xasync_bot.py258
-rwxr-xr-xbot.py6
-rwxr-xr-xconfig.json.example2
-rwxr-xr-xingrex/praser.py62
4 files changed, 149 insertions, 179 deletions
diff --git a/async_bot.py b/async_bot.py
index 3e6d9b0..3a5aadc 100755
--- a/async_bot.py
+++ b/async_bot.py
@@ -6,60 +6,53 @@ import time
6import json 6import json
7import logging 7import logging
8import urllib 8import urllib
9import sys
9from selenium import webdriver 10from selenium import webdriver
10 11
11import telegram 12import telegram
12from telegram.error import NetworkError
13from pymongo import MongoClient 13from pymongo import MongoClient
14 14
15import ingrex 15import ingrex
16 16
17Debug = True 17import aiohttp
18import asyncio
19import async_timeout
20
18bot = None 21bot = None
19BOT_TOKEN = '' 22BOT_TOKEN = ''
20CHANNEL_NAME = '' 23CHANNEL_NAME = ''
21Email = '' 24Email = ''
22Passwd = '' 25Passwd = ''
23PhantomjsPath = '' 26PhantomJSPath = ''
24DBName = '' 27DBName = ''
25DBUser = '' 28DBUser = ''
26DBPass = '' 29DBPass = ''
27DBHost = '' 30DBHost = ''
28BlockList = '' 31BlockList = ''
29
30LOG_FILENAME = 'voh.log' 32LOG_FILENAME = 'voh.log'
31logging.basicConfig(level=logging.DEBUG, 33TIME_ZONE='Asia/Shanghai'
32 filename=LOG_FILENAME,
33 filemode='w')
34console = logging.StreamHandler()
35console.setLevel(logging.INFO)
36# set a format which is simpler for console use
37formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
38# tell the handler to use this format
39console.setFormatter(formatter)
40# add the handler to the root logger
41logging.getLogger('').addHandler(console)
42 34
43 35
44class CookieException(Exception): 36class CookieException(Exception):
45 ''' CookieError ''' 37 pass
46 38
47 39
48def getTime(): 40def get_time():
49 return time.strftime('%x %X %Z') 41 return time.strftime('%x %X %Z')
50 42
51 43
52def readConfig(): 44def read_config():
53 global Email 45 global Email
54 global Passwd 46 global Passwd
55 global BOT_TOKEN 47 global BOT_TOKEN
56 global CHANNEL_NAME 48 global CHANNEL_NAME
57 global PhantomjsPath 49 global PhantomJSPath
58 global DBName 50 global DBName
59 global DBUser 51 global DBUser
60 global DBPass 52 global DBPass
61 global DBHost 53 global DBHost
62 global BlockList 54 global BlockList
55 global LOG_FILENAME
63 56
64 configfile = open("./config.json") 57 configfile = open("./config.json")
65 config = json.load(configfile) 58 config = json.load(configfile)
@@ -67,186 +60,121 @@ def readConfig():
67 Passwd = config["Passwd"] 60 Passwd = config["Passwd"]
68 BOT_TOKEN = config["BOT_TOKEN"] 61 BOT_TOKEN = config["BOT_TOKEN"]
69 CHANNEL_NAME = config["CHANNEL_NAME"] 62 CHANNEL_NAME = config["CHANNEL_NAME"]
70 PhantomjsPath = config["PhantomjsPath"] 63 PhantomJSPath = config["PhantomJSPath"]
71 DBName = config["DBName"] 64 DBName = config["DBName"]
72 DBUser = config["DBUser"] 65 DBUser = config["DBUser"]
73 DBPass = config["DBPass"] 66 DBPass = config["DBPass"]
74 DBHost = config["DBHost"] 67 DBHost = config["DBHost"]
75 BlockList = config["BlockList"] 68 BlockList = config["BlockList"]
76 69
77 os.environ['TZ'] = 'Asia/Shanghai' 70 os.environ['TZ'] = TIME_ZONE
78 time.tzset() 71 time.tzset()
79 72
73 logging.basicConfig(level=logging.DEBUG,
74 filename=LOG_FILENAME,
75 filemode='w')
76 console = logging.StreamHandler()
77 console.setLevel(logging.INFO)
78 formatter = logging.Formatter('%(name)-8s: %(levelname)-4s %(message)s')
79 console.setFormatter(formatter)
80 logging.getLogger('').addHandler(console)
80 81
81def fetchCookie():
82 global Debug
83 global Email
84 global Passwd
85 global PhantomjsPath
86 82
87 logger = logging.getLogger('fetchCookie') 83def fetch_cookie():
88 logger.info(getTime() + ': Fetching Cookie...') 84 logger = logging.getLogger('fetch_cookie')
85 logger.info(get_time() + ': Fetching Cookie...')
89 86
90 driver = webdriver.PhantomJS(PhantomjsPath) 87 driver = webdriver.PhantomJS(PhantomJSPath)
91 driver.get('https://www.ingress.com/intel') 88 driver.get('https://www.ingress.com/intel')
92 89
93 # get login page 90 # get login page
94 link = driver.find_elements_by_tag_name('a')[0].get_attribute('href') 91 link = driver.find_elements_by_tag_name('a')[0].get_attribute('href')
95 driver.get(link) 92 driver.get(link)
96 if Debug: 93 driver.get_screenshot_as_file('1.png')
97 driver.get_screenshot_as_file('1.png') 94
98 # simulate manual login 95 # simulate manual login
99 driver.set_page_load_timeout(10) 96 driver.set_page_load_timeout(10)
100 driver.set_script_timeout(20) 97 driver.set_script_timeout(20)
101 driver.find_element_by_id('Email').send_keys(Email) 98 driver.find_element_by_id('Email').send_keys(Email)
102 if Debug: 99 driver.get_screenshot_as_file('2.png')
103 driver.get_screenshot_as_file('2.png')
104 driver.find_element_by_css_selector('#next').click() 100 driver.find_element_by_css_selector('#next').click()
105 time.sleep(3) 101 time.sleep(3)
106 driver.find_element_by_id('Passwd').send_keys(Passwd) 102 driver.find_element_by_id('Passwd').send_keys(Passwd)
107 if Debug: 103 driver.get_screenshot_as_file('3.png')
108 driver.get_screenshot_as_file('3.png')
109 driver.find_element_by_css_selector('#signIn').click() 104 driver.find_element_by_css_selector('#signIn').click()
110 time.sleep(3) 105 time.sleep(3)
111 if Debug: 106 driver.get_screenshot_as_file('4.png')
112 driver.get_screenshot_as_file('3.png') 107
113 # get cookies 108 # get cookies
114 temp = driver.get_cookies() 109 cookies = driver.get_cookies()
115 110
116 csrftoken = '' 111 csrftoken = ''
117 SACSID = '' 112 SACSID = ''
118 for a in temp: 113 for key in cookies:
119 if (a['name'] == 'csrftoken'): 114 if key['name'] == 'csrftoken':
120 csrftoken = a['value'] 115 csrftoken = key['value']
121 if (a['name'] == 'SACSID'): 116 if key['name'] == 'SACSID':
122 SACSID = a['value'] 117 SACSID = key['value']
123 118
124 if csrftoken == '' or SACSID == '': 119 if csrftoken == '' or SACSID == '':
125 logger.error(getTime() + ': Fetch Cookie Failed')
126 raise CookieException 120 raise CookieException
127 121
128 with open('cookie', 'w') as file: 122 with open('cookie', 'w') as file:
129 cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17' 123 cookie = 'SACSID='+SACSID+'; csrftoken='+csrftoken+'; ingress.intelmap.shflt=viz; ingress.intelmap.lat=29.098418372855484; ingress.intelmap.lng=119.81689453125; ingress.intelmap.zoom=17'
130 file.write(cookie) 124 file.write(cookie)
131 125
126 logger.info(get_time() + ': Fetching Cookie Succeed')
132 driver.quit() 127 driver.quit()
133 logger.info(getTime() + ': Fetching Cookie Succeed')
134 return True 128 return True
135 129
136 130
137def sendMessge(bot, msg): 131def send_message(bot, message, monitor=False):
138 "sendMsg" 132 logger = logging.getLogger('send_message')
139
140 logger = logging.getLogger('sendMessage')
141 while True: 133 while True:
142 try: 134 try:
143 url = 'https://api.telegram.org/bot' 135 if monitor is True:
144 url += BOT_TOKEN 136 bot.sendMessage(chat_id="@voamonitor", text=message)
145 url += '/sendMessage?chat_id=' 137 else:
146 url += CHANNEL_NAME 138 bot.sendMessage(chat_id=CHANNEL_NAME, text=urllib.parse.quote(message))
147 url += '&text=' 139 logger.info(get_time() + ": sendMsg " + message)
148 url += urllib.parse.quote(msg)
149
150 req = urllib.request.Request(url, headers={'Content-Type': 'application/x-www-form-urlencoded'})
151 resp = urllib.request.urlopen(req)
152 data = resp.read()
153 logger.info(data)
154 logger.info(getTime() + ": sendMsg " + msg)
155 break 140 break
156 except NetworkError: 141 except telegram.TelegramError:
142 logger.error(get_time() + ": Send Message to Channel Failed")
157 time.sleep(1) 143 time.sleep(1)
158 144 except Exception:
159 145 logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0])
160def sendMonitor(bot, msg):
161 logger = logging.getLogger('sendMonitor')
162 while True:
163 try:
164 bot.sendMessage(chat_id="@voamonitor", text=msg)
165 logger.info(getTime() + ": sendMsg " + msg)
166 break
167 except NetworkError:
168 time.sleep(1) 146 time.sleep(1)
169 147
170 148
171def formatMessage(raw): 149def find_message_record(id):
172 pattern = re.compile(BlockList)
173 match = pattern.search(str(raw))
174 if match:
175 return "Blocked"
176
177 msg = ''
178 plext = raw[2]['plext']
179 markup = plext['markup']
180
181 for mark in markup:
182 if mark[0] == 'SECURE':
183 msg += mark[1]['plain']
184 elif mark[0] == 'SENDER':
185 player = mark[1]['plain']
186 team = mark[1]['team']
187
188 pattern = re.compile(':')
189 match = pattern.search(player)
190 if match:
191 if team == 'RESISTANCE':
192 player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:]
193 elif team == 'ENLIGHTENED':
194 player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:]
195 msg += player
196
197 elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER':
198 player = mark[1]['plain']
199 team = mark[1]['team']
200
201 msg += player
202 if team == 'RESISTANCE':
203 msg += ' 🐳'
204 elif team == 'ENLIGHTENED':
205 msg += ' 🐸'
206
207 elif mark[0] == 'TEXT':
208 msg += mark[1]['plain']
209
210 pattern = re.compile('\[secure\]')
211 match = pattern.search(msg)
212 if match:
213 if msg.find(':') != -1:
214 msg = msg[:9] + '@' + msg[9:]
215 else:
216 msg = msg[:10] + '@' + msg[10:]
217 else:
218 msg = '@' + msg
219
220 return msg
221
222
223def FindRecord(id):
224 uri = 'mongodb://' + DBHost 150 uri = 'mongodb://' + DBHost
225 Conn = MongoClient(uri) 151 conn = MongoClient(uri)
226 Conn.api.authenticate(DBUser, DBPass, DBName) 152 conn.api.authenticate(DBUser, DBPass, DBName)
227 database = Conn[DBName] 153 database = conn[DBName]
228 mycollection = database.entries 154 collection = database.entries
229 res = mycollection.find({"id": id}) 155 count = collection.find({"id": id}).count()
230 if res.count() == 0: 156 conn.close()
157 if count == 0:
231 return False 158 return False
232 else: 159 else:
233 return True 160 return True
234 161
235 162
236def insertDB(time, id, msg): 163def insert_message_to_database(time, id, msg):
237 uri = 'mongodb://' + DBHost 164 uri = 'mongodb://' + DBHost
238 Conn = MongoClient(uri) 165 conn = MongoClient(uri)
239 Conn.api.authenticate(DBUser, DBPass, DBName) 166 conn.api.authenticate(DBUser, DBPass, DBName)
240 database = Conn[DBName] 167 database = conn[DBName]
241 mycollection = database.entries 168 collection = database.entries
242 post = {"id": id, "time": time, "msg": msg} 169 post = {"id": id, "time": time, "msg": msg}
243 mycollection.insert(post) 170 collection.insert(post)
244 Conn.close() 171 conn.close()
245 172
246 173
247def main(): 174def main():
248 logger = logging.getLogger('main') 175 logger = logging.getLogger('main')
249 176
177 # Lat Lng of Hangzhou
250 field = { 178 field = {
251 'minLngE6': 119618783, 179 'minLngE6': 119618783,
252 'minLatE6': 29912919, 180 'minLatE6': 29912919,
@@ -255,71 +183,73 @@ def main():
255 } 183 }
256 184
257 mints = -1 185 mints = -1
258 maxts=-1 186 maxts = -1
259 reverse=False 187 reverse = False
260 tab='all' 188 tab = 'all'
261 189
190 # fetch cookie
262 while True: 191 while True:
263 try: 192 try:
264 if fetchCookie(): 193 if fetch_cookie():
265 break 194 break
266 except CookieException: 195 except CookieException:
196 logger.error(get_time() + ': Fetch Cookie Failed')
197 time.sleep(3)
198 except Exception:
199 logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0])
267 time.sleep(3) 200 time.sleep(3)
268 201
202 # fetch message
269 count = 0 203 count = 0
270 while True: 204 while True:
271 count += 1 205 count += 1
272 206
273 with open('cookie') as cookies: 207 with open('cookie') as cookies:
274 cookies = cookies.read().strip() 208 cookies = cookies.read().strip()
209 logger.info(get_time() + ": {} Fetching from Intel...".format(str(count)))
275 210
276 logger.info(getTime() + ": {} Fetching from Intel...".format(str(count))) 211 # fetch message per time
277
278 while True: 212 while True:
279 try: 213 try:
280
281 intel = ingrex.Intel(cookies, field) 214 intel = ingrex.Intel(cookies, field)
282 result = intel.fetch_msg(mints, maxts, reverse, tab) 215 result = intel.fetch_msg(mints, maxts, reverse, tab)
283
284 if result: 216 if result:
285 mints = result[0][1] + 1 217 mints = result[0][1] + 1
286 break 218 break
287
288 except CookieException: 219 except CookieException:
289 while True: 220 while True:
290 try: 221 try:
291 if fetchCookie(): 222 if fetch_cookie():
292 break 223 break
293 except CookieException: 224 except CookieException:
294 time.sleep(3) 225 time.sleep(3)
295 except Exception: 226 except Exception:
296 pass 227 logger.error(get_time() + ": Unexpected error: " + sys.exc_info()[0])
228 time.sleep(3)
297 229
298 for item in result[::-1]: 230 for item in result[::-1]:
299 message = ingrex.Message(item) 231 # Check spam message
232 pattern = re.compile(BlockList)
233 match = pattern.search(str(item))
234 if match:
235 continue
300 236
237 message = ingrex.Message(item)
301 if message.ptype == 'PLAYER_GENERATED': 238 if message.ptype == 'PLAYER_GENERATED':
302 # logger.info(getTime() + str(item)) 239 logger.info(get_time() + " " + message)
303 240 if find_message_record(message.guid) is False:
304 msg = formatMessage(item) 241 insert_message_to_database(message.time, message.guid, message)
305 if msg == 'Blocked': 242 send_message(bot, message, False)
306 logger.info(getTime() + " " + message.text)
307 else:
308 msg = message.time + " " + msg
309 # logger.info(getTime() + " " + msg)
310 if FindRecord(message.guid) is False:
311 insertDB(message.time, message.guid, msg)
312 # sendMonitor(bot, msg)
313 sendMessge(bot, msg)
314 243
315 time.sleep(10) 244 time.sleep(10)
316 245
317if __name__ == '__main__': 246if __name__ == '__main__':
318 readConfig() 247 read_config()
319 bot = telegram.Bot(BOT_TOKEN) 248 bot = telegram.Bot(BOT_TOKEN)
320 249
321 while True: 250 while True:
322 try: 251 try:
323 main() 252 main()
324 except Exception: 253 except Exception:
325 sendMonitor(bot, 'Main Error') 254 send_message(bot, 'Main Unexpected error' + sys.exc_info()[0], True)
255 time.sleep(3) \ No newline at end of file
diff --git a/bot.py b/bot.py
index 3e6d9b0..80ee4ed 100755
--- a/bot.py
+++ b/bot.py
@@ -20,7 +20,7 @@ BOT_TOKEN = ''
20CHANNEL_NAME = '' 20CHANNEL_NAME = ''
21Email = '' 21Email = ''
22Passwd = '' 22Passwd = ''
23PhantomjsPath = '' 23PhantomJSPath = ''
24DBName = '' 24DBName = ''
25DBUser = '' 25DBUser = ''
26DBPass = '' 26DBPass = ''
@@ -54,7 +54,7 @@ def readConfig():
54 global Passwd 54 global Passwd
55 global BOT_TOKEN 55 global BOT_TOKEN
56 global CHANNEL_NAME 56 global CHANNEL_NAME
57 global PhantomjsPath 57 global PhantomJSPath
58 global DBName 58 global DBName
59 global DBUser 59 global DBUser
60 global DBPass 60 global DBPass
@@ -82,7 +82,7 @@ def fetchCookie():
82 global Debug 82 global Debug
83 global Email 83 global Email
84 global Passwd 84 global Passwd
85 global PhantomjsPath 85 global PhantomJSPath
86 86
87 logger = logging.getLogger('fetchCookie') 87 logger = logging.getLogger('fetchCookie')
88 logger.info(getTime() + ': Fetching Cookie...') 88 logger.info(getTime() + ': Fetching Cookie...')
diff --git a/config.json.example b/config.json.example
index 7e61d75..ea29c95 100755
--- a/config.json.example
+++ b/config.json.example
@@ -3,7 +3,7 @@
3 "Passwd": "Your Ingress Account Password", 3 "Passwd": "Your Ingress Account Password",
4 "BOT_TOKEN": "Your Telegram Bot Token", 4 "BOT_TOKEN": "Your Telegram Bot Token",
5 "CHANNEL_NAME": "Your Channel ID", 5 "CHANNEL_NAME": "Your Channel ID",
6 "PhantomjsPath": "/usr/local/phantomjs/bin/phantomjs", 6 "PhantomJSPath": "/usr/local/phantomjs/bin/phantomjs",
7 "DBName": "", 7 "DBName": "",
8 "DBUser": "", 8 "DBUser": "",
9 "DBPass": "", 9 "DBPass": "",
diff --git a/ingrex/praser.py b/ingrex/praser.py
index a05ec1e..1d2c03d 100755
--- a/ingrex/praser.py
+++ b/ingrex/praser.py
@@ -1,26 +1,66 @@
1"Ingrex praser deal with message"
2from datetime import datetime, timedelta 1from datetime import datetime, timedelta
3import platform 2import re
4import os 3import os
5import time 4import time
6 5
7osname = platform.system()
8if osname == "Linux":
9 os.environ['TZ'] = 'Asia/Shanghai'
10 time.tzset()
11
12 6
13class Message(object): 7class Message(object):
14 "Message object"
15 def __init__(self, raw_msg): 8 def __init__(self, raw_msg):
9 os.environ['TZ'] = 'Asia/Shanghai'
10 time.tzset()
11
12 seconds, millis = divmod(raw_msg[1], 1000)
13 timestamp = datetime.fromtimestamp(seconds) + timedelta(milliseconds=millis)
14 self.time = timestamp.strftime('%Y/%m/%d %H:%M:%S:%f')[:-3]
15
16 self.raw = raw_msg 16 self.raw = raw_msg
17 self.guid = raw_msg[0] 17 self.guid = raw_msg[0]
18 self.timestamp = raw_msg[1] 18 self.timestamp = raw_msg[1]
19 seconds, millis = divmod(raw_msg[1], 1000)
20 time = datetime.fromtimestamp(seconds) + timedelta(milliseconds=millis)
21 self.time = time.strftime('%Y/%m/%d %H:%M:%S:%f')[:-3]
22 self.text = raw_msg[2]['plext']['text'] 19 self.text = raw_msg[2]['plext']['text']
23 self.ptype = raw_msg[2]['plext']['plextType'] 20 self.ptype = raw_msg[2]['plext']['plextType']
24 self.team = raw_msg[2]['plext']['team'] 21 self.team = raw_msg[2]['plext']['team']
25 self.type = raw_msg[2]['plext']['markup'][1][1]['plain'] 22 self.type = raw_msg[2]['plext']['markup'][1][1]['plain']
26 23
24 self.msg = ''
25 self.markup = raw_msg[2]['plext']['markup']
26
27 for mark in self.markup:
28 if mark[0] == 'SECURE':
29 self.msg += mark[1]['plain']
30 elif mark[0] == 'SENDER':
31 player = mark[1]['plain']
32 team = mark[1]['team']
33
34 pattern = re.compile(':')
35 match = pattern.search(player)
36 if match:
37 if team == 'RESISTANCE':
38 player = player[:match.span()[0]] + ' 🐳' + player[match.span()[0]:]
39 elif team == 'ENLIGHTENED':
40 player = player[:match.span()[0]] + ' 🐸' + player[match.span()[0]:]
41 self.msg += player
42
43 elif mark[0] == 'PLAYER' or mark[0] == 'AT_PLAYER':
44 player = mark[1]['plain']
45 team = mark[1]['team']
46
47 self.msg += player
48 if team == 'RESISTANCE':
49 self.msg += ' 🐳'
50 elif team == 'ENLIGHTENED':
51 self.msg += ' 🐸'
52
53 elif mark[0] == 'TEXT':
54 self.msg += mark[1]['plain']
55
56 pattern = re.compile('\[secure\]')
57 match = pattern.search(self.msg)
58 if match:
59 if self.msg.find(':') != -1:
60 self.msg = self.msg[:9] + '@' + self.msg[9:]
61 else:
62 self.msg = self.msg[:10] + '@' + self.msg[10:]
63 else:
64 self.msg = '@' + self.msg
65
66 self.msg = self.time + self.msg
Powered by cgit v1.2.3 (git 2.41.0)