update

app/__init__.py

@@ -1,3 +0,0 @@
-import logging
-
-logging.basicConfig(level=logging.INFO)

app/bnet_retriever/credential.py → bnet_retriever/credential.py

@@ -1,14 +1,14 @@
-import json
-import os
 import time
+import logging
 from typing import Dict, Optional
 
-from bson import ObjectId
+from .errors import LoginFailed
+from ._plugin import Plugin
+from .utils import serialize
+from .utils.login import login
+from .utils.db import Mongo
 
-from app.bnet_retriever._plugin import Plugin
-from app.bnet_retriever.utils import serialize
-from app.bnet_retriever.utils.login import login
-from app.utils.db import Mongo
+logger = logging.getLogger(__name__)
 
 
 class CredentialManager(Plugin):
@@ -25,9 +25,14 @@ class CredentialManager(Plugin):
         # Here, self.last_updated_at records if the bnet_user_cred outdated ( 1 hr expiration)
         # Different from the lastUpdatedAt field in mongodb
         if self._bnet_user_cred is None or time.time() - self.last_updated_at > 60*55:
-            bnet_user_cred, new_cred = login(self.raw_cred)
+            try:
+                bnet_user_cred, new_cred = login(self.raw_cred)
+            except LoginFailed:
+                logger.info('[%s] login failed')
+                return
             self._bnet_user_cred = bnet_user_cred
             self.update_raw_cred(new_cred)
+            logger.info('[%s] Updated credential', self._id)
             self.last_updated_at = time.time()
         return self._bnet_user_cred
 

bnet_retriever/errors.py

@@ -0,0 +1,2 @@
+class LoginFailed(Exception):
+    pass

bnet_retriever/producer.py

@@ -0,0 +1,59 @@
+import datetime
+import logging
+import threading
+import time
+
+from .runner import Runner
+from .utils.db import Mongo
+
+logger = logging.getLogger(__name__)
+
+
+class Producer(threading.Thread):
+    def __init__(self, queue):
+        self.queue = queue
+        self.load_users(need_initialize=True)
+        super(Producer, self).__init__()
+
+    def load_users(self, need_initialize=False) -> None:
+        if need_initialize:
+            self.users = {i['_id'] for i in Mongo.db.user.find({})}
+            self.updated_at = datetime.datetime.utcnow()
+        else:
+            new_users = {i['_id'] for i in Mongo.db.user.find({
+                'createdAt': {
+                    '$gt': self.updated_at
+                }
+            })}
+            if new_users:
+                logger.info('Loaded users: %s', new_users)
+                self.users.update(new_users)
+                self.updated_at = datetime.datetime.utcnow()
+
+    def run(self):
+        while True:
+            try:
+                for i in self.users:
+                    self.queue.put(i)
+                self.load_users()
+                logger.info('Got all users')
+            except Exception:
+                logger.exception('Producer Error')
+            time.sleep(180)
+
+
+class Consumer(threading.Thread):
+    def __init__(self, queue):
+        self.queue = queue
+        super(Consumer, self).__init__()
+
+    def run(self):
+        while True:
+            try:
+                _id = self.queue.get()
+                runner = Runner(_id)
+                logger.info('Got User [%s], starting Runner!', _id)
+                runner.run()
+            except Exception:
+                logger.exception('Consumer Error')
+            time.sleep(5)

app/bnet_retriever/profile.py → bnet_retriever/profile.py

@@ -1,12 +1,12 @@
-from app.bnet_retriever._plugin import Plugin
-from app.bnet_retriever import api
-from app.bnet_retriever.utils import reformat
-from app.utils.db import Mongo
-from typing import Dict
-from bson import ObjectId
 import logging
+
 import pymongo
 
+from . import api
+from ._plugin import Plugin
+from .utils.db import Mongo
+from .utils import reformat
+
 gamedata = api.get_gamedata()
 
 logger = logging.getLogger(__name__)

app/bnet_retriever/runner.py → bnet_retriever/runner.py

@@ -1,19 +1,18 @@
-from app.bnet_retriever.profile import Profile, RemoteProfile, DBProfile
-from app.bnet_retriever.credential import CredentialManager
-from app.bnet_retriever.stat import CompetitiveStat
-from app.utils.db import Mongo
+from .profile import RemoteProfile, DBProfile
+from .credential import CredentialManager
+from .stat import CompetitiveStat
+from .utils.db import Mongo
 from bson import ObjectId
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class Runner:
     callbacks = []
 
-    def __init__(self, username: str):
-        record = Mongo.db.user.find_one({'username': username})
-        if not record:
-            raise Exception('No Such User')
-        self.username: str = username
-        self._id: ObjectId = record['_id']
+    def __init__(self, _id: ObjectId):
+        self._id: ObjectId = _id
 
         # Plugs
         # self.profile: Profile = Profile(self)
@@ -26,5 +25,15 @@ class Runner:
         self.new_profile.refresh()
         self.latest_profile.refresh()
         if self.new_profile > self.latest_profile:
+            logger.info("[%s] Found New Profile!", self._id)
             Mongo.db.profile.insert(self.new_profile.to_db_record())
             self.competitive_stat.calc()
+
+
+class RunnerProxy(Runner):
+    def __init__(self, username: str):
+        record = Mongo.db.user.find_one({'username': username})
+        if not record:
+            raise Exception('No Such User')
+        self.username: str = username
+        super(RunnerProxy, self).__init__(record['_id'])

app/bnet_retriever/stat.py → bnet_retriever/stat.py

@@ -1,5 +1,5 @@
-from app.bnet_retriever._plugin import Plugin
-from app.utils.db import Mongo
+from ._plugin import Plugin
+from .utils.db import Mongo
 import logging
 
 logger = logging.getLogger(__name__)
@@ -9,12 +9,12 @@ class CompetitiveStat(Plugin):
 
     def get_stats(self):
         return ({
-            'gamePlayed': round(i['ranked']['所有英雄']['比赛场次']),
-            'won': round(i['ranked']['所有英雄']['比赛胜利']),
-            'lost': round(i['ranked']['所有英雄']['比赛战败']),
-            'draw': round(i['ranked']['所有英雄']['比赛战平']),
-            'skillRate': i['player']['ranked']['level'],
-        } for i in (self.root.latest_profile.formatted['career'], self.root.new_profile.formatted['career']))
+            'gamePlayed': round(i['career']['ranked']['所有英雄']['比赛场次']),
+            'won': round(i['career']['ranked']['所有英雄']['比赛胜利']),
+            'lost': round(i['career']['ranked']['所有英雄']['比赛战败']),
+            'draw': round(i['career']['ranked']['所有英雄']['比赛战平']),
+            'skillRate': i['career']['player']['ranked']['level'],
+        } for i in (self.root.latest_profile.formatted, self.root.new_profile.formatted))
 
     def calc(self):
         old, new = self.get_stats()

app/utils/db.py → bnet_retriever/utils/db.py

@@ -1,6 +1,6 @@
 from pymongo import MongoClient
 from .classproperty import classproperty
-from app.config import MONGO_URL
+from ..config import MONGO_URL
 
 
 class Mongo:

app/bnet_retriever/utils/login.py → bnet_retriever/utils/login.py

@@ -1,5 +1,6 @@
 import requests
 from .cookies import BnetCookieJar
+from .errors import LoginFailed
 
 
 def login(creds):
@@ -12,5 +13,7 @@ def login(creds):
         'Accept-Language': 'en-CN;q=1, zh-Hans-CN;q=0.9, ja-JP;q=0.8',
         'Accept-Encoding': 'gzip, deflate, br',
     })
-    sess.get(url)
+    r = sess.get(url)
+    if not r.url.startwith('http://ow.blizzard.cn'):
+        raise LoginFailed
     return jar.get('bnet_user_cred'), jar.dump_bnet_cookies()

crack-api.py

@@ -0,0 +1,46 @@
+# from app.credential import get_cred
+# from app import api
+
+# data = api.get_profile(get_cred('ray'))
+
+import requests
+from ipdb import set_trace
+from pprint import pprint
+from app.cookies import BnetCookieJar
+
+url = 'https://account.bnet.163.com/battlenet/login?inner_client_id=ow&inner_redirect_uri=http://ow.blizzard.cn/battlenet/login?redirect_url=http://ow.blizzard.cn/career/'
+
+creds = {
+    '_ntes_nuid': '5839d11e3af2940333f3a633ef92b178',
+    'MTK_BBID': 'be6/D4Bjz5/8jblp8/Xmzg==',
+    'opt': '1',
+    'web.id': 'CN-89e275ef-e4aa-4d59-bb9c-d7c1b6dd4d88',
+    'BA-tassadar-login.key': 'dce2ddd983c75b63d419fbc02105d298',
+    'login.key': 'dce2ddd983c75b63d419fbc02105d298',
+    'BA-tassadar': 'CN-e03cf811a4ac77682aabf30d99670de0-570729539',
+    'bnet.extra': 'AYfHjrAo5NopD2_eKr_51969kSI7MZRj-P2D18MdsOZm7TENj5-dN2dbb-fVu8V-56LQZU-Es35BaMr6ozvivETxBb2sLHThln6ch7BKYHJ4',
+}
+
+sess = requests.Session()
+sess.cookies = BnetCookieJar.load_bnet_cookies(creds)
+sess.headers.update({
+    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148',
+    'Accept-Language': 'en-CN;q=1, zh-Hans-CN;q=0.9, ja-JP;q=0.8',
+    'Accept-Encoding': 'gzip, deflate, br',
+})
+
+
+r = sess.get(url)
+
+for i in r.history:
+    print(i.url)
+
+
+def inspect(i):
+    r_0 = r.history[i]
+    pprint(r_0.url)
+    pprint({i[:i.find('=')].strip(): i[i.find('=')+1:] for i in r_0.request.headers['Cookie'].split(';')})
+    pprint(r_0.cookies)
+
+
+set_trace()

doc.md

@@ -0,0 +1,52 @@
+猜测:
+    流程
+        1. account.bnet.163
+            url: https://account.bnet.163.com/battlenet/login?inner_client_id=ow&inner_redirect_uri=http%3A%2F%2Fow.blizzard.cn%2Fbattlenet%2Flogin%3Fredirect_url%3Dhttp%253A%252F%252Fow.blizzard.cn%252Fcareer%252F
+            cookies: MTK_BBID, _ntes_nuid
+
+        2. 被forward到oauth
+            url: https://www.battlenet.com.cn/oauth/authorize?client_id=netease-hearthstone-site&response_type=code&scope=account.basic+account.full&redirect_uri=https%3A%2F%2Faccount.bnet.163.com%2Fbattlenet%2Flogin%3Finner_client_id%3Dow%26inner_redirect_uri%3Dhttp%253A%252F%252Fow.blizzard.cn%252Fbattlenet%252Flogin%253Fredirect_url%253Dhttp%25253A%25252F%25252Fow.blizzard.cn%25252Fcareer%25252F
+            cookies: BA-tessadar-login.key, login.key, opt, web.inner_client_id
+            此时被set-cookie
+                loginChecked=1
+
+        3. bnet.com.cn/login
+            url: https://www.battlenet.com.cn/login/en/?ref=https://www.battlenet.com.cn/oauth/authorize?client_id%3Dnetease-hearthstone-site%26response_type%3Dcode%26scope%3Daccount.basic%2Baccount.full%26redirect_uri%3Dhttps%253A%252F%252Faccount.bnet.163.com%252Fbattlenet%252Flogin%253Finner_client_id%253Dow%2526inner_redirect_uri%253Dhttp%25253A%25252F%25252Fow.blizzard.cn%25252Fbattlenet%25252Flogin%25253Fredirect_url%25253Dhttp%2525253A%2525252F%2525252Fow.blizzard.cn%2525252Fcareer%2525252F&app=oauth&opt
+            cookies: BA-tassadar, bnet.extra (path=/login)
+            同时被set-cookies:
+                BA-tassadar
+                BA-tassadar-cl
+                cl
+                BA-tassadar-login.key
+                login.key
+                opt=1
+
+        4. 被返回到oauth, params里携带了ST, 被set-cookies JSESSIONID
+
+        5. 被返回到oauth, 在redirect中添加了code
+
+        6. 被返回到account.bnet, 在携带code时被set-cookies
+            MTK_BBID
+            account_user_cred
+            同时加上了inner code
+
+        7. 返回ow.bnet, 以及innercode, 被设了bnet_user_cred和battletag, 一小时后过期
+
+结构:
+    User -> Denotes a user / maybe a bnet ID or just a user
+            See further iteration
+        本质上是user collection里的一条唯一记录
+        以插件的方式来做一些交互
+        以线程/协程的方式跑起来
+
+        Profile: 用来做信息的获取, format 和比较的结构
+        LatestProfile: 简单的handle, 用以取数据库中最新的Profile
+        NewProfile: 用以取最新接口获取回来的Profile
+
+        CredentialManager: 管理登录时的cred及其更新
+        Updator: 完成具体的更新逻辑 (可能在User的上层), 并且更新完成后调用相关其他Plugin的callback
+
+        Stats: 统计更新发生后的数据变动 (分数变动等细节计算)
+
+        如何完成第一次初始化?
+        Updator中定义细节逻辑: 在某个时间后, 获取LatestProfile和NewProfile, 做比较, 决定是否更新数据库, 并将比较结果给Stats(作为Stats)记录, 

failed_login.py

@@ -0,0 +1,44 @@
+from bnet_retriever.utils.cookies import BnetCookieJar
+from bnet_retriever.utils import serialize
+
+import requests
+import getpass
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+import time
+import ipdb
+
+url = 'https://www.battlenet.com.cn/login/zh/?ref=https://www.battlenet.com.cn/oauth/authorize?client_id%3Dnetease-hearthstone-site%26response_type%3Dcode%26scope%3Daccount.basic%2Baccount.full%26redirect_uri%3Dhttps%253A%252F%252Faccount.bnet.163.com%252Fbattlenet%252Flogin%253Finner_client_id%253Dow%2526inner_redirect_uri%253Dhttps%25253A%25252F%25252Fow.blizzard.cn%25252Fbattlenet%25252Flogin%25253Fredirect_url%25253Dhttps%2525253A%2525252F%2525252Fow.blizzard.cn%2525252Fcareer%2525252F&app=oauth'
+
+tup = ('accountName', 'password')
+
+username = input('Your BNet Username:')
+password = getpass.getpass('Your BNet Password:')
+
+opts = Options()
+opts.add_argument('--window-size=375,812')
+opts.add_argument('--window-position=0,0')
+browser = webdriver.Chrome(options=opts)
+browser.get(url)
+submit_btn = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.ID, 'submit')))
+browser.find_element_by_id('accountName').clear()
+browser.find_element_by_id('password').clear()
+browser.find_element_by_id('accountName').send_keys(username)
+browser.find_element_by_id('password').send_keys(password)
+submit_btn.click()
+time.sleep(2)
+
+keys = {key for key, *_ in BnetCookieJar._template}
+
+browser.get('http://163.com')
+cookies = browser.get_cookies()
+final = {i['name']: i['value'] for i in cookies if i['name'] in keys}
+browser.get('https://shop.battlenet.com.cn/zh-cn')
+cookies = browser.get_cookies()
+final.update({i['name']: i['value'] for i in cookies if i['name'] in keys})
+
+print('Your Login token is:')
+print(serialize.tostr(final))

get_token.py

@@ -6,6 +6,7 @@ import glob
 from requests.utils import dict_from_cookiejar
 from base64 import b64encode
 from typing import Dict, List, AnyStr
+from urllib.parse import unquote
 
 
 def tostr(obj: Dict) -> str:
@@ -33,6 +34,8 @@ def get_cookies() -> Dict:
     path = generate_path()
     _163_cookies = dict_from_cookiejar(bc.chrome(cookie_file=path, domain_name='.163.com'))
     _bnet_cookies = dict_from_cookiejar(bc.chrome(cookie_file=path, domain_name='.battlenet.com.cn'))
+    _ow_cookies = dict_from_cookiejar(bc.chrome(cookie_file=path, domain_name='ow.blizzard.cn'))
+    battletag = unquote(_ow_cookies.get('battletag'))
 
     keys = {
         '_ntes_nuid',
@@ -45,13 +48,15 @@ def get_cookies() -> Dict:
         'bnet.extra',
     }
 
-    return {k: v for k, v in {**_163_cookies, **_bnet_cookies}.items() if k in keys}
+    return battletag, {k: v for k, v in {**_163_cookies, **_bnet_cookies}.items() if k in keys}
 
 
 if __name__ == '__main__':
-    cookies = get_cookies()
+    btag, cookies = get_cookies()
     if not cookies:
         print('Cookie not found! Please Sign in to Battle.net with Chrome')
     else:
-        print('Congrats! Here\'s your token: \n')
+        print('Congrats!')
+        print('Your BattleTag: ', btag)
+        print('Here\'s your token: \n')
         print(tostr(cookies))

run.py

@@ -0,0 +1,16 @@
+import logging
+from queue import Queue
+from bnet_retriever.producer import Producer, Consumer
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(module)s - %(levelname)s - %(message)s')
+
+if __name__ == '__main__':
+    id_queue = Queue()
+    producer = Producer(id_queue)
+    consumer_list = [Consumer(id_queue) for i in range(5)]
+    producer.start()
+    for i in consumer_list:
+        i.start()
+    producer.join()
+    for i in consumer_list:
+        i.join()