add InfoScanner referred at #10

remove encode path in service.common.generate_video_link

CHANGELOG.md

@@ -1,3 +1,15 @@
+#0.6.0-alpha
+
+##Features Add
+
+- InfoScanner for scanning missing information (name, name_cn, duration) and auto fill those information from bangumi.tv, note that the name_cn is not always filled.
+
+##Bug fix
+
+- rollback session when db connection lost
+
+NOTE: this release require use to update their config.yml.
+
 #0.4.0-alpha
 
 ##Features Add

README.md

@@ -57,6 +57,10 @@ $ python tools.py --user-add admin 1234 # admin is username 1234 is password
 $ python tools.py --user-promote admin 3 # admin is username 3 is the level, currently means super user
 ```
 
+### set your server locale
+
+To avoid some unicode issues, it is recommended to set locale of your server
+
 ### Docker
 you can also set up the development environment with [Docker](https://www.docker.com/)
 

Scheduler.py

@@ -40,6 +40,9 @@
 from sqlalchemy.sql import func
 import traceback
 
+from taskrunner.InfoScanner import info_scanner
+
+
 class Scheduler:
 
  def __init__(self):
@@ -173,6 +176,7 @@ def scan_bangumi(self):
 def on_connected(result):
  # logger.info(result)
  scheduler.start()
+ info_scanner.start()
 
 def on_connect_fail(result):
  logger.error(result)

config/config-sample.yml

@@ -13,6 +13,9 @@ deluge:
 
 task:
  interval: 15 # unit minute
+ info_scanner:
+ scan_time: # A info scanner will scheduled for once a day at this particular time. it is not an accurate time, default is 23:00
+ scan_time_format: # default is '%H:%M' see https://docs.python.org/2/library/datetime.html#strftime-strptime-behavior
 
 download:
  location: / # change this location

service/common.py

@@ -37,7 +37,7 @@ def generate_cover_link(self, bangumi):
  return cover_url
 
  def generate_video_link(self, bangumi_id, path):
- video_link = '/video/{0}/{1}'.format(bangumi_id, path.encode('utf-8'))
+ video_link = '/video/{0}/{1}'.format(bangumi_id, path)
  if self.video_domain is not None:
  video_link = self.video_domain + video_link
  return video_link

taskrunner/InfoScanner.py

@@ -0,0 +1,115 @@
+from utils.SessionManager import SessionManager
+from domain.Bangumi import Bangumi
+from domain.Episode import Episode
+from sqlalchemy.sql.expression import or_
+from sqlalchemy import exc
+from twisted.internet import threads
+from twisted.internet.task import LoopingCall
+from datetime import datetime
+import time
+import yaml
+import requests
+import logging
+import traceback
+
+logger = logging.getLogger(__name__)
+
+class InfoScanner:
+
+ def __init__(self):
+ fr = open('./config/config.yml', 'r')
+ config = yaml.load(fr)
+ if 'info_scanner' in config['task']:
+ scan_time = '16:00'
+ scan_time_format = '%H:%M'
+ if 'scan_time' in config['task']['info_scanner'] and config['task']['info_scanner']['scan_time'] is not None:
+ scan_time = config['task']['info_scanner']['scan_time']
+
+ if 'scan_time_format' in config['task']['info_scanner'] and config['task']['info_scanner']['scan_time_format'] is not None:
+ scan_time_format = config['task']['info_scanner']['scan_time_format']
+
+ self.scan_time = datetime.strptime(scan_time, scan_time_format)
+
+ self.scanner_running = False
+ self.last_scan_date = None
+
+ def start(self):
+ lc = LoopingCall(self.check_time)
+ lc.start(60)
+
+ def check_time(self):
+ if self.scanner_running:
+ return
+ current_time = datetime.now()
+ if self.last_scan_date is not None and self.last_scan_date == current_time.date():
+ return
+ if (not self.scanner_running) and (self.scan_time.hour == current_time.hour):
+ self.scanner_running = True
+ self.scan_episode()
+ self.last_scan_date = current_time.date()
+ self.scanner_running = False
+
+ def get_bgm_info(self, bgm_id):
+ bangumi_tv_url_base = 'http://api.bgm.tv/subject/'
+ bangumi_tv_url_param = '?responseGroup=large'
+ bangumi_tv_url = bangumi_tv_url_base + str(bgm_id) + bangumi_tv_url_param
+ r = requests.get(bangumi_tv_url, timeout=60)
+ if r.status_code < 400:
+ return (r.status_code, r.json())
+ else:
+ return (r.status_code, {})
+
+
+ def __scan_episode_in_thread(self):
+ logger.info('start scan info of episode')
+ session = SessionManager.Session
+ try:
+ # we don't scan the episode those name_cn is missing
+ # because many of them don't have name_cn
+ result = session.query(Episode, Bangumi).\
+ join(Bangumi).\
+ filter(or_(Episode.name == '', Episode.duration == ''))
+
+ bgm_episode_dict = {}
+
+ for episode, bangumi in result:
+ if not (bangumi.bgm_id in bgm_episode_dict):
+ # if this is not the first call for get_bgm_info, a delay should be added to prevent access the bgm api
+ # too frequently
+ if bgm_episode_dict:
+ time.sleep(20)
+ logger.info('try to get info for bangumi of %s' % str(bangumi.bgm_id))
+ (status_code, bangumi_info) = self.get_bgm_info(bangumi.bgm_id)
+ if status_code < 400:
+ bgm_episode_dict[bangumi.bgm_id] = bangumi_info
+
+ if not (bangumi.bgm_id in bgm_episode_dict):
+ continue
+
+ bangumi_info = bgm_episode_dict[bangumi.bgm_id]
+
+ for eps in bangumi_info['eps']:
+ if eps['id'] == episode.bgm_eps_id:
+ if episode.name == '':
+ episode.name = eps['name']
+ if episode.name_cn == '':
+ episode.name_cn = eps['name_cn']
+ if episode.duration == '':
+ episode.duration = eps['duration']
+ break
+
+ session.commit()
+ logger.info('scan finished, will scan at next day')
+ except exc.DBAPIError as db_error:
+ logger.error(db_error)
+ # if connection is invalid rollback the session
+ if db_error.connection_invalidated:
+ session.rollback()
+ except Exception as error:
+ logger.error(error)
+ traceback.print_exc()
+
+ def scan_episode(self):
+ threads.deferToThread(self.__scan_episode_in_thread)
+
+info_scanner = InfoScanner()