Twitch support

tubesync/sync/admin.py

@@ -16,10 +16,14 @@ class SourceAdmin(admin.ModelAdmin):
 class MediaAdmin(admin.ModelAdmin):
 
  ordering = ('-created',)
- list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'downloaded')
+ list_display = ('uuid', 'key', 'source', 'can_download', 'skip', 'is_live', 'downloaded')
  readonly_fields = ('uuid', 'created')
  search_fields = ('uuid', 'source__key', 'key')
 
+ @admin.display(boolean=True)
+ def is_live(self, obj):
+ return obj.is_live
+
 
 @admin.register(MediaServer)
 class MediaServerAdmin(admin.ModelAdmin):

tubesync/sync/migrations/0018_twitch_support.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.18 on 2023-03-28 15:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('sync', '0017_alter_source_sponsorblock_categories'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='media',
+ name='last_crawl',
+ field=models.DateTimeField(blank=True, db_index=True, help_text='Date and time the metadata of the media was last crawled', null=True, verbose_name='last crawl'),
+ ),
+ migrations.AlterField(
+ model_name='source',
+ name='source_type',
+ field=models.CharField(choices=[('c', 'YouTube channel'), ('i', 'YouTube channel by ID'), ('p', 'YouTube playlist'), ('t', 'Twitch channel')], db_index=True, default='c', help_text='Source type', max_length=1, verbose_name='source type'),
+ ),
+ ]

tubesync/sync/models.py

@@ -14,8 +14,9 @@
 from common.errors import NoFormatException
 from common.utils import clean_filename
 from .youtube import (get_media_info as get_youtube_media_info,
- download_media as download_youtube_media)
-from .utils import seconds_to_timestr, parse_media_format
+ download_media as download_youtube_media,
+ get_twitch_media_info)
+from .utils import seconds_to_timestr, parse_media_format, parse_twitch_media_format
 from .matching import (get_best_combined_format, get_best_audio_format, 
  get_best_video_format)
 from .mediaservers import PlexMediaServer
@@ -32,12 +33,14 @@ class Source(models.Model):
  SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
  SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
+ SOURCE_TYPE_TWITCH_CHANNEL = 't'
  SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
- SOURCE_TYPE_YOUTUBE_PLAYLIST)
+ SOURCE_TYPE_YOUTUBE_PLAYLIST, SOURCE_TYPE_TWITCH_CHANNEL)
  SOURCE_TYPE_CHOICES = (
  (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
  (SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
  (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
+ (SOURCE_TYPE_TWITCH_CHANNEL, _('Twitch channel')),
  )
 
  SOURCE_RESOLUTION_360P = '360p'
@@ -151,30 +154,35 @@ class Source(models.Model):
  SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
  SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
+ SOURCE_TYPE_TWITCH_CHANNEL: '<i class="fab fa-twitch"></i>',
  }
  # Format to use to display a URL for the source
  URLS = {
  SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
  SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+ SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos',
  }
  # Format used to create indexable URLs
  INDEX_URLS = {
  SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}/videos',
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}/videos',
  SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
+ SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/{key}/videos?filter=all&sort=time',
  }
  # Callback functions to get a list of media from the source
  INDEXERS = {
  SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
  SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+ SOURCE_TYPE_TWITCH_CHANNEL: get_twitch_media_info,
  }
  # Field names to find the media ID used as the key when storing media
  KEY_FIELD = {
  SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
  SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
  SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
+ SOURCE_TYPE_TWITCH_CHANNEL: 'id',
  }
 
  class CapChoices(models.IntegerChoices):
@@ -546,29 +554,41 @@ class Media(models.Model):
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'https://www.twitch.tv/videos/{key}',
  }
  # Callback functions to get a list of media from the source
  INDEXERS = {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: get_youtube_media_info,
+ }
+ # Callback functions to get a list of media from the source
+ FORMAT_PARSER = {
+ Source.SOURCE_TYPE_YOUTUBE_CHANNEL: parse_media_format,
+ Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: parse_media_format,
+ Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: parse_media_format,
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: parse_twitch_media_format,
  }
  # Maps standardised names to names used in source metdata
  METADATA_FIELDS = {
  'upload_date': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'upload_date',
  },
  'title': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'title',
  },
  'thumbnail': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'thumbnail',
  },
  'description': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
@@ -579,53 +599,69 @@ class Media(models.Model):
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'duration',
  },
  'formats': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'formats',
  },
  'categories': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'categories',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'categories',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'categories',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'categories',
  },
  'rating': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'average_rating',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'average_rating',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'average_rating',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'average_rating',
  },
  'age_limit': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'age_limit',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'age_limit',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'age_limit',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'age_limit',
  },
  'uploader': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'uploader',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'uploader',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'uploader',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'uploader',
  },
  'upvotes': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'like_count',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'like_count',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'like_count',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'like_count',
  },
  'downvotes': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'dislike_count',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'dislike_count',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'dislike_count',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'dislike_count',
  },
  'playlist_title': {
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'playlist_title',
  Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'playlist_title',
  Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'playlist_title',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'playlist_title',
+ },
+ 'is_live': {
+ Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'is_live',
+ Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'is_live',
+ Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'is_live',
+ Source.SOURCE_TYPE_TWITCH_CHANNEL: 'is_live',
  },
  }
  STATE_UNKNOWN = 'unknown'
  STATE_SCHEDULED = 'scheduled'
  STATE_DOWNLOADING = 'downloading'
  STATE_DOWNLOADED = 'downloaded'
  STATE_SKIPPED = 'skipped'
+ STATE_LIVE = 'live'
  STATE_DISABLED_AT_SOURCE = 'source-disabled'
  STATE_ERROR = 'error'
  STATES = (STATE_UNKNOWN, STATE_SCHEDULED, STATE_DOWNLOADING, STATE_DOWNLOADED,
@@ -638,6 +674,7 @@ class Media(models.Model):
  STATE_SKIPPED: '<i class="fas fa-exclamation-circle" title="Skipped"></i>',
  STATE_DISABLED_AT_SOURCE: '<i class="fas fa-stop-circle" title="Media downloading disabled at source"></i>',
  STATE_ERROR: '<i class="fas fa-exclamation-triangle" title="Error downloading"></i>',
+ STATE_LIVE: '<i class="fas fa-clock" title="Live, download postponed"></i>',
  }
 
  uuid = models.UUIDField(
@@ -798,6 +835,13 @@ class Media(models.Model):
  null=True,
  help_text=_('Size of the downloaded media in bytes')
  )
+ last_crawl = models.DateTimeField(
+ _('last crawl'),
+ db_index=True,
+ null=True,
+ blank=True,
+ help_text=_('Date and time the metadata of the media was last crawled')
+ )
 
  def __str__(self):
  return self.key
@@ -814,8 +858,9 @@ def get_metadata_field(self, field):
  return fields.get(self.source.source_type, '')
 
  def iter_formats(self):
+ parser = self.FORMAT_PARSER.get(self.source.source_type)
  for fmt in self.formats:
- yield parse_media_format(fmt)
+ yield parser(fmt)
 
  def get_best_combined_format(self):
  return get_best_combined_format(self)
@@ -1018,13 +1063,20 @@ def url(self):
 
  @property
  def description(self):
+ if self.source.source_type == Source.SOURCE_TYPE_TWITCH_CHANNEL:
+ chapters = self.loaded_metadata.get('chapters', [])
+ descriptions = [chapter['title'] for chapter in chapters]
+ return ' - '.join(descriptions)
+
  field = self.get_metadata_field('description')
- return self.loaded_metadata.get(field, '').strip()
+ description = self.loaded_metadata.get(field) or ''
+ return description.strip()
 
  @property
  def title(self):
  field = self.get_metadata_field('title')
- return self.loaded_metadata.get(field, '').strip()
+ title = self.loaded_metadata.get(field) or ''
+ return title.strip()
 
  @property
  def slugtitle(self):
@@ -1034,7 +1086,8 @@ def slugtitle(self):
  @property
  def thumbnail(self):
  field = self.get_metadata_field('thumbnail')
- return self.loaded_metadata.get(field, '').strip()
+ thumbnail = self.loaded_metadata.get(field) or ''
+ return thumbnail.strip()
 
  @property
  def name(self):
@@ -1207,6 +1260,13 @@ def content_type(self):
  else:
  return 'video/mp4'
 
+ @property
+ def is_live(self):
+ field = self.get_metadata_field('is_live')
+ is_live = self.loaded_metadata.get(field)
+ is_live = True if is_live else False
+ return is_live
+
  @property
  def nfoxml(self):
  '''
@@ -1318,6 +1378,8 @@ def get_download_state(self, task=None):
  return self.STATE_SKIPPED
  if not self.source.download_media:
  return self.STATE_DISABLED_AT_SOURCE
+ if self.is_live:
+ return self.STATE_LIVE
  return self.STATE_UNKNOWN
 
  def get_download_state_icon(self, task=None):

tubesync/sync/signals.py

@@ -1,8 +1,10 @@
+from datetime import timedelta
 import os
 from django.conf import settings
 from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
 from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
 from background_task.signals import task_failed
 from background_task.models import Task
 from common.logger import log
@@ -150,20 +152,28 @@ def media_post_save(sender, instance, created, **kwargs):
  post_save.disconnect(media_post_save, sender=Media)
  instance.save()
  post_save.connect(media_post_save, sender=Media)
- # If the media is missing metadata schedule it to be downloaded
- if not instance.metadata:
- log.info(f'Scheduling task to download metadata for: {instance.url}')
+ # If the media is missing metadata schedule it to be downloaded,
+ # or refresh it if it was live during last run
+ if not instance.metadata or instance.is_live:
+ kwargs = {}
+ if instance.is_live:
+ kwargs['schedule'] = timedelta(hours=1)
+
+ log.info(f'Recheduling task to download metadata for live stream in 1h: {instance.url}')
+ else:
+ log.info(f'Scheduling task to download metadata for: {instance.url}')
  verbose_name = _('Downloading metadata for "{}"')
  download_media_metadata(
  str(instance.pk),
  priority=10,
  verbose_name=verbose_name.format(instance.pk),
- remove_existing_tasks=True
+ remove_existing_tasks=True,
+ **kwargs
  )
  # If the media is missing a thumbnail schedule it to be downloaded
  if not instance.thumb_file_exists:
  instance.thumb = None
- if not instance.thumb:
+ if not instance.thumb and not instance.is_live:
  thumbnail_url = instance.thumbnail
  if thumbnail_url:
  log.info(f'Scheduling task to download thumbnail for: {instance.name} '
@@ -177,12 +187,13 @@ def media_post_save(sender, instance, created, **kwargs):
  verbose_name=verbose_name.format(instance.name),
  remove_existing_tasks=True
  )
+
  # If the media has not yet been downloaded schedule it to be downloaded
  if not instance.media_file_exists:
  instance.downloaded = False
  instance.media_file = None
  if (not instance.downloaded and instance.can_download and not instance.skip
- and instance.source.download_media):
+ and instance.source.download_media and not instance.is_live):
  delete_task_by_media('sync.tasks.download_media', (str(instance.pk),))
  verbose_name = _('Downloading media for "{}"')
  download_media(

tubesync/sync/tasks.py

@@ -229,6 +229,7 @@ def download_media_metadata(media_id):
  source = media.source
  metadata = media.index_metadata()
  media.metadata = json.dumps(metadata, default=json_serial)
+ media.last_crawl = timezone.now()
  upload_date = media.upload_date
  # Media must have a valid upload date
  if upload_date:

tubesync/sync/templates/sync/sources.html

@@ -19,6 +19,9 @@ <h1 class="truncate">Sources</h1>
  <div class="col m12 xl4 margin-bottom">
  <a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
  </div>
+ <div class="col m12 xl4 margin-bottom">
+ <a href="{% url 'sync:validate-source' source_type='twitch-channel' %}" class="btn">Add a Twitch channel <i class="fab fa-twitch"></i></a>
+ </div>
 </div>
 <div class="row no-margin-bottom">
  <div class="col s12">