[Feature] optionally load only configured indexers.

src/Jackett.Common/Content/custom.js

@@ -99,6 +99,7 @@ function loadJackettSettings() {
  $("#jackett-allowupdate").attr('checked', data.updatedisabled);
  $("#jackett-prerelease").attr('checked', data.prerelease);
  $("#jackett-logging").attr('checked', data.logging);
+ $("#jackett-loadonlyconfiguredindexers").attr('checked', data.loadonlyconfiguredindexers);
  $("#jackett-omdbkey").val(data.omdbkey);
  $("#jackett-omdburl").val(data.omdburl);
  var password = data.password;
@@ -1212,6 +1213,7 @@ function bindUIButtons() {
  var jackett_update = $("#jackett-allowupdate").is(':checked');
  var jackett_prerelease = $("#jackett-prerelease").is(':checked');
  var jackett_logging = $("#jackett-logging").is(':checked');
+ var jackett_loadonlyconfiguredindexers = $("#jackett-loadonlyconfiguredindexers").is(':checked');
  var jackett_omdb_key = $("#jackett-omdbkey").val();
  var jackett_omdb_url = $("#jackett-omdburl").val();
 
@@ -1228,6 +1230,7 @@ function bindUIButtons() {
  prerelease: jackett_prerelease,
  blackholedir: $("#jackett-savedir").val(),
  logging: jackett_logging,
+ loadonlyconfiguredindexers:jackett_loadonlyconfiguredindexers,
  basepathoverride: jackett_basepathoverride,
  omdbkey: jackett_omdb_key,
  omdburl: jackett_omdb_url,

src/Jackett.Common/Content/index.html

@@ -196,6 +196,10 @@ <h3>Jackett Configuration</h3>
  <span class="input-header">OMDB API Url: </span>
  <input id="jackett-omdburl" class="form-control input-right" type="text" value="" placeholder="Blank for default">
  </div>
+ <div class="input-area">
+ <span class="input-header">Load only configured indexers (conserve memory, requires restart): </span>
+ <input id="jackett-loadonlyconfiguredindexers" class="form-control input-right" type="checkbox" />
+ </div>
  <hr />
  <div id="footer">
  <a href="https://github.com/Jackett/Jackett" target="_blank" title="Jackett on GitHub">Jackett</a> Version <span id="app-version"></span>

src/Jackett.Common/Models/Config/ServerConfig.cs

@@ -30,7 +30,7 @@ public ServerConfig(RuntimeSettings runtimeSettings)
  public string OmdbApiUrl { get; set; }
 
  /// <summary>
- /// Ignore as we don't really want to be saving settings specified in the command line. 
+ /// Ignore as we don't really want to be saving settings specified in the command line.
  /// This is a bit of a hack, but in future it might not be all that bad to be able to override config values using settings that were provided at runtime. (and save them if required)
  /// </summary>
  [JsonIgnore]
@@ -44,6 +44,8 @@ public ServerConfig(RuntimeSettings runtimeSettings)
 
  public bool ProxyIsAnonymous => string.IsNullOrWhiteSpace(ProxyUsername) || string.IsNullOrWhiteSpace(ProxyPassword);
 
+ public bool LoadOnlyConfiguredIndexers { get; set; }
+
  public string GetProxyAuthString() =>
  !ProxyIsAnonymous
  ? $"{ProxyUsername}:{ProxyPassword}"

src/Jackett.Common/Models/DTO/ServerConfig.cs

@@ -36,6 +36,9 @@ public class ServerConfig
  [DataMember]
  public bool can_run_netcore { get; set; }
 
+ [DataMember]
+ public bool loadonlyconfiguredindexers { get; set; }
+
  [DataMember]
  public ProxyType proxy_type { get; set; }
  [DataMember]
@@ -65,6 +68,7 @@ public ServerConfig(IEnumerable<string> notices, Models.Config.ServerConfig conf
  omdburl = config.OmdbApiUrl;
  app_version = version;
  can_run_netcore = canRunNetCore;
+ loadonlyconfiguredindexers = config.LoadOnlyConfiguredIndexers;
 
  proxy_type = config.ProxyType;
  proxy_url = config.ProxyUrl;

src/Jackett.Common/Services/IndexerConfigurationService.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Jackett.Common.Indexers;
 using Jackett.Common.Services.Interfaces;
 using Newtonsoft.Json.Linq;
@@ -10,6 +12,7 @@ namespace Jackett.Common.Services
 
  public class IndexerConfigurationService : IIndexerConfigurationService
  {
+ private const string ConfigFileSuffix = ".json";
 
  //public override void LoadFromSavedConfiguration(JToken jsonConfig)
  //{
@@ -141,8 +144,22 @@ public void Save(IIndexer indexer, JToken obj)
  }
  }
 
+ public IEnumerable<string> FindConfiguredIndexerIds()
+ {
+ var directoryInfo = new DirectoryInfo(configService.GetIndexerConfigDir());
+
+ if (!directoryInfo.Exists)
+ {
+ return new List<string>();
+ }
+
+ var fileNames = directoryInfo
+ .GetFiles().Select(file => file.Name).Where(fileName => fileName.EndsWith(ConfigFileSuffix)).ToList();
+ return fileNames.Select(file => file.Split('.')[0]).ToList();
+ }
+
  public string GetIndexerConfigFilePath(string indexerId)
- => Path.Combine(configService.GetIndexerConfigDir(), indexerId + ".json");
+ => Path.Combine(configService.GetIndexerConfigDir(), indexerId + ConfigFileSuffix);
 
  private readonly IConfigurationService configService;
  private readonly Logger logger;

src/Jackett.Common/Services/IndexerManagerService.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using AngleSharp.Text;
 using Jackett.Common.Indexers;
 using Jackett.Common.Indexers.Meta;
 using Jackett.Common.Models;
@@ -63,11 +64,52 @@ public IndexerManagerService(IIndexerConfigurationService config, IProtectionSer
  public void InitIndexers(IEnumerable<string> path)
  {
  MigrateRenamedIndexers();
- InitIndexers();
- InitCardigannIndexers(path);
+ InitIndexers(GetIndexersToLoad());
+ InitCardigannIndexers(GetRelevantIndexedDefinitionFiles(new HashSet<string>(path)));
  InitAggregateIndexer();
  }
 
+ private ISet<string> GetIndexersToLoad()
+ {
+ if (serverConfig.LoadOnlyConfiguredIndexers)
+ {
+ return new HashSet<string>(configService.FindConfiguredIndexerIds());
+ }
+
+ return null;
+ }
+
+ private bool GetIndexerShouldBeLoaded(string indexerId, ISet<string> configuredIndexers)
+ {
+ if (!serverConfig.LoadOnlyConfiguredIndexers)
+ {
+ return true;
+ }
+
+ return configuredIndexers != null && configuredIndexers.Contains(indexerId);
+ }
+
+ private IEnumerable<FileInfo> GetRelevantIndexedDefinitionFiles(ISet<string> path)
+ {
+ logger.Info("Loading Cardigann definitions from: " + string.Join(", ", path));
+ if (!serverConfig.LoadOnlyConfiguredIndexers)
+ {
+ logger.Info("Loading all Indexer definitions");
+ return GetIndexerDefinitionFiles(path);
+ }
+
+ logger.Info("Loading only configured Indexer definitions");
+ var configuredIndexers = configService.FindConfiguredIndexerIds();
+ return GetIndexerDefinitionFiles(path, new HashSet<string>(configuredIndexers));
+ }
+
+ private IEnumerable<FileInfo> GetIndexerDefinitionFiles(IEnumerable<string> path,
+ ISet<string> configuredIndexers)
+ {
+ return GetIndexerDefinitionFiles(path)
+ .Where(file => configuredIndexers.Contains(file.Name.SplitWithTrimming('.')[0])).ToList();
+ }
+
  private void MigrateRenamedIndexers()
  {
  foreach (var oldId in renamedIndexers.Keys)
@@ -95,7 +137,7 @@ private void MigrateRenamedIndexers()
  }
  }
 
- private void InitIndexers()
+ private void InitIndexers(ISet<string> configuredIndexers)
  {
  logger.Info("Using HTTP Client: " + webClient.GetType().Name);
 
@@ -115,38 +157,37 @@ private void InitIndexers()
 
  var arguments = new object[] { configService, indexerWebClientInstance, logger, protectionService };
  var indexer = (IIndexer)constructor.Invoke(arguments);
- return indexer;
+
+ if (GetIndexerShouldBeLoaded(indexer.Id, configuredIndexers))
+ {
+ return indexer;
+ }
+
+ return null;
  }
  else
  {
  logger.Error("Cannot instantiate " + type.Name);
  }
  return null;
- });
+ }).Where(indexer => indexer != null);
 
  foreach (var idx in ixs)
  {
- if (idx == null)
- continue;
  indexers.Add(idx.Id, idx);
  configService.Load(idx);
  }
  }
 
- private void InitCardigannIndexers(IEnumerable<string> path)
+ private void InitCardigannIndexers(IEnumerable<FileInfo> files)
  {
- logger.Info("Loading Cardigann definitions from: " + string.Join(", ", path));
-
  var deserializer = new DeserializerBuilder()
  .WithNamingConvention(CamelCaseNamingConvention.Instance)
 // .IgnoreUnmatchedProperties()
  .Build();
 
  try
  {
- var directoryInfos = path.Select(p => new DirectoryInfo(p));
- var existingDirectories = directoryInfos.Where(d => d.Exists);
- var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
  var definitions = files.Select(file =>
  {
  logger.Debug("Loading Cardigann definition " + file.FullName);
@@ -191,14 +232,22 @@ private void InitCardigannIndexers(IEnumerable<string> path)
 
  indexers.Add(indexer.Id, indexer);
  }
- logger.Info("Cardigann definitions loaded: " + string.Join(", ", indexers.Keys));
+ logger.Info("Cardigann definitions loaded ("+indexers.Count+"): " + string.Join(", ", indexers.Keys));
  }
  catch (Exception ex)
  {
  logger.Error(ex, "Error while loading Cardigann definitions: " + ex.Message);
  }
  }
 
+ private static IEnumerable<FileInfo> GetIndexerDefinitionFiles(IEnumerable<string> path)
+ {
+ var directoryInfos = path.Select(p => new DirectoryInfo(p));
+ var existingDirectories = directoryInfos.Where(d => d.Exists);
+ var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
+ return files;
+ }
+
  public void InitAggregateIndexer()
  {
  var omdbApiKey = serverConfig.OmdbApiKey;

src/Jackett.Common/Services/Interfaces/IIndexerConfigurationService.cs

@@ -1,3 +1,4 @@
+using System.Collections.Generic;
 using Jackett.Common.Indexers;
 using Newtonsoft.Json.Linq;
 
@@ -9,5 +10,6 @@ public interface IIndexerConfigurationService
  void Save(IIndexer indexer, JToken config);
  void Delete(IIndexer indexer);
  string GetIndexerConfigFilePath(string indexerId);
+ IEnumerable<string> FindConfiguredIndexerIds();
  }
 }

src/Jackett.Server/Controllers/ServerConfigurationController.cs

@@ -79,6 +79,7 @@ public IActionResult UpdateConfig([FromBody]Common.Models.DTO.ServerConfig confi
  var updateDisabled = config.updatedisabled;
  var preRelease = config.prerelease;
  var logging = config.logging;
+ var loadOnlyConfiguredIndexers = config.loadonlyconfiguredindexers;
  var basePathOverride = config.basepathoverride;
  if (basePathOverride != null)
  {
@@ -95,6 +96,7 @@ public IActionResult UpdateConfig([FromBody]Common.Models.DTO.ServerConfig confi
  webHostRestartNeeded = true;
  }
 
+ serverConfig.LoadOnlyConfiguredIndexers = loadOnlyConfiguredIndexers;
  serverConfig.UpdateDisabled = updateDisabled;
  serverConfig.UpdatePrerelease = preRelease;
  serverConfig.BasePathOverride = basePathOverride;