Merge branch 'miniflux:main' into dev-keep-rss-content-and-web-content

ChangeLog

@@ -1,3 +1,37 @@
+Version 2.0.47 (August 20, 2023)
+--------------------------------
+
+* Update rules for `webtoons.com`
+* Use HTTP client from the standard library for third-party integrations
+* Rename internal `url` package to `urllib` to avoid overlap with `net/url`
+* Add Shaarli integration
+* Add Shiori integration
+* Add Apprise integration
+* Add Readwise Reader integration
+* Consider base path when generating third-party services API endpoint
+* Use podcast duration tag as reading time
+* Move internal packages to an `internal` folder
+ * For reference: <https://go.dev/doc/go1.4#internalpackages>
+* Rename Miniflux package name to follow Go module naming convention
+ * For reference: <https://go.dev/ref/mod#major-version-suffixes>
+* Update RockyLinux image from 8 to 9 (used to build RPM package)
+* Add force refresh in feed edit and feed entries page
+* Use Odysee video duration as read time
+* Upgrade to Go 1.21
+* Use details disclosure element to show the list of third-party services
+* Use Web Share API for sharing entry
+* Add a workaround for parsing some invalid date format
+* Add Thunder Client API collection into contrib folder
+* Add new API endpoint: `/entries/{entryID}/save`
+* Trigger Docker and packages workflows only for semantic tags
+ * Go module versioning expect Git tags to start with the letter v.
+ * The goal is to keep the existing naming convention for generated artifacts and 
+ have proper versioning for the Go module.
+* Bump `golang.org/x/*` dependencies
+* Bump `github.com/yuin/goldmark`
+* Bump `github.com/tdewolff/minify/v2`
+* Bump `github.com/mccutchen/go-httpbin/v2`
+
 Version 2.0.46 (July 21, 2023)
 ------------------------------
 

internal/config/config_test.go

@@ -724,6 +724,41 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
  }
 }
 
+func TestDefautSchedulerEntryFrequencyFactorValue(t *testing.T) {
+ os.Clearenv()
+
+ parser := NewParser()
+ opts, err := parser.ParseEnvironmentVariables()
+ if err != nil {
+ t.Fatalf(`Parsing failure: %v`, err)
+ }
+
+ expected := defaultSchedulerEntryFrequencyFactor
+ result := opts.SchedulerEntryFrequencyFactor()
+
+ if result != expected {
+ t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected)
+ }
+}
+
+func TestDefautSchedulerEntryFrequencyFactor(t *testing.T) {
+ os.Clearenv()
+ os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", "2")
+
+ parser := NewParser()
+ opts, err := parser.ParseEnvironmentVariables()
+ if err != nil {
+ t.Fatalf(`Parsing failure: %v`, err)
+ }
+
+ expected := 2
+ result := opts.SchedulerEntryFrequencyFactor()
+
+ if result != expected {
+ t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected)
+ }
+}
+
 func TestPollingParsingErrorLimit(t *testing.T) {
  os.Clearenv()
  os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100")

internal/config/options.go

@@ -30,6 +30,7 @@ const (
  defaultPollingScheduler = "round_robin"
  defaultSchedulerEntryFrequencyMinInterval = 5
  defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
+ defaultSchedulerEntryFrequencyFactor = 1
  defaultPollingParsingErrorLimit = 3
  defaultRunMigrations = false
  defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
@@ -118,6 +119,7 @@ type Options struct {
  pollingScheduler string
  schedulerEntryFrequencyMinInterval int
  schedulerEntryFrequencyMaxInterval int
+ schedulerEntryFrequencyFactor int
  pollingParsingErrorLimit int
  workerPoolSize int
  createAdmin bool
@@ -191,6 +193,7 @@ func NewOptions() *Options {
  pollingScheduler: defaultPollingScheduler,
  schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
  schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
+ schedulerEntryFrequencyFactor: defaultSchedulerEntryFrequencyFactor,
  pollingParsingErrorLimit: defaultPollingParsingErrorLimit,
  workerPoolSize: defaultWorkerPoolSize,
  createAdmin: defaultCreateAdmin,
@@ -368,6 +371,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int {
  return o.schedulerEntryFrequencyMinInterval
 }
 
+// SchedulerEntryFrequencyFactor returns the factor for the entry frequency scheduler.
+func (o *Options) SchedulerEntryFrequencyFactor() int {
+ return o.schedulerEntryFrequencyFactor
+}
+
 // PollingParsingErrorLimit returns the limit of errors when to stop polling.
 func (o *Options) PollingParsingErrorLimit() int {
  return o.pollingParsingErrorLimit
@@ -628,6 +636,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
  "RUN_MIGRATIONS": o.runMigrations,
  "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval,
  "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval,
+ "SCHEDULER_ENTRY_FREQUENCY_FACTOR": o.schedulerEntryFrequencyFactor,
  "SCHEDULER_SERVICE": o.schedulerService,
  "SERVER_TIMING_HEADER": o.serverTimingHeader,
  "WATCHDOG": o.watchdog,

internal/config/parser.go

@@ -135,6 +135,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
  p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
  case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
  p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
+ case "SCHEDULER_ENTRY_FREQUENCY_FACTOR":
+ p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor)
  case "POLLING_PARSING_ERROR_LIMIT":
  p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
  // kept for compatibility purpose

internal/googlereader/handler.go

@@ -815,7 +815,7 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) {
  Updated: time.Now().Unix(),
  Self: []contentHREF{
  {
- HREF: config.Opts.BaseURL() + route.Path(h.router, "StreamItemsContents"),
+ HREF: config.Opts.RootURL() + route.Path(h.router, "StreamItemsContents"),
  },
  },
  Author: user.Username,

internal/model/feed.go

@@ -115,10 +115,10 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int) {
  if weeklyCount == 0 {
  intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
  } else {
- intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount)))
+ intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor())))
+ intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
+ intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
  }
- intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
- intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
  f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
  default:
  f.NextCheckAt = time.Now()

internal/model/feed_test.go

@@ -152,3 +152,28 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) {
  t.Error(`The next_check_at should not be before the now + min interval`)
  }
 }
+
+func TestFeedScheduleNextCheckEntryFrequencyFactor(t *testing.T) {
+ factor := 2
+ os.Clearenv()
+ os.Setenv("POLLING_SCHEDULER", "entry_frequency")
+ os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", fmt.Sprintf("%d", factor))
+
+ var err error
+ parser := config.NewParser()
+ config.Opts, err = parser.ParseEnvironmentVariables()
+ if err != nil {
+ t.Fatalf(`Parsing failure: %v`, err)
+ }
+ feed := &Feed{}
+ weeklyCount := 7
+ feed.ScheduleNextCheck(weeklyCount)
+
+ if feed.NextCheckAt.IsZero() {
+ t.Error(`The next_check_at must be set`)
+ }
+
+ if feed.NextCheckAt.After(time.Now().Add(time.Minute * time.Duration(config.Opts.SchedulerEntryFrequencyMaxInterval()/factor))) {
+ t.Error(`The next_check_at should not be after the now + factor * count`)
+ }
+}

internal/reader/scraper/rules.go

@@ -50,7 +50,7 @@ var predefinedRules = map[string]string{
  "universfreebox.com": "#corps_corps",
  "version2.dk": "section.body",
  "wdwnt.com": "div.entry-content",
- "webtoons.com": ".viewer_img",
+ "webtoons.com": ".viewer_img,p.author_text",
  "wired.com": "main figure, article",
  "zeit.de": ".summary, .article-body",
  "zdnet.com": "div.storyBody",

internal/storage/entry_query_builder.go

@@ -267,6 +267,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
  f.crawler,
  f.user_agent,
  f.cookie,
+ f.hide_globally,
  f.no_media_player,
  fi.icon_id,
  u.timezone
@@ -333,6 +334,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
  &entry.Feed.Crawler,
  &entry.Feed.UserAgent,
  &entry.Feed.Cookie,
+ &entry.Feed.HideGlobally,
  &entry.Feed.NoMediaPlayer,
  &iconID,
  &tz,

internal/ui/integration_pocket.go

@@ -32,7 +32,7 @@ func (h *handler) pocketAuthorize(w http.ResponseWriter, r *http.Request) {
 
  sess := session.New(h.store, request.SessionID(r))
  connector := pocket.NewConnector(config.Opts.PocketConsumerKey(integration.PocketConsumerKey))
- redirectURL := config.Opts.BaseURL() + route.Path(h.router, "pocketCallback")
+ redirectURL := config.Opts.RootURL() + route.Path(h.router, "pocketCallback")
  requestToken, err := connector.RequestToken(redirectURL)
  if err != nil {
  logger.Error("[Pocket:Authorize] %v", err)

miniflux.1

@@ -181,6 +181,11 @@ Minimum interval in minutes for the entry frequency scheduler\&.
 .br
 Default is 5 minutes\&.
 .TP
+.B SCHEDULER_ENTRY_FREQUENCY_FACTOR
+Factor to increase refresh frequency for the entry frequency scheduler\&.
+.br
+Default is 1\&.
+.TP
 .B POLLING_PARSING_ERROR_LIMIT
 The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited.
 .br