Skip to content

Commit

Permalink
Support multiple link tags in both rss channel, items and in the univ…
Browse files Browse the repository at this point in the history
…ersal feed.
  • Loading branch information
mmcdole committed Jan 29, 2023
1 parent 26a6aec commit ea1dcfa
Show file tree
Hide file tree
Showing 22 changed files with 220 additions and 47 deletions.
23 changes: 17 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
module github.com/mmcdole/gofeed

go 1.14
go 1.19

require (
github.com/PuerkitoBio/goquery v1.5.1
github.com/json-iterator/go v1.1.10
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf
github.com/PuerkitoBio/goquery v1.8.0
github.com/json-iterator/go v1.1.12
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354
github.com/stretchr/testify v1.3.0
github.com/urfave/cli v1.22.3
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/text v0.3.2
golang.org/x/net v0.4.0
golang.org/x/text v0.5.0
)

require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
)
41 changes: 21 additions & 20 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 h1:Z6i7ND25ixRtXFBylIUggqpvLMV1I15yprcqMVB7WZA=
github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
Expand All @@ -28,15 +29,15 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/urfave/cli v1.22.3 h1:FpNT6zq26xNpHZy08emi755QwzLPs6Pukqjlc7RfOMU=
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
2 changes: 2 additions & 0 deletions rss/feed.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
type Feed struct {
Title string `json:"title,omitempty"`
Link string `json:"link,omitempty"`
Links []string `json:"links,omitempty"`
Description string `json:"description,omitempty"`
Language string `json:"language,omitempty"`
Copyright string `json:"copyright,omitempty"`
Expand Down Expand Up @@ -46,6 +47,7 @@ func (f Feed) String() string {
type Item struct {
Title string `json:"title,omitempty"`
Link string `json:"link,omitempty"`
Links []string `json:"links,omitempty"`
Description string `json:"description,omitempty"`
Content string `json:"content,omitempty"`
Author string `json:"author,omitempty"`
Expand Down
31 changes: 26 additions & 5 deletions rss/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ func (rp *Parser) parseRoot(p *xpp.XMLPullParser) (*Feed, error) {
}

func (rp *Parser) parseChannel(p *xpp.XMLPullParser) (rss *Feed, err error) {

if err = p.Expect(xpp.StartTag, "channel"); err != nil {
return nil, err
}
Expand All @@ -128,6 +127,7 @@ func (rp *Parser) parseChannel(p *xpp.XMLPullParser) (rss *Feed, err error) {

extensions := ext.Extensions{}
categories := []*Category{}
links := []string{}

for {
tok, err := rp.base.NextTag(p)
Expand Down Expand Up @@ -162,11 +162,12 @@ func (rp *Parser) parseChannel(p *xpp.XMLPullParser) (rss *Feed, err error) {
}
rss.Description = result
} else if name == "link" {
result, err := shared.ParseText(p)
result, err := rp.parseLink(p)
if err != nil {
return nil, err
}
rss.Link = result
links = append(links, result)
} else if name == "language" {
result, err := shared.ParseText(p)
if err != nil {
Expand Down Expand Up @@ -295,6 +296,10 @@ func (rp *Parser) parseChannel(p *xpp.XMLPullParser) (rss *Feed, err error) {
rss.Categories = categories
}

if len(links) > 0 {
rss.Links = links
}

if len(extensions) > 0 {
rss.Extensions = extensions

Expand All @@ -311,7 +316,6 @@ func (rp *Parser) parseChannel(p *xpp.XMLPullParser) (rss *Feed, err error) {
}

func (rp *Parser) parseItem(p *xpp.XMLPullParser) (item *Item, err error) {

if err = p.Expect(xpp.StartTag, "item"); err != nil {
return nil, err
}
Expand All @@ -320,6 +324,7 @@ func (rp *Parser) parseItem(p *xpp.XMLPullParser) (item *Item, err error) {
extensions := ext.Extensions{}
categories := []*Category{}
enclosures := []*Enclosure{}
links := []string{}

for {
tok, err := rp.base.NextTag(p)
Expand Down Expand Up @@ -363,11 +368,12 @@ func (rp *Parser) parseItem(p *xpp.XMLPullParser) (item *Item, err error) {
item.Content = result
}
} else if name == "link" {
result, err := shared.ParseText(p)
result, err := rp.parseLink(p)
if err != nil {
return nil, err
}
item.Link = result
links = append(links, result)
} else if name == "author" {
result, err := shared.ParseText(p)
if err != nil {
Expand Down Expand Up @@ -437,6 +443,10 @@ func (rp *Parser) parseItem(p *xpp.XMLPullParser) (item *Item, err error) {
item.Categories = categories
}

if len(links) > 0 {
item.Links = links
}

if len(extensions) > 0 {
item.Extensions = extensions

Expand All @@ -456,6 +466,18 @@ func (rp *Parser) parseItem(p *xpp.XMLPullParser) (item *Item, err error) {
return item, nil
}

func (rp *Parser) parseLink(p *xpp.XMLPullParser) (url string, err error) {
href := p.Attribute("href")
url, err = shared.ParseText(p)
if err != nil {
return
}
if url == "" && href != "" {
url = href
}
return url, err
}

func (rp *Parser) parseSource(p *xpp.XMLPullParser) (source *Source, err error) {
if err = p.Expect(xpp.StartTag, "source"); err != nil {
return nil, err
Expand Down Expand Up @@ -590,7 +612,6 @@ func (rp *Parser) parseGUID(p *xpp.XMLPullParser) (guid *GUID, err error) {
}

func (rp *Parser) parseCategory(p *xpp.XMLPullParser) (cat *Category, err error) {

if err = p.Expect(xpp.StartTag, "category"); err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions testdata/parser/rss/rdf_channel_link.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"link": "http://example.org",
"links": [ "http://example.org" ],
"items": [],
"version": "1.0"
}
6 changes: 6 additions & 0 deletions testdata/parser/rss/rdf_channel_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"link": "http://example3.org",
"links": ["http://example.org", "http://example2.org", "http://example3.org"],
"items": [],
"version": "1.0"
}
10 changes: 10 additions & 0 deletions testdata/parser/rss/rdf_channel_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<!--
Description: rdf channel links
-->
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel rdf:about="http://example.org/index.rdf">
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</channel>
</rdf:RDF>
3 changes: 2 additions & 1 deletion testdata/parser/rss/rdf_item_link.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
{
"items": [
{
"link": "http://example.org"
"link": "http://example.org",
"links": [ "http://example.org" ]
}
],
"version": "1.0"
Expand Down
13 changes: 13 additions & 0 deletions testdata/parser/rss/rdf_item_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"items": [
{
"link": "http://example3.org",
"links": [
"http://example.org",
"http://example2.org",
"http://example3.org"
]
}
],
"version": "1.0"
}
17 changes: 17 additions & 0 deletions testdata/parser/rss/rdf_item_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<!--
Description: rdf item links
-->
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel rdf:about="http://example.org/index.rdf">
<items>
<rdf:Seq>
<rdf:li resource="http://example.org/entry/1" />
</rdf:Seq>
</items>
</channel>
<item rdf:about="http://example.org/entry/1">
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</item>
</rdf:RDF>
13 changes: 13 additions & 0 deletions testdata/parser/rss/rss_channel_item_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"items": [
{
"link": "http://example3.org",
"links": [
"http://example.org",
"http://example2.org",
"http://example3.org"
]
}
],
"version": "2.0"
}
12 changes: 12 additions & 0 deletions testdata/parser/rss/rss_channel_item_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<!--
Description: rss item link
-->
<rss version="2.0">
<channel>
<item>
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</item>
</channel>
</rss>
1 change: 1 addition & 0 deletions testdata/parser/rss/rss_channel_link.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"link": "http://example.org",
"links": [ "http://example.org" ],
"items": [],
"version": "2.0"
}
6 changes: 6 additions & 0 deletions testdata/parser/rss/rss_channel_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"link": "http://example3.org",
"links": ["http://example.org", "http://example2.org", "http://example3.org"],
"items": [],
"version": "2.0"
}
10 changes: 10 additions & 0 deletions testdata/parser/rss/rss_channel_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<!--
Description: rss channel links
-->
<rss version="2.0">
<channel>
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</channel>
</rss>
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"feedType": "rss",
"feedVersion": "2.0",
"items": [
{
"link": "http://example3.org",
"links": ["http://example.org", "http://example2.org", "http://example3.org"]
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<!--
Description: item link
-->
<rss version="2.0">
<channel>
<item>
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</item>
</channel>
</rss>
7 changes: 7 additions & 0 deletions testdata/translator/rss/feed_links_-_rdf_channel_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"feedType": "rss",
"feedVersion": "1.0",
"items": [],
"link": "http://example3.org",
"links": ["http://example.org", "http://example2.org", "http://example3.org"]
}
10 changes: 10 additions & 0 deletions testdata/translator/rss/feed_links_-_rdf_channel_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<!--
Description: rdf channel link(s)
-->
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel rdf:about="http://example.org/index.rdf">
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</channel>
</rdf:RDF>
7 changes: 7 additions & 0 deletions testdata/translator/rss/feed_links_-_rss_channel_links.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"feedType": "rss",
"feedVersion": "2.0",
"items": [],
"link": "http://example3.org",
"links": ["http://example.org", "http://example2.org", "http://example3.org"]
}
10 changes: 10 additions & 0 deletions testdata/translator/rss/feed_links_-_rss_channel_links.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<!--
Description: normal link(s)
-->
<rss version="2.0">
<channel>
<link>http://example.org</link>
<link>http://example2.org</link>
<link href="http://example3.org" />
</channel>
</rss>

0 comments on commit ea1dcfa

Please sign in to comment.