Add robots.txt and sitemap.xml generators (#5)

cmd/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+ "errors"
  "flag"
  "fmt"
  "html/template"
@@ -129,10 +130,15 @@ var staticFiles = map[string]string{
  "/favicon.ico": "./web/assets/favicon.ico",
 }
 
+var generatedFiles = map[string]string{
+ "/robots.txt": "",
+ "/sitemap.xml": "",
+}
+
 func AssetSkipper(c echo.Context) bool {
  path := c.Request().URL.Path
  parts := strings.Split(path, "/")
- collections := []map[string]string{staticFiles, staticDirs}
+ collections := []map[string]string{staticFiles, generatedFiles, staticDirs}
  for _, col := range collections {
  for k := range col {
  if parts[1] == k[1:] {
@@ -187,9 +193,24 @@ func ResolveRelativePaths() echo.MiddlewareFunc {
 }
 
 func main() {
+ protocol := "https"
+ flag.Func("protocol", "webserver protocol: [http, https] (default \"https\")", func(value string) error {
+ switch value {
+ case "http":
+ fallthrough
+ case "https":
+ protocol = value
+ return nil
+ default:
+ return errors.New("unexpected value, expected one of [http, https]")
+ }
+ })
+ hostname := flag.String("hostname", "aidenhale.dev", "hostname")
  portPtr := flag.Int("port", 31337, "port to bind to")
  flag.Parse()
 
+ urlPrefix := monke.SanitizeUrl(fmt.Sprintf("%s://%s/", protocol, *hostname))
+
  tmpls := make(map[string]TemplateEntry)
 
  snippets := []string{
@@ -206,6 +227,15 @@ func main() {
  Light,
  )
 
+ tmpls["500.html"] = newTemplateEntry(
+ []string{
+ "./web/templates/500.html",
+ "./web/templates/base.html",
+ },
+ "base.html",
+ Light,
+ )
+
  tmpls["article.html"] = newTemplateEntry(
  append([]string{
  "./web/templates/article.html",
@@ -252,6 +282,14 @@ func main() {
  Light,
  )
 
+ tmpls["robots.txt"] = newTemplateEntry(
+ []string{
+ "./web/templates/robots.txt",
+ },
+ "robots.txt",
+ "",
+ )
+
  err := monke.InitDb("./web/data")
 
  if err != nil {
@@ -269,6 +307,10 @@ func main() {
  Skipper: AssetSkipper,
  RedirectCode: http.StatusMovedPermanently,
  }))
+ e.Pre(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{
+ Skipper: func(c echo.Context) bool { return !AssetSkipper(c) },
+ RedirectCode: http.StatusMovedPermanently,
+ }))
 
  for k, v := range staticFiles {
  e.File(k, v)
@@ -280,6 +322,18 @@ func main() {
 
  e.HTTPErrorHandler = customHTTPErrorHandler
  e.GET("/", pages.IndexPage)
+ e.GET("/sitemap.xml", func(c echo.Context) error {
+ return pages.RawXML(c, func() ([]byte, error) {
+ return monke.SitemapXml(urlPrefix)
+ })
+ })
+ e.GET("/robots.txt", func(c echo.Context) error {
+ return c.Render(200, "robots.txt", struct {
+ UrlPrefix string
+ }{
+ UrlPrefix: urlPrefix,
+ })
+ })
  e.GET("/about/", pages.StaticPage(200, "about.html"))
  e.GET("/articles/", pages.ArticlesSnippet)
  e.GET("/blog/:category/:article/", pages.ArticlePage)

pkg/monke/article.go

@@ -105,7 +105,7 @@ func NewArticle(f string, c string, urlPrefix string, tags []string) (*Article,
  article.Category = c
  article.Root = dir.Name()
  article.Id = filepath.Base(article.Root)
- article.Url = sanitizeUrl(fmt.Sprintf("%s/%s/", urlPrefix, article.Id))
+ article.Url = SanitizePath(fmt.Sprintf("%s/%s/", urlPrefix, article.Id))
  article.ReadmePath = readme
  article.Summary = string(summary)
  article.ArticleData = meta.Data
@@ -136,11 +136,3 @@ func NewArticle(f string, c string, urlPrefix string, tags []string) (*Article,
 
  return article, nil
 }
-
-func sanitizeUrl(s string) string {
- s, err := filepath.Abs(s)
- if err != nil {
- return ""
- }
- return s + "/"
-}

pkg/monke/sitemap.go

@@ -0,0 +1,48 @@
+package monke
+
+import (
+ "encoding/xml"
+ "fmt"
+ "html"
+)
+
+type urlEntry struct {
+ Loc string `xml:"loc"`
+ Lastmod string `xml:"lastmod"`
+ Priority float32 `xml:"priority"`
+}
+
+type urlsetEntry struct {
+ XMLName xml.Name `xml:"urlset"`
+ Xmlns string `xml:"xmlns,attr"`
+ Urls []urlEntry `xml:"url"`
+}
+
+func SitemapXml(urlPrefix string) ([]byte, error) {
+ articles := Db.Articles.GetArticles(ArticleFilter{"", "", 0}, 0, 0)
+ urls := make([]urlEntry, 0, len(articles))
+ for _, article := range articles {
+ articleLoc := html.EscapeString(SanitizeUrl(urlPrefix + article.Url))
+ urls = append(urls, urlEntry{
+ Loc: articleLoc,
+ Lastmod: article.Created.Format("2006-01-02"),
+ Priority: 0.8,
+ })
+
+ extras := article.Extras
+ for k := range extras {
+ urls = append(urls, urlEntry{
+ Loc: html.EscapeString(SanitizeUrl(fmt.Sprintf("%s/extra/%s/", articleLoc, k))),
+ Lastmod: article.Created.Format("2006-01-02"),
+ Priority: 0.5,
+ })
+ }
+ }
+ urlset := urlsetEntry{
+ Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
+ Urls: urls,
+ }
+
+ return xml.MarshalIndent(urlset, "", " ")
+
+}

pkg/monke/util.go

@@ -0,0 +1,32 @@
+package monke
+
+import (
+ "net/url"
+ "path/filepath"
+)
+
+func SanitizePath(s string) string {
+ if len(s) == 1 {
+ return s
+ }
+ trailingSlash := s[len(s)-1] == '/'
+ s, err := filepath.Abs(s)
+ if err != nil {
+ return ""
+ }
+ switch trailingSlash {
+ case true:
+ return s + "/"
+ default:
+ return s
+ }
+}
+
+func SanitizeUrl(s string) string {
+ u, err := url.Parse(s)
+ if err != nil {
+ return ""
+ }
+ u.Path = SanitizePath(u.Path)
+ return u.String()
+}

pkg/pages/raw.go

@@ -0,0 +1,17 @@
+package pages
+
+import (
+ "github.com/labstack/echo/v4"
+ "github.com/labstack/gommon/log"
+)
+
+type RawProviderFunc func() ([]byte, error)
+
+func RawXML(c echo.Context, provider RawProviderFunc) error {
+ data, err := provider()
+ if err != nil {
+ log.Warnf("failed to generate XML response for %+v: %+v", c.Request().URL, err)
+ return StaticPage(500, "500.html")(c)
+ }
+ return c.XMLBlob(200, data)
+}

web/templates/500.html

@@ -0,0 +1,5 @@
+{{- define "title" -}}Internal Server Error | Error{{- end -}}
+
+{{- define "body" }}
+<h1>500 something went wrong</h1>
+{{- end }}

web/templates/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: {{ .Data.UrlPrefix }}sitemap.xml