-
-
Notifications
You must be signed in to change notification settings - Fork 125
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This used to be the repo https://github.com/cuducos/minha-receita-mirror
- Loading branch information
Showing
12 changed files
with
390 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/cuducos/minha-receita/mirror" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| const mirrorHelper = ` | ||
| Mirror of CNPJ files from the Federal Revenue. | ||
| Minha Receita maintains a mirror of data from the Federal Revenue CNPJ, in | ||
| addition to the executables. This is the wbe interface for the bucket of these | ||
| files.` | ||
|
|
||
| var mirrorCmd = &cobra.Command{ | ||
| Use: "mirror", | ||
| Long: mirrorHelper, | ||
| Short: "Starts the files mirror web interface.", | ||
| RunE: func(_ *cobra.Command, _ []string) error { | ||
| if port == "" { | ||
| port = os.Getenv("PORT") | ||
| } | ||
| if port == "" { | ||
| port = defaultPort | ||
| } | ||
| return mirror.Mirror(port) | ||
| }, | ||
| } | ||
|
|
||
| func mirrorCLI() *cobra.Command { | ||
| mirrorCmd.Flags().StringVarP( | ||
| &port, | ||
| "port", | ||
| "p", | ||
| "", | ||
| fmt.Sprintf("web server port (default PORT environment variable or %s)", defaultPort), | ||
| ) | ||
| return mirrorCmd | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package mirror | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "html/template" | ||
| "time" | ||
| _ "embed" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/credentials" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| ) | ||
|
|
||
| const ( | ||
| cacheExpiration = 12 * time.Hour | ||
| ) | ||
|
|
||
| //go:embed index.html | ||
| var home string | ||
| type Cache struct { | ||
| settings settings | ||
| createdAt time.Time | ||
| template *template.Template | ||
| HTML []byte | ||
| JSON []byte | ||
| } | ||
|
|
||
| func (c *Cache) isExpired() bool { | ||
| return time.Since(c.createdAt) > cacheExpiration | ||
| } | ||
|
|
||
| type JSONResponse struct { | ||
| Data []Group `json:"data"` | ||
| } | ||
|
|
||
| func (c *Cache) refresh() error { | ||
| var fs []File | ||
| sess, err := session.NewSession(&aws.Config{ | ||
| Region: aws.String(c.settings.region), | ||
| Endpoint: aws.String(c.settings.endpointURL), | ||
| S3ForcePathStyle: aws.Bool(true), | ||
| Credentials: credentials.NewStaticCredentials( | ||
| c.settings.accessKey, | ||
| c.settings.secretAccessKey, | ||
| "", | ||
| ), | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var token *string | ||
| loadPage := func(t *string) ([]File, *string, error) { | ||
| var fs []File | ||
| sdk := s3.New(sess) | ||
| r, err := sdk.ListObjectsV2(&s3.ListObjectsV2Input{ | ||
| Bucket: aws.String(c.settings.bucket), | ||
| ContinuationToken: t, | ||
| }) | ||
| if err != nil { | ||
| return []File{}, nil, err | ||
| } | ||
| for _, obj := range r.Contents { | ||
| url := fmt.Sprintf("%s%s", c.settings.publicDomain, *obj.Key) | ||
| fs = append(fs, File{url, *obj.Size, *obj.Key, *obj.LastModified}) | ||
| } | ||
| if *r.IsTruncated { | ||
| return fs, r.NextContinuationToken, nil | ||
| } | ||
| return fs, nil, nil | ||
| } | ||
| for { | ||
| r, nxt, err := loadPage(token) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| fs = append(fs, r...) | ||
| if nxt == nil { | ||
| break | ||
| } | ||
| token = nxt | ||
| } | ||
|
|
||
| data := newGroups(fs) | ||
| var h bytes.Buffer | ||
| c.template.Execute(&h, data) | ||
| c.HTML = h.Bytes() | ||
|
|
||
| var j bytes.Buffer | ||
| if err := json.NewEncoder(&j).Encode(JSONResponse{data}); err != nil { | ||
| return err | ||
| } | ||
| c.JSON = j.Bytes() | ||
|
|
||
| c.createdAt = time.Now() | ||
| return nil | ||
| } | ||
|
|
||
| func newCache(s settings) (*Cache, error) { | ||
| t, err := template.New("home").Parse(home) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| c := Cache{s, time.Now(), t, []byte{}, []byte{}} | ||
| if err := c.refresh(); err != nil { | ||
| return nil, err | ||
| } | ||
| return &c, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package mirror | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "sort" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| const unit = 1024 | ||
|
|
||
| type File struct { | ||
| URL string `json:"url"` | ||
| Size int64 `json:"size"` | ||
| name string | ||
| lastModifiedAt time.Time | ||
| } | ||
|
|
||
| func (f *File) HumanReadableSize() string { | ||
| if f.Size < unit { | ||
| return fmt.Sprintf("%d B", f.Size) | ||
| } | ||
| div, exp := int64(unit), 0 | ||
| for n := f.Size / unit; n >= unit; n /= unit { | ||
| div *= unit | ||
| exp++ | ||
| } | ||
| return fmt.Sprintf("%.1f %cB", float64(f.Size)/float64(div), "KMGTPE"[exp]) | ||
| } | ||
|
|
||
| func (f *File) ShortName() string { | ||
| p := strings.Split(f.name, "/") | ||
| return p[len(p)-1] | ||
| } | ||
|
|
||
| func (f *File) group() string { | ||
| p := strings.Split(f.name, "/") | ||
| if len(p) == 1 { | ||
| return "Binários" | ||
| } | ||
| return p[0] | ||
| } | ||
|
|
||
| type Group struct { | ||
| Name string `json:"name"` | ||
| Files []File `json:"urls"` | ||
| } | ||
|
|
||
| func newGroups(fs []File) []Group { | ||
| var m = make(map[string][]File) | ||
| for _, f := range fs { | ||
| n := f.group() | ||
| m[n] = append(m[n], f) | ||
| } | ||
| ks := []string{} | ||
| for k := range m { | ||
| ks = append(ks, k) | ||
| } | ||
| sort.Sort(sort.Reverse(sort.StringSlice(ks))) | ||
| var gs []Group | ||
| for _, k := range ks { | ||
| gs = append(gs, Group{k, m[k]}) | ||
| } | ||
| return gs | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| <!DOCTYPE html> | ||
| <html lang="pt-BR"> | ||
| <head> | ||
| <meta charset="UTF-8"> | ||
| <meta name="viewport" content="width=device-width, initial-scale=1"> | ||
| <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/semantic-ui/dist/semantic.min.css"> | ||
| <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script> | ||
| <script src="https://cdn.jsdelivr.net/npm/semantic-ui/dist/semantic.min.js"></script> | ||
| <title>Espelho de dados — Minha Receita</title> | ||
| </head> | ||
| <body> | ||
| <div class="ui container"> | ||
| <h1 class="ui center aligned header"> | ||
| <img alt="Minha Receita" src="https://docs.minhareceita.org/minha-receita.svg" style="width: auto; height: 7rem; margin: 2rem 0"><br> | ||
| Espelho de Dados | ||
| </h1> | ||
| <div class="ui accordion"> | ||
| {{ range . }} | ||
| <div class="title"> | ||
| <i class="dropdown icon"></i> | ||
| {{ .Name }} | ||
| </div> | ||
| <div class="content"> | ||
| <table class="ui single line table"> | ||
| <thead> | ||
| <tr> | ||
| <th>Name</th> | ||
| <th>Size</th> | ||
| </tr> | ||
| </thead> | ||
| <tbody> | ||
| {{ range .Files }} | ||
| <tr> | ||
| <td><a href="{{ .URL }}">{{ .ShortName }}</a></td> | ||
| <td>{{ .HumanReadableSize }}</td> | ||
| </tr> | ||
| {{ end }} | ||
| </tbody> | ||
| </table> | ||
| </div> | ||
| {{ end }} | ||
| </div> | ||
| <div class="ui divider"></div> | ||
| <div> | ||
| <p class="right aligned"> | ||
| <i class="github icon"></i> | ||
| Código-fonte: | ||
| <a href="https://github.com/cuducos/minha-receita">Minha Receita</a> | ||
| </p> | ||
| </div> | ||
| </div> | ||
| <script> | ||
| $(document).ready(function(){ | ||
| $('.ui.accordion').accordion(); | ||
| }); | ||
| </script> | ||
| </body> | ||
| </html> |
Oops, something went wrong.