Go + Colly/v2

[coloraven]

Give the curl command
And output as blow:

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/debug"
)

// creat a new collector
var c = colly.NewCollector(
	colly.Async(true), // Enable Async
	colly.AllowURLRevisit(),
	colly.Debugger(&debug.LogDebugger{}),
)

func init() {
	// The headers
	Headers := map[string]string{
		"User-Agent":    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36",
		"Accept-Custom": "en-US,en;q=0.9",
	}

	// Set proxy
	err := c.SetProxy("http://127.0.0.1:xxx")
	if err != nil {
		log.Fatal("set proxy failed:", err)
	}

	c.Limit(&colly.LimitRule{
		DomainGlob:  "*",
		Parallelism: 5,
		RandomDelay: 5 * time.Second,
	})

	c.OnRequest(func(r *colly.Request) {
		if r.Method == "GET" {
		} else if r.Method == "POST" {
		}

		// Set headers
		for k, v := range Headers {
			r.Headers.Set(k, v)
		}
	})

}
func main() {
	url := "https://httpbin.org/html"
	c.OnResponse(func(r *colly.Response) {
		fmt.Println("Response Text:", string(r.Body))
	})
	err := c.Visit(url)
	if err != nil {
		fmt.Println("Visit error:", err)
	}

	c.Wait()
}