Simple Concurrent Web Scraping in Go with Geziyor, GoQuery, and Headless Chrome

go get github.com/geziyor/geziyor

// DataExporter is a generic struct that can export data of any type.
type DataExporter[T any] struct {
	Data []T
}

// Export adds scraped data to the DataExporter's Data slice.
// It receives data through a channel and appends it to the Data slice.
func (de *DataExporter[T]) Export(scraped chan any) error {
	if len(de.Data) == 0 {
		de.Data = []T{}
	}
	for pd := range scraped {
		d := pd.(T)
		de.Data = append(de.Data, d)
	}
	return nil
}

// AutherNText represents a quote with its author and text.
type AutherNText struct {
	Author string `json:"author"`
	Text   string `json:"text"`
}

// quotesParse is a parsing function for quote pages.
// It extracts author and text information from each quote on the page,
// exports the data as AutherNText structs, and follows pagination links.
func quotesParse(g *geziyor.Geziyor, r *client.Response) {
	// Find all quote elements and process each one
	r.HTMLDoc.Find("div.quote").Each(func(i int, s *goquery.Selection) {
		// Extract author and text from the quote
		g.Exports <- AutherNText{
			Author: s.Find("small.author").Text(),
			Text:   s.Find("span.text").Text(),
		}
	})

	// Check for a "next" pagination link
	if href, ok := r.HTMLDoc.Find("li.next > a").Attr("href"); ok {
		// If found, follow the link and continue parsing
		g.Get(r.JoinURL(href), quotesParse)
	}
}

func main() {
    exporter := DataExporter[AutherNText]{}
    g := geziyor.NewGeziyor(&geziyor.Options{
        StartURLs: []string{"http://quotes.toscrape.com/"},
        ParseFunc: quotesParse,
        Exporters: []export.Exporter{&exporter},
    })
    g.Start()
    slog.Info("Quotes scraped", "count", len(exporter.Data))
}

// Turtle represents a turtle species with its name, description, and image URL.
type Turtle struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Image       string `json:"image"`
}

// turtleScraper is a parsing function for turtle family pages.
// It extracts the turtle's name, description, and image URL from the HTML
// and exports the data as a Turtle struct.
func turtleScraper(g *geziyor.Geziyor, r *client.Response) {
	g.Exports <- Turtle{
		Name:        r.HTMLDoc.Find(".turtle-family-detail > h3").Text(),
		Description: r.HTMLDoc.Find(".turtle-family-detail > p").Text(),
		Image:       r.HTMLDoc.Find(".turtle-family-detail > img").AttrOr("src", ""),
	}
}

func main() {
    turtleExporter := DataExporter[Turtle]{}
    urls := []string{
        "https://www.scrapethissite.com/pages/frames/?frame=i&family=Cheloniidae",
        "https://www.scrapethissite.com/pages/frames/?frame=i&family=Chelydridae",
        "https://www.scrapethissite.com/pages/frames/?frame=i&family=Carettochelyidae",
    }
    
    g := geziyor.NewGeziyor(&geziyor.Options{
        StartURLs: urls,
        ParseFunc: turtleScraper,
        Exporters: []export.Exporter{&turtleExporter},
    })
    g.Start()
    
    slog.Info("Turtle information scraped", "count", len(turtleExporter.Data))
}

docker run -d -p 9222:9222 --rm --name headless-shell chromedp/headless-shell

// AliExpressProduct represents a product from AliExpress with its name and price.
type AliExpressProduct struct {
	Name  string `json:"name"`
	Price string `json:"price"`
}

// aliexpressProduct is a parsing function for AliExpress product pages.
// It extracts the product title and price from the rendered HTML and exports
// the data as an AliExpressProduct struct.
// Note: This function uses JavaScript rendering to access dynamic content.
func aliexpressProduct(g *geziyor.Geziyor, r *client.Response) {
	// Extract the product title from the h1 element with data-pl="product-title"
	title := r.HTMLDoc.Find("h1[data-pl=product-title]").Text()

	// Extract the product price from the span element with class "product-price-value"
	price := r.HTMLDoc.Find("span.product-price-value").Text()

	// Export the extracted data as an AliExpressProduct struct
	g.Exports <- AliExpressProduct{Name: title, Price: price}
}

func main() {
    aliExport := DataExporter[AliExpressProduct]{}
    urls := []string{
        "https://www.aliexpress.com/item/1005006959851087.html",
        "https://www.aliexpress.com/item/1005007265735821.html",
    }
    
    g := geziyor.NewGeziyor(&geziyor.Options{
        StartRequestsFunc: func(g *geziyor.Geziyor) {
            var wg sync.WaitGroup
            for _, url := range urls {
                wg.Add(1)
                go func(url string) {
                    defer wg.Done()
                    g.GetRendered(url, aliexpressProduct)
                }(url)
            }
            wg.Wait()
        },
        Exporters:       []export.Exporter{&aliExport},
        BrowserEndpoint: "ws://127.0.0.1:9222",
    })
    g.Start()
    
    slog.Info("AliExpress products scraped", "count", len(aliExport.Data))
}