Fragment Description:



This example (by Russ Cox) demonstrates many features of Go:
- function passed around as value...
parameter to other functions, - decoding a json file content into a go struct{}, - accessing the Net using http.Get(), - concurrency with:
go, channel (data or signal/timeout), select, - use of 'time' to measure a connection duration A quite dense introduction, worth your time.
The video of the talk here:
https://www.youtube.com/watch?t=6&v=ytEkHepK08c

httpCrawler

Last update, on 2015, Fri 9 Oct, 16:15:38

/* ... <== see fragment description ... */

package main

import (
    "encoding/json"
    "fmt"
    "io"
    "io/ioutil"
    "log"
    "net/http"
    // "os"  // see file case
    "strings" // see Reader case
    "time"
)

type Lang struct {
    Name string
    Year int
    URL  string
}

// function as parameter... first class citizen
// In Go, functions can be passed around just like any other value.
// A function's type signature describes the types of its arguments
// and return values.
// here do() takes one argument which is a function taking itself one
// argument
// that is a Lang type. It returns nothing, just executes some tasks.
//
func do(f func(Lang)) {
    // Alternative to Reader reading see further down
    // reading a json file
    // input, err := os.Open("lang.json")
    // if err != nil {
    // log.Fatal(err)
    // }
    // setting a json decoder on the input (file example)
    // dec := json.NewDecoder(input)

    // Alternative to file reading, reader...
    const jsonStream = `
{"Name" : "Python", "Year" : 1991, "URL" : "http://python.org/"}
{"Name" : "Ruby", "Year" : 1995, "URL" : "http://ruby-lang.org/"}
{"Name" : "Scala", "Year" : 2003, "URL" : "http://scala.org/"}
{"Name" : "Go", "Year" : 2009, "URL" : "http://golang.org/"}
{"Name" : "Gofragments :)", "Year" : 2015, "URL" : "http://gofragments.net/"}
   `
    // setting a json decoder on the jsonStream (Reader example)
    dec := json.NewDecoder(strings.NewReader(jsonStream))

    // for to iterate on the lang items... there are 4 in this example (see json)
    for {
        // lang of type Lang
        var lang Lang
        // decoding the json file content and writing it into the "lang" struct
        err := dec.Decode(&lang)
        if err != nil {
            // the EOF situation is under control
            if err == io.EOF {
                break
            }
            log.Fatal(err)
        }
        // after the prepatory work, let's execute this parameter!
        f(lang)
    }
}
func main() {
    c := make(chan string)
    n := 0
    // this do(f) function receives here func(lang Lang){} as its parameter
    // This parameter will be "executed" after the preparatory work of do()
    // will have been done (see above)
    //
    do(func(lang Lang) {
        n++
        go count(lang.Name, lang.URL, c)
    })
    // time.After() returns a Time channel
    // sign: func After(d Duration) <-chan Time
    //
    timeout := time.After(1 * time.Second)
    // Let's receive from channel c the data: result or timeout
    //
    for i := 0; i < n; i++ {
        // a communication pseudo-random switch
        select {
        // either we receive a result from the channel
        case result := <-c:
            fmt.Print(result)
            // or we receive the timeout signal from the channel
        case <-timeout:
            fmt.Print("Timed out\n")
            return
        }
    }
}
func count(name, url string, c chan<- string) {
    // start is of type Time
    start := time.Now()
    // sign: func Get(url string) (resp *Response, err error)
    r, err := http.Get(url)
    if err != nil {
        // sending on channel c in case of error
        c <- fmt.Sprintf("%s: %s\n", name, err)
        return
    }
    // sign: func Copy(dst Writer, src Reader) (written int64, err error)
    // the Body received is discarded
    n, _ := io.Copy(ioutil.Discard, r.Body)
    // any Reader when used must be closed
    r.Body.Close()
    // sending on channel c the results grabbed with the http.Get
    // time.Since() returns a Duration (int64), the time elapsed since start here
    // Seconds() is a method of the Duration type
    //
    c <- fmt.Sprintf("%s %d [%.2fs]\n", name, n, time.Since(start).Seconds())
}



Comments