package main import ( "bufio" "errors" "flag" "fmt" "log" "net/http" "os" "strings" "sync" ) var ( list = flag.String("list", "sitemap.txt", "Link list file") urls = make(map[string]string) ) func fetchStatus(url string) string { client := new(http.Client) if u, ok := urls[url]; ok && u != "" { client.CheckRedirect = redir } resp, err := client.Get(url) if err != nil { return err.Error() } defer resp.Body.Close() return resp.Status } func readList(file string) { fd, err := os.Open(file) if err != nil { log.Fatal(err) } defer fd.Close() scanner := bufio.NewScanner(fd) for scanner.Scan() { s := strings.Fields(scanner.Text()) switch len(s) { case 2: urls[s[0]] = s[1] case 1: urls[s[0]] = "" } } } func redir(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("too many redirects") } to := req.URL.String() from := via[0].URL.String() if u, ok := urls[from]; ok { if u == to { return nil } return errors.New("expected " + u + " got " + to) } return errors.New("missing target") } func main() { flag.Parse() readList(*list) wg := new(sync.WaitGroup) defer wg.Wait() for k, _ := range urls { wg.Add(1) go func(u string, w *sync.WaitGroup) { defer w.Done() fmt.Println(fetchStatus(u), u) }(k, wg) } }