From 611e0eaa2e7eae70ab038ea18f2bd2a973acdf9f Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Wed, 9 Sep 2015 18:04:11 +0200 Subject: Add redir tester --- linkchecker.go | 57 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/linkchecker.go b/linkchecker.go index 81abe96..6faaf83 100644 --- a/linkchecker.go +++ b/linkchecker.go @@ -2,56 +2,79 @@ package main import ( "bufio" + "errors" "flag" "fmt" "log" "net/http" "os" + "strings" "sync" - "text/tabwriter" ) -var list = flag.String("list", "sitemap.txt", "Link list file") +var ( + list = flag.String("list", "sitemap.txt", "Link list file") + urls = make(map[string]string) +) func fetchStatus(url string) string { - resp, err := http.Get(url) + client := new(http.Client) + if u, ok := urls[url]; ok && u != "" { + client.CheckRedirect = redir + } + resp, err := client.Get(url) if err != nil { - log.Fatal(err) + return err.Error() } defer resp.Body.Close() return resp.Status } -func readList(file string) (s []string) { +func readList(file string) { fd, err := os.Open(file) if err != nil { log.Fatal(err) } defer fd.Close() - scanner := bufio.NewScanner(fd) for scanner.Scan() { - s = append(s, scanner.Text()) + s := strings.Fields(scanner.Text()) + switch len(s) { + case 2: + urls[s[0]] = s[1] + case 1: + urls[s[0]] = "" + } } +} - return s +func redir(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("too many redirects") + } + to := req.URL.String() + from := via[0].URL.String() + if u, ok := urls[from]; ok { + if u == to { + return nil + } + return errors.New("expected " + u + " got " + to) + } + return errors.New("missing target") } func main() { flag.Parse() + readList(*list) wg := new(sync.WaitGroup) defer wg.Wait() - w := new(tabwriter.Writer) - w.Init(os.Stdout, 16, 8, 0, '\t', 0) - - for _, url := range readList(*list) { + for k, _ := range urls { wg.Add(1) - go func(url string) { - defer wg.Done() - fmt.Fprintf(w, "%s\t%s\n", fetchStatus(url), url) - w.Flush() - }(url) + go func(u string, w *sync.WaitGroup) { + defer w.Done() + fmt.Println(fetchStatus(u), u) + }(k, wg) } } -- cgit v1.2.3