summaryrefslogtreecommitdiff
path: root/linkchecker.go
blob: 6faaf83b419ff936d6734635823afa38b0ce014a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
)

var (
	list = flag.String("list", "sitemap.txt", "Link list file")
	urls = make(map[string]string)
)

func fetchStatus(url string) string {
	client := new(http.Client)
	if u, ok := urls[url]; ok && u != "" {
		client.CheckRedirect = redir
	}
	resp, err := client.Get(url)
	if err != nil {
		return err.Error()
	}
	defer resp.Body.Close()
	return resp.Status
}

func readList(file string) {
	fd, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer fd.Close()
	scanner := bufio.NewScanner(fd)
	for scanner.Scan() {
		s := strings.Fields(scanner.Text())
		switch len(s) {
		case 2:
			urls[s[0]] = s[1]
		case 1:
			urls[s[0]] = ""
		}
	}
}

func redir(req *http.Request, via []*http.Request) error {
	if len(via) >= 10 {
		return errors.New("too many redirects")
	}
	to := req.URL.String()
	from := via[0].URL.String()
	if u, ok := urls[from]; ok {
		if u == to {
			return nil
		}
		return errors.New("expected " + u + " got " + to)
	}
	return errors.New("missing target")
}

func main() {
	flag.Parse()
	readList(*list)

	wg := new(sync.WaitGroup)
	defer wg.Wait()

	for k, _ := range urls {
		wg.Add(1)
		go func(u string, w *sync.WaitGroup) {
			defer w.Done()
			fmt.Println(fetchStatus(u), u)
		}(k, wg)
	}
}