From 5fe432b7c1d6d626699e8091005a0de788d8168a Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Tue, 20 Dec 2016 23:05:09 +0100 Subject: links --- href.go | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'href.go') diff --git a/href.go b/href.go index 0799fad..e1a687b 100644 --- a/href.go +++ b/href.go @@ -28,7 +28,7 @@ func title(n *html.Node) (s string) { return "" } -func findTitle(uri string) (string, error) { +func getTitle(uri string) (string, error) { resp, err := http.Get(uri) if err != nil { return "", err @@ -53,26 +53,30 @@ func findTitle(uri string) (string, error) { return title(doc), nil } -func ExtractLinks(conn *irc.Conn, line *irc.Line) { - for _, v := range strings.Fields(line.Text()) { +func links(s string) (ret []string) { + for _, v := range strings.Fields(s) { switch { case strings.HasPrefix(v, "www"): v = "http://" + v fallthrough - case strings.HasPrefix(v, "http"): - go func(uri string) { - log.Println(line.Nick, uri) - t, err := findTitle(uri) - if err != nil { - log.Println(err) - } - if t != "" { - if len(t) > maxLen { - t = t[:maxLen] + "..." - } - conn.Notice(line.Target(), "Title: "+t) - } - }(v) + case strings.HasPrefix(v, "http:"), strings.HasPrefix(v, "https:"): + ret = append(ret, v) + } + } + return +} + +func Links(conn *irc.Conn, line *irc.Line) { + for _, v := range links(line.Text()) { + t, err := getTitle(v) + if err != nil { + log.Println(err) + } + if t != "" { + if len(t) > maxLen { + t = t[:maxLen] + "..." + } + conn.Notice(line.Target(), "Title: "+t) } } } -- cgit v1.2.3 From a037c25326bceabc471f679ab2203d99e248ab0e Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Tue, 20 Dec 2016 23:18:45 +0100 Subject: maxLen --- href.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'href.go') diff --git a/href.go b/href.go index e1a687b..918a2f2 100644 --- a/href.go +++ b/href.go @@ -13,6 +13,8 @@ import ( var errNotHTML = errors.New("not HTML content") +const maxLen = 500 + func title(n *html.Node) (s string) { if n.Type == html.ElementNode && n.Data == "title" { for c := n.FirstChild; c != nil; c = c.NextSibling { -- cgit v1.2.3 From 429466e258c9228483ae8928f66fd52cec1178eb Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Wed, 21 Dec 2016 00:25:57 +0100 Subject: privmsg --- href.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'href.go') diff --git a/href.go b/href.go index 918a2f2..5efa71c 100644 --- a/href.go +++ b/href.go @@ -2,19 +2,17 @@ package main import ( "errors" + "fmt" "log" "net/http" "strings" - irc "github.com/fluffle/goirc/client" "golang.org/x/net/html" "golang.org/x/net/html/charset" ) var errNotHTML = errors.New("not HTML content") -const maxLen = 500 - func title(n *html.Node) (s string) { if n.Type == html.ElementNode && n.Data == "title" { for c := n.FirstChild; c != nil; c = c.NextSibling { @@ -55,7 +53,7 @@ func getTitle(uri string) (string, error) { return title(doc), nil } -func links(s string) (ret []string) { +func getLinks(s string) (ret []string) { for _, v := range strings.Fields(s) { switch { case strings.HasPrefix(v, "www"): @@ -68,17 +66,20 @@ func links(s string) (ret []string) { return } -func Links(conn *irc.Conn, line *irc.Line) { - for _, v := range links(line.Text()) { - t, err := getTitle(v) - if err != nil { - log.Println(err) - } - if t != "" { - if len(t) > maxLen { - t = t[:maxLen] + "..." +func links(msg chan string) chan string { + c := make(chan string) + go func() { + for l := range c { + for _, v := range getLinks(l) { + t, err := getTitle(v) + if err != nil { + log.Println(err) + } + if t != "" { + msg <- fmt.Sprint("Title:", t) + } } - conn.Notice(line.Target(), "Title: "+t) } - } + }() + return c } -- cgit v1.2.3 From 432d01e3872d050663379006a7e66dc2932a89d7 Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Wed, 21 Dec 2016 00:31:46 +0100 Subject: Fix title --- href.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'href.go') diff --git a/href.go b/href.go index 5efa71c..9c67bb0 100644 --- a/href.go +++ b/href.go @@ -76,7 +76,7 @@ func links(msg chan string) chan string { log.Println(err) } if t != "" { - msg <- fmt.Sprint("Title:", t) + msg <- fmt.Sprintf("Title: %v", t) } } } -- cgit v1.2.3 From d3c97e413a8731bf61dba3f5a0fa9d8a62def0e5 Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Wed, 21 Dec 2016 01:07:35 +0100 Subject: add ssl flag --- href.go | 1 + 1 file changed, 1 insertion(+) (limited to 'href.go') diff --git a/href.go b/href.go index 9c67bb0..88c9680 100644 --- a/href.go +++ b/href.go @@ -71,6 +71,7 @@ func links(msg chan string) chan string { go func() { for l := range c { for _, v := range getLinks(l) { + log.Println("URL:", v) t, err := getTitle(v) if err != nil { log.Println(err) -- cgit v1.2.3 From 58981abe098c4e7062020de2332acaeb5fc6d5b1 Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Wed, 21 Dec 2016 13:34:30 +0100 Subject: cleanup --- href.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'href.go') diff --git a/href.go b/href.go index 88c9680..0743807 100644 --- a/href.go +++ b/href.go @@ -11,7 +11,7 @@ import ( "golang.org/x/net/html/charset" ) -var errNotHTML = errors.New("not HTML content") +var errNotHTML = errors.New("Not HTML") func title(n *html.Node) (s string) { if n.Type == html.ElementNode && n.Data == "title" { -- cgit v1.2.3 From 1c780ab0000890760bc923fc9322dd8462e83fa1 Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Sun, 25 Dec 2016 14:31:37 +0100 Subject: Split last --- href.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'href.go') diff --git a/href.go b/href.go index 0743807..b18b3f3 100644 --- a/href.go +++ b/href.go @@ -66,18 +66,18 @@ func getLinks(s string) (ret []string) { return } -func links(msg chan string) chan string { - c := make(chan string) +func linker(out chan string) chan string { + c := make(chan string, 1) go func() { for l := range c { for _, v := range getLinks(l) { - log.Println("URL:", v) + log.Println("URL", v) t, err := getTitle(v) if err != nil { log.Println(err) } if t != "" { - msg <- fmt.Sprintf("Title: %v", t) + out <- fmt.Sprintf("Title: %v", t) } } } -- cgit v1.2.3 From 384585ae8848a5f16d570c5febf16eaa7b2084c6 Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Sun, 25 Dec 2016 15:15:38 +0100 Subject: minor fixes --- href.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'href.go') diff --git a/href.go b/href.go index b18b3f3..bd4cec6 100644 --- a/href.go +++ b/href.go @@ -56,7 +56,7 @@ func getTitle(uri string) (string, error) { func getLinks(s string) (ret []string) { for _, v := range strings.Fields(s) { switch { - case strings.HasPrefix(v, "www"): + case strings.HasPrefix(v, "www."): v = "http://" + v fallthrough case strings.HasPrefix(v, "http:"), strings.HasPrefix(v, "https:"): -- cgit v1.2.3 From e663cdca3df5471ffbf7bfc2d329e7780e5ccf6f Mon Sep 17 00:00:00 2001 From: Dimitri Sokolyuk Date: Mon, 1 Jan 2018 09:48:16 +0100 Subject: 10M limit --- href.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'href.go') diff --git a/href.go b/href.go index bd4cec6..a4cc5c9 100644 --- a/href.go +++ b/href.go @@ -11,7 +11,10 @@ import ( "golang.org/x/net/html/charset" ) -var errNotHTML = errors.New("Not HTML") +var ( + errNotHTML = errors.New("Not HTML") + errTooBig = errors.New("Content too big") +) func title(n *html.Node) (s string) { if n.Type == html.ElementNode && n.Data == "title" { @@ -40,6 +43,10 @@ func getTitle(uri string) (string, error) { return "", errNotHTML } + if resp.ContentLength > 10*1024*1024 { + return "", errTooBig + } + r, err := charset.NewReader(resp.Body, ct) if err != nil { return "", err -- cgit v1.2.3