poppen · August 11, 2016 01:13 · Aug 11, 2016 · Aug 11, 2016 · Aug 11, 2016
diff --git a/a.go → webcrawler.go b/a.go → webcrawler.go
diff --git a/a.go b/a.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"sync"
 )
 
 type Fetcher interface {
@@ -10,12 +11,21 @@ type Fetcher interface {
 	Fetch(url string) (body string, urls []string, err error)
 }
 
-func Crawl(url string, depth int, fetcher Fetcher, ret chan string) {
+type SafeCounter struct {
+	fetched map[string]int
+	mux sync.Mutex
+}
+
+func Crawl(url string, depth int, fetcher Fetcher, counter SafeCounter, ret chan string) {
 	defer close(ret)
 	if depth <= 0 {
 		return
 	}
 
+	counter.mux.Lock()
+	counter.fetched[url] = 1
+	counter.mux.Unlock()
+
 	body, urls, err := fetcher.Fetch(url)
 	if err != nil {
 		ret <- err.Error()
@@ -26,13 +36,17 @@ func Crawl(url string, depth int, fetcher Fetcher, ret chan string) {
 
 	result := make([]chan string, len(urls))
 	for i, u := range urls {
-		result[i] = make(chan string)
-		go Crawl(u, depth-1, fetcher, result[i])
+		if _, ok := counter.fetched[u]; ok == false {
+			result[i] = make(chan string)
+			go Crawl(u, depth-1, fetcher, counter, result[i])
+		}
 	}
 
 	for i := range result {
-		for s := range result[i] {
-			ret <- s
+		if result[i] != nil {
+			for s := range result[i] {
+				ret <- s
+			}
 		}
 	}
 
@@ -41,7 +55,9 @@ func Crawl(url string, depth int, fetcher Fetcher, ret chan string) {
 
 func main() {
 	ch := make(chan string)
-	go Crawl("http://golang.org/", 4, fetcher, ch)
+	s := SafeCounter{}
+	s.fetched = make(map[string]int)
+	go Crawl("http://golang.org/", 4, fetcher, s, ch)
 
 	for s := range ch {
 		fmt.Println(s)

diff --git a/a.go b/a.go
@@ -0,0 +1,98 @@
+package main
+
+import (
+	"fmt"
+)
+
+type Fetcher interface {
+	// Fetch returns the body of URL and
+	// a slice of URLs found on that page.
+	Fetch(url string) (body string, urls []string, err error)
+}
+
+func Crawl(url string, depth int, fetcher Fetcher, ret chan string) {
+	defer close(ret)
+	if depth <= 0 {
+		return
+	}
+
+	body, urls, err := fetcher.Fetch(url)
+	if err != nil {
+		ret <- err.Error()
+		return
+	}
+
+	ret <- fmt.Sprintf("found: %s %q", url, body)
+
+	result := make([]chan string, len(urls))
+	for i, u := range urls {
+		result[i] = make(chan string)
+		go Crawl(u, depth-1, fetcher, result[i])
+	}
+
+	for i := range result {
+		for s := range result[i] {
+			ret <- s
+		}
+	}
+
+	return
+}
+
+func main() {
+	ch := make(chan string)
+	go Crawl("http://golang.org/", 4, fetcher, ch)
+
+	for s := range ch {
+		fmt.Println(s)
+	}
+}
+
+// fakeFetcher is Fetcher that returns canned results.
+type fakeFetcher map[string]*fakeResult
+
+type fakeResult struct {
+	body string
+	urls []string
+}
+
+func (f fakeFetcher) Fetch(url string) (string, []string, error) {
+	if res, ok := f[url]; ok {
+		return res.body, res.urls, nil
+	}
+	return "", nil, fmt.Errorf("not found: %s", url)
+}
+
+// fetcher is a populated fakeFetcher.
+var fetcher = fakeFetcher{
+	"http://golang.org/": &fakeResult{
+		"The Go Programming Language",
+		[]string{
+			"http://golang.org/pkg/",
+			"http://golang.org/cmd/",
+		},
+	},
+	"http://golang.org/pkg/": &fakeResult{
+		"Packages",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/cmd/",
+			"http://golang.org/pkg/fmt/",
+			"http://golang.org/pkg/os/",
+		},
+	},
+	"http://golang.org/pkg/fmt/": &fakeResult{
+		"Package fmt",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/pkg/",
+		},
+	},
+	"http://golang.org/pkg/os/": &fakeResult{
+		"Package os",
+		[]string{
+			"http://golang.org/",
+			"http://golang.org/pkg/",
+		},
+	},
+}