// Perform an HTTP request to load a page and search for a string
package main

import (
   "fmt"
   "io/ioutil"
   "log"
   "net/http"
   "os"
   "strings"
   "time"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 3 {
      fmt.Println("Search for a keyword in the contents of a URL")
      fmt.Println("Usage: " + os.Args[0] + " <url> <keyword>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com NanoDano")
      os.Exit(1)
   }
   url := os.Args[1]
   needle := os.Args[2] // Like searching for a needle in a haystack

   // Create a custom http client to override default settings. Optional
   // Use http.Get() instead of client.Get() to use default client.
   client := &http.Client{
      Timeout: 30 * time.Second, // Default is forever!
      // CheckRedirect - Policy for following HTTP redirects
      // Jar - Cookie jar holding cookies
      // Transport - Change default method for making request
   }

   response, err := client.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   // Read response body
   body, err := ioutil.ReadAll(response.Body)
   if err != nil {
      log.Fatal("Error reading HTTP body. ", err)
   }

   // Search for string
   if strings.Contains(string(body), needle) {
      fmt.Println("Match found for " + needle + " in URL " + url)
   } else {
      fmt.Println("No match found for " + needle + " in URL " + url)
   }
} 

<a href="mailto:nanodano@devdungeon.com">
<a href="mailto:nanodano@devdungeon.com?subject=Hello">

// Search through a URL and find mailto links with email addresses
package main

import (
   "fmt"
   "io/ioutil"
   "log"
   "net/http"
   "os"
   "regexp"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("Search for emails in a URL")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   // Read the response
   body, err := ioutil.ReadAll(response.Body)
   if err != nil {
      log.Fatal("Error reading HTTP body. ", err)
   }

   // Look for mailto: links using a regular expression
   re := regexp.MustCompile("\"mailto:.*?[?\"]")
   matches := re.FindAllString(string(body), -1)
   if matches == nil {
      // Clean exit if no matches found
      fmt.Println("No emails found.")
      os.Exit(0)
   }

   // Print all emails found
   for _, match := range matches {
      // Remove "mailto prefix and the trailing quote or question mark
      // by performing a slice operation to extract the substring
      cleanedMatch := match[8 : len(match)-1]
      fmt.Println(cleanedMatch)
   }
} 

// Perform an HTTP HEAD request on a URL and print out headers
package main

import (
   "fmt"
   "log"
   "net/http"
   "os"
)

func main() {
   // Load URL from command line arguments
   if len(os.Args) != 2 {
      fmt.Println(os.Args[0] + " - Perform an HTTP HEAD request to a URL")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Perform HTTP HEAD
   response, err := http.Head(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   // Print out each header key and value pair
   for key, value := range response.Header {
      fmt.Printf("%s: %s\n", key, value[0])
   }
} 

Set-Cookie: preferred_background=blue
Set-Cookie: session_id=PZRNVYAMDFECHBGDSSRLH

Cookie: preferred_background=blue; session_id=PZRNVYAMDFECHBGDSSRLH

package main

import (
   "fmt"
   "io/ioutil"
   "log"
   "net/http"
)

var url = "https://www.example.com"

func main() {
   // Create the HTTP request
   request, err := http.NewRequest("GET", url, nil)
   if err != nil {
      log.Fatal("Error creating HTTP request. ", err)
   }

   // Set cookie
   request.Header.Set("Cookie", "session_id=<SESSION_TOKEN>")

   // Create the HTTP client, make request and print response
   httpClient := &http.Client{}
   response, err := httpClient.Do(request)
   data, err := ioutil.ReadAll(response.Body)
   fmt.Printf("%s\n", data)
} 

// Search through a URL and find HTML comments
package main

import (
   "fmt"
   "io/ioutil"
   "log"
   "net/http"
   "os"
   "regexp"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("Search for HTML comments in a URL")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL and get response
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }
   body, err := ioutil.ReadAll(response.Body)
   if err != nil {
      log.Fatal("Error reading HTTP body. ", err)
   }

   // Look for HTML comments using a regular expression
   re := regexp.MustCompile("<!--(.|\n)*?-->")
   matches := re.FindAllString(string(body), -1)
   if matches == nil {
      // Clean exit if no matches found
      fmt.Println("No HTML comments found.")
      os.Exit(0)
   }

   // Print all HTML comments found
   for _, match := range matches {
      fmt.Println(match)
   }
} 

// Look for unlisted files on a domain
package main

import (
   "bufio"
   "fmt"
   "log"
   "net/http"
   "net/url"
   "os"
   "strconv"
)

// Given a base URL (protocol+hostname) and a filepath (relative URL)
// perform an HTTP HEAD and see if the path exists.
// If the path returns a 200 OK print out the path
func checkIfUrlExists(baseUrl, filePath string, doneChannel chan bool) {
   // Create URL object from raw string
   targetUrl, err := url.Parse(baseUrl)
   if err != nil {
      log.Println("Error parsing base URL. ", err)
   }
   // Set the part of the URL after the host name
   targetUrl.Path = filePath

   // Perform a HEAD only, checking status without
   // downloading the entire file
   response, err := http.Head(targetUrl.String())
   if err != nil {
      log.Println("Error fetching ", targetUrl.String())
   }

   // If server returns 200 OK file can be downloaded
   if response.StatusCode == 200 {
      log.Println(targetUrl.String())
   }

   // Signal completion so next thread can start
   doneChannel <- true
}

func main() {
   // Load command line arguments
   if len(os.Args) != 4 {
      fmt.Println(os.Args[0] + " - Perform an HTTP HEAD request to a URL")
      fmt.Println("Usage: " + os.Args[0] + 
         " <wordlist_file> <url> <maxThreads>")
      fmt.Println("Example: " + os.Args[0] + 
         " wordlist.txt https://www.devdungeon.com 10")
      os.Exit(1)
   }
   wordlistFilename := os.Args[1]
   baseUrl := os.Args[2]
   maxThreads, err := strconv.Atoi(os.Args[3])
   if err != nil {
      log.Fatal("Error converting maxThread value to integer. ", err)
   }

   // Track how many threads are active to avoid
   // flooding a web server
   activeThreads := 0
   doneChannel := make(chan bool)

   // Open word list file for reading
   wordlistFile, err := os.Open(wordlistFilename)
   if err != nil {
      log.Fatal("Error opening wordlist file. ", err)
   }

   // Read each line and do an HTTP HEAD
   scanner := bufio.NewScanner(wordlistFile)
   for scanner.Scan() {
      go checkIfUrlExists(baseUrl, scanner.Text(), doneChannel)
      activeThreads++

      // Wait until a done signal before next if max threads reached
      if activeThreads >= maxThreads {
         <-doneChannel
         activeThreads -= 1
      }
   }

   // Wait for all threads before repeating and fetching a new batch
   for activeThreads > 0 {
      <-doneChannel
      activeThreads -= 1
   }

   // Scanner errors must be checked manually
   if err := scanner.Err(); err != nil {
      log.Fatal("Error reading wordlist file. ", err)
   }
} 

// Change HTTP user agent
package main

import (
   "log"
   "net/http"
)

func main() {
   // Create the request for use later
   client := &http.Client{}
   request, err := http.NewRequest("GET", 
      "https://www.devdungeon.com", nil)
   if err != nil {
      log.Fatal("Error creating request. ", err)
   }

   // Override the user agent
   request.Header.Set("User-Agent", "_Custom User Agent_")

   // Perform the request, ignore response.
   _, err = client.Do(request)
   if err != nil {
      log.Fatal("Error making request. ", err)
   }
} 

go get https://github.com/PuerkitoBio/goquery  

<a href="https://www.devdungeon.com">DevDungeon</a>  

// Load a URL and list all links found
package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "os"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("Find all links in a web page")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   // Extract all links
   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      log.Fatal("Error loading HTTP response body. ", err)
   }

   // Find and print all links
   doc.Find("a").Each(func(i int, s *goquery.Selection) {
      href, exists := s.Attr("href")
      if exists {
         fmt.Println(href)
      }
   })
} 

// Load a URL and list all documents 
package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "os"
   "strings"
)

var documentExtensions = []string{"doc", "docx", "pdf", "csv", 
   "xls", "xlsx", "zip", "gz", "tar"}

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("Find all links in a web page")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   // Extract all links
   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      log.Fatal("Error loading HTTP response body. ", err)
   }

   // Find and print all links that contain a document
   doc.Find("a").Each(func(i int, s *goquery.Selection) {
      href, exists := s.Attr("href")
      if exists && linkContainsDocument(href) {
         fmt.Println(href)
      }
   })
} 

func linkContainsDocument(url string) bool {
   // Split URL into pieces
   urlPieces := strings.Split(url, ".")
   if len(urlPieces) < 2 {
      return false
   }

   // Check last item in the split string slice (the extension)
   for _, extension := range documentExtensions {
      if urlPieces[len(urlPieces)-1] == extension {
         return true
      }
   }
   return false
} 

package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "os"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("List all headings (h1-h6) in a web page")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      log.Fatal("Error loading HTTP response body. ", err)
   }

   // Print title before headings
   title := doc.Find("title").Text()
   fmt.Printf("== Title ==\n%s\n", title)

   // Find and list all headings h1-h6
   headingTags := [6]string{"h1", "h2", "h3", "h4", "h5", "h6"}
   for _, headingTag := range headingTags {
      fmt.Printf("== %s ==\n", headingTag)
      doc.Find(headingTag).Each(func(i int, heading *goquery.Selection) {
         fmt.Println(" * " + heading.Text())
      })
   }

} 

package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "os"
   "strings"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("List all words by frequency from a web page")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      log.Fatal("Error loading HTTP response body. ", err)
   }

   // Find and list all headings h1-h6
   wordCountMap := make(map[string]int)
   doc.Find("p").Each(func(i int, body *goquery.Selection) {
      fmt.Println(body.Text())
      words := strings.Split(body.Text(), " ")
      for _, word := range words {
         trimmedWord := strings.Trim(word, " \t\n\r,.?!")
         if trimmedWord == "" {
            continue
         }
         wordCountMap[strings.ToLower(trimmedWord)]++

      }
   })

   // Print all words along with the number of times the word was seen
   for word, count := range wordCountMap {
      fmt.Printf("%d | %s\n", count, word)
   }

} 

<script src="/ajax/libs/jquery/3.2.1/jquery.min.js"></script>  

/ajax/libs/jquery/3.2.1/jquery.min.js

package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "os"
)

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("List all JavaScript files in a webpage")
      fmt.Println("Usage: " + os.Args[0] + " <url>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   url := os.Args[1]

   // Fetch the URL
   response, err := http.Get(url)
   if err != nil {
      log.Fatal("Error fetching URL. ", err)
   }

   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      log.Fatal("Error loading HTTP response body. ", err)
   }

   // Find and list all external scripts in page
   fmt.Println("Scripts found in", url)
   fmt.Println("==========================")
   doc.Find("script").Each(func(i int, script *goquery.Selection) {

      // By looking only at the script src we are limiting
      // the search to only externally loaded JavaScript files.
      // External files might be hosted on the same domain
      // or hosted remotely
      src, exists := script.Attr("src")
      if exists {
         fmt.Println(src)
      }

      // script.Text() will contain the raw script text
      // if the JavaScript code is written directly in the
      // HTML source instead of loaded from a separate file
   })
} 

// Crawl a website, depth-first, listing all unique paths found
package main

import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "net/http"
   "net/url"
   "os"
   "time"
)

var (
   foundPaths  []string
   startingUrl *url.URL
   timeout     = time.Duration(8 * time.Second)
)

func crawlUrl(path string) {
   // Create a temporary URL object for this request
   var targetUrl url.URL
   targetUrl.Scheme = startingUrl.Scheme
   targetUrl.Host = startingUrl.Host
   targetUrl.Path = path

   // Fetch the URL with a timeout and parse to goquery doc
   httpClient := http.Client{Timeout: timeout}
   response, err := httpClient.Get(targetUrl.String())
   if err != nil {
      return
   }
   doc, err := goquery.NewDocumentFromReader(response.Body)
   if err != nil {
      return
   }

   // Find all links and crawl if new path on same host
   doc.Find("a").Each(func(i int, s *goquery.Selection) {
      href, exists := s.Attr("href")
      if !exists {
         return
      }

      parsedUrl, err := url.Parse(href)
      if err != nil { // Err parsing URL. Ignore
         return
      }

      if urlIsInScope(parsedUrl) {
         foundPaths = append(foundPaths, parsedUrl.Path)
         log.Println("Found new path to crawl: " +
            parsedUrl.String())
         crawlUrl(parsedUrl.Path)
      }
   })
}

// Determine if path has already been found
// and if it points to the same host
func urlIsInScope(tempUrl *url.URL) bool {
   // Relative url, same host
   if tempUrl.Host != "" && tempUrl.Host != startingUrl.Host {
      return false // Link points to different host
   }

   if tempUrl.Path == "" {
      return false
   }

   // Already found?
   for _, existingPath := range foundPaths {
      if existingPath == tempUrl.Path {
         return false // Match
      }
   }
   return true // No match found
}

func main() {
   // Load command line arguments
   if len(os.Args) != 2 {
      fmt.Println("Crawl a website, depth-first")
      fmt.Println("Usage: " + os.Args[0] + " <startingUrl>")
      fmt.Println("Example: " + os.Args[0] + 
         " https://www.devdungeon.com")
      os.Exit(1)
   }
   foundPaths = make([]string, 0)

   // Parse starting URL
   startingUrl, err := url.Parse(os.Args[1])
   if err != nil {
      log.Fatal("Error parsing starting URL. ", err)
   }
   log.Println("Crawling: " + startingUrl.String())

   crawlUrl(startingUrl.Path)

   for _, path := range foundPaths {
      fmt.Println(path)
   }
   log.Printf("Total unique paths crawled: %d\n", len(foundPaths))
}