User-agent: *
Disallow: /

User-Agent: *
Disallow: /m/
Disallow: /me/
Disallow: /@me$
Disallow: /@me/
Disallow: /*/edit$
Disallow: /*/*/edit$
Allow: /_/
Allow: /_/api/users/*/meta
Allow: /_/api/users/*/profile/stream
Allow: /_/api/posts/*/responses
Allow: /_/api/posts/*/responsesStream
Allow: /_/api/posts/*/related
Sitemap: https://medium.com/sitemap/sitemap.xml

User-agent: *
Disallow: /*null*
Disallow: /*Cart-MiniAddProduct
Disallow: /jp/apps/shoplocator*
Disallow: /com/apps/claimfreedom*
Disallow: /us/help-topics-affiliates.html
Disallow: /on/Demandware.store/Sites-adidas-US-Site/en_US/
User-Agent: bingbot
Crawl-delay: 1
Sitemap: https://www.adidas.com/on/demandware.static/-/Sites-CustomerFileStore/default/adidas-US/en_US/sitemaps/adidas-US-sitemap.xml
Sitemap: https://www.adidas.com/on/demandware.static/-/Sites-CustomerFileStore/default/adidas-MLT/en_PT/sitemaps/adidas-MLT-sitemap.xml

Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0

GET /index.html HTTP/1.1
Host: example.com
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0

go get github.com/temoto/robotstxt

package main

import (
  "net/http"

  "github.com/temoto/robotstxt"
)

func main() {
  // Get the contents of robots.txt from packtpub.com
  resp, err := http.Get("https://www.packtpub.com/robots.txt")
  if err != nil {
    panic(err)
  }
  // Process the response using temoto/robotstxt
  data, err := robotstxt.FromResponse(resp)
  if err != nil {
    panic(err)
  }
  // Look for the definition in the robots.txt file that matches the default Go User-Agent string
  grp := data.FindGroup("Go-http-client/1.1")
  if grp != nil {
    testUrls := []string{
      // These paths are all permissable
      "/all",
      "/all?search=Go",
      "/bundles",

      // These paths are not
      "/contact/",
      "/search/",
      "/user/password/",
    }

    for _, url := range testUrls {
      print("checking " + url + "...")

      // Test the path against the User-Agent group
      if grp.Test(url) == true {
        println("OK")
      } else {
        println("X")
      }
    }
  }
}

package main

import (
  "fmt"
  "net/http"
  "time"
)

func main() {
  // Tracks the timestamp of the last request to the webserver
  var lastRequestTime time.Time

  // The maximum number of requests we will make to the webserver
  maximumNumberOfRequests := 5

  // Our scrape rate at 1 page per 5 seconds
  pageDelay := 5 * time.Second

  for i := 0; i < maximumNumberOfRequests; i++ {
    // Calculate the time difference since our last request
    elapsedTime := time.Now().Sub(lastRequestTime)
    fmt.Printf("Elapsed Time: %.2f (s)\n", elapsedTime.Seconds())
    //Check if there has been enough time
    if elapsedTime < pageDelay {
      // Sleep the difference between the pageDelay and elapsedTime
      var timeDiff time.Duration = pageDelay - elapsedTime
      fmt.Printf("Sleeping for %.2f (s)\n", timeDiff.Seconds())
      time.Sleep(pageDelay - elapsedTime)
    }

    // Just for this example, we are not processing the response
    println("GET example.com/index.html")
    _, err := http.Get("http://www.example.com/index.html")
    if err != nil {
      panic(err)
    }

    // Update the last request time
    lastRequestTime = time.Now()
  }
}

var lastRequestMap map[string]time.Time = map[string]time.Time{
  "example.com": time.Time{},
  "packtpub.com": time.Time{},
}

// Check if "i" is an even number
if i%2 == 0 {
  // Use the Packt Publishing site and elapsed time
  webpage = packtPage
  elapsedTime = time.Now().Sub(lastRequestMap["packtpub.com"])
} else {
  // Use the example.com elapsed time
  elapsedTime = time.Now().Sub(lastRequestMap["example.com"])
}

// Update the last request time
if i%2 == 0 {
  // Use the Packt Publishing elapsed time
  lastRequestMap["packtpub.com"] = time.Now()
} else {
  // Use the example.com elapsed time
  lastRequestMap["example.com"] = time.Now()
}

HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: max-age=604800
Content-Type: text/html; charset=UTF-8
Date: Mon, 29 Oct 2018 13:31:23 GMT
Etag: "1541025663"
Expires: Mon, 05 Nov 2018 13:31:23 GMT
Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
Server: ECS (dca/53DB)
Vary: Accept-Encoding
X-Cache: HIT
Content-Length: 1270
...

Cache-Control: max-age=604800

Date: Mon, 29 Oct 2018 13:31:23 GMT
Expires: Mon, 05 Nov 2018 13:31:23 GMT

HTTP/1.1 304 Not Modified
Accept-Ranges: bytes
Cache-Control: max-age=604800
Date: Fri, 02 Nov 2018 14:37:16 GMT
Etag: "1541025663"
Expires: Fri, 09 Nov 2018 14:37:16 GMT
Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
Server: ECS (dca/53DB)
Vary: Accept-Encoding
X-Cache: HIT

package main

import (
  "io/ioutil"

  "github.com/gregjones/httpcache"
  "github.com/gregjones/httpcache/diskcache"
)

func main() {
  // Set up the local disk cache
  storage := diskcache.New("./cache")
  cache := httpcache.NewTransport(storage)

  // Set this to true to inform us if the responses are being read from a cache
  cache.MarkCachedResponses = true
  cachedClient := cache.Client()

  // Make the initial request
  println("Caching: http://www.example.com/index.html")
  resp, err := cachedClient.Get("http://www.example.com/index.html")
  if err != nil {
    panic(err)
  }

  // httpcache requires you to read the body in order to cache the response
  ioutil.ReadAll(resp.Body)
  resp.Body.Close()

  // Request index.html again
  println("Requesting: http://www.example.com/index.html")
  resp, err = cachedClient.Get("http://www.example.com/index.html")
  if err != nil {
    panic(err)
  }

  // Look for the flag added by httpcache to show the result is read from the cache
  _, ok = resp.Header["X-From-Cache"]
  if ok {
    println("Result was pulled from the cache!")
  }
}