User-agent: * Allow: .js Allow: .css Disallow: /generic Disallow: /var Disallow: /person Disallow: /photoalbum Disallow: /pay Disallow: /thank-you Disallow: /plans Disallow: /create-account Disallow: /account/choose-favorites Disallow: /collections/tag/* User-agent: dotbot Crawl-delay: 10 User-agent: AhrefsBot Crawl-delay: 10 User-agent: Bytedance Disallow: / User-agent: Bytespider Disallow: /