{
 "published": true,
 "published_at": "2026-11-04",
 "site": "https://lab2.crawlcheck.io",
 "clean_twin": "https://lab2-clean.crawlcheck.io",
 "write_up": "https://crawlcheck.io/research/best-ai-audit-tool-round-2",
 "note": "Every fault below is on the faulty host and absent from the clean twin. R1 depends on the robots.txt parser: Google's reference parser blocks ClaudeBot site-wide; others allow it.",
 "faults": [
  {
   "id": "R1",
   "category": "robots",
   "fault": "ClaudeBot's robots group holds only a Crawl-delay, so its access depends on the parser: Google's reference parser merges it into the next group and blocks it site-wide (line 11); others read an empty group and allow everything",
   "standard": "RFC 9309 s2.2.1; google/robotstxt"
  },
  {
   "id": "R2",
   "category": "robots",
   "fault": "/assets/ (the stylesheet and the script every page needs) is disallowed for all crawlers",
   "standard": "Google: do not block resources needed to render"
  },
  {
   "id": "R3",
   "category": "robots",
   "fault": "Googlebot-Image is disallowed from the whole site",
   "standard": "Google image crawler docs"
  },
  {
   "id": "R4",
   "category": "robots",
   "fault": "Storebot-Google is disallowed from the whole site",
   "standard": "Google shopping crawler docs"
  },
  {
   "id": "R5",
   "category": "robots",
   "fault": "A group for \"Perplexity-Bot\" (not PerplexityBot) never applies to Perplexity's crawler",
   "standard": "Perplexity crawler docs"
  },
  {
   "id": "R6",
   "category": "robots",
   "fault": "The Sitemap line is a relative path; it must be an absolute URL",
   "standard": "sitemaps.org / RFC 9309"
  },
  {
   "id": "R7",
   "category": "robots",
   "fault": "A Noindex: line in robots.txt, which Google has ignored since 2019",
   "standard": "Google Search Central"
  },
  {
   "id": "S1",
   "category": "server",
   "fault": "PerplexityBot is answered with HTTP 403 on every page",
   "standard": "Perplexity crawler docs"
  },
  {
   "id": "S2",
   "category": "server",
   "fault": "Applebot gets HTTP 200 with a 'Just a moment...' challenge page instead of content",
   "standard": "Apple crawler docs"
  },
  {
   "id": "S3",
   "category": "server",
   "fault": "OAI-SearchBot gets pages with the services, areas and prices sections removed",
   "standard": "OpenAI crawler docs"
  },
  {
   "id": "S4",
   "category": "server",
   "fault": "ClaudeBot is redirected (302) from every page to /unavailable",
   "standard": "Anthropic crawler docs"
  },
  {
   "id": "S5",
   "category": "server",
   "fault": "GPTBot gets HTTP 429 on every request",
   "standard": "OpenAI crawler docs"
  },
  {
   "id": "S6",
   "category": "server",
   "fault": "Googlebot user agents get an extra keyword block no other visitor sees (cloaking)",
   "standard": "Google spam policies"
  },
  {
   "id": "S7",
   "category": "server",
   "fault": "Bingbot gets HTTP 503 on /pricing",
   "standard": "Bing Webmaster Guidelines"
  },
  {
   "id": "I1",
   "category": "indexing",
   "fault": "/services/gutter-guards carries meta robots noindex",
   "standard": "Google robots meta docs"
  },
  {
   "id": "I2",
   "category": "indexing",
   "fault": "/pricing is sent with an X-Robots-Tag: noindex header",
   "standard": "Google robots meta docs"
  },
  {
   "id": "I3",
   "category": "indexing",
   "fault": "/areas/riverbend has a canonical pointing to a URL that returns 404",
   "standard": "Google canonical docs"
  },
  {
   "id": "I4",
   "category": "indexing",
   "fault": "/services/downspout-repair has a canonical pointing to a different service page",
   "standard": "Google canonical docs"
  },
  {
   "id": "I5",
   "category": "indexing",
   "fault": "The homepage carries meta robots nosnippet",
   "standard": "Google robots meta docs"
  },
  {
   "id": "I6",
   "category": "indexing",
   "fault": "/faq has two conflicting robots meta tags (index,follow and noindex)",
   "standard": "Google robots meta docs"
  },
  {
   "id": "I7",
   "category": "indexing",
   "fault": "/about has a canonical to the http:// version of itself",
   "standard": "Google canonical docs"
  },
  {
   "id": "J1",
   "category": "rendering",
   "fault": "/pricing prices exist only after JavaScript runs (the HTML says 'Loading prices...')",
   "standard": "Google JavaScript SEO basics"
  },
  {
   "id": "J2",
   "category": "rendering",
   "fault": "The homepage links to the area pages only through JavaScript click handlers, not <a href>",
   "standard": "Google link best practices"
  },
  {
   "id": "J3",
   "category": "rendering",
   "fault": "/reviews content is rendered entirely by JavaScript into an empty container",
   "standard": "Google JavaScript SEO basics"
  },
  {
   "id": "J4",
   "category": "rendering",
   "fault": "/faq answers are fetched from /api/faq.json after load and are not in the HTML",
   "standard": "Google JavaScript SEO basics"
  },
  {
   "id": "J5",
   "category": "rendering",
   "fault": "/gallery photos are CSS background images with no <img>, so they cannot be indexed",
   "standard": "Google image SEO docs"
  },
  {
   "id": "J6",
   "category": "rendering",
   "fault": "/careers content is inside an iframe from /embed/careers",
   "standard": "Google iframe guidance"
  },
  {
   "id": "D1",
   "category": "structured",
   "fault": "The homepage JSON-LD does not parse (trailing comma)",
   "standard": "JSON / schema.org"
  },
  {
   "id": "D2",
   "category": "structured",
   "fault": "/contact LocalBusiness has no address",
   "standard": "Google LocalBusiness structured data"
  },
  {
   "id": "D3",
   "category": "structured",
   "fault": "/contact schema telephone differs from the phone number printed on the page",
   "standard": "Google structured data guidelines"
  },
  {
   "id": "D4",
   "category": "structured",
   "fault": "/reviews marks up self-serving AggregateRating on the business itself",
   "standard": "Google review snippet guidelines"
  },
  {
   "id": "D5",
   "category": "structured",
   "fault": "/about has two Organization nodes with different names",
   "standard": "schema.org / Google"
  },
  {
   "id": "D6",
   "category": "structured",
   "fault": "sameAs on /about points to a profile URL that returns 404",
   "standard": "schema.org sameAs"
  },
  {
   "id": "D7",
   "category": "structured",
   "fault": "Blog posts mark up Article without datePublished or author",
   "standard": "Google Article structured data"
  },
  {
   "id": "M1",
   "category": "machine",
   "fault": "llms.txt does not start with an H1 title",
   "standard": "llmstxt.org"
  },
  {
   "id": "M2",
   "category": "machine",
   "fault": "llms.txt links to two pages that return 404",
   "standard": "llmstxt.org"
  },
  {
   "id": "M3",
   "category": "machine",
   "fault": "security.txt Expires is in the past (2025-01-01)",
   "standard": "RFC 9116"
  },
  {
   "id": "M4",
   "category": "machine",
   "fault": "security.txt has no Contact field",
   "standard": "RFC 9116"
  },
  {
   "id": "M5",
   "category": "machine",
   "fault": "security.txt is served as text/html",
   "standard": "RFC 9116"
  },
  {
   "id": "X1",
   "category": "sitemap",
   "fault": "The sitemap index lists /sitemap-blog.xml, which returns 404",
   "standard": "sitemaps.org"
  },
  {
   "id": "X2",
   "category": "sitemap",
   "fault": "The sitemap lists a noindex URL (/services/gutter-guards)",
   "standard": "Google sitemap docs"
  },
  {
   "id": "X3",
   "category": "sitemap",
   "fault": "The sitemap lists a redirecting URL (/old-gutter-page)",
   "standard": "Google sitemap docs"
  },
  {
   "id": "X4",
   "category": "sitemap",
   "fault": "The sitemap lists a URL that returns 404 (/services/roof-repair)",
   "standard": "Google sitemap docs"
  },
  {
   "id": "X5",
   "category": "sitemap",
   "fault": "Two lastmod values are not valid W3C dates",
   "standard": "sitemaps.org"
  },
  {
   "id": "X6",
   "category": "sitemap",
   "fault": "The sitemap lists a URL on another host",
   "standard": "sitemaps.org"
  },
  {
   "id": "L1",
   "category": "links",
   "fault": "The homepage links to /services/roof-repair, which returns 404",
   "standard": "Google link best practices"
  },
  {
   "id": "L2",
   "category": "links",
   "fault": "/old-gutter-page redirects three times before reaching a page",
   "standard": "Google redirect docs"
  },
  {
   "id": "L3",
   "category": "links",
   "fault": "/promo-a and /promo-b redirect to each other (loop), linked from a blog post",
   "standard": "Google redirect docs"
  },
  {
   "id": "L4",
   "category": "links",
   "fault": "/specials returns 200 with 'Page not found' text (soft 404), linked from the footer",
   "standard": "Google soft 404 docs"
  },
  {
   "id": "L5",
   "category": "links",
   "fault": "/areas/lakeside is in the sitemap but linked from no page (orphan)",
   "standard": "Google link best practices"
  },
  {
   "id": "L6",
   "category": "links",
   "fault": "/faq and /faq/ both return 200 with no redirect between them",
   "standard": "Google duplicate URL docs"
  },
  {
   "id": "L7",
   "category": "links",
   "fault": "The about page links internally to an http:// URL",
   "standard": "Google HTTPS docs"
  },
  {
   "id": "P1",
   "category": "onpage",
   "fault": "/careers has no <title>",
   "standard": "Google title link docs"
  },
  {
   "id": "P2",
   "category": "onpage",
   "fault": "/areas/millbrook and /areas/cedar-park have the same title",
   "standard": "Google title link docs"
  },
  {
   "id": "P3",
   "category": "onpage",
   "fault": "/services/fascia-repair has no meta description",
   "standard": "Google snippet docs"
  },
  {
   "id": "P4",
   "category": "onpage",
   "fault": "/services/ice-dam-prevention has two <h1> elements",
   "standard": "Accessibility / heading structure"
  },
  {
   "id": "P5",
   "category": "onpage",
   "fault": "The three homepage images have no alt text",
   "standard": "WCAG 1.1.1 / Google image SEO"
  },
  {
   "id": "P6",
   "category": "onpage",
   "fault": "/privacy has no lang attribute",
   "standard": "WCAG 3.1.1"
  },
  {
   "id": "P7",
   "category": "onpage",
   "fault": "/terms has no viewport meta tag",
   "standard": "Google mobile-friendly docs"
  },
  {
   "id": "T1",
   "category": "security",
   "fault": "No Strict-Transport-Security header on any page",
   "standard": "RFC 6797"
  },
  {
   "id": "T2",
   "category": "security",
   "fault": "/about loads an image over http:// (mixed content)",
   "standard": "MDN mixed content"
  },
  {
   "id": "T3",
   "category": "security",
   "fault": "A blog post links to a staging host (staging.lab2.crawlcheck.io)",
   "standard": "OWASP information exposure"
  },
  {
   "id": "T4",
   "category": "security",
   "fault": "The homepage sets a session cookie without Secure or HttpOnly",
   "standard": "OWASP session management"
  }
 ]
}