{"@context":{"@language":"en","@vocab":"https://schema.org/","arrayShape":"cr:arrayShape","citeAs":"cr:citeAs","column":"cr:column","conformsTo":"dct:conformsTo","containedIn":"cr:containedIn","cr":"http://mlcommons.org/croissant/","rai":"http://mlcommons.org/croissant/RAI/","data":{"@id":"cr:data","@type":"@json"},"dataType":{"@id":"cr:dataType","@type":"@vocab"},"dct":"http://purl.org/dc/terms/","description":{"@container":"@language"},"equivalentProperty":"cr:equivalentProperty","examples":{"@id":"cr:examples","@type":"@json"},"extract":"cr:extract","field":"cr:field","fileProperty":"cr:fileProperty","fileObject":"cr:fileObject","fileSet":"cr:fileSet","format":"cr:format","includes":"cr:includes","isArray":"cr:isArray","isLiveDataset":"cr:isLiveDataset","jsonPath":"cr:jsonPath","key":"cr:key","md5":"cr:md5","name":{"@container":"@language"},"parentField":"cr:parentField","path":"cr:path","recordSet":"cr:recordSet","references":"cr:references","regex":"cr:regex","repeated":"cr:repeated","replace":"cr:replace","samplingRate":"cr:samplingRate","sc":"https://schema.org/","separator":"cr:separator","source":"cr:source","subField":"cr:subField","transform":"cr:transform"},"@type":"sc:Dataset","conformsTo":"http://mlcommons.org/croissant/1.1","name":"US law-firm websites and AI crawlers: crawler-file rules, text before JavaScript, snippet limits and Spanish pages, by state and market","description":"What US law-firm websites show AI crawlers, measured from the public files and pages of the websites themselves: which of eight named AI crawlers and controls (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-User, PerplexityBot, Google-Extended, meta-webindexer) each site's crawler file (the plain-text file at its root that tells crawlers what they may visit) names and whose access to the home page it blocks, whether the apex and www host names lead to one site, how much home-page text a reader gets before JavaScript runs, and how their Spanish pages present themselves. It also counts FAQPage and Organization structured data and nosnippet and max-snippet tags on the same pages. Aggregates by state and market only; no firm or website is named or scored.","url":"https://probativegrowth.com/data/law-firm-ai-readiness/","identifier":"https://doi.org/10.5281/zenodo.24015038","sameAs":["https://doi.org/10.5281/zenodo.24015038","https://doi.org/10.5281/zenodo.24015037","https://zenodo.org/records/24015038"],"version":"1.0.0","datePublished":"2026-11-06","dateModified":"2026-11-06","license":"https://creativecommons.org/licenses/by/4.0/","creator":{"@type":"sc:Organization","name":"Probative Growth","url":"https://probativegrowth.com"},"publisher":{"@type":"sc:Organization","name":"Probative Growth","url":"https://probativegrowth.com"},"citeAs":"Probative. (2026). US law-firm websites and AI crawlers: crawler-file rules, text before JavaScript, snippet limits and Spanish pages, by state and market (Version 1.0.0) [Data set]. Zenodo. https://doi.org/10.5281/zenodo.24015038","keywords":["law firm AI crawler rules","law firms blocking GPTBot","law firm website JavaScript AI crawlers","law firm FAQ schema share","law firm Spanish website","law firm websites AI crawlers data"],"inLanguage":"en","isAccessibleForFree":true,"isLiveDataset":false,"temporalCoverage":"2026-11-03/2026-11-04","spatialCoverage":{"@type":"sc:Place","name":"United States"},"distribution":[{"@type":"cr:FileObject","@id":"law-firm-ai-readiness.csv","name":"law-firm-ai-readiness.csv","description":"Every record, one row each (RFC 4180, UTF-8). Columns are described in README.txt.","contentUrl":"https://probativegrowth.com/data/v1/law-firm-ai-readiness/law-firm-ai-readiness.csv","contentSize":"83640 B","encodingFormat":"text/csv","sha256":"f3b7e7f8130ea7bab7ca20790d54c3459fd4b303b2668b7a49dd58aeb9c2ec2a"},{"@type":"cr:FileObject","@id":"law-firm-ai-readiness.json","name":"law-firm-ai-readiness.json","description":"Every record with its full sources, under a meta envelope (credit line, DOI, license).","contentUrl":"https://probativegrowth.com/data/v1/law-firm-ai-readiness/law-firm-ai-readiness.json","contentSize":"194036 B","encodingFormat":"application/json","sha256":"eac09d692faad9eb2a65ec09993597739ab3a73b1035c51b76011d258568536a"},{"@type":"cr:FileObject","@id":"law-firm-ai-readiness-values.csv","name":"law-firm-ai-readiness-values.csv","description":"The columns listed in README.txt as being in law-firm-ai-readiness-values.csv, one row per id, column and value; join it to law-firm-ai-readiness.csv on id.","contentUrl":"https://probativegrowth.com/data/v1/law-firm-ai-readiness/law-firm-ai-readiness-values.csv","contentSize":"2757 B","encodingFormat":"text/csv","sha256":"91edd3395f0dc84fcef239e28ef0309749691cf750a6ad52de049b213d6a02fd"}],"recordSet":[{"@type":"cr:RecordSet","@id":"law-firm-ai-readiness","name":"law-firm-ai-readiness","description":"The records of US law-firm websites and AI crawlers: crawler-file rules, text before JavaScript, snippet limits and Spanish pages, by state and market, one per CSV row.","key":{"@id":"law-firm-ai-readiness/id"},"field":[{"@type":"cr:Field","@id":"law-firm-ai-readiness/id","name":"id","description":"Stable record id (never reused)","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"id"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/jurisdiction","name":"jurisdiction","description":"Two-letter state code, or US; a market row carries its state's code","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"jurisdiction"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/level","name":"level","description":"us, state or metro (a market on this site, by its core counties)","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"level"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/areaName","name":"areaName","description":"The place the row covers","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"areaName"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/status","name":"status","description":"P primary source, S secondary source","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"status"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/text","name":"text","description":"The record in one plain-English sentence","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"text"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/value","name":"value","description":"The headline share: % of websites blocking the home page for at least one of the eight AI crawlers and controls","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"value"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/unit","name":"unit","description":"What value measures","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"unit"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/websites","name":"websites","description":"US law-firm websites read in the group (N); every state and market has at least 100","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"websites"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/robots_read","name":"robots_read","description":"Websites whose crawler file answer was read: the base of the crawler-file shares","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"robots_read"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_robots_file","name":"share_robots_file","description":"% of those with a crawler file holding text","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_robots_file"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_any","name":"share_block_any","description":"% blocking the home page for at least one of the eight crawlers","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_any"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_gptbot","name":"share_block_gptbot","description":"% whose crawler file blocks the home page for GPTBot (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_gptbot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_oai_searchbot","name":"share_block_oai_searchbot","description":"% whose crawler file blocks the home page for OAI-SearchBot (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_oai_searchbot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_chatgpt_user","name":"share_block_chatgpt_user","description":"% whose crawler file blocks the home page for ChatGPT-User (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_chatgpt_user"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_claudebot","name":"share_block_claudebot","description":"% whose crawler file blocks the home page for ClaudeBot (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_claudebot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_claude_user","name":"share_block_claude_user","description":"% whose crawler file blocks the home page for Claude-User (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_claude_user"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_perplexitybot","name":"share_block_perplexitybot","description":"% whose crawler file blocks the home page for PerplexityBot (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_perplexitybot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_google_extended","name":"share_block_google_extended","description":"% whose crawler file blocks the home page for Google-Extended (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_google_extended"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_block_meta_webindexer","name":"share_block_meta_webindexer","description":"% whose crawler file blocks the home page for meta-webindexer (its own group, else the group for all crawlers; longest match wins, an allow wins a tie)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_block_meta_webindexer"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_gptbot","name":"share_named_gptbot","description":"% whose crawler file names GPTBot in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_gptbot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_oai_searchbot","name":"share_named_oai_searchbot","description":"% whose crawler file names OAI-SearchBot in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_oai_searchbot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_chatgpt_user","name":"share_named_chatgpt_user","description":"% whose crawler file names ChatGPT-User in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_chatgpt_user"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_claudebot","name":"share_named_claudebot","description":"% whose crawler file names ClaudeBot in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_claudebot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_claude_user","name":"share_named_claude_user","description":"% whose crawler file names Claude-User in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_claude_user"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_perplexitybot","name":"share_named_perplexitybot","description":"% whose crawler file names PerplexityBot in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_perplexitybot"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_google_extended","name":"share_named_google_extended","description":"% whose crawler file names Google-Extended in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_google_extended"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_named_meta_webindexer","name":"share_named_meta_webindexer","description":"% whose crawler file names meta-webindexer in a User-agent line","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_named_meta_webindexer"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/apex_read","name":"apex_read","description":"Websites with an apex and a www host name asked: the base of the host shares","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"apex_read"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_apex_same","name":"share_apex_same","description":"% where the other host name redirects to the site's host","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_apex_same"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_apex_split","name":"share_apex_split","description":"% where the other host name answers its own page (200, no redirect)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_apex_split"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_apex_other","name":"share_apex_other","description":"% where the other host name gives any other answer (an error status, or a redirect elsewhere)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_apex_other"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_apex_none","name":"share_apex_none","description":"% where the other host name gives no answer (no DNS, no connection or a certificate that fails)","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_apex_none"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_static_under_200","name":"share_static_under_200","description":"% of websites whose home page shows under 200 characters of visible text in the raw HTML, before JavaScript runs","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_static_under_200"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/median_static_chars","name":"median_static_chars","description":"Median characters of visible text on the home page before JavaScript runs","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"median_static_chars"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_faq_schema","name":"share_faq_schema","description":"% with FAQPage structured data on the pages read","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_faq_schema"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_org_schema","name":"share_org_schema","description":"% with Organization, LocalBusiness, LegalService, Attorney or a close type on the pages read","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_org_schema"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_snippet_limit","name":"share_snippet_limit","description":"% with a search meta tag saying nosnippet or max-snippet of 0 or more on the pages read","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_snippet_limit"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_snippet_unlimited","name":"share_snippet_unlimited","description":"% with max-snippet:-1 (no limit) on the pages read","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_snippet_unlimited"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/spanish_read","name":"spanish_read","description":"Websites whose Spanish status could be classed from the pages read: the base of the Spanish shares","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"spanish_read"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_spanish_own","name":"share_spanish_own","description":"% with a page read in Spanish whose HTML names no translation service","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_spanish_own"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_spanish_machine","name":"share_spanish_machine","description":"% with a page read in Spanish whose HTML names a translation service","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_spanish_machine"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_spanish_widget","name":"share_spanish_widget","description":"% with a translation widget on the home page and no Spanish page","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_spanish_widget"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/share_spanish_none","name":"share_spanish_none","description":"% with no Spanish page, link or translation widget","dataType":"sc:Float","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"share_spanish_none"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/static_threshold_chars","name":"static_threshold_chars","description":"The threshold, in characters of visible text, behind share_static_under_200","dataType":"sc:Integer","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"static_threshold_chars"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/period","name":"period","description":"First and last read date","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"period"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/lastChecked","name":"lastChecked","description":"The day it was last confirmed (YYYY-MM-DD)","dataType":"sc:Date","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"lastChecked"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/sourceLabels","name":"sourceLabels","description":"Sources, separated by \" | \"","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"sourceLabels"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/sourceUrls","name":"sourceUrls","description":"Source URLs, in the same order","dataType":"sc:URL","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"sourceUrls"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/confidence","name":"confidence","description":"high, medium or low: computed from status, a second check, age against the refresh cadence and sampling error (rule: /data/#confidence)","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"confidence"}}},{"@type":"cr:Field","@id":"law-firm-ai-readiness/confidenceBasis","name":"confidenceBasis","description":"The reasons for the confidence level, in a short phrase","dataType":"sc:Text","source":{"fileObject":{"@id":"law-firm-ai-readiness.csv"},"extract":{"column":"confidenceBasis"}}}]}]}