robots.txt
See which AI crawlers other sites block →# Mostly Right — robots.txt
# We sell data to AI agents. Agents are welcome here.
# If you are a bot and this page returns 403, our CDN is misconfigured. Try again.
User-agent: *
Allow: /
Disallow: /page/
Disallow: /root-landing/
# Content Signals (contentsignals.org, IETF draft-romm-aipref-contentsignals).
# Our business is selling data to agents. Anything an agent can reach here is
# fair game for training, search indexing, and inference-time retrieval.
Content-Signal: search=yes, ai-train=yes, ai-input=yes
# Explicit allowlist for AI crawlers. Some CDNs block these by default.
# We override that at the edge. If you are one of these agents, you are welcome.
User-agent: GPTBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: ChatGPT-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: OAI-SearchBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: ClaudeBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Claude-Web
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Claude-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Claude-SearchBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: anthropic-ai
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: PerplexityBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Perplexity-User
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: CCBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Google-Extended
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Googlebot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Bingbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: DuckDuckBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Applebot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Applebot-Extended
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Amazonbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Bytespider
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: FacebookBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: cohere-ai
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: cohere-training-data-crawler
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Diffbot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: YouBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: Timpibot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: ImagesiftBot
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
User-agent: omgili
Allow: /
Disallow: /page/
Disallow: /root-landing/
Content-Signal: search=yes, ai-train=yes, ai-input=yes
# LLM-friendly discovery files. Fetch these first if you want the site in one shot.
# /llms.txt — short index with canonical URLs
# /llms-full.txt — the same index plus every public dataset, its
# tables, and the API paths that read them
# /datasets/{slug}/llms.txt — one dataset as Markdown
# /api/v2/public/openapi.json — the complete HTTP contract
#
# If you are an AI client with tools rather than a crawler, there is an MCP
# server at /api/mcp. Searching, schemas and sample rows need no account.
# Past the first 500 rows of a table, reading needs an account and is metered.
# /.well-known/mcp.json — server card
# /.well-known/oauth-protected-resource/api/mcp — protected resource metadata
Sitemap: https://mostlyright.md/sitemap-index.xml
# The app's own sitemap: published dataset and publisher pages, generated from
# the public share index with a real lastmod (dashboard/app/sitemap.ts).
# Deliberately a SECOND line rather than a replacement — this file is the one
# crawler policy for the whole origin and is extended, never regenerated.
Sitemap: https://mostlyright.md/sitemap.xml
llms.txt
Compare with other sites' llms.txt files →# Mostly Right
Mostly Right is where you publish and discover datasets with AI agents. You describe a dataset. Agents find the sources and configure the ingest engine. Mostly Right runs the build, keeps the dataset updated, and serves each version as Apache Parquet or as JSON through the API and MCP.
The catalog covers AI work: specialized and frontier models, data ETL, agentic engineering, and the records those workflows use. Examples include LLM benchmark scores by model since 2019, Y Combinator company status and Form D filings, Czech companies with KYB risk flags, EU financial firm licences and regulator warnings, and movies and TV shows with plots, cast and ratings. Every dataset carries a documented column schema, column profiles, an explicit licence and a named publisher.
There are three tiers. Reading metadata, schemas and sample rows needs no account, and neither does a bounded query of up to 100 rows a page (POST /api/public/tables/{table_id}/query). Querying a whole table — up to 10000 rows a page, paged to the end — and downloading a full snapshot need a free account and an API key from Settings → Access; the first request connects the dataset to your workspace. Building a new dataset — writing a recipe, running it and publishing the result — needs a paid subscription on that workspace.
- Dataset directory: /datasets
- Every public dataset and table, in one file: /llms-full.txt
- One dataset as Markdown: /datasets/{slug}/llms.txt
- Documentation index (Markdown): https://mostlyright.md/docs/llms.txt
- Full documentation (Markdown): https://mostlyright.md/docs/llms-full.txt
- Search API: https://mostlyright.md/docs/reference/search/
- JavaScript SDK and native fetch: https://mostlyright.md/docs/guides/javascript/
- Public Dataset API: https://mostlyright.md/docs/reference/public-dataset-api/
- Use a public dataset: https://mostlyright.md/docs/guides/use-public-datasets/
- Quickstart for agents: https://mostlyright.md/docs/guides/for-agents/
- Connect an AI tool with MCP: https://mostlyright.md/docs/guides/mcp/
- MCP server: /api/mcp (server card: /.well-known/mcp.json)
- Agent skills: /.well-known/agent-skills/index.json
- How to build a dataset: /.well-known/agent-skills/mostlyright-build/SKILL.md
- OpenAPI: /api/v2/public/openapi.json
Use the traditional HTTP API for application code. GET /api/v2/public/datasets searches the catalogue and needs no key. GET /api/v2/public/tables/{tableId}/schema and GET /api/v2/public/tables/{tableId}/preview need no key. GET /api/v2/public/tables/{tableId}/current returns application/vnd.apache.parquet. POST /api/v2/public/tables/{tableId}/query returns bounded JSON. The last two take a workspace key as x-api-key: mr_use_…. Do not assume arbitrary SQL is available.
Agents can use the MCP server at /api/mcp instead: Streamable HTTP, protocol 2025-11-25 (2025-06-18, 2025-03-26 and 2024-11-05 also negotiate), stateless, no session id. Server card: /.well-known/mcp.json. search, fetch, search_datasets, get_dataset, list_tables, get_table_schema, sample_rows, get_download_instructions and get_access_instructions answer with NO credential; search and fetch are the two-tool connector contract, where search returns {id, title, url} per dataset and fetch returns that dataset as Markdown. query_table, search_table, catalog_search and list_connected_datasets take a read-only mr_use_ workspace key as Authorization: Bearer, or an OAuth connection. connect_dataset changes the workspace and needs the OAuth connection. catalog_search is a different question from the rest: it searches a sealed snapshot of public data SOURCES — one provider, Data.gov, partially swept — rather than the datasets published here.
The same server also builds datasets, on a paid workspace, over an OAuth connection carrying the datasets:build scope: create_dataset, update_dataset, configure_dataset_layout, configure_table_view, generate_cover, set_dataset_visibility, archive_dataset, list_my_datasets, get_my_dataset, list_dataset_events, list_recipes, get_recipe, register_recipe, explain_recipe, forecast_recipe, test_recipe, propose_revision, replay_run, start_run, confirm_run, approve_full_run, cancel_run, get_run, run_events, list_runs, compare_runs, query_run, run_artifacts, get_artifact_download, write_note, list_notes, list_table_parts, verify_run, promote_table, reschedule_table, demote_table, refresh_table, resync_table, archive_table, rename_table, get_table, diagnose_table, list_source_credentials, list_models, suggest_enrichments, normalize_reader_options, start_source_inspection, get_source_inspection, probe_source and get_source_probe, get_run_attempts, get_recipe_readiness, inspect_table_search, configure_table_search, build_table_search, activate_table_search, get_search_job, cancel_search_job, sample_table, list_column_values, preview_table_search, get_search_evaluation, add_search_cases, register_stream_document, create_stream_registry, start_stream, get_stream, control_stream_subscription. An mr_use_ key cannot build. Source credentials are pasted by a person at /dashboard/settings/secrets and referenced from a recipe by name; no secret passes through MCP. The loop is written out at /.well-known/agent-skills/mostlyright-build/SKILL.md and served as the mostlyright://build-guide resource.
Results should cite the dataset, table, immutable version, content digest, and query time.
## Build a dataset
Building runs from the command line as well as over MCP. Install it with pip install mostlyright-data, which puts the mr-data command on the path, then run mr-data login once per machine. The first mr-data command of any kind also places the mr-data-build agent skill at ~/.claude/skills/mr-data-build, or under ~/.codex/skills for Codex, and leaves an edited copy alone; MOSTLYRIGHT_SKILL_AUTOINSTALL=0 switches that off. That skill drives the CLI. The MCP build skill is a separate document: /.well-known/agent-skills/index.json lists it, and it is served at /.well-known/agent-skills/mostlyright-build/SKILL.md and as the mostlyright://build-guide resource.
## Documentation
Start here.
- https://mostlyright.md/docs/ — what Mostly Right builds, and which page to read next.
- https://mostlyright.md/docs/start/install/ — install mr-data and sign in.
- https://mostlyright.md/docs/start/first-dataset/ — build one dataset end to end.
- https://mostlyright.md/docs/start/how-it-works/ — the lifecycle: dataset, recipe, run, sealed table version, refresh.
- https://mostlyright.md/docs/guides/for-agents/ — the curl ladder from a catalogue search to a cited result.
Build a dataset.
- https://mostlyright.md/docs/build/probe-sources/ — ask one source one question before writing a recipe.
- https://mostlyright.md/docs/build/write-a-recipe/ — the recipe document, with a complete example.
- https://mostlyright.md/docs/build/run-and-inspect/ — run under ceilings, follow the events, read the rows.
- https://mostlyright.md/docs/build/publish-and-refresh/ — what a succeeded run publishes, and how refresh cadence is recorded.
- https://mostlyright.md/docs/build/credentials/ — store a source credential by name and reference it from a recipe.
- https://mostlyright.md/docs/build/live-streams/ — record a public wss venue and build a table from the recording.
- https://mostlyright.md/docs/build/collections/ — one source that fans out over many pages.
- https://mostlyright.md/docs/build/documents/ — documents and files as rows.
- https://mostlyright.md/docs/build/troubleshooting/ — read a failed run and fix what it names.
Recipe examples, each one a complete document.
- https://mostlyright.md/docs/recipes/ — the index.
- https://mostlyright.md/docs/recipes/city-temperatures/ — daily city temperatures.
- https://mostlyright.md/docs/recipes/csv-snapshot/ — one CSV file to one table.
- https://mostlyright.md/docs/recipes/json-api/ — a JSON API with a records pointer and pagination.
- https://mostlyright.md/docs/recipes/html-collection/ — many HTML pages to one table.
- https://mostlyright.md/docs/recipes/document-extraction/ — a PDF or other document projected into rows.
- https://mostlyright.md/docs/recipes/weather-grib/ — weather GRIB2 through the scientific reader.
- https://mostlyright.md/docs/recipes/websocket-stream/ — a WebSocket venue to a stream table.
- https://mostlyright.md/docs/recipes/multi-source-join/ — two sources joined, with checks.
- https://mostlyright.md/docs/recipes/benchmark-scores-join/ — LLM benchmark scores joined to model release dates.
- https://mostlyright.md/docs/recipes/forecast-vs-observation/ — a weather forecast joined to the observation it predicted.
- https://mostlyright.md/docs/recipes/snapshot-window/ — a listing replaced as one snapshot on each refresh.
- https://mostlyright.md/docs/recipes/many-station-weather/ — twenty station feeds unioned into one hourly table.
- https://mostlyright.md/docs/recipes/stream-to-bars/ — recorded trade frames aggregated into one-minute bars.
Reference.
- https://mostlyright.md/docs/reference/cli/ — every mr-data command, one line each.
- https://mostlyright.md/docs/reference/cli/auth/ — login, auth, whoami, open, clarify.
- https://mostlyright.md/docs/reference/cli/dataset/ — dataset and cover.
- https://mostlyright.md/docs/reference/cli/recipe/ — recipe, probe, catalog, connections, keys.
- https://mostlyright.md/docs/reference/cli/run/ — run, runs, status, watch, checks, receipt, note, diff, verify.
- https://mostlyright.md/docs/reference/cli/read/ — peek, query, download, parts.
- https://mostlyright.md/docs/reference/cli/table/ — table, promote, reschedule, pin, unpin, demote.
- https://mostlyright.md/docs/reference/cli/stream/ — stream venues and recordings.
- https://mostlyright.md/docs/reference/recipe/ — the recipe document, field by field.
- https://mostlyright.md/docs/reference/sources/ — source kinds and connectors.
- https://mostlyright.md/docs/reference/readers/ — readers.
- https://mostlyright.md/docs/reference/transforms/ — transforms.
- https://mostlyright.md/docs/reference/checks/ — checks.
- https://mostlyright.md/docs/reference/units/ — units and vocabulary.
- https://mostlyright.md/docs/reference/run-states-and-errors/ — run states, events, refusal and worker codes.
- https://mostlyright.md/docs/reference/limits/ — limits and ceilings.
Reading a dataset somebody else published is covered by the three pages named at the top of this file: the Public Dataset API, Use a public dataset, and Connect an AI tool with MCP.