{"openapi":"3.1.0","info":{"title":"LangSmith Evaluation Config (BYOK) API","version":"0.1.0","description":"LangSmith Evaluation Config (BYOK) lets an AI assistant read your own LangSmith Evaluation Config data through the LangSmith Evaluation Config API with a key you supply (LangSmith API key (personal access token or service key)). Read-only: 6 tools (list_evaluators, get_evaluator, get_evaluator_spend, list_feedback_configs, list_feedback_formulas, list_run_rules). The BYOK key travels with each...\n\nEvery operation is read-only or compute-only. The same tools are available over MCP (streamable HTTP) at https://byok-langsmith-evaluation-config.connectorbay.com/mcp.","contact":{"name":"Unify Dynamics LLC","email":"agent@connectorbay.com","url":"https://connectorbay.com/byok-langsmith-evaluation-config/docs"},"termsOfService":"https://connectorbay.com/byok-langsmith-evaluation-config/terms"},"servers":[{"url":"https://byok-langsmith-evaluation-config.connectorbay.com"}],"externalDocs":{"url":"https://connectorbay.com/byok-langsmith-evaluation-config/docs"},"security":[{"upstreamKey":[]}],"components":{"securitySchemes":{"bearer":{"type":"http","scheme":"bearer","description":"Optional. A Pro API key raises rate limits; no credentials are required."},"upstreamKey":{"type":"apiKey","in":"header","name":"X-Upstream-Key","description":"Your own key for the service this connector wraps (bring your own key). Used for that one call only; never stored, logged or cached."}},"schemas":{"Error":{"type":"object","properties":{"ok":{"const":false},"error":{"type":"object"}}}}},"paths":{"/v1/tools/list_evaluators":{"post":{"operationId":"list_evaluators","summary":"List the evaluators of the workspace (LLM-as-judge and code evaluators) with feedback keys, managed flag, dates and thei","description":"List the evaluators of the workspace (LLM-as-judge and code evaluators) with feedback keys, managed flag, dates and their definition. Use for \"Which evaluators are defined in my LangSmith workspace\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"name_contains":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Only evaluators whose name contains this text.","title":"Name Contains"},"type":{"anyOf":[{"enum":["llm","code"],"type":"string"},{"type":"null"}],"default":null,"description":"Only evaluators of this type: llm or code.","title":"Type"},"feedback_key":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Only evaluators that write this feedback key.","title":"Feedback Key"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"How many evaluators to return (1 to 100).","title":"Limit"},"offset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":0,"description":"Number of evaluators to skip, for paging.","title":"Offset"}},"title":"list_evaluatorsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluator":{"post":{"operationId":"get_evaluator","summary":"Get one evaluator: type, feedback keys, LLM judge settings (prompt repository, commit or tag, few-shot count) or code se","description":"Get one evaluator: type, feedback keys, LLM judge settings (prompt repository, commit or tag, few-shot count) or code settings, and the run rules that use it. Use for \"Show the LangSmith evaluator 3f1c2a9e-8b7d-4c1e-9a55-0d6f2b7a1c10\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"evaluator_id":{"description":"Evaluator id (a UUID), as returned by list_evaluators.","maxLength":200,"title":"Evaluator Id","type":"string"}},"required":["evaluator_id"],"title":"get_evaluatorArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluator_spend":{"post":{"operationId":"get_evaluator_spend","summary":"Get evaluator spend in USD and number of traces evaluated for a period, grouped by evaluator and rule, with the previous","description":"Get evaluator spend in USD and number of traces evaluated for a period, grouped by evaluator and rule, with the previous period for comparison. Use for \"How much did my LangSmith evaluators cost since the start of this month\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"period_start":{"description":"Start of the period as an ISO 8601 time, for example 2026-10-01T00:00:00Z.","maxLength":40,"title":"Period Start","type":"string"},"evaluator_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only this evaluator (UUID).","title":"Evaluator Id"},"session_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only this tracing project (UUID).","title":"Session Id"},"dataset_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only this dataset (UUID).","title":"Dataset Id"}},"required":["period_start"],"title":"get_evaluator_spendArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_feedback_configs":{"post":{"operationId":"list_feedback_configs","summary":"List the feedback key schemas (continuous range, categorical categories or freeform) with whether a lower score is bette","description":"List the feedback key schemas (continuous range, categorical categories or freeform) with whether a lower score is better and whether the schema is enforced. Use for \"Which feedback keys and score ranges are configured in LangSmith\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"name_contains":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Only feedback keys whose name contains this text.","title":"Name Contains"},"is_enforced":{"anyOf":[{"type":"boolean"},{"type":"null"}],"default":null,"description":"True for only enforced schemas, false for only unenforced ones.","title":"Is Enforced"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"How many configs to return (1 to 100).","title":"Limit"},"offset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":0,"description":"Number of configs to skip, for paging.","title":"Offset"}},"title":"list_feedback_configsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_feedback_formulas":{"post":{"operationId":"list_feedback_formulas","summary":"List the composite feedback formulas (aggregation type and weighted parts) defined for datasets or projects. Use for \"Wh","description":"List the composite feedback formulas (aggregation type and weighted parts) defined for datasets or projects. Use for \"Which composite feedback formulas exist in LangSmith\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"dataset_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only formulas of this dataset (UUID).","title":"Dataset Id"},"session_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only formulas of this tracing project (UUID).","title":"Session Id"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many formulas to return (1 to 100).","title":"Limit"},"offset":{"anyOf":[{"minimum":0,"type":"integer"},{"type":"null"}],"default":0,"description":"Number of formulas to skip, for paging.","title":"Offset"}},"title":"list_feedback_formulasArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_run_rules":{"post":{"operationId":"list_run_rules","summary":"List the online evaluation and automation rules (sampling rate, filters, evaluator, target dataset or annotation queue, ","description":"List the online evaluation and automation rules (sampling rate, filters, evaluator, target dataset or annotation queue, spend limit, trace count and spend). Alert and webhook settings are not returned. Use for \"Which automatic evaluation rules run in my LangSmith workspace\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"type":{"anyOf":[{"enum":["session","dataset"],"type":"string"},{"type":"null"}],"default":null,"description":"Only rules attached to a project (session) or a dataset.","title":"Type"},"name_contains":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Only rules whose name contains this text.","title":"Name Contains"},"session_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only rules of this tracing project (UUID).","title":"Session Id"},"dataset_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only rules of this dataset (UUID).","title":"Dataset Id"},"evaluator_id":{"anyOf":[{"maxLength":60,"type":"string"},{"type":"null"}],"default":null,"description":"Only rules that use this evaluator (UUID).","title":"Evaluator Id"}},"title":"list_run_rulesArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}}}}