From 2393bef34897cb93061f9b9b7092991cbf3467f5 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 27 Jan 2025 08:38:05 -0800 Subject: [PATCH 1/6] wip --- .../{tracing => }/access_current_span.mdx | 0 .../{tracing => }/add_metadata_tags.mdx | 0 .../{tracing => }/annotate_code.mdx | 0 .../calculate_token_based_costs.mdx | 0 .../{tracing => }/compare_traces.mdx | 0 .../{monitoring => }/dashboards.mdx | 0 .../{tracing => }/data_export.mdx | 0 .../{tracing => }/distributed_tracing.mdx | 0 .../{tracing => }/export_traces.mdx | 0 .../filter_traces_in_application.mdx | 0 docs/observability/how_to_guides/index.md | 76 +++++++++--------- .../{tracing => }/log_llm_trace.mdx | 0 .../{tracing => }/log_multimodal_traces.mdx | 0 .../{tracing => }/log_retriever_trace.mdx | 0 .../{tracing => }/log_traces_to_project.mdx | 0 .../{tracing => }/mask_inputs_outputs.mdx | 0 .../{tracing => }/nest_traces.mdx | 0 .../{monitoring => }/online_evaluations.mdx | 0 .../{tracing => }/output_detailed_logs.mdx | 0 .../how_to_guides/{monitoring => }/rules.mdx | 0 .../{tracing => }/sample_traces.mdx | 0 .../{tracing => }/serverless_environments.mdx | 0 .../{tracing => }/share_trace.mdx | 0 .../static/add_automation_rule.png | Bin .../{monitoring => }/static/add_chart.png | Bin .../{monitoring => }/static/add_dashboard.png | Bin .../{monitoring => }/static/ai_query.png | Bin .../static/annotate_code_trace.gif | Bin .../{monitoring => }/static/automations.png | Bin .../static/aws-comprehend-anonymized.png | Bin .../static/aws-comprehend-not-anonymized.png | Bin .../{monitoring => }/static/bar_chart.png | Bin .../{monitoring => }/static/chart_filters.png | Bin .../chart_filters_for_node_decision.png | Bin .../{tracing => }/static/chat_model.png | Bin .../{monitoring => }/static/child_runs.png | Bin .../static/compare_metrics.png | Bin .../static/compare_traces/compare_button.png | Bin .../static/compare_traces/compare_trace.png | Bin .../static/compare_traces/select_trace.png | Bin .../{monitoring => }/static/convo.png | Bin .../{monitoring => }/static/convo_tab.png | Bin .../{monitoring => }/static/copy_filter.png | Bin .../static/dashboard_table.png | Bin .../static/decision_at_node.png | Bin .../{monitoring => }/static/drill_monitor.png | Bin .../static/evaluator_secrets.png | Bin .../static/expanded_chart.png | Bin .../{monitoring => }/static/filter.png | Bin .../static/filter_full_text.png | Bin .../static/filter_runs_in_trace_view.png | Bin .../static/filter_shortcuts.png | Bin .../static/give_rule_name.png | Bin .../{tracing => }/static/hello_llm.png | Bin .../static/hide_inputs_outputs.png | Bin .../{tracing => }/static/langchain_trace.png | Bin .../static/langgraph_with_langchain_trace.png | Bin .../langgraph_without_langchain_trace.png | Bin .../static/mapping_variables.png | Bin .../static/modal_langsmith_secret.png | Bin .../{tracing => }/static/model_costs.png | Bin .../{tracing => }/static/model_price_map.png | Bin .../static/more_actions_bar.png | Bin .../{tracing => }/static/multimodal.png | Bin .../static/multiple_data_series.png | Bin .../static/new_price_map_entry.png | Bin .../static/online-eval-custom-code.png | Bin .../static/presidio-anonymized.png | Bin .../static/presidio-not-anonymized.png | Bin .../static/prompt_preview.png | Bin .../{monitoring => }/static/raw_query.png | Bin .../{tracing => }/static/regex-anonymized.png | Bin .../static/regex-not-anonymized.png | Bin .../{tracing => }/static/retriever_trace.png | Bin .../{monitoring => }/static/rules_filter.png | Bin .../{monitoring => }/static/rules_logs.gif | Bin .../{monitoring => }/static/rules_logs.png | Bin .../static/rules_past_runs.png | Bin .../static/rules_past_runs_logs.png | Bin .../{monitoring => }/static/save_a_filter.png | Bin .../static/search_kv_filter_shortcut.png | Bin .../static/search_kv_input.png | Bin .../static/search_kv_output.png | Bin .../static/selecting_a_filter.png | Bin .../{tracing => }/static/share_trace.png | Bin .../static/subsets_monitor.png | Bin .../{monitoring => }/static/trace_filter.png | Bin .../static/trace_tree_manual_tracing.png | Bin .../static/trace_tree_python_interop.png | Bin .../static/trace_with_attachments.png | Bin .../{tracing => }/static/unshare_trace.png | Bin .../static/unshare_trace_list.png | Bin .../{monitoring => }/static/webhook.png | Bin .../static/webhook_headers.png | Bin .../{monitoring => }/threads.mdx | 0 .../{tracing => }/toggle_tracing.mdx | 0 .../trace_generator_functions.mdx | 0 .../{tracing => }/trace_with_api.mdx | 0 .../{tracing => }/trace_with_instructor.mdx | 0 .../{tracing => }/trace_with_langchain.mdx | 0 .../{tracing => }/trace_with_langgraph.mdx | 0 .../trace_with_opentelemetry.mdx | 0 .../trace_with_vercel_ai_sdk.mdx | 0 .../{tracing => }/trace_without_env_vars.mdx | 0 .../upload_files_with_traces.mdx | 0 .../use_monitoring_charts.mdx | 0 .../{monitoring => }/webhooks.mdx | 0 vercel.json | 8 ++ 108 files changed, 46 insertions(+), 38 deletions(-) rename docs/observability/how_to_guides/{tracing => }/access_current_span.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/add_metadata_tags.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/annotate_code.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/calculate_token_based_costs.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/compare_traces.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/dashboards.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/data_export.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/distributed_tracing.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/export_traces.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/filter_traces_in_application.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/log_llm_trace.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/log_multimodal_traces.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/log_retriever_trace.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/log_traces_to_project.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/mask_inputs_outputs.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/nest_traces.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/online_evaluations.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/output_detailed_logs.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/rules.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/sample_traces.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/serverless_environments.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/share_trace.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/static/add_automation_rule.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/add_chart.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/add_dashboard.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/ai_query.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/annotate_code_trace.gif (100%) rename docs/observability/how_to_guides/{monitoring => }/static/automations.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/aws-comprehend-anonymized.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/aws-comprehend-not-anonymized.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/bar_chart.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/chart_filters.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/chart_filters_for_node_decision.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/chat_model.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/child_runs.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/compare_metrics.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/compare_traces/compare_button.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/compare_traces/compare_trace.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/compare_traces/select_trace.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/convo.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/convo_tab.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/copy_filter.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/dashboard_table.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/decision_at_node.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/drill_monitor.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/evaluator_secrets.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/expanded_chart.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/filter.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/filter_full_text.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/filter_runs_in_trace_view.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/filter_shortcuts.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/give_rule_name.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/hello_llm.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/hide_inputs_outputs.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/langchain_trace.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/langgraph_with_langchain_trace.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/langgraph_without_langchain_trace.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/mapping_variables.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/modal_langsmith_secret.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/model_costs.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/model_price_map.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/more_actions_bar.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/multimodal.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/multiple_data_series.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/new_price_map_entry.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/online-eval-custom-code.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/presidio-anonymized.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/presidio-not-anonymized.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/prompt_preview.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/raw_query.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/regex-anonymized.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/regex-not-anonymized.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/retriever_trace.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/rules_filter.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/rules_logs.gif (100%) rename docs/observability/how_to_guides/{monitoring => }/static/rules_logs.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/rules_past_runs.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/rules_past_runs_logs.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/save_a_filter.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/search_kv_filter_shortcut.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/search_kv_input.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/search_kv_output.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/selecting_a_filter.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/share_trace.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/subsets_monitor.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/trace_filter.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/trace_tree_manual_tracing.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/trace_tree_python_interop.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/trace_with_attachments.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/unshare_trace.png (100%) rename docs/observability/how_to_guides/{tracing => }/static/unshare_trace_list.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/webhook.png (100%) rename docs/observability/how_to_guides/{monitoring => }/static/webhook_headers.png (100%) rename docs/observability/how_to_guides/{monitoring => }/threads.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/toggle_tracing.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_generator_functions.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_api.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_instructor.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_langchain.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_langgraph.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_opentelemetry.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_with_vercel_ai_sdk.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/trace_without_env_vars.mdx (100%) rename docs/observability/how_to_guides/{tracing => }/upload_files_with_traces.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/use_monitoring_charts.mdx (100%) rename docs/observability/how_to_guides/{monitoring => }/webhooks.mdx (100%) diff --git a/docs/observability/how_to_guides/tracing/access_current_span.mdx b/docs/observability/how_to_guides/access_current_span.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/access_current_span.mdx rename to docs/observability/how_to_guides/access_current_span.mdx diff --git a/docs/observability/how_to_guides/tracing/add_metadata_tags.mdx b/docs/observability/how_to_guides/add_metadata_tags.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/add_metadata_tags.mdx rename to docs/observability/how_to_guides/add_metadata_tags.mdx diff --git a/docs/observability/how_to_guides/tracing/annotate_code.mdx b/docs/observability/how_to_guides/annotate_code.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/annotate_code.mdx rename to docs/observability/how_to_guides/annotate_code.mdx diff --git a/docs/observability/how_to_guides/tracing/calculate_token_based_costs.mdx b/docs/observability/how_to_guides/calculate_token_based_costs.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/calculate_token_based_costs.mdx rename to docs/observability/how_to_guides/calculate_token_based_costs.mdx diff --git a/docs/observability/how_to_guides/tracing/compare_traces.mdx b/docs/observability/how_to_guides/compare_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/compare_traces.mdx rename to docs/observability/how_to_guides/compare_traces.mdx diff --git a/docs/observability/how_to_guides/monitoring/dashboards.mdx b/docs/observability/how_to_guides/dashboards.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/dashboards.mdx rename to docs/observability/how_to_guides/dashboards.mdx diff --git a/docs/observability/how_to_guides/tracing/data_export.mdx b/docs/observability/how_to_guides/data_export.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/data_export.mdx rename to docs/observability/how_to_guides/data_export.mdx diff --git a/docs/observability/how_to_guides/tracing/distributed_tracing.mdx b/docs/observability/how_to_guides/distributed_tracing.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/distributed_tracing.mdx rename to docs/observability/how_to_guides/distributed_tracing.mdx diff --git a/docs/observability/how_to_guides/tracing/export_traces.mdx b/docs/observability/how_to_guides/export_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/export_traces.mdx rename to docs/observability/how_to_guides/export_traces.mdx diff --git a/docs/observability/how_to_guides/monitoring/filter_traces_in_application.mdx b/docs/observability/how_to_guides/filter_traces_in_application.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/filter_traces_in_application.mdx rename to docs/observability/how_to_guides/filter_traces_in_application.mdx diff --git a/docs/observability/how_to_guides/index.md b/docs/observability/how_to_guides/index.md index 29deb0b3..ce635c2b 100644 --- a/docs/observability/how_to_guides/index.md +++ b/docs/observability/how_to_guides/index.md @@ -8,62 +8,62 @@ Set up LangSmith tracing to get visibility into your production applications. ### Basic configuration -- [Set your tracing project](./how_to_guides/tracing/log_traces_to_project) -- [Enable or disable tracing](./how_to_guides/tracing/toggle_tracing) -- [Trace any Python or JS Code](./how_to_guides/tracing/annotate_code) -- [Trace using the LangSmith REST API](./how_to_guides/tracing/trace_with_api) -- [Trace without environment variables](./how_to_guides/tracing/trace_without_env_vars) +- [Set your tracing project](./how_to_guides/log_traces_to_project) +- [Enable or disable tracing](./how_to_guides/toggle_tracing) +- [Trace any Python or JS Code](./how_to_guides/annotate_code) +- [Trace using the LangSmith REST API](./how_to_guides/trace_with_api) +- [Trace without environment variables](./how_to_guides/trace_without_env_vars) ### Trace natively supported libraries -- [Trace with `LangChain`](./how_to_guides/tracing/trace_with_langchain) -- [Trace with `LangGraph`](./how_to_guides/tracing/trace_with_langgraph) -- [Trace the OpenAI API client](./how_to_guides/tracing/annotate_code#wrap-the-openai-client) -- [Trace with `Instructor` (Python only)](./how_to_guides/tracing/trace_with_instructor) -- [Trace with the Vercel `AI SDK` (JS only)](./how_to_guides/tracing/trace_with_vercel_ai_sdk) -- [Trace with OpenTelemetry](./how_to_guides/tracing/trace_with_opentelemetry) +- [Trace with `LangChain`](./how_to_guides/trace_with_langchain) +- [Trace with `LangGraph`](./how_to_guides/trace_with_langgraph) +- [Trace the OpenAI API client](./how_to_guides/annotate_code#wrap-the-openai-client) +- [Trace with `Instructor` (Python only)](./how_to_guides/trace_with_instructor) +- [Trace with the Vercel `AI SDK` (JS only)](./how_to_guides/trace_with_vercel_ai_sdk) +- [Trace with OpenTelemetry](./how_to_guides/trace_with_opentelemetry) ### Advanced configuration -- [Configure threads](./how_to_guides/monitoring/threads) -- [Set a sampling rate for traces](./how_to_guides/tracing/sample_traces) -- [Add metadata and tags to traces](./how_to_guides/tracing/add_metadata_tags) -- [Implement distributed tracing](./how_to_guides/tracing/distributed_tracing) -- [Access the current span within a traced function](./how_to_guides/tracing/access_current_span) -- [Log multimodal traces](./how_to_guides/tracing/log_multimodal_traces) -- [Log retriever traces](./how_to_guides/tracing/log_retriever_trace) -- [Log custom LLM traces / provide custom token counts](./how_to_guides/tracing/log_llm_trace) -- [Prevent logging of sensitive data in traces](./how_to_guides/tracing/mask_inputs_outputs) -- [Trace generator functions](./how_to_guides/tracing/trace_generator_functions) -- [Calculate token-based costs for traces](./how_to_guides/tracing/calculate_token_based_costs) -- [Trace JS functions in serverless environments](./how_to_guides/tracing/serverless_environments) -- [Troubleshoot trace testing](./how_to_guides/tracing/nest_traces) -- [Upload files with traces](./how_to_guides/tracing/upload_files_with_traces) -- [Print out logs from the LangSmith SDK (Python Only)](./how_to_guides/tracing/output_detailed_logs) +- [Configure threads](./how_to_guides/threads) +- [Set a sampling rate for traces](./how_to_guides/sample_traces) +- [Add metadata and tags to traces](./how_to_guides/add_metadata_tags) +- [Implement distributed tracing](./how_to_guides/distributed_tracing) +- [Access the current span within a traced function](./how_to_guides/access_current_span) +- [Log multimodal traces](./how_to_guides/log_multimodal_traces) +- [Log retriever traces](./how_to_guides/log_retriever_trace) +- [Log custom LLM traces / provide custom token counts](./how_to_guides/log_llm_trace) +- [Prevent logging of sensitive data in traces](./how_to_guides/mask_inputs_outputs) +- [Trace generator functions](./how_to_guides/trace_generator_functions) +- [Calculate token-based costs for traces](./how_to_guides/calculate_token_based_costs) +- [Trace JS functions in serverless environments](./how_to_guides/serverless_environments) +- [Troubleshoot trace testing](./how_to_guides/nest_traces) +- [Upload files with traces](./how_to_guides/upload_files_with_traces) +- [Print out logs from the LangSmith SDK (Python Only)](./how_to_guides/output_detailed_logs) ## Tracing projects UI & API View and interact with your traces to debug your applications. -- [Filter traces in a project](./how_to_guides/monitoring/filter_traces_in_application) -- [Save a filter for your project](./how_to_guides/monitoring/filter_traces_in_application#saved-filters) -- [Export traces using the SDK (low volume)](./how_to_guides/tracing/export_traces) -- [Bulk exporting traces (high volume)](./how_to_guides/tracing/data_export) -- [Share or unshare a trace publicly](./how_to_guides/tracing/share_trace) -- [Compare traces](./how_to_guides/tracing/compare_traces) -- [View threads](./how_to_guides/monitoring/threads#view-threads) +- [Filter traces in a project](./how_to_guides/filter_traces_in_application) +- [Save a filter for your project](./how_to_guides/filter_traces_in_application#saved-filters) +- [Export traces using the SDK (low volume)](./how_to_guides/export_traces) +- [Bulk exporting traces (high volume)](./how_to_guides/data_export) +- [Share or unshare a trace publicly](./how_to_guides/share_trace) +- [Compare traces](./how_to_guides/compare_traces) +- [View threads](./how_to_guides/threads#view-threads) ## Dashboards Use LangSmith custom and built-in dashboards to gain insight into your production systems. -- [Create and use custom dashboards](./how_to_guides/monitoring/dashboards) -- [Use built-in monitoring dashboards](./how_to_guides/monitoring/use_monitoring_charts) +- [Create and use custom dashboards](./how_to_guides/dashboards) +- [Use built-in monitoring dashboards](./how_to_guides/use_monitoring_charts) ## Automations Leverage LangSmith's powerful monitoring, automation, and online evaluation features to make sense of your production data. -- [Set up automation rules](./how_to_guides/monitoring/rules) -- [Set up webhook notifications for rules](./how_to_guides/monitoring/webhooks) -- [Perform online evaluations](./how_to_guides/monitoring/online_evaluations) +- [Set up automation rules](./how_to_guides/rules) +- [Set up webhook notifications for rules](./how_to_guides/webhooks) +- [Perform online evaluations](./how_to_guides/online_evaluations) diff --git a/docs/observability/how_to_guides/tracing/log_llm_trace.mdx b/docs/observability/how_to_guides/log_llm_trace.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/log_llm_trace.mdx rename to docs/observability/how_to_guides/log_llm_trace.mdx diff --git a/docs/observability/how_to_guides/tracing/log_multimodal_traces.mdx b/docs/observability/how_to_guides/log_multimodal_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/log_multimodal_traces.mdx rename to docs/observability/how_to_guides/log_multimodal_traces.mdx diff --git a/docs/observability/how_to_guides/tracing/log_retriever_trace.mdx b/docs/observability/how_to_guides/log_retriever_trace.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/log_retriever_trace.mdx rename to docs/observability/how_to_guides/log_retriever_trace.mdx diff --git a/docs/observability/how_to_guides/tracing/log_traces_to_project.mdx b/docs/observability/how_to_guides/log_traces_to_project.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/log_traces_to_project.mdx rename to docs/observability/how_to_guides/log_traces_to_project.mdx diff --git a/docs/observability/how_to_guides/tracing/mask_inputs_outputs.mdx b/docs/observability/how_to_guides/mask_inputs_outputs.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/mask_inputs_outputs.mdx rename to docs/observability/how_to_guides/mask_inputs_outputs.mdx diff --git a/docs/observability/how_to_guides/tracing/nest_traces.mdx b/docs/observability/how_to_guides/nest_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/nest_traces.mdx rename to docs/observability/how_to_guides/nest_traces.mdx diff --git a/docs/observability/how_to_guides/monitoring/online_evaluations.mdx b/docs/observability/how_to_guides/online_evaluations.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/online_evaluations.mdx rename to docs/observability/how_to_guides/online_evaluations.mdx diff --git a/docs/observability/how_to_guides/tracing/output_detailed_logs.mdx b/docs/observability/how_to_guides/output_detailed_logs.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/output_detailed_logs.mdx rename to docs/observability/how_to_guides/output_detailed_logs.mdx diff --git a/docs/observability/how_to_guides/monitoring/rules.mdx b/docs/observability/how_to_guides/rules.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/rules.mdx rename to docs/observability/how_to_guides/rules.mdx diff --git a/docs/observability/how_to_guides/tracing/sample_traces.mdx b/docs/observability/how_to_guides/sample_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/sample_traces.mdx rename to docs/observability/how_to_guides/sample_traces.mdx diff --git a/docs/observability/how_to_guides/tracing/serverless_environments.mdx b/docs/observability/how_to_guides/serverless_environments.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/serverless_environments.mdx rename to docs/observability/how_to_guides/serverless_environments.mdx diff --git a/docs/observability/how_to_guides/tracing/share_trace.mdx b/docs/observability/how_to_guides/share_trace.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/share_trace.mdx rename to docs/observability/how_to_guides/share_trace.mdx diff --git a/docs/observability/how_to_guides/monitoring/static/add_automation_rule.png b/docs/observability/how_to_guides/static/add_automation_rule.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/add_automation_rule.png rename to docs/observability/how_to_guides/static/add_automation_rule.png diff --git a/docs/observability/how_to_guides/monitoring/static/add_chart.png b/docs/observability/how_to_guides/static/add_chart.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/add_chart.png rename to docs/observability/how_to_guides/static/add_chart.png diff --git a/docs/observability/how_to_guides/monitoring/static/add_dashboard.png b/docs/observability/how_to_guides/static/add_dashboard.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/add_dashboard.png rename to docs/observability/how_to_guides/static/add_dashboard.png diff --git a/docs/observability/how_to_guides/monitoring/static/ai_query.png b/docs/observability/how_to_guides/static/ai_query.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/ai_query.png rename to docs/observability/how_to_guides/static/ai_query.png diff --git a/docs/observability/how_to_guides/tracing/static/annotate_code_trace.gif b/docs/observability/how_to_guides/static/annotate_code_trace.gif similarity index 100% rename from docs/observability/how_to_guides/tracing/static/annotate_code_trace.gif rename to docs/observability/how_to_guides/static/annotate_code_trace.gif diff --git a/docs/observability/how_to_guides/monitoring/static/automations.png b/docs/observability/how_to_guides/static/automations.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/automations.png rename to docs/observability/how_to_guides/static/automations.png diff --git a/docs/observability/how_to_guides/tracing/static/aws-comprehend-anonymized.png b/docs/observability/how_to_guides/static/aws-comprehend-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/aws-comprehend-anonymized.png rename to docs/observability/how_to_guides/static/aws-comprehend-anonymized.png diff --git a/docs/observability/how_to_guides/tracing/static/aws-comprehend-not-anonymized.png b/docs/observability/how_to_guides/static/aws-comprehend-not-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/aws-comprehend-not-anonymized.png rename to docs/observability/how_to_guides/static/aws-comprehend-not-anonymized.png diff --git a/docs/observability/how_to_guides/monitoring/static/bar_chart.png b/docs/observability/how_to_guides/static/bar_chart.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/bar_chart.png rename to docs/observability/how_to_guides/static/bar_chart.png diff --git a/docs/observability/how_to_guides/monitoring/static/chart_filters.png b/docs/observability/how_to_guides/static/chart_filters.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/chart_filters.png rename to docs/observability/how_to_guides/static/chart_filters.png diff --git a/docs/observability/how_to_guides/monitoring/static/chart_filters_for_node_decision.png b/docs/observability/how_to_guides/static/chart_filters_for_node_decision.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/chart_filters_for_node_decision.png rename to docs/observability/how_to_guides/static/chart_filters_for_node_decision.png diff --git a/docs/observability/how_to_guides/tracing/static/chat_model.png b/docs/observability/how_to_guides/static/chat_model.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/chat_model.png rename to docs/observability/how_to_guides/static/chat_model.png diff --git a/docs/observability/how_to_guides/monitoring/static/child_runs.png b/docs/observability/how_to_guides/static/child_runs.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/child_runs.png rename to docs/observability/how_to_guides/static/child_runs.png diff --git a/docs/observability/how_to_guides/monitoring/static/compare_metrics.png b/docs/observability/how_to_guides/static/compare_metrics.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/compare_metrics.png rename to docs/observability/how_to_guides/static/compare_metrics.png diff --git a/docs/observability/how_to_guides/tracing/static/compare_traces/compare_button.png b/docs/observability/how_to_guides/static/compare_traces/compare_button.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/compare_traces/compare_button.png rename to docs/observability/how_to_guides/static/compare_traces/compare_button.png diff --git a/docs/observability/how_to_guides/tracing/static/compare_traces/compare_trace.png b/docs/observability/how_to_guides/static/compare_traces/compare_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/compare_traces/compare_trace.png rename to docs/observability/how_to_guides/static/compare_traces/compare_trace.png diff --git a/docs/observability/how_to_guides/tracing/static/compare_traces/select_trace.png b/docs/observability/how_to_guides/static/compare_traces/select_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/compare_traces/select_trace.png rename to docs/observability/how_to_guides/static/compare_traces/select_trace.png diff --git a/docs/observability/how_to_guides/monitoring/static/convo.png b/docs/observability/how_to_guides/static/convo.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/convo.png rename to docs/observability/how_to_guides/static/convo.png diff --git a/docs/observability/how_to_guides/monitoring/static/convo_tab.png b/docs/observability/how_to_guides/static/convo_tab.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/convo_tab.png rename to docs/observability/how_to_guides/static/convo_tab.png diff --git a/docs/observability/how_to_guides/monitoring/static/copy_filter.png b/docs/observability/how_to_guides/static/copy_filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/copy_filter.png rename to docs/observability/how_to_guides/static/copy_filter.png diff --git a/docs/observability/how_to_guides/monitoring/static/dashboard_table.png b/docs/observability/how_to_guides/static/dashboard_table.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/dashboard_table.png rename to docs/observability/how_to_guides/static/dashboard_table.png diff --git a/docs/observability/how_to_guides/monitoring/static/decision_at_node.png b/docs/observability/how_to_guides/static/decision_at_node.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/decision_at_node.png rename to docs/observability/how_to_guides/static/decision_at_node.png diff --git a/docs/observability/how_to_guides/monitoring/static/drill_monitor.png b/docs/observability/how_to_guides/static/drill_monitor.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/drill_monitor.png rename to docs/observability/how_to_guides/static/drill_monitor.png diff --git a/docs/observability/how_to_guides/monitoring/static/evaluator_secrets.png b/docs/observability/how_to_guides/static/evaluator_secrets.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/evaluator_secrets.png rename to docs/observability/how_to_guides/static/evaluator_secrets.png diff --git a/docs/observability/how_to_guides/monitoring/static/expanded_chart.png b/docs/observability/how_to_guides/static/expanded_chart.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/expanded_chart.png rename to docs/observability/how_to_guides/static/expanded_chart.png diff --git a/docs/observability/how_to_guides/monitoring/static/filter.png b/docs/observability/how_to_guides/static/filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/filter.png rename to docs/observability/how_to_guides/static/filter.png diff --git a/docs/observability/how_to_guides/monitoring/static/filter_full_text.png b/docs/observability/how_to_guides/static/filter_full_text.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/filter_full_text.png rename to docs/observability/how_to_guides/static/filter_full_text.png diff --git a/docs/observability/how_to_guides/monitoring/static/filter_runs_in_trace_view.png b/docs/observability/how_to_guides/static/filter_runs_in_trace_view.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/filter_runs_in_trace_view.png rename to docs/observability/how_to_guides/static/filter_runs_in_trace_view.png diff --git a/docs/observability/how_to_guides/monitoring/static/filter_shortcuts.png b/docs/observability/how_to_guides/static/filter_shortcuts.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/filter_shortcuts.png rename to docs/observability/how_to_guides/static/filter_shortcuts.png diff --git a/docs/observability/how_to_guides/monitoring/static/give_rule_name.png b/docs/observability/how_to_guides/static/give_rule_name.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/give_rule_name.png rename to docs/observability/how_to_guides/static/give_rule_name.png diff --git a/docs/observability/how_to_guides/tracing/static/hello_llm.png b/docs/observability/how_to_guides/static/hello_llm.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/hello_llm.png rename to docs/observability/how_to_guides/static/hello_llm.png diff --git a/docs/observability/how_to_guides/tracing/static/hide_inputs_outputs.png b/docs/observability/how_to_guides/static/hide_inputs_outputs.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/hide_inputs_outputs.png rename to docs/observability/how_to_guides/static/hide_inputs_outputs.png diff --git a/docs/observability/how_to_guides/tracing/static/langchain_trace.png b/docs/observability/how_to_guides/static/langchain_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/langchain_trace.png rename to docs/observability/how_to_guides/static/langchain_trace.png diff --git a/docs/observability/how_to_guides/tracing/static/langgraph_with_langchain_trace.png b/docs/observability/how_to_guides/static/langgraph_with_langchain_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/langgraph_with_langchain_trace.png rename to docs/observability/how_to_guides/static/langgraph_with_langchain_trace.png diff --git a/docs/observability/how_to_guides/tracing/static/langgraph_without_langchain_trace.png b/docs/observability/how_to_guides/static/langgraph_without_langchain_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/langgraph_without_langchain_trace.png rename to docs/observability/how_to_guides/static/langgraph_without_langchain_trace.png diff --git a/docs/observability/how_to_guides/monitoring/static/mapping_variables.png b/docs/observability/how_to_guides/static/mapping_variables.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/mapping_variables.png rename to docs/observability/how_to_guides/static/mapping_variables.png diff --git a/docs/observability/how_to_guides/monitoring/static/modal_langsmith_secret.png b/docs/observability/how_to_guides/static/modal_langsmith_secret.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/modal_langsmith_secret.png rename to docs/observability/how_to_guides/static/modal_langsmith_secret.png diff --git a/docs/observability/how_to_guides/tracing/static/model_costs.png b/docs/observability/how_to_guides/static/model_costs.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/model_costs.png rename to docs/observability/how_to_guides/static/model_costs.png diff --git a/docs/observability/how_to_guides/tracing/static/model_price_map.png b/docs/observability/how_to_guides/static/model_price_map.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/model_price_map.png rename to docs/observability/how_to_guides/static/model_price_map.png diff --git a/docs/observability/how_to_guides/monitoring/static/more_actions_bar.png b/docs/observability/how_to_guides/static/more_actions_bar.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/more_actions_bar.png rename to docs/observability/how_to_guides/static/more_actions_bar.png diff --git a/docs/observability/how_to_guides/tracing/static/multimodal.png b/docs/observability/how_to_guides/static/multimodal.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/multimodal.png rename to docs/observability/how_to_guides/static/multimodal.png diff --git a/docs/observability/how_to_guides/monitoring/static/multiple_data_series.png b/docs/observability/how_to_guides/static/multiple_data_series.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/multiple_data_series.png rename to docs/observability/how_to_guides/static/multiple_data_series.png diff --git a/docs/observability/how_to_guides/tracing/static/new_price_map_entry.png b/docs/observability/how_to_guides/static/new_price_map_entry.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/new_price_map_entry.png rename to docs/observability/how_to_guides/static/new_price_map_entry.png diff --git a/docs/observability/how_to_guides/monitoring/static/online-eval-custom-code.png b/docs/observability/how_to_guides/static/online-eval-custom-code.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/online-eval-custom-code.png rename to docs/observability/how_to_guides/static/online-eval-custom-code.png diff --git a/docs/observability/how_to_guides/tracing/static/presidio-anonymized.png b/docs/observability/how_to_guides/static/presidio-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/presidio-anonymized.png rename to docs/observability/how_to_guides/static/presidio-anonymized.png diff --git a/docs/observability/how_to_guides/tracing/static/presidio-not-anonymized.png b/docs/observability/how_to_guides/static/presidio-not-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/presidio-not-anonymized.png rename to docs/observability/how_to_guides/static/presidio-not-anonymized.png diff --git a/docs/observability/how_to_guides/monitoring/static/prompt_preview.png b/docs/observability/how_to_guides/static/prompt_preview.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/prompt_preview.png rename to docs/observability/how_to_guides/static/prompt_preview.png diff --git a/docs/observability/how_to_guides/monitoring/static/raw_query.png b/docs/observability/how_to_guides/static/raw_query.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/raw_query.png rename to docs/observability/how_to_guides/static/raw_query.png diff --git a/docs/observability/how_to_guides/tracing/static/regex-anonymized.png b/docs/observability/how_to_guides/static/regex-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/regex-anonymized.png rename to docs/observability/how_to_guides/static/regex-anonymized.png diff --git a/docs/observability/how_to_guides/tracing/static/regex-not-anonymized.png b/docs/observability/how_to_guides/static/regex-not-anonymized.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/regex-not-anonymized.png rename to docs/observability/how_to_guides/static/regex-not-anonymized.png diff --git a/docs/observability/how_to_guides/tracing/static/retriever_trace.png b/docs/observability/how_to_guides/static/retriever_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/retriever_trace.png rename to docs/observability/how_to_guides/static/retriever_trace.png diff --git a/docs/observability/how_to_guides/monitoring/static/rules_filter.png b/docs/observability/how_to_guides/static/rules_filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/rules_filter.png rename to docs/observability/how_to_guides/static/rules_filter.png diff --git a/docs/observability/how_to_guides/monitoring/static/rules_logs.gif b/docs/observability/how_to_guides/static/rules_logs.gif similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/rules_logs.gif rename to docs/observability/how_to_guides/static/rules_logs.gif diff --git a/docs/observability/how_to_guides/monitoring/static/rules_logs.png b/docs/observability/how_to_guides/static/rules_logs.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/rules_logs.png rename to docs/observability/how_to_guides/static/rules_logs.png diff --git a/docs/observability/how_to_guides/monitoring/static/rules_past_runs.png b/docs/observability/how_to_guides/static/rules_past_runs.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/rules_past_runs.png rename to docs/observability/how_to_guides/static/rules_past_runs.png diff --git a/docs/observability/how_to_guides/monitoring/static/rules_past_runs_logs.png b/docs/observability/how_to_guides/static/rules_past_runs_logs.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/rules_past_runs_logs.png rename to docs/observability/how_to_guides/static/rules_past_runs_logs.png diff --git a/docs/observability/how_to_guides/monitoring/static/save_a_filter.png b/docs/observability/how_to_guides/static/save_a_filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/save_a_filter.png rename to docs/observability/how_to_guides/static/save_a_filter.png diff --git a/docs/observability/how_to_guides/monitoring/static/search_kv_filter_shortcut.png b/docs/observability/how_to_guides/static/search_kv_filter_shortcut.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/search_kv_filter_shortcut.png rename to docs/observability/how_to_guides/static/search_kv_filter_shortcut.png diff --git a/docs/observability/how_to_guides/monitoring/static/search_kv_input.png b/docs/observability/how_to_guides/static/search_kv_input.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/search_kv_input.png rename to docs/observability/how_to_guides/static/search_kv_input.png diff --git a/docs/observability/how_to_guides/monitoring/static/search_kv_output.png b/docs/observability/how_to_guides/static/search_kv_output.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/search_kv_output.png rename to docs/observability/how_to_guides/static/search_kv_output.png diff --git a/docs/observability/how_to_guides/monitoring/static/selecting_a_filter.png b/docs/observability/how_to_guides/static/selecting_a_filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/selecting_a_filter.png rename to docs/observability/how_to_guides/static/selecting_a_filter.png diff --git a/docs/observability/how_to_guides/tracing/static/share_trace.png b/docs/observability/how_to_guides/static/share_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/share_trace.png rename to docs/observability/how_to_guides/static/share_trace.png diff --git a/docs/observability/how_to_guides/monitoring/static/subsets_monitor.png b/docs/observability/how_to_guides/static/subsets_monitor.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/subsets_monitor.png rename to docs/observability/how_to_guides/static/subsets_monitor.png diff --git a/docs/observability/how_to_guides/monitoring/static/trace_filter.png b/docs/observability/how_to_guides/static/trace_filter.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/trace_filter.png rename to docs/observability/how_to_guides/static/trace_filter.png diff --git a/docs/observability/how_to_guides/tracing/static/trace_tree_manual_tracing.png b/docs/observability/how_to_guides/static/trace_tree_manual_tracing.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/trace_tree_manual_tracing.png rename to docs/observability/how_to_guides/static/trace_tree_manual_tracing.png diff --git a/docs/observability/how_to_guides/tracing/static/trace_tree_python_interop.png b/docs/observability/how_to_guides/static/trace_tree_python_interop.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/trace_tree_python_interop.png rename to docs/observability/how_to_guides/static/trace_tree_python_interop.png diff --git a/docs/observability/how_to_guides/tracing/static/trace_with_attachments.png b/docs/observability/how_to_guides/static/trace_with_attachments.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/trace_with_attachments.png rename to docs/observability/how_to_guides/static/trace_with_attachments.png diff --git a/docs/observability/how_to_guides/tracing/static/unshare_trace.png b/docs/observability/how_to_guides/static/unshare_trace.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/unshare_trace.png rename to docs/observability/how_to_guides/static/unshare_trace.png diff --git a/docs/observability/how_to_guides/tracing/static/unshare_trace_list.png b/docs/observability/how_to_guides/static/unshare_trace_list.png similarity index 100% rename from docs/observability/how_to_guides/tracing/static/unshare_trace_list.png rename to docs/observability/how_to_guides/static/unshare_trace_list.png diff --git a/docs/observability/how_to_guides/monitoring/static/webhook.png b/docs/observability/how_to_guides/static/webhook.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/webhook.png rename to docs/observability/how_to_guides/static/webhook.png diff --git a/docs/observability/how_to_guides/monitoring/static/webhook_headers.png b/docs/observability/how_to_guides/static/webhook_headers.png similarity index 100% rename from docs/observability/how_to_guides/monitoring/static/webhook_headers.png rename to docs/observability/how_to_guides/static/webhook_headers.png diff --git a/docs/observability/how_to_guides/monitoring/threads.mdx b/docs/observability/how_to_guides/threads.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/threads.mdx rename to docs/observability/how_to_guides/threads.mdx diff --git a/docs/observability/how_to_guides/tracing/toggle_tracing.mdx b/docs/observability/how_to_guides/toggle_tracing.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/toggle_tracing.mdx rename to docs/observability/how_to_guides/toggle_tracing.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_generator_functions.mdx b/docs/observability/how_to_guides/trace_generator_functions.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_generator_functions.mdx rename to docs/observability/how_to_guides/trace_generator_functions.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_api.mdx b/docs/observability/how_to_guides/trace_with_api.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_api.mdx rename to docs/observability/how_to_guides/trace_with_api.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_instructor.mdx b/docs/observability/how_to_guides/trace_with_instructor.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_instructor.mdx rename to docs/observability/how_to_guides/trace_with_instructor.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_langchain.mdx b/docs/observability/how_to_guides/trace_with_langchain.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_langchain.mdx rename to docs/observability/how_to_guides/trace_with_langchain.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_langgraph.mdx b/docs/observability/how_to_guides/trace_with_langgraph.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_langgraph.mdx rename to docs/observability/how_to_guides/trace_with_langgraph.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_opentelemetry.mdx b/docs/observability/how_to_guides/trace_with_opentelemetry.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_opentelemetry.mdx rename to docs/observability/how_to_guides/trace_with_opentelemetry.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_with_vercel_ai_sdk.mdx b/docs/observability/how_to_guides/trace_with_vercel_ai_sdk.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_with_vercel_ai_sdk.mdx rename to docs/observability/how_to_guides/trace_with_vercel_ai_sdk.mdx diff --git a/docs/observability/how_to_guides/tracing/trace_without_env_vars.mdx b/docs/observability/how_to_guides/trace_without_env_vars.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/trace_without_env_vars.mdx rename to docs/observability/how_to_guides/trace_without_env_vars.mdx diff --git a/docs/observability/how_to_guides/tracing/upload_files_with_traces.mdx b/docs/observability/how_to_guides/upload_files_with_traces.mdx similarity index 100% rename from docs/observability/how_to_guides/tracing/upload_files_with_traces.mdx rename to docs/observability/how_to_guides/upload_files_with_traces.mdx diff --git a/docs/observability/how_to_guides/monitoring/use_monitoring_charts.mdx b/docs/observability/how_to_guides/use_monitoring_charts.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/use_monitoring_charts.mdx rename to docs/observability/how_to_guides/use_monitoring_charts.mdx diff --git a/docs/observability/how_to_guides/monitoring/webhooks.mdx b/docs/observability/how_to_guides/webhooks.mdx similarity index 100% rename from docs/observability/how_to_guides/monitoring/webhooks.mdx rename to docs/observability/how_to_guides/webhooks.mdx diff --git a/vercel.json b/vercel.json index 91455610..a302f6fd 100644 --- a/vercel.json +++ b/vercel.json @@ -221,6 +221,14 @@ { "source": "/evaluation/how_to_guides/unit_testing(/?)", "destination": "/evauation/how_to_guides/pytest" + }, + { + "source": "/observability/how_to_guides/tracing/:path*", + "destination": "/observability/how_to_guides/:path*" + }, + { + "source": "/observability/how_to_guides/monitoring/:path*", + "destination": "/observability/how_to_guides/:path*" } ], "builds": [ From 43ea936c5117ceabc1797add7e346d0cee90527a Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 27 Jan 2025 13:36:37 -0800 Subject: [PATCH 2/6] prompt engineering --- .../how_to_guides/monitoring/_category_.json | 5 -- .../how_to_guides/tracing/_category_.json | 6 --- .../{prompts => }/create_a_prompt.mdx | 0 .../custom_openai_compliant_model.mdx | 0 .../custom_tls_certificates.mdx | 0 .../prompt_engineering/how_to_guides/index.md | 18 +++---- .../{prompts => }/langchain_hub.mdx | 0 .../manage_prompts_programatically.mdx | 0 .../managing_model_configurations.mdx | 0 .../open_a_prompt_from_a_trace.mdx | 0 .../how_to_guides/playground/_category_.json | 5 -- .../playground/custom_endpoint.mdx | 46 ------------------ .../{prompts => }/prompt_tags.mdx | 0 .../how_to_guides/prompts/_category_.json | 5 -- .../static/azure_playground.png | Bin .../static/blank_prompts_page.png | Bin .../static/create_prompt_playground.png | Bin .../{playground => }/static/dataset_table.png | Bin .../static/edit_in_playground.png | Bin .../{prompts => }/static/edit_prompt.png | Bin .../static/evaluator_pane.png | Bin .../{prompts => }/static/langchain_hub.png | Bin .../static/metadata_edit_button.png | Bin .../static/openai_proxy_provider.png | Bin .../static/playground_custom_model.png | Bin .../static/prompt_commits_tab.png | Bin .../static/prompt_playground_edit_commit.png | Bin .../static/prompt_settings.png | Bin .../{prompts => }/static/prompt_table.png | Bin .../static/prompt_tags/commits_tab.png | Bin .../prompt_tags/create_new_prompt_tag.png | Bin .../static/prompt_tags/move_prompt_tag.png | Bin .../{prompts => }/static/prompts_tab.png | Bin .../static/renamed_config.mp4 | Bin .../{prompts => }/static/run_metadata.png | Bin .../{prompts => }/static/save_prompt.png | Bin .../{playground => }/static/saving_config.mp4 | Bin .../static/select_dataset.png | Bin .../static/set_default_config.png | Bin .../static/trace_with_prompt_link.png | Bin .../static/update_prompt_form.png | Bin .../{playground => }/testing_over_dataset.mdx | 0 .../{prompts => }/update_a_prompt.mdx | 0 vercel.json | 8 +++ 44 files changed, 17 insertions(+), 76 deletions(-) delete mode 100644 docs/observability/how_to_guides/monitoring/_category_.json delete mode 100644 docs/observability/how_to_guides/tracing/_category_.json rename docs/prompt_engineering/how_to_guides/{prompts => }/create_a_prompt.mdx (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/custom_openai_compliant_model.mdx (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/custom_tls_certificates.mdx (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/langchain_hub.mdx (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/manage_prompts_programatically.mdx (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/managing_model_configurations.mdx (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/open_a_prompt_from_a_trace.mdx (100%) delete mode 100644 docs/prompt_engineering/how_to_guides/playground/_category_.json delete mode 100644 docs/prompt_engineering/how_to_guides/playground/custom_endpoint.mdx rename docs/prompt_engineering/how_to_guides/{prompts => }/prompt_tags.mdx (100%) delete mode 100644 docs/prompt_engineering/how_to_guides/prompts/_category_.json rename docs/prompt_engineering/how_to_guides/{playground => }/static/azure_playground.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/blank_prompts_page.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/create_prompt_playground.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/dataset_table.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/edit_in_playground.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/edit_prompt.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/evaluator_pane.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/langchain_hub.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/metadata_edit_button.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/openai_proxy_provider.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/playground_custom_model.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_commits_tab.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_playground_edit_commit.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_settings.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_table.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_tags/commits_tab.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_tags/create_new_prompt_tag.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompt_tags/move_prompt_tag.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/prompts_tab.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/renamed_config.mp4 (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/run_metadata.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/save_prompt.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/saving_config.mp4 (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/select_dataset.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/set_default_config.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/trace_with_prompt_link.png (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/static/update_prompt_form.png (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/testing_over_dataset.mdx (100%) rename docs/prompt_engineering/how_to_guides/{prompts => }/update_a_prompt.mdx (100%) diff --git a/docs/observability/how_to_guides/monitoring/_category_.json b/docs/observability/how_to_guides/monitoring/_category_.json deleted file mode 100644 index f5141dd6..00000000 --- a/docs/observability/how_to_guides/monitoring/_category_.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label": "Monitoring and automations", - "collapsed": true, - "collapsible": true -} diff --git a/docs/observability/how_to_guides/tracing/_category_.json b/docs/observability/how_to_guides/tracing/_category_.json deleted file mode 100644 index 767c513f..00000000 --- a/docs/observability/how_to_guides/tracing/_category_.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "label": "Tracing", - "collapsed": true, - "collapsible": true, - "position": 2 -} diff --git a/docs/prompt_engineering/how_to_guides/prompts/create_a_prompt.mdx b/docs/prompt_engineering/how_to_guides/create_a_prompt.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/create_a_prompt.mdx rename to docs/prompt_engineering/how_to_guides/create_a_prompt.mdx diff --git a/docs/prompt_engineering/how_to_guides/playground/custom_openai_compliant_model.mdx b/docs/prompt_engineering/how_to_guides/custom_openai_compliant_model.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/custom_openai_compliant_model.mdx rename to docs/prompt_engineering/how_to_guides/custom_openai_compliant_model.mdx diff --git a/docs/prompt_engineering/how_to_guides/playground/custom_tls_certificates.mdx b/docs/prompt_engineering/how_to_guides/custom_tls_certificates.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/custom_tls_certificates.mdx rename to docs/prompt_engineering/how_to_guides/custom_tls_certificates.mdx diff --git a/docs/prompt_engineering/how_to_guides/index.md b/docs/prompt_engineering/how_to_guides/index.md index c575b17c..68d22108 100644 --- a/docs/prompt_engineering/how_to_guides/index.md +++ b/docs/prompt_engineering/how_to_guides/index.md @@ -6,20 +6,20 @@ Step-by-step guides that cover key tasks and operations for doing prompt enginee Organize and manage prompts in LangSmith to streamline your LLM development workflow. -- [Create a prompt](./how_to_guides/prompts/create_a_prompt) -- [Update a prompt](./how_to_guides/prompts/update_a_prompt) -- [Manage prompts programmatically](./how_to_guides/prompts/manage_prompts_programatically) -- [LangChain Hub](./how_to_guides/prompts/langchain_hub) +- [Create a prompt](./how_to_guides/create_a_prompt) +- [Update a prompt](./how_to_guides/update_a_prompt) +- [Manage prompts programmatically](./how_to_guides/manage_prompts_programatically) +- [LangChain Hub](./how_to_guides/langchain_hub) ## Playground Quickly iterate on prompts and models in the LangSmith Playground. -- [Run the playground against a custom LangServe model server](./how_to_guides/playground/custom_endpoint) -- [Run the playground against an OpenAI-compliant model provider/proxy](./how_to_guides/playground/custom_openai_compliant_model) -- [Use custom TLS certificates](./how_to_guides/playground/custom_tls_certificates) -- [Manage prompt settings](./how_to_guides/playground/managing_model_configurations) -- [Testing over a dataset](./how_to_guides/playground/testing_over_dataset) +- [Run the playground against a custom LangServe model server](./how_to_guides/custom_endpoint) +- [Run the playground against an OpenAI-compliant model provider/proxy](./how_to_guides/custom_openai_compliant_model) +- [Use custom TLS certificates](./how_to_guides/custom_tls_certificates) +- [Manage prompt settings](./how_to_guides/managing_model_configurations) +- [Testing over a dataset](./how_to_guides/testing_over_dataset) ## Few shot prompting diff --git a/docs/prompt_engineering/how_to_guides/prompts/langchain_hub.mdx b/docs/prompt_engineering/how_to_guides/langchain_hub.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/langchain_hub.mdx rename to docs/prompt_engineering/how_to_guides/langchain_hub.mdx diff --git a/docs/prompt_engineering/how_to_guides/prompts/manage_prompts_programatically.mdx b/docs/prompt_engineering/how_to_guides/manage_prompts_programatically.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/manage_prompts_programatically.mdx rename to docs/prompt_engineering/how_to_guides/manage_prompts_programatically.mdx diff --git a/docs/prompt_engineering/how_to_guides/playground/managing_model_configurations.mdx b/docs/prompt_engineering/how_to_guides/managing_model_configurations.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/managing_model_configurations.mdx rename to docs/prompt_engineering/how_to_guides/managing_model_configurations.mdx diff --git a/docs/prompt_engineering/how_to_guides/prompts/open_a_prompt_from_a_trace.mdx b/docs/prompt_engineering/how_to_guides/open_a_prompt_from_a_trace.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/open_a_prompt_from_a_trace.mdx rename to docs/prompt_engineering/how_to_guides/open_a_prompt_from_a_trace.mdx diff --git a/docs/prompt_engineering/how_to_guides/playground/_category_.json b/docs/prompt_engineering/how_to_guides/playground/_category_.json deleted file mode 100644 index af4db0c7..00000000 --- a/docs/prompt_engineering/how_to_guides/playground/_category_.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label": "Playground", - "collapsed": true, - "collapsible": true -} diff --git a/docs/prompt_engineering/how_to_guides/playground/custom_endpoint.mdx b/docs/prompt_engineering/how_to_guides/playground/custom_endpoint.mdx deleted file mode 100644 index 141a368c..00000000 --- a/docs/prompt_engineering/how_to_guides/playground/custom_endpoint.mdx +++ /dev/null @@ -1,46 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Run the playground against a custom LangServe model server - -The LangSmith playground allows you to use your own custom models. You can deploy a model server that exposes your model's API via [LangServe](https://github.com/langchain-ai/langserve), an open source library for serving LangChain applications. -Behind the scenes, the playground will interact with your model server to generate responses. - -## Deploy a custom model server - -For your convenience, we have provided a sample model server that you can use as a reference. You can find the sample model server [here](https://github.com/langchain-ai/langsmith-model-server) -We highly recommend using the sample model server as a starting point. - -Depending on your model is an instruct-style or chat-style model, you will need to implement either `custom_model.py` or `custom_chat_model.py` respectively. - -## Adding configurable fields - -It is often useful to configure your model with different parameters. These might include temperature, model_name, max_tokens, etc. - -To make your model configurable in the LangSmith playground, you need to add configurable fields to your model server. These fields can be used to change model parameters from the playground. - -You can add configurable fields by implementing the `with_configurable_fields` function in the `config.py` file. You can - -```python -def with_configurable_fields(self) -> Runnable: - """Expose fields you want to be configurable in the playground. We will automatically expose these to the - playground. If you don't want to expose any fields, you can remove this method.""" - return self.configurable_fields(n=ConfigurableField( - id="n", - name="Num Characters", - description="Number of characters to return from the input prompt.", - )) -``` - -## Use the model in the LangSmith Playground - -Once you have deployed a model server, you can use it in the LangSmith Playground. Enter the playground and select either the `ChatCustomModel` or the `CustomModel` provider for chat-style model or instruct-style models. - -Enter the `URL`. The playground will automatically detect the available endpoints and configurable fields. You can then invoke the model with the desired parameters. - -![ChatCustomModel in Playground](./static//playground_custom_model.png) - -If everything is set up correctly, you should see the model's response in the playground as well as the configurable fields specified in the `with_configurable_fields`. - -See how to store your model configuration for later use [here](./managing_model_configurations). diff --git a/docs/prompt_engineering/how_to_guides/prompts/prompt_tags.mdx b/docs/prompt_engineering/how_to_guides/prompt_tags.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/prompt_tags.mdx rename to docs/prompt_engineering/how_to_guides/prompt_tags.mdx diff --git a/docs/prompt_engineering/how_to_guides/prompts/_category_.json b/docs/prompt_engineering/how_to_guides/prompts/_category_.json deleted file mode 100644 index 8e9c04ef..00000000 --- a/docs/prompt_engineering/how_to_guides/prompts/_category_.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "label": "Prompts", - "collapsed": true, - "collapsible": true -} diff --git a/docs/prompt_engineering/how_to_guides/playground/static/azure_playground.png b/docs/prompt_engineering/how_to_guides/static/azure_playground.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/azure_playground.png rename to docs/prompt_engineering/how_to_guides/static/azure_playground.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/blank_prompts_page.png b/docs/prompt_engineering/how_to_guides/static/blank_prompts_page.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/blank_prompts_page.png rename to docs/prompt_engineering/how_to_guides/static/blank_prompts_page.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/create_prompt_playground.png b/docs/prompt_engineering/how_to_guides/static/create_prompt_playground.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/create_prompt_playground.png rename to docs/prompt_engineering/how_to_guides/static/create_prompt_playground.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/dataset_table.png b/docs/prompt_engineering/how_to_guides/static/dataset_table.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/dataset_table.png rename to docs/prompt_engineering/how_to_guides/static/dataset_table.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/edit_in_playground.png b/docs/prompt_engineering/how_to_guides/static/edit_in_playground.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/edit_in_playground.png rename to docs/prompt_engineering/how_to_guides/static/edit_in_playground.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/edit_prompt.png b/docs/prompt_engineering/how_to_guides/static/edit_prompt.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/edit_prompt.png rename to docs/prompt_engineering/how_to_guides/static/edit_prompt.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/evaluator_pane.png b/docs/prompt_engineering/how_to_guides/static/evaluator_pane.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/evaluator_pane.png rename to docs/prompt_engineering/how_to_guides/static/evaluator_pane.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/langchain_hub.png b/docs/prompt_engineering/how_to_guides/static/langchain_hub.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/langchain_hub.png rename to docs/prompt_engineering/how_to_guides/static/langchain_hub.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/metadata_edit_button.png b/docs/prompt_engineering/how_to_guides/static/metadata_edit_button.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/metadata_edit_button.png rename to docs/prompt_engineering/how_to_guides/static/metadata_edit_button.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/openai_proxy_provider.png b/docs/prompt_engineering/how_to_guides/static/openai_proxy_provider.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/openai_proxy_provider.png rename to docs/prompt_engineering/how_to_guides/static/openai_proxy_provider.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/playground_custom_model.png b/docs/prompt_engineering/how_to_guides/static/playground_custom_model.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/playground_custom_model.png rename to docs/prompt_engineering/how_to_guides/static/playground_custom_model.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_commits_tab.png b/docs/prompt_engineering/how_to_guides/static/prompt_commits_tab.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_commits_tab.png rename to docs/prompt_engineering/how_to_guides/static/prompt_commits_tab.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_playground_edit_commit.png b/docs/prompt_engineering/how_to_guides/static/prompt_playground_edit_commit.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_playground_edit_commit.png rename to docs/prompt_engineering/how_to_guides/static/prompt_playground_edit_commit.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_settings.png b/docs/prompt_engineering/how_to_guides/static/prompt_settings.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_settings.png rename to docs/prompt_engineering/how_to_guides/static/prompt_settings.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_table.png b/docs/prompt_engineering/how_to_guides/static/prompt_table.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_table.png rename to docs/prompt_engineering/how_to_guides/static/prompt_table.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/commits_tab.png b/docs/prompt_engineering/how_to_guides/static/prompt_tags/commits_tab.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/commits_tab.png rename to docs/prompt_engineering/how_to_guides/static/prompt_tags/commits_tab.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/create_new_prompt_tag.png b/docs/prompt_engineering/how_to_guides/static/prompt_tags/create_new_prompt_tag.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/create_new_prompt_tag.png rename to docs/prompt_engineering/how_to_guides/static/prompt_tags/create_new_prompt_tag.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/move_prompt_tag.png b/docs/prompt_engineering/how_to_guides/static/prompt_tags/move_prompt_tag.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompt_tags/move_prompt_tag.png rename to docs/prompt_engineering/how_to_guides/static/prompt_tags/move_prompt_tag.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/prompts_tab.png b/docs/prompt_engineering/how_to_guides/static/prompts_tab.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/prompts_tab.png rename to docs/prompt_engineering/how_to_guides/static/prompts_tab.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/renamed_config.mp4 b/docs/prompt_engineering/how_to_guides/static/renamed_config.mp4 similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/renamed_config.mp4 rename to docs/prompt_engineering/how_to_guides/static/renamed_config.mp4 diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/run_metadata.png b/docs/prompt_engineering/how_to_guides/static/run_metadata.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/run_metadata.png rename to docs/prompt_engineering/how_to_guides/static/run_metadata.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/save_prompt.png b/docs/prompt_engineering/how_to_guides/static/save_prompt.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/save_prompt.png rename to docs/prompt_engineering/how_to_guides/static/save_prompt.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/saving_config.mp4 b/docs/prompt_engineering/how_to_guides/static/saving_config.mp4 similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/saving_config.mp4 rename to docs/prompt_engineering/how_to_guides/static/saving_config.mp4 diff --git a/docs/prompt_engineering/how_to_guides/playground/static/select_dataset.png b/docs/prompt_engineering/how_to_guides/static/select_dataset.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/select_dataset.png rename to docs/prompt_engineering/how_to_guides/static/select_dataset.png diff --git a/docs/prompt_engineering/how_to_guides/playground/static/set_default_config.png b/docs/prompt_engineering/how_to_guides/static/set_default_config.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/set_default_config.png rename to docs/prompt_engineering/how_to_guides/static/set_default_config.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/trace_with_prompt_link.png b/docs/prompt_engineering/how_to_guides/static/trace_with_prompt_link.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/trace_with_prompt_link.png rename to docs/prompt_engineering/how_to_guides/static/trace_with_prompt_link.png diff --git a/docs/prompt_engineering/how_to_guides/prompts/static/update_prompt_form.png b/docs/prompt_engineering/how_to_guides/static/update_prompt_form.png similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/static/update_prompt_form.png rename to docs/prompt_engineering/how_to_guides/static/update_prompt_form.png diff --git a/docs/prompt_engineering/how_to_guides/playground/testing_over_dataset.mdx b/docs/prompt_engineering/how_to_guides/testing_over_dataset.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/testing_over_dataset.mdx rename to docs/prompt_engineering/how_to_guides/testing_over_dataset.mdx diff --git a/docs/prompt_engineering/how_to_guides/prompts/update_a_prompt.mdx b/docs/prompt_engineering/how_to_guides/update_a_prompt.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/prompts/update_a_prompt.mdx rename to docs/prompt_engineering/how_to_guides/update_a_prompt.mdx diff --git a/vercel.json b/vercel.json index a302f6fd..0f9b9445 100644 --- a/vercel.json +++ b/vercel.json @@ -229,6 +229,14 @@ { "source": "/observability/how_to_guides/monitoring/:path*", "destination": "/observability/how_to_guides/:path*" + }, + { + "source": "/prompt_engineering/how_to_guides/prompts/:path*", + "destination": "/prompt_engineering/how_to_guides/:path*" + }, + { + "source": "/prompt_engineering/how_to_guides/playground/:path*", + "destination": "/prompt_engineering/how_to_guides/:path*" } ], "builds": [ From c3620d0da3df06ac4edb58b3b85eac9529c1a7b8 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Wed, 29 Jan 2025 17:47:56 -0800 Subject: [PATCH 3/6] links --- .../manage_organization_by_api.mdx | 2 +- docs/administration/tutorials/manage_spend.mdx | 4 ++-- docs/evaluation/concepts/index.mdx | 4 ++-- .../evaluation/how_to_guides/annotation_queues.mdx | 2 +- .../how_to_guides/attach_user_feedback.mdx | 4 ++-- .../how_to_guides/bind_evaluator_to_dataset.mdx | 2 +- .../how_to_guides/create_few_shot_evaluators.mdx | 4 ++-- .../how_to_guides/evaluate_llm_application.mdx | 2 +- .../evaluation/how_to_guides/evaluate_pairwise.mdx | 2 +- .../how_to_guides/evaluate_with_attachments.mdx | 2 +- .../manage_datasets_in_application.mdx | 6 +++--- .../manage_datasets_programmatically.mdx | 4 ++-- docs/evaluation/tutorials/evaluation.mdx | 2 +- docs/index.mdx | 4 ++-- docs/observability/concepts/index.mdx | 6 +++--- .../how_to_guides/serverless_environments.mdx | 2 +- .../how_to_guides/custom_tls_certificates.mdx | 2 +- .../data_formats/feedback_data_format.mdx | 2 +- .../evaluation/dataset_transformations.mdx | 14 +++++++------- docs/self_hosting/release_notes.mdx | 4 ++-- 20 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/administration/how_to_guides/organization_management/manage_organization_by_api.mdx b/docs/administration/how_to_guides/organization_management/manage_organization_by_api.mdx index 6d099d79..7ce975ed 100644 --- a/docs/administration/how_to_guides/organization_management/manage_organization_by_api.mdx +++ b/docs/administration/how_to_guides/organization_management/manage_organization_by_api.mdx @@ -144,7 +144,7 @@ If the header is not present, operations will default to the workspace the API k ## Security Settings :::note -"Shared resources" in this context refer to [public prompts](../../../prompt_engineering/how_to_guides/prompts/create_a_prompt#save-your-prompt), [shared runs](../../../observability/how_to_guides/tracing/share_trace), and [shared datasets](../../../evaluation/how_to_guides/share_dataset.mdx). +"Shared resources" in this context refer to [public prompts](../../../prompt_engineering/how_to_guides/create_a_prompt#save-your-prompt), [shared runs](../../../observability/how_to_guides/share_trace), and [shared datasets](../../../evaluation/how_to_guides/share_dataset.mdx). ::: - Add Rule`, and configure the rule as follows: ![](./static/P2SampleTraces.png) @@ -140,7 +140,7 @@ be thought of as a tree of runs making up an API call. When a run rule matches a upgrades to be retained for 400 days. Therefore, to make sure we have the proper sampling rate on traces, we take advantage of the -[filtering](../../observability/how_to_guides/monitoring/rules#step-2-define-the-filter) functionality of run rules. +[filtering](../../observability/how_to_guides/rules#step-2-define-the-filter) functionality of run rules. We add add a filter condition to only match the "root" run in the run tree. This is distinct per trace, so our 10% sampling will upgrade 10% of traces, rather 10% of runs, which could correspond to more than 10% of traces. If desired, we can optionally add diff --git a/docs/evaluation/concepts/index.mdx b/docs/evaluation/concepts/index.mdx index a738dda2..323dc07f 100644 --- a/docs/evaluation/concepts/index.mdx +++ b/docs/evaluation/concepts/index.mdx @@ -98,7 +98,7 @@ There are a number of ways to define and run evaluators: - **Custom code**: Define [custom evaluators](/evaluation/how_to_guides/custom_evaluator) as Python or TypeScript functions and run them client-side using the SDKs or server-side via the UI. - **Built-in evaluators**: LangSmith has a number of built-in evaluators that you can configure and run via the UI. -You can run evaluators using the LangSmith SDK ([Python](https://docs.smith.langchain.com/reference/python) and [TypeScript](https://docs.smith.langchain.com/reference/js)), via the [Prompt Playground](../../prompt_engineering/concepts#prompt-playground), or by configuring [Rules](../../observability/how_to_guides/monitoring/rules) to automatically run them on particular tracing projects or datasets. +You can run evaluators using the LangSmith SDK ([Python](https://docs.smith.langchain.com/reference/python) and [TypeScript](https://docs.smith.langchain.com/reference/js)), via the [Prompt Playground](../../prompt_engineering/concepts#prompt-playground), or by configuring [Rules](../../observability/how_to_guides/rules) to automatically run them on particular tracing projects or datasets. #### Evaluation techniques @@ -162,7 +162,7 @@ It is offline because we're evaluating on a pre-compiled set of data. An online evaluation, on the other hand, is one in which we evaluate a deployed application's outputs on real traffic, in near realtime. Offline evaluations are used for testing a version(s) of your application pre-deployment. -You can run offline evaluations client-side using the LangSmith SDK ([Python](https://docs.smith.langchain.com/reference/python) and [TypeScript](https://docs.smith.langchain.com/reference/js)). You can run them server-side via the [Prompt Playground](../../prompt_engineering/concepts#prompt-playground) or by configuring [automations](/observability/how_to_guides/monitoring/rules) to run certain evaluators on every new experiment against a specific dataset. +You can run offline evaluations client-side using the LangSmith SDK ([Python](https://docs.smith.langchain.com/reference/python) and [TypeScript](https://docs.smith.langchain.com/reference/js)). You can run them server-side via the [Prompt Playground](../../prompt_engineering/concepts#prompt-playground) or by configuring [automations](/observability/how_to_guides/rules) to run certain evaluators on every new experiment against a specific dataset. ![Offline](./static/offline.png) diff --git a/docs/evaluation/how_to_guides/annotation_queues.mdx b/docs/evaluation/how_to_guides/annotation_queues.mdx index f42f70ba..c54069f7 100644 --- a/docs/evaluation/how_to_guides/annotation_queues.mdx +++ b/docs/evaluation/how_to_guides/annotation_queues.mdx @@ -71,7 +71,7 @@ To assign runs to an annotation queue, either: 2. Select multiple runs in the runs table then click **Add to Annotation Queue** at the bottom of the page. ![](./static/multi_select_annotation_queue.png) -3. [Set up an automation rule](../../../observability/how_to_guides/monitoring/rules) that automatically assigns runs which pass a certain filter and sampling condition to an annotation queue. +3. [Set up an automation rule](../../../observability/how_to_guides/rules) that automatically assigns runs which pass a certain filter and sampling condition to an annotation queue. 4. Select one or multiple experiments from the dataset page and click **Annotate**. From the resulting popup, you may either create a new queue or add the runs to an existing one: ![](./static/annotate_experiment.png) diff --git a/docs/evaluation/how_to_guides/attach_user_feedback.mdx b/docs/evaluation/how_to_guides/attach_user_feedback.mdx index 9a220bba..84798746 100644 --- a/docs/evaluation/how_to_guides/attach_user_feedback.mdx +++ b/docs/evaluation/how_to_guides/attach_user_feedback.mdx @@ -20,10 +20,10 @@ Before diving into this content, it might be helpful to read the following: In many applications, but even more so for LLM applications, it is important to collect user feedback to understand how your application is performing in real-world scenarios. The ability to observe user feedback along with trace data can be very powerful to drill down into the most interesting datapoints, then send those datapoints for further review, automatic evaluation, or even datasets. -To learn more about how to filter traces based on various attributes, including user feedback, see [this guide](../../../observability/how_to_guides/monitoring/filter_traces_in_application) +To learn more about how to filter traces based on various attributes, including user feedback, see [this guide](../../../observability/how_to_guides/filter_traces_in_application) LangSmith makes it easy to attach user feedback to traces. -It's often helpful to expose a simple mechanism (such as a thumbs-up, thumbs-down button) to collect user feedback for your application responses. You can then use the LangSmith SDK or API to send feedback for a trace. To get the `run_id` of a logged run, see [this guide](../../../observability/how_to_guides/tracing/access_current_span). +It's often helpful to expose a simple mechanism (such as a thumbs-up, thumbs-down button) to collect user feedback for your application responses. You can then use the LangSmith SDK or API to send feedback for a trace. To get the `run_id` of a logged run, see [this guide](../../../observability/how_to_guides/access_current_span). :::note diff --git a/docs/evaluation/how_to_guides/bind_evaluator_to_dataset.mdx b/docs/evaluation/how_to_guides/bind_evaluator_to_dataset.mdx index d289071c..70fbd2c8 100644 --- a/docs/evaluation/how_to_guides/bind_evaluator_to_dataset.mdx +++ b/docs/evaluation/how_to_guides/bind_evaluator_to_dataset.mdx @@ -7,7 +7,7 @@ sidebar_position: 2 While you can specify evaluators to grade the results of your experiments programmatically (see [this guide](./evaluate_llm_application) for more information), you can also bind evaluators to a dataset in the UI. This allows you to configure automatic evaluators that grade your experiment results. We have support for both LLM-based evaluators, and custom python code evaluators. -The process for configuring this is very similar to the process for configuring an [online evaluator](../../../observability/how_to_guides/monitoring/online_evaluations) for traces. +The process for configuring this is very similar to the process for configuring an [online evaluator](../../../observability/how_to_guides/online_evaluations) for traces. :::note Only affects subsequent experiment runs When you configure an evaluator for a dataset, it will only affect the experiment runs that are created after the evaluator is configured. It will not affect the evaluation of experiment runs that were created before the evaluator was configured. diff --git a/docs/evaluation/how_to_guides/create_few_shot_evaluators.mdx b/docs/evaluation/how_to_guides/create_few_shot_evaluators.mdx index ecdbbb78..e1cb664d 100644 --- a/docs/evaluation/how_to_guides/create_few_shot_evaluators.mdx +++ b/docs/evaluation/how_to_guides/create_few_shot_evaluators.mdx @@ -11,7 +11,7 @@ you to automatically collect human corrections on evaluator prompts, which are t :::tip Recommended Reading Before learning how to create few-shot evaluators, it might be helpful to learn how to setup automations (both online and offline) and how to leave corrections on evaluator scores: -- [Set up online evaluations](../../../observability/how_to_guides/monitoring/online_evaluations) +- [Set up online evaluations](../../../observability/how_to_guides/online_evaluations) - [Bind an evaluator to a dataset in the UI (offline evaluation)](./bind_evaluator_to_dataset) - [Audit evaluator scores](./audit_evaluator_scores) @@ -24,7 +24,7 @@ The default maximum few-shot examples to use in the prompt is 5. Examples are pu ::: -When creating an [online](../../../observability/how_to_guides/monitoring/online_evaluations) or [offline](./bind_evaluator_to_dataset) evaluator - from a tracing project or a dataset, respectively - you will see the option to use corrections as few-shot examples. Note that these types of evaluators +When creating an [online](../../../observability/how_to_guides/online_evaluations) or [offline](./bind_evaluator_to_dataset) evaluator - from a tracing project or a dataset, respectively - you will see the option to use corrections as few-shot examples. Note that these types of evaluators are only supported when using mustache prompts - you will not be able to click this option if your prompt uses f-string formatting. When you select this, we will auto-create a few-shot prompt for you. Each individual few-shot example will be formatted according to this prompt, and inserted into your main prompt in place of the `{{Few-shot examples}}` template variable which will be auto-added above. Your few-shot prompt should contain the same variables as your main prompt, plus a `few_shot_explanation` and a score variable which should have the same name diff --git a/docs/evaluation/how_to_guides/evaluate_llm_application.mdx b/docs/evaluation/how_to_guides/evaluate_llm_application.mdx index e8ac1937..76dc3cc9 100644 --- a/docs/evaluation/how_to_guides/evaluate_llm_application.mdx +++ b/docs/evaluation/how_to_guides/evaluate_llm_application.mdx @@ -92,7 +92,7 @@ First we need an application to evaluate. Let's create a simple toxicity classif /> We've optionally enabled tracing to capture the inputs and outputs of each step in the pipeline. -To understand how to annotate your code for tracing, please refer to [this guide](../../../observability/how_to_guides/tracing/annotate_code). +To understand how to annotate your code for tracing, please refer to [this guide](../../../observability/how_to_guides/annotate_code). ## Create or select a dataset diff --git a/docs/evaluation/how_to_guides/evaluate_pairwise.mdx b/docs/evaluation/how_to_guides/evaluate_pairwise.mdx index 375791c0..d3b46174 100644 --- a/docs/evaluation/how_to_guides/evaluate_pairwise.mdx +++ b/docs/evaluation/how_to_guides/evaluate_pairwise.mdx @@ -88,7 +88,7 @@ which asks the LLM to decide which is better between two AI assistant responses. :::info Optional LangChain Usage -In the Python example below, we are pulling [this structured prompt](https://smith.langchain.com/hub/langchain-ai/pairwise-evaluation-2) from the [LangChain Hub](../../../prompt_engineering/how_to_guides/prompts/langchain_hub) and using it with a LangChain chat model wrapper. +In the Python example below, we are pulling [this structured prompt](https://smith.langchain.com/hub/langchain-ai/pairwise-evaluation-2) from the [LangChain Hub](../../../prompt_engineering/how_to_guides/langchain_hub) and using it with a LangChain chat model wrapper. **Usage of LangChain is totally optional.** To illustrate this point, the TypeScript example uses the OpenAI SDK directly. diff --git a/docs/evaluation/how_to_guides/evaluate_with_attachments.mdx b/docs/evaluation/how_to_guides/evaluate_with_attachments.mdx index 6581577f..96b9cbfd 100644 --- a/docs/evaluation/how_to_guides/evaluate_with_attachments.mdx +++ b/docs/evaluation/how_to_guides/evaluate_with_attachments.mdx @@ -7,7 +7,7 @@ import { # Run an evaluation with large file inputs -In addition to supporting [file attachments with traces](../../../observability/how_to_guides/tracing/upload_files_with_traces), LangSmith supports arbitrary file attachments with your examples, which you can consume when you run experiments. +In addition to supporting [file attachments with traces](../../../observability/how_to_guides/upload_files_with_traces), LangSmith supports arbitrary file attachments with your examples, which you can consume when you run experiments. This is particularly useful when working with LLM applications that require multimodal inputs or outputs. diff --git a/docs/evaluation/how_to_guides/manage_datasets_in_application.mdx b/docs/evaluation/how_to_guides/manage_datasets_in_application.mdx index c99bf18a..ee53b095 100644 --- a/docs/evaluation/how_to_guides/manage_datasets_in_application.mdx +++ b/docs/evaluation/how_to_guides/manage_datasets_in_application.mdx @@ -41,7 +41,7 @@ For the full list of available transformations, see [our reference](/reference/e :::note If you plan to collect production traces in your dataset from LangChain [ChatModels](https://python.langchain.com/docs/concepts/chat_models/) -or from OpenAI calls using the [LangSmith OpenAI wrapper](/observability/how_to_guides/tracing/annotate_code#wrap-the-openai-client), we offer a prebuilt Chat Model schema that converts messages and tools into industry standard openai formats that can be used downstream with any model for testing. You can also customize the template settings to match your use case. +or from OpenAI calls using the [LangSmith OpenAI wrapper](/observability/how_to_guides/annotate_code#wrap-the-openai-client), we offer a prebuilt Chat Model schema that converts messages and tools into industry standard openai formats that can be used downstream with any model for testing. You can also customize the template settings to match your use case. Please see the [dataset transformations reference](/reference/evaluation/dataset_transformations) for more information. ::: @@ -57,7 +57,7 @@ through our tracing projects to find the runs we want to add to the dataset. The :::tip An extremely powerful technique to build datasets is to drill-down into the most interesting traces, such as traces that were tagged with poor user feedback, and add them to a dataset. -For tips on how to filter traces, see the [filtering traces](../../../observability/how_to_guides/monitoring/filter_traces_in_application) guide. +For tips on how to filter traces, see the [filtering traces](../../../observability/how_to_guides/filter_traces_in_application) guide. ::: There are two ways to add data from tracing projects to datasets. @@ -78,7 +78,7 @@ the run before adding it to the dataset. ### Automatically add runs to a dataset -You can use [run rules](../../../observability/how_to_guides/monitoring/rules) to automatically add traces to a dataset based on certain conditions. For example, you could add all traces that have a certain tag to a dataset. +You can use [run rules](../../../observability/how_to_guides/rules) to automatically add traces to a dataset based on certain conditions. For example, you could add all traces that have a certain tag to a dataset. ### Add runs from an annotation queue diff --git a/docs/evaluation/how_to_guides/manage_datasets_programmatically.mdx b/docs/evaluation/how_to_guides/manage_datasets_programmatically.mdx index e56dfe2c..38b844cc 100644 --- a/docs/evaluation/how_to_guides/manage_datasets_programmatically.mdx +++ b/docs/evaluation/how_to_guides/manage_datasets_programmatically.mdx @@ -86,7 +86,7 @@ await client.createExamples({ ### Create a dataset from traces To create datasets from the runs (spans) of your traces, you can use the same approach. -For **many** more examples of how to fetch and filter runs, see the [export traces](../../../observability/how_to_guides/tracing/export_traces) guide. +For **many** more examples of how to fetch and filter runs, see the [export traces](../../../observability/how_to_guides/export_traces) guide. Below is an example:
Array[Message] | Converts any incoming data from LangChain's internal serialization format to OpenAI's standard message format using langchain's [convert_to_openai_messages](https://python.langchain.com/api_reference/core/messages/langchain_core.messages.utils.convert_to_openai_messages.html).



If the target field is marked as required, and no matching message is found upon entry, it will attempt to extract a message (or list of messages) from several well-known LangSmith tracing formats (e.g., any traced LangChain [BaseChatModel](https://python.langchain.com/api_reference/core/language_models/langchain_core.language_models.chat_models.BaseChatModel.html) run or traced run from the [LangSmith OpenAI wrapper](/observability/how_to_guides/tracing/annotate_code#wrap-the-openai-client)), and remove the original key containing the message. | -| convert_to_openai_tool | Array[Tool]



Only available on top level fields in the inputs dictionary. | Converts any incoming data into OpenAI standard tool formats here using langchain's [convert_to_openai_tool](https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.convert_to_openai_tool.html)



Will extract tool definitions from a run's invocation parameters if present / no tools are found at the specified key. This is useful because LangChain chat models trace tool definitions to the `extra.invocation_params` field of the run rather than inputs. | -| remove_extra_fields | Object | Removes any field not defined in the schema for this target object. | +| Transformation Type | Target Types | Functionality | +| ------------------------- | ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| remove_system_messages | Array[Message] | Filters a list of messages to remove any system messages. | +| convert_to_openai_message | Message

Array[Message] | Converts any incoming data from LangChain's internal serialization format to OpenAI's standard message format using langchain's [convert_to_openai_messages](https://python.langchain.com/api_reference/core/messages/langchain_core.messages.utils.convert_to_openai_messages.html).



If the target field is marked as required, and no matching message is found upon entry, it will attempt to extract a message (or list of messages) from several well-known LangSmith tracing formats (e.g., any traced LangChain [BaseChatModel](https://python.langchain.com/api_reference/core/language_models/langchain_core.language_models.chat_models.BaseChatModel.html) run or traced run from the [LangSmith OpenAI wrapper](/observability/how_to_guides/annotate_code#wrap-the-openai-client)), and remove the original key containing the message. | +| convert_to_openai_tool | Array[Tool]



Only available on top level fields in the inputs dictionary. | Converts any incoming data into OpenAI standard tool formats here using langchain's [convert_to_openai_tool](https://python.langchain.com/api_reference/core/utils/langchain_core.utils.function_calling.convert_to_openai_tool.html)



Will extract tool definitions from a run's invocation parameters if present / no tools are found at the specified key. This is useful because LangChain chat models trace tool definitions to the `extra.invocation_params` field of the run rather than inputs. | +| remove_extra_fields | Object | Removes any field not defined in the schema for this target object. | ## Chat Model prebuilt schema @@ -34,7 +34,7 @@ input messages when using our Chat Model schema, which will prevent you from sav The LLM run collection schema is built to collect data from LangChain [BaseChatModel](https://python.langchain.com/api_reference/core/language_models/langchain_core.language_models.chat_models.BaseChatModel.html) -runs or traced runs from the [LangSmith OpenAI wrapper](/observability/how_to_guides/tracing/annotate_code#wrap-the-openai-client). +runs or traced runs from the [LangSmith OpenAI wrapper](/observability/how_to_guides/annotate_code#wrap-the-openai-client). Please reach out to support@langchain.dev if you have an LLM run you are tracing that is not compatible and we can extend support. diff --git a/docs/self_hosting/release_notes.mdx b/docs/self_hosting/release_notes.mdx index adf7bfc2..54880659 100644 --- a/docs/self_hosting/release_notes.mdx +++ b/docs/self_hosting/release_notes.mdx @@ -81,7 +81,7 @@ LangSmith v0.6 improves run rules performance and reliability, adds support for - Workspaces in LangSmith for improved collaboration & organization. [Learn More...](https://blog.langchain.dev/week-of-6-10-langchain-release-notes/#workspaces) - Enter the playground from scratch instead of from a trace or a prompt. [Learn More...](https://blog.langchain.dev/week-of-6-10-langchain-release-notes/#playground-from-scratch) - Variable mapping for online evaluator prompts. [Learn More...](https://blog.langchain.dev/week-of-6-10-langchain-release-notes/#variable-mapping) -- Custom Model support in Playground. [Learn More...](https://docs.smith.langchain.com/how_to_guides/playground/custom_endpoint) +- Custom Model support in Playground. [Learn More...](https://docs.smith.langchain.com/how_to_guides/custom_endpoint) ### Performance and Reliability Changes @@ -101,7 +101,7 @@ LangSmith v0.6 improves run rules performance and reliability, adds support for - Added support for Workspaces. See the [Admin concepts guide](/administration/concepts#workspaces) for more details. - Added global setting `orgCreationDisabled` to `values.yaml` to disable creation of new Organizations. -- Added support for custom TLS certificates for the for the Azure OpenAI model provider. See the [how-to guide](../prompt_engineering/how_to_guides/playground/custom_tls_certificates) for more details. +- Added support for custom TLS certificates for the for the Azure OpenAI model provider. See the [how-to guide](../prompt_engineering/how_to_guides/custom_tls_certificates) for more details. ### Deprecation notices From 8cfc9248f52888ab357b60fc9e617cbeddd7e791 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Fri, 31 Jan 2025 09:20:33 -0800 Subject: [PATCH 4/6] links --- docs/observability/how_to_guides/add_metadata_tags.mdx | 2 +- docs/observability/how_to_guides/export_traces.mdx | 2 +- .../how_to_guides/filter_traces_in_application.mdx | 4 ++-- docs/observability/how_to_guides/threads.mdx | 2 +- .../observability/how_to_guides/use_monitoring_charts.mdx | 2 +- docs/observability/index.mdx | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/observability/how_to_guides/add_metadata_tags.mdx b/docs/observability/how_to_guides/add_metadata_tags.mdx index e297c6d0..3917ada9 100644 --- a/docs/observability/how_to_guides/add_metadata_tags.mdx +++ b/docs/observability/how_to_guides/add_metadata_tags.mdx @@ -15,7 +15,7 @@ LangSmith supports sending arbitrary metadata and tags along with traces. Tags are strings that can be used to categorize or label a trace. Metadata is a dictionary of key-value pairs that can be used to store additional information about a trace. Both are useful for associating additional information with a trace, such as the environment in which it was executed, the user who initiated it, or an internal correlation ID. -For more information on tags and metadata, see the [Concepts](/observability/concepts#tags) page. For information on how to query traces and runs by metadata and tags, see the [Filter traces in the application](../monitoring/filter_traces_in_application) page. +For more information on tags and metadata, see the [Concepts](/observability/concepts#tags) page. For information on how to query traces and runs by metadata and tags, see the [Filter traces in the application](../filter_traces_in_application) page. Date: Fri, 31 Jan 2025 11:49:07 -0800 Subject: [PATCH 5/6] links --- docs/evaluation/how_to_guides/index.md | 6 +++--- docs/index.mdx | 4 ++-- .../how_to_guides/add_metadata_tags.mdx | 2 +- .../how_to_guides/filter_traces_in_application.mdx | 2 +- .../how_to_guides/trace_with_langchain.mdx | 2 +- docs/prompt_engineering/how_to_guides/index.md | 12 ++++++------ .../how_to_guides/managing_model_configurations.mdx | 4 ++-- .../prompt_engineering/quickstarts/quickstart_ui.mdx | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/evaluation/how_to_guides/index.md b/docs/evaluation/how_to_guides/index.md index bfb8faee..df83996b 100644 --- a/docs/evaluation/how_to_guides/index.md +++ b/docs/evaluation/how_to_guides/index.md @@ -42,7 +42,7 @@ Evaluate and improve your application before deploying it. - [Evaluate with repetitions](./how_to_guides/repetition) - [Handle model rate limits](./how_to_guides/rate_limiting) -- [Print detailed logs (Python only)](../../observability/how_to_guides/tracing/output_detailed_logs) +- [Print detailed logs (Python only)](../../observability/how_to_guides/output_detailed_logs) - [Run an evaluation locally (beta, Python only)](./how_to_guides/local) ## Testing integrations @@ -56,8 +56,8 @@ Run evals using your favorite testing tools: Evaluate and monitor your system's live performance on production data. -- [Set up an LLM-as-judge online evaluator](../../observability/how_to_guides/monitoring/online_evaluations#configure-llm-as-judge-evaluators) -- [Set up a custom code online evaluator](../../observability/how_to_guides/monitoring/online_evaluations#configure-custom-code-evaluators) +- [Set up an LLM-as-judge online evaluator](../../observability/how_to_guides/online_evaluations#configure-llm-as-judge-evaluators) +- [Set up a custom code online evaluator](../../observability/how_to_guides/online_evaluations#configure-custom-code-evaluators) - [Create a few-shot evaluator](./how_to_guides/create_few_shot_evaluators) ## Automatic evaluation diff --git a/docs/index.mdx b/docs/index.mdx index f38f08d8..fa00ceef 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -33,7 +33,7 @@ It allows you to closely monitor and evaluate your application, so you can ship LangSmith integrates seamlessly with LangChain's open source frameworks [`langchain`](https://python.langchain.com) and [`langgraph`](https://langchain-ai.github.io/langgraph/), with no extra instrumentation needed. -If you're already using either of these, see the how-to guide for [setting up LangSmith with LangChain](./observability/how_to_guides/tracing/trace_with_langchain) or [setting up LangSmith with LangGraph](https://docs.smith.langchain.com/observability/how_to_guides/tracing/trace_with_langgraph). +If you're already using either of these, see the how-to guide for [setting up LangSmith with LangChain](./observability/how_to_guides/trace_with_langchain) or [setting up LangSmith with LangGraph](https://docs.smith.langchain.com/observability/how_to_guides/tracing/trace_with_langgraph). ::: ## Observability @@ -43,7 +43,7 @@ Observability is important for any software application, but especially so for L This is where LangSmith can help! LangSmith has LLM-native observability, allowing you to get meaningful insights from your application. LangSmith’s observability features have you covered throughout all stages of application development - from prototyping, to beta testing, to production. - Get started by [adding tracing](./observability) to your application. -- [Create dashboards](./observability/how_to_guides/monitoring/dashboards) to view key metrics like RPS, error rates and costs. +- [Create dashboards](./observability/how_to_guides/dashboards) to view key metrics like RPS, error rates and costs. ## Evals diff --git a/docs/observability/how_to_guides/add_metadata_tags.mdx b/docs/observability/how_to_guides/add_metadata_tags.mdx index 3917ada9..cf23d6c1 100644 --- a/docs/observability/how_to_guides/add_metadata_tags.mdx +++ b/docs/observability/how_to_guides/add_metadata_tags.mdx @@ -15,7 +15,7 @@ LangSmith supports sending arbitrary metadata and tags along with traces. Tags are strings that can be used to categorize or label a trace. Metadata is a dictionary of key-value pairs that can be used to store additional information about a trace. Both are useful for associating additional information with a trace, such as the environment in which it was executed, the user who initiated it, or an internal correlation ID. -For more information on tags and metadata, see the [Concepts](/observability/concepts#tags) page. For information on how to query traces and runs by metadata and tags, see the [Filter traces in the application](../filter_traces_in_application) page. +For more information on tags and metadata, see the [Concepts](/observability/concepts#tags) page. For information on how to query traces and runs by metadata and tags, see the [Filter traces in the application](./filter_traces_in_application) page. Date: Fri, 31 Jan 2025 11:58:06 -0800 Subject: [PATCH 6/6] links --- docs/index.mdx | 2 +- .../how_to_guides/custom_endpoint.mdx | 46 ++++++++++++++++++ .../{playground => }/prompt_canvas.mdx | 0 .../prompt_canvas_custom_quick_action.gif | Bin .../static/prompt_canvas_diff.gif | Bin .../static/prompt_canvas_open.gif | Bin .../static/prompt_canvas_quick_actions.gif | Bin .../static/prompt_canvas_rewrite.gif | Bin .../static/prompt_canvas_save.gif | Bin 9 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 docs/prompt_engineering/how_to_guides/custom_endpoint.mdx rename docs/prompt_engineering/how_to_guides/{playground => }/prompt_canvas.mdx (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_custom_quick_action.gif (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_diff.gif (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_open.gif (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_quick_actions.gif (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_rewrite.gif (100%) rename docs/prompt_engineering/how_to_guides/{playground => }/static/prompt_canvas_save.gif (100%) diff --git a/docs/index.mdx b/docs/index.mdx index fa00ceef..bc59418b 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -60,4 +60,4 @@ While traditional software applications are built by writing code, AI applicatio - Get started by [creating your first prompt](./prompt_engineering/tutorials/optimize_classifier). - Iterate on models and prompts using the [Playground](./prompt_engineering/how_to_guides#playground). -- [Manage prompts programmatically](https://docs.smith.langchain.com/prompt_engineering/how_to_guides/prompts/manage_prompts_programatically) in your application. \ No newline at end of file +- [Manage prompts programmatically](https://docs.smith.langchain.com/prompt_engineering/how_to_guides/prompts/manage_prompts_programatically) in your application. diff --git a/docs/prompt_engineering/how_to_guides/custom_endpoint.mdx b/docs/prompt_engineering/how_to_guides/custom_endpoint.mdx new file mode 100644 index 00000000..141a368c --- /dev/null +++ b/docs/prompt_engineering/how_to_guides/custom_endpoint.mdx @@ -0,0 +1,46 @@ +--- +sidebar_position: 1 +--- + +# Run the playground against a custom LangServe model server + +The LangSmith playground allows you to use your own custom models. You can deploy a model server that exposes your model's API via [LangServe](https://github.com/langchain-ai/langserve), an open source library for serving LangChain applications. +Behind the scenes, the playground will interact with your model server to generate responses. + +## Deploy a custom model server + +For your convenience, we have provided a sample model server that you can use as a reference. You can find the sample model server [here](https://github.com/langchain-ai/langsmith-model-server) +We highly recommend using the sample model server as a starting point. + +Depending on your model is an instruct-style or chat-style model, you will need to implement either `custom_model.py` or `custom_chat_model.py` respectively. + +## Adding configurable fields + +It is often useful to configure your model with different parameters. These might include temperature, model_name, max_tokens, etc. + +To make your model configurable in the LangSmith playground, you need to add configurable fields to your model server. These fields can be used to change model parameters from the playground. + +You can add configurable fields by implementing the `with_configurable_fields` function in the `config.py` file. You can + +```python +def with_configurable_fields(self) -> Runnable: + """Expose fields you want to be configurable in the playground. We will automatically expose these to the + playground. If you don't want to expose any fields, you can remove this method.""" + return self.configurable_fields(n=ConfigurableField( + id="n", + name="Num Characters", + description="Number of characters to return from the input prompt.", + )) +``` + +## Use the model in the LangSmith Playground + +Once you have deployed a model server, you can use it in the LangSmith Playground. Enter the playground and select either the `ChatCustomModel` or the `CustomModel` provider for chat-style model or instruct-style models. + +Enter the `URL`. The playground will automatically detect the available endpoints and configurable fields. You can then invoke the model with the desired parameters. + +![ChatCustomModel in Playground](./static//playground_custom_model.png) + +If everything is set up correctly, you should see the model's response in the playground as well as the configurable fields specified in the `with_configurable_fields`. + +See how to store your model configuration for later use [here](./managing_model_configurations). diff --git a/docs/prompt_engineering/how_to_guides/playground/prompt_canvas.mdx b/docs/prompt_engineering/how_to_guides/prompt_canvas.mdx similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/prompt_canvas.mdx rename to docs/prompt_engineering/how_to_guides/prompt_canvas.mdx diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_custom_quick_action.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_custom_quick_action.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_custom_quick_action.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_custom_quick_action.gif diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_diff.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_diff.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_diff.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_diff.gif diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_open.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_open.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_open.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_open.gif diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_quick_actions.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_quick_actions.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_quick_actions.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_quick_actions.gif diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_rewrite.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_rewrite.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_rewrite.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_rewrite.gif diff --git a/docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_save.gif b/docs/prompt_engineering/how_to_guides/static/prompt_canvas_save.gif similarity index 100% rename from docs/prompt_engineering/how_to_guides/playground/static/prompt_canvas_save.gif rename to docs/prompt_engineering/how_to_guides/static/prompt_canvas_save.gif