langfuse and local adapters for trace

lib/llm_eval_ruby/api_clients/langfuse.rb

@@ -71,7 +71,8 @@ def update_generation(params = {})
         body = {
           id: params[:id],
           output: params[:output],
-          endTime: params[:end_time]
+          endTime: params[:end_time],
+          usage: convert_keys_to_camel_case(params[:usage])
         }
         create_event(type: "generation-update", body:)
       end
@@ -91,6 +92,15 @@ def create_event(type:, body:)
 
         self.class.post("/ingestion", body: payload.to_json)
       end
+
+      private
+
+      def convert_keys_to_camel_case(hash)
+        hash.each_with_object({}) do |(key, value), new_hash|
+          camel_case_key = key.gsub(/_([a-z])/) { ::Regexp.last_match(1).upcase }
+          new_hash[camel_case_key] = value
+        end
+      end
     end
   end
 end

lib/llm_eval_ruby/prompt_types/base.rb

@@ -7,6 +7,7 @@ class Base
 
       def initialize(adapter:, content:, role:)
         @adapter = adapter
+        @adapter = adapter.safe_constantize if adapter.is_a?(String)
         @role = role
         @content = content
       end

lib/llm_eval_ruby/trace_adapters/base.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module LlmEvalRuby
+  module TraceAdapters
+    class Base
+    end
+  end
+end

lib/llm_eval_ruby/trace_adapters/langfuse.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../api_clients/langfuse"
+require_relative "../trace_types"
+
+module LlmEvalRuby
+  module TraceAdapters
+    class Langfuse < Base
+      class << self
+        def trace(**)
+          trace = TraceTypes::Trace.new(id: SecureRandom.uuid, **)
+          response = client.create_trace(trace.to_h)
+
+          Rails.logger.warn "Failed to create generation" if response["successes"].blank?
+
+          trace
+        end
+
+        def span(**)
+          span = TraceTypes::Span.new(id: SecureRandom.uuid, **)
+          response = client.create_span(span.to_h)
+
+          Rails.logger.warn "Failed to create span" if response["successes"].blank?
+
+          if block_given?
+            result = yield
+
+            span.end_time = Time.now.utc.iso8601
+            span.output = result
+
+            client.update_span(span.to_h)
+          else
+            span
+          end
+        end
+
+        def update_generation(**)
+          generation = TraceTypes::Generation.new(**)
+          response = client.update_generation(generation.to_h)
+
+          Rails.logger.warn "Failed to create generation" if response["successes"].blank?
+
+          generation
+        end
+
+        def generation(**)
+          generation = TraceTypes::Generation.new(id: SecureRandom.uuid, tracer: self, **)
+          response = client.create_generation(generation.to_h)
+          Rails.logger.warn "Failed to create generation" if response["successes"].blank?
+
+          if block_given?
+            result = yield generation
+
+            generation.end_time = Time.now.utc.iso8601
+            generation.output = result.dig("choices", 0, "message", "content")
+            generation.usage = result["usage"]
+
+            client.update_generation(generation.to_h)
+          else
+            generation
+          end
+        end
+
+        private
+
+        def client
+          @client ||= ApiClients::Langfuse.new(**LlmEvalRuby.config.langfuse_options)
+        end
+      end
+    end
+  end
+end

lib/llm_eval_ruby/trace_adapters/local.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module LlmEvalRuby
+  module TraceAdapters
+    class Local < Base
+      class << self
+        def trace(**)
+          trace = TraceTypes::Trace.new(id: SecureRandom.uuid, **)
+
+          logger.info("Trace created: #{JSON.pretty_generate(trace.to_h)}")
+
+          trace
+        end
+
+        def span(**)
+          span = TraceTypes::Span.new(id: SecureRandom.uuid, **)
+
+          logger.info("Span created: #{JSON.pretty_generate(span.to_h)}")
+
+          if block_given?
+            result = yield
+
+            span.end_time = Time.now.utc.iso8601
+            span.output = result
+
+            logger.info("Span updated: #{JSON.pretty_generate(span.to_h)}")
+          else
+            span
+          end
+        end
+
+        def update_generation(**)
+          generation = TraceTypes::Generation.new(**)
+
+          logger.info("Generation updated: #{JSON.pretty_generate(generation.to_h)}")
+
+          generation
+        end
+
+        def generation(**)
+          generation = TraceTypes::Generation.new(id: SecureRandom.uuid, tracer: self, **)
+
+          logger.info("Generation created: #{JSON.pretty_generate(generation.to_h)}")
+
+          if block_given?
+            result = yield
+
+            generation.end_time = Time.now.utc.iso8601
+            generation.output = result
+
+            logger.info("Generation updated: #{JSON.pretty_generate(generation.to_h)}")
+          else
+            generation
+          end
+        end
+
+        def logger
+          @logger ||= ActiveSupport::Logger.new(LlmEvalRuby.config.local_options[:traces_path])
+        end
+      end
+    end
+  end
+end

lib/llm_eval_ruby/trace_types.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module LlmEvalRuby
+  module TraceTypes
+    Trace = Struct.new(:id, :name, :session_id, keyword_init: true)
+
+    Span = Struct.new(:id, :name, :trace_id, :output, :end_time, keyword_init: true)
+
+    Generation = Struct.new(:tracer,
+                            :id,
+                            :name,
+                            :trace_id,
+                            :input,
+                            :output,
+                            :end_time,
+                            :prompt_name,
+                            :prompt_version,
+                            :usage,
+                            keyword_init: true) do
+      def end(output:, usage: nil)
+        self.output = output
+        self.end_time = Time.now.utc.iso8601
+        self.usage = convert_keys_to_camel_case(usage) if usage
+
+        tracer.update_generation(**to_h)
+      end
+
+      def convert_keys_to_camel_case(hash)
+        hash.each_with_object({}) do |(key, value), new_hash|
+          camel_case_key = key.gsub(/_([a-z])/) { ::Regexp.last_match(1).upcase }
+          new_hash[camel_case_key] = value
+        end
+      end
+    end
+  end
+end

lib/llm_eval_ruby/tracer.rb

@@ -1,85 +1,52 @@
 # frozen_string_literal: true
 
+require_relative "trace_adapters/langfuse"
+require_relative "trace_adapters/local"
 module LlmEvalRuby
   class Tracer
-    attr_reader :client
+    attr_reader :adapter
 
-    Generation = Struct.new(:tracer,
-                            :id,
-                            :name,
-                            :trace_id,
-                            :input,
-                            :output,
-                            :end_time,
-                            :prompt_name,
-                            :prompt_version,
-                            keyword_init: true) do
-      def end(output:)
-        self.output = output
-        self.end_time = Time.now.utc.iso8601
-
-        tracer.update_generation(**to_h)
-      end
+    def self.trace(...)
+      new(adapter: LlmEvalRuby.config.adapter).trace(...)
     end
 
-    Trace = Struct.new(:id, :name, :session_id, keyword_init: true)
-    Span = Struct.new(:id, :name, :trace_id, :output, :end_time, keyword_init: true)
-
-    def initialize(client = ApiClients::Langfuse.new(**LlmEvalRuby.config.langfuse_options))
-      @client = client
+    def self.span(...)
+      new(adapter: LlmEvalRuby.config.adapter).span(...)
     end
 
-    def trace(**)
-      trace = Trace.new(id: SecureRandom.uuid, **)
-      response = client.create_trace(trace.to_h)
-
-      Rails.logger.warn "Failed to create generation" if response["successes"].blank?
-
-      trace
+    def self.generation(...)
+      new(adapter: LlmEvalRuby.config.adapter).generation(...)
     end
 
-    def span(**)
-      span = Span.new(id: SecureRandom.uuid, **)
-      response = client.create_span(span.to_h)
-
-      Rails.logger.warn "Failed to create span" if response["successes"].blank?
-
-      if block_given?
-        result = yield
-
-        span.end_time = Time.now.utc.iso8601
-        span.output = result
+    def self.update_generation(...)
+      new(adapter: LlmEvalRuby.config.adapter).update_generation(...)
+    end
 
-        client.update_span(span.to_h)
+    def initialize(adapter:)
+      case adapter
+      when :langfuse
+        @adapter = TraceAdapters::Langfuse
+      when :local
+        @adapter = TraceAdapters::Local
       else
-        span
+        raise "Unsupported adapter #{adapter}"
       end
     end
 
-    def update_generation(**)
-      generation = Generation.new(**)
-      response = client.update_generation(generation.to_h)
-
-      Rails.logger.warn "Failed to create generation" if response["successes"].blank?
-
-      generation
+    def trace(...)
+      adapter.trace(...)
     end
 
-    def generation(**)
-      generation = Generation.new(id: SecureRandom.uuid, tracer: self, **)
-      response = client.create_generation(generation.to_h)
-      Rails.logger.warn "Failed to create generation" if response["successes"].blank?
-
-      if block_given?
-        result = yield
+    def span(...)
+      adapter.span(...)
+    end
 
-        generation.end_time = Time.now.utc.iso8601
-        generation.output = result
+    def generation(...)
+      adapter.generation(...)
+    end
 
-        client.update_generation(generation.to_h)
-      else
-        generation
-      end
+    def update_generation(...)
+      adapter.update_generation(...)
     end
   end
 end