Merge branch 'main' into ido/opik-1386-guardrails-ingestion

Author: andrescrz

Diff for: .github/workflows/span_cost_upload_daily.yml

@@ -0,0 +1,46 @@
+name: Update Span Cost Daily
+
+on:
+  schedule:
+    - cron: '0 0 * * *' # Runs daily at midnight UTC
+  workflow_dispatch: # Allows manual trigger
+
+jobs:
+  update-span-cost:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        # with:
+        #   persist-credentials: false # Prevent issues with GitHub token
+
+      - name: Set branch name and update the file
+        run: |
+          echo "BRANCH_NAME=span-cost-update-$(date +%Y-%m-%d)" >> $GITHUB_ENV
+          curl -o apps/opik-backend/src/main/resources/model_prices_and_context_window.json https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json
+
+      - name: Create branch
+        uses: JosiahSiegel/[email protected]
+        with:
+          branch: ${{ env.BRANCH_NAME }}
+
+      - name: Push updated file
+        uses: comet-ml/push-files@main
+        with:
+          branch: ${{ env.BRANCH_NAME }}
+          files: apps/opik-backend/src/main/resources/model_prices_and_context_window.json:apps/opik-backend/src/main/resources/model_prices_and_context_window.json
+          github_token: ${{ secrets.GH_PAT_TO_ACCESS_GITHUB_API }}
+          repository: 'comet-ml/opik'
+          commit_user_email: [email protected]
+          commit_user_name: 'Github Actions (${{ github.actor }})'
+          commit_message: "Update span cost file"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ env.BRANCH_NAME }}
+          title: "Update span cost from LiteLLM"
+          body: "Automated update of model_prices_and_context_window.json file, which includes span prices information from the LiteLLM repo."
+          base: main

Diff for: .gitignore

@@ -63,3 +63,9 @@ deployment/helm_chart/opik/charts
 deployment/helm_chart/opik/values-*.yaml
 temp
 opik-sandbox-executor-python.tar.gz
+
+# init
+.opik_install_reported
+
+# Update span cost workflow
+response.txt

Diff for: apps/opik-backend/src/main/java/com/comet/opik/api/Identifier.java renamed to apps/opik-backend/src/main/java/com/comet/opik/api/ExperimentStreamRequest.java

@@ -3,11 +3,23 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import jakarta.validation.constraints.Max;
+import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotBlank;
 import lombok.Builder;
 
+import java.util.UUID;
+
 @Builder(toBuilder = true)
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
-public record Identifier(@NotBlank String name) {
+public record ExperimentStreamRequest(
+        @NotBlank String name,
+        @Min(1) @Max(2000) Integer limit,
+        UUID lastRetrievedId) {
+
+    @Override
+    public Integer limit() {
+        return limit == null ? 500 : limit;
+    }
 }

Diff for: apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ExperimentsResource.java

@@ -8,10 +8,10 @@
 import com.comet.opik.api.ExperimentItemsBatch;
 import com.comet.opik.api.ExperimentItemsDelete;
 import com.comet.opik.api.ExperimentSearchCriteria;
+import com.comet.opik.api.ExperimentStreamRequest;
 import com.comet.opik.api.ExperimentsDelete;
 import com.comet.opik.api.FeedbackDefinition;
 import com.comet.opik.api.FeedbackScoreNames;
-import com.comet.opik.api.Identifier;
 import com.comet.opik.api.resources.v1.priv.validate.IdParamsValidator;
 import com.comet.opik.api.sorting.ExperimentSortingFactory;
 import com.comet.opik.api.sorting.SortingField;
@@ -190,25 +190,26 @@ public Response deleteExperimentsById(
     }
 
     @POST
-    @Path("/retrieve")
-    @Operation(operationId = "getExperimentByName", summary = "Get experiment by name", description = "Get experiment by name", responses = {
-            @ApiResponse(responseCode = "200", description = "Experiments resource", content = @Content(schema = @Schema(implementation = Experiment.class))),
-            @ApiResponse(responseCode = "404", description = "Not found", content = @Content(schema = @Schema(implementation = ErrorMessage.class)))
+    @Path("/stream")
+    @Produces(MediaType.APPLICATION_OCTET_STREAM)
+    @Operation(operationId = "streamExperiments", summary = "Stream experiments", description = "Stream experiments", responses = {
+            @ApiResponse(responseCode = "200", description = "Experiments stream or error during process", content = @Content(array = @ArraySchema(schema = @Schema(anyOf = {
+                    Experiment.class,
+                    ErrorMessage.class
+            }), maxItems = 2000)))
     })
     @JsonView(Experiment.View.Public.class)
-    public Response getExperimentByName(
-            @RequestBody(content = @Content(schema = @Schema(implementation = Identifier.class))) @NotNull @Valid Identifier identifier) {
-
-        String workspaceId = requestContext.get().getWorkspaceId();
-        String name = identifier.name();
-
-        log.info("Finding experiment by name '{}' on workspace_id '{}'", name, workspaceId);
-        var experiment = experimentService.getByName(name)
-                .contextWrite(ctx -> setRequestContext(ctx, requestContext))
-                .block();
-        log.info("Found experiment by name '{}' on workspace_id '{}'", name, workspaceId);
-
-        return Response.ok(experiment).build();
+    public ChunkedOutput<JsonNode> streamExperiments(
+            @RequestBody(content = @Content(schema = @Schema(implementation = ExperimentStreamRequest.class))) @NotNull @Valid ExperimentStreamRequest request) {
+        var workspaceId = requestContext.get().getWorkspaceId();
+        var userName = requestContext.get().getUserName();
+        log.info("Streaming experiments by '{}', workspaceId '{}', userName '{}'", request, workspaceId, userName);
+        var experiments = experimentService.get(request)
+                .contextWrite(ctx -> ctx.put(RequestContext.USER_NAME, userName)
+                        .put(RequestContext.WORKSPACE_ID, workspaceId));
+        var stream = streamer.getOutputStream(experiments);
+        log.info("Streamed experiments by '{}', workspaceId '{}', userName '{}'", request, workspaceId, userName);
+        return stream;
     }
 
     // Experiment Item Resources

Diff for: apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ServiceTogglesResource.java

@@ -2,8 +2,11 @@
 
 import com.codahale.metrics.annotation.Timed;
 import com.comet.opik.infrastructure.OpikConfiguration;
-import io.swagger.v3.oas.annotations.Hidden;
+import com.comet.opik.infrastructure.ServiceTogglesConfig;
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import jakarta.inject.Inject;
 import jakarta.ws.rs.GET;
@@ -20,14 +23,14 @@
 @Timed
 @Slf4j
 @RequiredArgsConstructor(onConstructor_ = @Inject)
-@Tag(name = "Service Toggles", description = "Service Toggles")
+@Tag(name = "Service Toggles", description = "Service Toggles resources")
 public class ServiceTogglesResource {
 
     private final @NonNull OpikConfiguration config;
 
     @GET
-    @Operation(operationId = "getServiceToggles", summary = "Get Service Toggles", description = "Get Service Toggles")
-    @Hidden
+    @Operation(operationId = "getServiceToggles", summary = "Get Service Toggles", description = "Get Service Toggles", responses = {
+            @ApiResponse(responseCode = "200", description = "Service Toggles", content = @Content(schema = @Schema(implementation = ServiceTogglesConfig.class)))})
     public Response getToggles() {
         return Response.ok()
                 .entity(config.getServiceToggles())

Diff for: apps/opik-backend/src/main/java/com/comet/opik/domain/AutomationRuleEvaluatorService.java

@@ -10,7 +10,7 @@
 import com.comet.opik.api.LogCriteria;
 import com.comet.opik.api.error.EntityAlreadyExistsException;
 import com.comet.opik.api.error.ErrorMessage;
-import com.comet.opik.infrastructure.ServiceTogglesConfig;
+import com.comet.opik.infrastructure.OpikConfiguration;
 import com.comet.opik.infrastructure.cache.CacheEvict;
 import com.comet.opik.infrastructure.cache.Cacheable;
 import com.google.inject.ImplementedBy;
@@ -24,7 +24,6 @@
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.core.statement.UnableToExecuteStatementException;
 import reactor.core.publisher.Mono;
-import ru.vyarus.dropwizard.guice.module.yaml.bind.Config;
 import ru.vyarus.guicey.jdbi3.tx.TransactionTemplate;
 
 import java.sql.SQLIntegrityConstraintViolationException;
@@ -70,7 +69,7 @@ class AutomationRuleEvaluatorServiceImpl implements AutomationRuleEvaluatorServi
     private final @NonNull IdGenerator idGenerator;
     private final @NonNull TransactionTemplate template;
     private final @NonNull AutomationRuleEvaluatorLogsDAO logsDAO;
-    private final @NonNull @Config("serviceToggles") ServiceTogglesConfig serviceTogglesConfig;
+    private final @NonNull OpikConfiguration opikConfiguration;
 
     @Override
     @CacheEvict(name = "automation_rule_evaluators_find_all", key = "$projectId + '-' + $workspaceId")
@@ -97,7 +96,7 @@ public <E, T extends AutomationRuleEvaluator<E>> T save(@NonNull T inputRuleEval
                     yield AutomationModelEvaluatorMapper.INSTANCE.map(definition);
                 }
                 case AutomationRuleEvaluatorUserDefinedMetricPython userDefinedMetricPython -> {
-                    if (!serviceTogglesConfig.isPythonEvaluatorEnabled()) {
+                    if (!opikConfiguration.getServiceToggles().isPythonEvaluatorEnabled()) {
                         throw new ServerErrorException("Python evaluator is disabled", 501);
                     }
                     var definition = userDefinedMetricPython.toBuilder()
@@ -153,7 +152,7 @@ public void update(@NonNull UUID id, @NonNull UUID projectId, @NonNull String wo
                                 .lastUpdatedBy(userName)
                                 .build();
                     case AutomationRuleEvaluatorUpdateUserDefinedMetricPython evaluatorUpdateUserDefinedMetricPython -> {
-                        if (!serviceTogglesConfig.isPythonEvaluatorEnabled()) {
+                        if (!opikConfiguration.getServiceToggles().isPythonEvaluatorEnabled()) {
                             throw new ServerErrorException("Python evaluator is disabled", 501);
                         }
                         yield UserDefinedMetricPythonAutomationRuleEvaluatorModel.builder()