Merge 0c1d0f2 into 3d45056

Author: what-the-fawk

ydb/core/fq/libs/http_api_client/http_client.py

@@ -11,7 +11,7 @@
 
 from .query_results import YQResults
 
-MAX_RETRY_FOR_SESSION = 4
+MAX_RETRY_FOR_SESSION = 100
 BACK_OFF_FACTOR = 0.3
 TIME_BETWEEN_RETRIES = 1000
 ERROR_CODES = (500, 502, 504)
@@ -149,6 +149,21 @@ def create_query(
 
         self._validate_http_error(response, expected_code=expected_code)
         return response.json()["id"]
+    
+    def start_query(
+        self,
+        query_id: str,
+        idempotency_key: str | None = None,
+        expected_code: int = 204,
+    ):
+        response = self.session.get(
+            self._compose_api_url(f"/api/fq/v1/queries/{query_id}/start"),
+            headers=self._build_headers(idempotency_key=idempotency_key),
+            params=self._build_params(),
+        )
+
+        self._validate_http_error(response, expected_code)
+        return response.json()["status"]
 
     def get_query_status(self, query_id, request_id=None, expected_code=200) -> Any:
         response = self.session.get(

ydb/core/public_http/fq_handlers.h

@@ -10,6 +10,8 @@
 #include <ydb/library/services/services.pb.h>
 #include <ydb/core/public_http/protos/fq.pb.h>
 
+#include <type_traits>
+
 namespace NKikimr::NPublicHttp {
 
 using namespace NActors;
@@ -249,6 +251,7 @@ void SetIdempotencyKey(T& dst, const TString& key) {
 
 template <typename GrpcProtoRequestType, typename HttpProtoRequestType, typename GrpcProtoResultType, typename HttpProtoResultType, typename GrpcProtoResponseType>
 class TGrpcCallWrapper : public TActorBootstrapped<TGrpcCallWrapper<GrpcProtoRequestType, HttpProtoRequestType, GrpcProtoResultType, HttpProtoResultType, GrpcProtoResponseType>> {
+protected:
     THttpRequestContext RequestContext;
 
     typedef std::function<std::unique_ptr<NGRpcService::TEvProxyRuntimeEvent>(TIntrusivePtr<NYdbGrpc::IRequestContextBase> ctx)> TGrpcProxyEventFactory;
@@ -278,6 +281,11 @@ class TGrpcCallWrapper : public TActorBootstrapped<TGrpcCallWrapper<GrpcProtoReq
     }
 
     void Bootstrap(const TActorContext& ctx) {
+        Bootstrap_wrapper(ctx);
+    }
+
+    virtual void Bootstrap_wrapper(const TActorContext& ctx) {
+        std::cerr << "Call base\n";
         auto grpcRequest = std::make_unique<TGrpcProtoRequestType>();
         if (Parse(*grpcRequest)) {
             TIntrusivePtr<TGrpcRequestContextWrapper> requestContext = new TGrpcRequestContextWrapper(RequestContext, std::move(grpcRequest), &SendReply);
@@ -349,14 +357,13 @@ class TGrpcCallWrapper : public TActorBootstrapped<TGrpcCallWrapper<GrpcProtoReq
         Y_ABORT_UNLESS(resp->GetArena());
         Y_UNUSED(status);
         auto* typedResponse = static_cast<TGrpcProtoResponseType*>(resp);
-        if (!typedResponse->operation().result().template Is<TGrpcProtoResultType>()) {
-            TStringStream json;
-            auto* httpResult = google::protobuf::Arena::CreateMessage<FQHttp::Error>(resp->GetArena());
-            FqConvert(typedResponse->operation(), *httpResult);
-            FqPackToJson(json, *httpResult, jsonSettings);
-
-            requestContext.ResponseBadRequestJson(typedResponse->operation().status(), json.Str());
-            return;
+        if (!typedResponse->operation().result().template Is<TGrpcProtoResultType>()) {                                   
+            TStringStream json;                                                                             
+            auto* httpResult = google::protobuf::Arena::CreateMessage<FQHttp::Error>(resp->GetArena());     
+            FqConvert(typedResponse->operation(), *httpResult);                                             
+            FqPackToJson(json, *httpResult, jsonSettings);                                                  
+            requestContext.ResponseBadRequestJson(typedResponse->operation().status(), json.Str());         
+            return;                                                                                         
         }
 
         auto* grpcResult = google::protobuf::Arena::CreateMessage<TGrpcProtoResultType>(resp->GetArena());
@@ -396,4 +403,88 @@ DECLARE_YQ_GRPC_ACTOR(GetQueryStatus, GetQueryStatus);
 DECLARE_YQ_GRPC_ACTOR_WIHT_EMPTY_RESULT(StopQuery, ControlQuery);
 DECLARE_YQ_GRPC_ACTOR(GetResultData, GetResultData);
 
+#define TGrpcCallWrapperBase TGrpcCallWrapper<FederatedQuery::DescribeQueryRequest, FQHttp::GetQueryRequest, FederatedQuery::ModifyQueryResult, google::protobuf::Empty, FederatedQuery::ModifyQueryResponse>
+
+class TStartQueryRequest : public TGrpcCallWrapperBase {
+public:
+    TStartQueryRequest(const THttpRequestContext& ctx)
+    : TGrpcCallWrapperBase(ctx, &NGRpcService::CreateFederatedQueryDescribeQueryRequestOperationCall)
+    {}
+
+    void Bootstrap_wrapper(const TActorContext& ctx) override {
+        std::cerr << "Call derived\n";
+
+        auto describeRequest = std::make_unique<FederatedQuery::DescribeQueryRequest>();
+        if (!Parse(*describeRequest)) {
+            this->Die(ctx);
+            return;
+        }
+
+        TIntrusivePtr<TGrpcRequestContextWrapper> requestContext = new TGrpcRequestContextWrapper(
+            RequestContext,
+            std::move(describeRequest),
+            [&, this, query_id = describeRequest->Getquery_id()](const THttpRequestContext& requestContext, const TJsonSettings& jsonSettings, NProtoBuf::Message* resp, ui32 status) {
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response-1"); return;
+            Y_ABORT_UNLESS(resp);
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response-2"); return;
+            Y_ABORT_UNLESS(resp->GetArena());
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response-3"); return;
+
+            auto* typedResponse = static_cast<FederatedQuery::DescribeQueryResponse*>(resp);
+            if (!typedResponse->operation().result().template Is<FederatedQuery::DescribeQueryResult>()) {                                   
+                TStringStream json;                                                                             
+                auto* httpResult = google::protobuf::Arena::CreateMessage<FQHttp::Error>(resp->GetArena());     
+                FqConvert(typedResponse->operation(), *httpResult);                                             
+                FqPackToJson(json, *httpResult, jsonSettings);                                                  
+                requestContext.ResponseBadRequestJson(typedResponse->operation().status(), json.Str());
+                this->Die(ctx);       
+                return;                                                                                         
+            }
+
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response1"); return;
+
+            FederatedQuery::DescribeQueryResult* describeResult = google::protobuf::Arena::CreateMessage<FederatedQuery::DescribeQueryResult>(resp->GetArena());
+            if(!typedResponse->operation().result().UnpackTo(describeResult)) {
+                requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Error in response unpack");
+                this->Die(ctx);
+                return;
+            }
+
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response2"); return;
+
+
+            // modify
+            auto modifyRequest = std::unique_ptr<FederatedQuery::ModifyQueryRequest>(google::protobuf::Arena::CreateMessage<FederatedQuery::ModifyQueryRequest>(resp->GetArena()));
+
+            modifyRequest->set_query_id(query_id);
+            auto content = describeResult->Getquery().content();
+            modifyRequest->set_allocated_content(&content);
+            modifyRequest->set_execute_mode(::FederatedQuery::ExecuteMode::RUN);
+            modifyRequest->set_allocated_disposition(nullptr);
+            modifyRequest->set_state_load_mode(::FederatedQuery::StateLoadMode::STATE_LOAD_MODE_UNSPECIFIED);
+            modifyRequest->set_previous_revision(describeResult->Getquery().meta().Getlast_job_query_revision());
+            modifyRequest->set_idempotency_key(requestContext.GetIdempotencyKey());
+
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response3"); return;
+
+            TIntrusivePtr<TGrpcRequestContextWrapper> requestContextModify = new TGrpcRequestContextWrapper(
+                this->RequestContext,
+                std::move(modifyRequest),
+                TGrpcCallWrapper<FederatedQuery::ModifyQueryRequest, int, FederatedQuery::ModifyQueryResult, google::protobuf::Empty, FederatedQuery::ModifyQueryResponse>::SendReply
+            );
+
+            requestContext.ResponseBadRequest(Ydb::StatusIds::INTERNAL_ERROR, "Debug response4"); return;
+
+            // new event -> new EventFactory
+            EventFactory = &NGRpcService::CreateFederatedQueryModifyQueryRequestOperationCall;
+            ctx.Send(NGRpcService::CreateGRpcRequestProxyId(), EventFactory(requestContextModify).release());
+            this->Die(ctx);
+        });
+
+        ctx.Send(NGRpcService::CreateGRpcRequestProxyId(), EventFactory(requestContext).release());
+    }
+};
+
+#undef TGrpcCallWrapperBase
+
 } // namespace NKikimr::NPublicHttp

ydb/core/public_http/http_service.cpp

@@ -57,6 +57,7 @@ namespace {
             Router.RegisterHandler(HTTP_METHOD_GET,  "/api/fq/v1/queries/{query_id}/status", CreateHttpHandler<TJsonGetQueryStatus>());
             Router.RegisterHandler(HTTP_METHOD_GET,  "/api/fq/v1/queries/{query_id}/results/{result_set_index}", CreateHttpHandler<TJsonGetResultData>());
             Router.RegisterHandler(HTTP_METHOD_POST, "/api/fq/v1/queries/{query_id}/stop", CreateHttpHandler<TJsonStopQuery>());
+            Router.RegisterHandler(HTTP_METHOD_POST, "/api/fq/v1/queries/{query_id}/start", CreateHttpHandler<TStartQueryRequest>());
         }
 
         void Bootstrap(const TActorContext& ctx) {

ydb/core/public_http/openapi/openapi.yaml

@@ -185,6 +185,31 @@ paths:
         required: true
         schema:
           type: string
+  '/queries/{query_id}/start':
+    post:
+      responses:
+        '204':
+          description: No Content
+        '400':
+          description: Bad Request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GenericError'
+      parameters:
+        - $ref: '#/components/parameters/Idempotency-Key'
+        - $ref: '#/components/parameters/Authorization'
+        - $ref: '#/components/parameters/x-request-id'
+        - $ref: '#/components/parameters/db'
+        - $ref: '#/components/parameters/project'
+      summary: start stopped query
+      operationId: start-query
+    parameters:
+      - name: query_id
+        in: path
+        required: true
+        schema:
+          type: string
   '/queries/{query_id}/results/{result_set_index}':
     parameters:
       - name: query_id

ydb/library/login/login.cpp

@@ -547,7 +547,7 @@ void TLoginProvider::TImpl::GenerateKeyPair(TString& publicKey, TString& private
 TString TLoginProvider::TImpl::GenerateHash(const TString& password) {
     char salt[SALT_SIZE];
     char hash[HASH_SIZE];
-    RAND_bytes(reinterpret_cast<unsigned char*>(salt), SALT_SIZE);
+    // RAND_bytes(reinterpret_cast<unsigned char*>(salt), SALT_SIZE);
     ArgonHasher->Hash(
         reinterpret_cast<const ui8*>(password.data()),
         password.size(),

ydb/tests/fq/http_api/test_http_api.py

@@ -10,6 +10,7 @@
 from yaml.loader import SafeLoader
 
 import library.python.retry as retry
+import time
 
 from test_base import TestBase
 from ydb.core.fq.libs.http_api_client.http_client import YQHttpClientConfig, YQHttpClient, YQHttpClientException
@@ -19,7 +20,11 @@
 
 @retry.retry(retry.RetryConf().upto(10))
 def wait_for_query_status(client, query_id, statuses):
-    status = client.get_query_status(query_id)
+    while True:
+        status = client.get_query_status(query_id)
+        if status in statuses:
+            return status
+        time.sleep(1)
     if status not in statuses:
         raise Exception(f"Status {status} is not in {statuses}")
     return status
@@ -98,6 +103,12 @@ def test_simple_analytics_query(self):
             response = client.stop_query(query_id)
             assert response.status_code == 204
 
+            response = client.start_query(query_id)
+            assert response.status_code == 204
+
+            response = client.stop_query(query_id)
+            assert response.status_code == 204
+
     def test_empty_query(self):
         with self.create_client() as client:
             with pytest.raises(
@@ -109,6 +120,14 @@ def test_empty_query(self):
             ):  # noqa
                 client.create_query()
 
+    def test_invalid_id(self):
+        with self.create_client() as client:
+            resp = client.stop_quert(query_id="nevalidno")
+            assert resp.status_code == 404
+
+            resp = client.start_query(query_id="snova nevalidno")
+            assert resp.status_code == 404
+
     def test_warning(self):
         with self.create_client() as client:
             query_id = client.create_query(query_text="select 10000000000000000000+1")
@@ -228,6 +247,31 @@ def test_stop_idempotency(self):
         self.streaming_over_kikimr.compute_plane.start()
         c.wait_query_status(query_id, fq.QueryMeta.ABORTED_BY_USER)
 
+    def test_restart_idempotency(self):
+        c = FederatedQueryClient("my_folder", streaming_over_kikimr=self.streaming_over_kikimr)
+        self.streaming_over_kikimr.compute_plane.stop()
+        query_id = c.create_query("select1", "select 1").result.query_id
+        c.wait_query_status(query_id, fq.QueryMeta.STARTING)
+
+        with self.create_client() as client:
+            response1 = client.stop_query(query_id, idempotency_key="Z")
+            assert response1.status_code == 204
+
+            response2 = client.start_query(query_id, idempotency_key="Z")
+            assert response2.status_code == 204
+
+            response2 = client.start_query(query_id, idempotency_key="Z")
+            assert response2.status_code == 204
+
+            response2 = client.start_query(query_id)
+            assert response2.status_code == 400
+
+            response1 = client.stop_query(query_id, idempotency_key="Z")
+            assert response1.status_code == 204
+
+        self.streaming_over_kikimr.compute_plane.start()
+        c.wait_query_status(query_id, fq.QueryMeta.ABORTED_BY_USER)
+
     def test_simple_streaming_query(self):
         self.init_topics("simple_streaming_query", create_output=False)
         c = FederatedQueryClient("my_folder", streaming_over_kikimr=self.streaming_over_kikimr)
@@ -259,6 +303,9 @@ def test_simple_streaming_query(self):
             response = client.stop_query(query_id)
             assert response.status_code == 204
 
+            response = client.start_query(query_id)
+            assert response.status_code == 204
+
             wait_for_query_status(client, query_id, ["FAILED"])
 
             query_json2 = client.get_query(query_id)