-
Notifications
You must be signed in to change notification settings - Fork 73
/
Copy pathLoop Flow.step
1 lines (1 loc) · 52.3 KB
/
Loop Flow.step
1
{"creationTimeStamp":"2024-03-13T07:44:43.388Z","modifiedTimeStamp":"2024-03-19T09:36:41.587Z","createdBy":"sdktjj","modifiedBy":"sdktjj","name":"Loop Flow.step","displayName":"Loop Flow.step","localDisplayName":"Loop Flow.step","properties":{},"links":[{"method":"GET","rel":"self","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","type":"application/vnd.sas.data.flow.step"},{"method":"GET","rel":"alternate","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","type":"application/vnd.sas.data.flow.step.summary"},{"method":"GET","rel":"up","href":"/dataFlows/steps","uri":"/dataFlows/steps","type":"application/vnd.sas.collection","itemType":"application/vnd.sas.data.flow.step.summary"},{"method":"PUT","rel":"update","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","type":"application/vnd.sas.data.flow.step","responseType":"application/vnd.sas.data.flow.step"},{"method":"DELETE","rel":"delete","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0"},{"method":"GET","rel":"transferExport","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","responseType":"application/vnd.sas.transfer.object"},{"method":"PUT","rel":"transferImportUpdate","href":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","uri":"/dataFlows/steps/a17a5050-254e-4ca2-813d-1edd3cb8a7b0","type":"application/vnd.sas.transfer.object","responseType":"application/vnd.sas.summary"}],"metadataVersion":0.0,"version":2,"type":"code","flowMetadata":{"inputPorts":[{"name":"_stepControlTable","displayName":"_stepControlTable","localDisplayName":"_stepControlTable","minEntries":1,"maxEntries":1,"defaultEntries":0,"type":"table"}],"outputPorts":[{"name":"_stepStatusTable","displayName":"_stepStatusTable","localDisplayName":"_stepStatusTable","minEntries":1,"maxEntries":1,"defaultEntries":0,"type":"table","columnDelta":{"updateColumnArguments":["etls_handleName","etls_machineId","etls_startTime","etls_endTime","etls_status","etls_jobRC"],"automaticMapping":[{"inputPort":"_stepControlTable","inputPortName":"_stepControlTable"}]},"supportsView":false,"requiresStructure":false}]},"ui":"{\n\t\"showPageContentOnly\": true,\n\t\"pages\": [\n\t\t{\n\t\t\t\"id\": \"ParallelOptions\",\n\t\t\t\"type\": \"page\",\n\t\t\t\"label\": \"Options\",\n\t\t\t\"children\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"_stepControlTable\",\n\t\t\t\t\t\"type\": \"inputtable\",\n\t\t\t\t\t\"label\": \"Control table:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"_stepStatusTable\",\n\t\t\t\t\t\"type\": \"outputtable\",\n\t\t\t\t\t\"label\": \"Status table\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"section1\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Basic options\",\n\t\t\t\t\t\"open\": true,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepFlowSourcePath\",\n\t\t\t\t\t\t\t\"type\": \"textfield\",\n\t\t\t\t\t\t\t\"label\": \"Full path to flow to execute:\",\n\t\t\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepAbortOnError\",\n\t\t\t\t\t\t\t\"type\": \"checkbox\",\n\t\t\t\t\t\t\t\"label\": \"Abort with a condition code if execution fails during iterations\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepLogOutputPath\",\n\t\t\t\t\t\t\t\"type\": \"textfield\",\n\t\t\t\t\t\t\t\"label\": \"Location on host for log and output files:\",\n\t\t\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\t\t\"required\": false,\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text1\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"If you leave the field blank, the log from the sub-processes will be streamed back into the main log\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"section2\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Advanced options\",\n\t\t\t\t\t\"open\": true,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepMaxProcesses\",\n\t\t\t\t\t\t\t\"type\": \"numberfield\",\n\t\t\t\t\t\t\t\"label\": \"Maximum number of sub-process:\",\n\t\t\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\t\t\"max\": null,\n\t\t\t\t\t\t\t\"min\": 1,\n\t\t\t\t\t\t\t\"excludemin\": false,\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text10\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"If you specify 1, the loop will be executed in sequential mode\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepUseOptimalProcesses\",\n\t\t\t\t\t\t\t\"type\": \"checkbox\",\n\t\t\t\t\t\t\t\"label\": \"Overwrite specified number of sub-processes and use environment maximum (PARALL_LOOP_LOCAL_NUMSUBPROCS)\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepControlName\",\n\t\t\t\t\t\t\t\"type\": \"textfield\",\n\t\t\t\t\t\t\t\"label\": \"Prefix for handle names on iterations:\",\n\t\t\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"_stepConnectContext\",\n\t\t\t\t\t\t\t\"type\": \"textfield\",\n\t\t\t\t\t\t\t\"label\": \"SAS Connect Server Context to use:\",\n\t\t\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\t\t\"required\": false,\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t]\n\t\t},\n\t\t{\n\t\t\t\"id\": \"page1\",\n\t\t\t\"type\": \"page\",\n\t\t\t\"label\": \"Columns Created\",\n\t\t\t\"children\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_handleName\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"Name of handle to remote session:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_machineId\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"Name of machine executing the task:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_startTime\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"Start time of task:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_endTime\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"End time of task:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_status\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"Current status of task:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"etls_jobRC\",\n\t\t\t\t\t\"type\": \"newcolumn\",\n\t\t\t\t\t\"label\": \"Return code of task:\",\n\t\t\t\t\t\"required\": true,\n\t\t\t\t\t\"placeholder\": \"\",\n\t\t\t\t\t\"hideproperties\": false,\n\t\t\t\t\t\"readonly\": true\n\t\t\t\t}\n\t\t\t]\n\t\t},\n\t\t{\n\t\t\t\"id\": \"about\",\n\t\t\t\"type\": \"page\",\n\t\t\t\"label\": \"About\",\n\t\t\t\"children\": [\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"text2\",\n\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\"text\": \"Loop Flow\\n=======\\n\\nThe Loop Flow Step enables a SAS Studio Flow to iteratively execute another flow, much like how Loop/Loop End is used in SAS Data Integration Studio.\\n\\nFeatures supported:\\n- The flow referenced by this step must be stored on the file system. \\n- Loop Flow requires a control table as input. The flow that is iterated will be executed as many times as there are rows in the control table. All columns in the table will automatically be made available as macro variables in the flow that is iterated.\\n- The output table contains all the rows in the original input table plus status information for each iteration.\\n- All iterations will execute in its own SAS Connect Session and can utilize workload management capabilities provided by SAS Workload Orchestrator.\\n- The number of concurrent processes to use for running iterations can be specified.\\n- A libname SWORK will automatically be made available in the SAS Connect Session so the iterated flow can access SAS Work of the parent flow.\\n- The entire flow can be configured to abort if one of the iterations fails with ERROR.\\n- The SAS-log for each iteration can be stored on the file system or be included in the main log.\",\n\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"requirements\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Requirements\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text3\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"The Custom Step requires SAS Viya to be configured for Python Integration.\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"basic_options\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Basic options\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text4\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"* Full path to flow to execute: \\nEnter the full path of the flow you want to execute iteratively. The flw-file must be stored on the file system. Macro variables can be used as part of the path. \\n\\n* Abort with a condition code if execution fails during iterations:\\nCheck this option if you want your flow to terminate immediately with '%abort return 2' if one of your iterations fails with an ERROR. Your flow will then show up as status = Cancelled In 'Jobs and Flows' in Environment Manager. Please note that other concurrent running iterations will continue until all statements have executed. If this option is not checked your flow will continue as normal regardless of any ERRORS in your iterations.\\n\\n* Location on host for log and output files:\\nSpecify a directory where you want to store the Output and SAS-log for each iteration. A name is assigned to these files that is unique across iterations. if you leave this field blank, the log from iterations will be streamed back into the main log.\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"advanced_options\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Advanced options\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text5\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"* Maximum number of sub-process:\\nSpecifies an exact number of concurrent processes to use for running iterations. Default is 3 processes. If you specify 1, the loop will be executed in sequential mode. All iterations will execute in its own SAS Connect Session. Each iteration can utilize the automatically created libname SWORK that references SAS Work of the parent flow.\\n\\n* Overwrite specified number of sub-processes and use environment maximum (PARALL_LOOP_LOCAL_NUMSUBPROCS):\\nInstructs the Loop Flow Step to look for a global macro variable PARALL_LOOP_LOCAL_NUMSUBPROCS. If this is specified, e.g. in the autoexec, the value given to this macro variable will override the value specified in 'Maximum number of sub-processes'. This option will only have effect if 'SAS Connect Server Context to use' is left blank. \\n\\n* Prefix for handle names on iterations:\\nSpecifies a prefix for handle names on loop iterations. Technically, the handle name identifies a distinct iteration and is also used to name Output- and Log-files if a value is specified in 'Location on host for log and output files'. The shorter the prefix, the more loop iterations are permitted. Minimum length is one character, which permits 9,999,999 iterations of the Loop Flow Step. The maximum length of the prefix is seven characters, which permits nine iterations.\\n\\n* SAS Connect Server Context to use:\\nIdentifies the SAS Connect Context to use for launching SAS Connect Sessions. Default value is 'default-launcher' and should work out of the box in Viya 4. The name of the context can be seen by selecting 'Connect Context' in Environment Manager - Context. When a valid value is specified, the launch of Connect sessions will be administered by SAS Workload Orchestrator, in much the same way as SAS Grid Computing on SAS 9. If you leave this value blank, all iterative sessions will be launched inside the same Pod, by-passing the load balancing capabilities of SAS Workload Orchestrator (this can be compared to the use of local MP Connect in SAS 9.4). As a rule, leave this value blank, if you have many iterations of a low resource (lightweight) flow, and want to avoid the overhead of launching sessions using Workload Orchestrator. The trade-off is that all concurrent running SAS Connect sessions, including the parent session, will be restricted to the memory limit defined in sas.launcher.max in Environment Manager. This memory limit is the ceiling of RAM usage that a Pod created by SAS Launcher can reach before it is forcefully terminated by Kubernetes. \\n\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"input_port\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Input port\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text6\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"The input table controls the number of iterations to perform, as well as the parameters passed to each iteration. The flow will be iterated as many times as there are rows in the control table, and all columns in the table will automatically be made available as macro variables in the flow that is iterated. \\n\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"output_port\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Output port\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text7\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"The output table contains all the rows in the original input table plus the following columns:\\n\\n* etls_handleName: Name of handle to remote session\\n* etls_machineId: Name of machine executing the iteration\\n* etls_startTime: Start time of the iteration\\n* etls_endTime: End time of the iteration\\n* etls_status: Current status of the iteration\\n* etls_jobRC: Return code of the iteration. Captures SYSCC Automatic Macro Variable right after the iteration have finished executing. \\n\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"special_considerations\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Special considerations\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text9\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"* You can freely utilize nested loops: That is, you can have a flow that uses a Loop Flow to call a flow that uses Loop Flow! However, don't create a circular reference (!), and IF you specify a SAS Connect Context, SWORK is only supported by the top most Loop Flow.\\n* Code for the flow to loop will be generated at run-time. Make sure that all steps in the iterated flow is without errors, and that you can generate code for it in SAS Studio, \\n\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"id\": \"change_log\",\n\t\t\t\t\t\"type\": \"section\",\n\t\t\t\t\t\"label\": \"Change log\",\n\t\t\t\t\t\"open\": false,\n\t\t\t\t\t\"visible\": \"\",\n\t\t\t\t\t\"children\": [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"id\": \"text8\",\n\t\t\t\t\t\t\t\"type\": \"text\",\n\t\t\t\t\t\t\t\"text\": \"Version: 1.0 (08MAR2024)\\n * Initial version\\n\",\n\t\t\t\t\t\t\t\"visible\": \"\"\n\t\t\t\t\t\t}\n\t\t\t\t\t]\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t],\n\t\"syntaxversion\": \"1.3.0\",\n\t\"values\": {\n\t\t\"_stepControlTable\": {\n\t\t\t\"library\": \"\",\n\t\t\t\"table\": \"\"\n\t\t},\n\t\t\"_stepStatusTable\": {\n\t\t\t\"library\": \"\",\n\t\t\t\"table\": \"\"\n\t\t},\n\t\t\"_stepFlowSourcePath\": \"\",\n\t\t\"_stepAbortOnError\": false,\n\t\t\"_stepLogOutputPath\": \"\",\n\t\t\"_stepMaxProcesses\": 3,\n\t\t\"_stepUseOptimalProcesses\": false,\n\t\t\"_stepControlName\": \"L41_\",\n\t\t\"_stepConnectContext\": \"default-launcher\",\n\t\t\"etls_handleName\": {\n\t\t\t\"value\": \"etls_handleName\",\n\t\t\t\"label\": \"Name of handle to remote session\",\n\t\t\t\"length\": 32\n\t\t},\n\t\t\"etls_machineId\": {\n\t\t\t\"value\": \"etls_machineId\",\n\t\t\t\"label\": \"Name of machine executing the task\",\n\t\t\t\"length\": 32\n\t\t},\n\t\t\"etls_startTime\": {\n\t\t\t\"value\": \"etls_startTime\",\n\t\t\t\"label\": \"Start time of task\",\n\t\t\t\"type\": \"n\",\n\t\t\t\"length\": 8,\n\t\t\t\"format\": \"nldatmap.\"\n\t\t},\n\t\t\"etls_endTime\": {\n\t\t\t\"value\": \"etls_endTime\",\n\t\t\t\"label\": \"End time of task\",\n\t\t\t\"type\": \"n\",\n\t\t\t\"length\": 8,\n\t\t\t\"format\": \"nldatmap.\"\n\t\t},\n\t\t\"etls_status\": {\n\t\t\t\"value\": \"etls_status\",\n\t\t\t\"label\": \"Current status of task\",\n\t\t\t\"length\": 32\n\t\t},\n\t\t\"etls_jobRC\": {\n\t\t\t\"value\": \"etls_status\",\n\t\t\t\"label\": \"Return code of task\",\n\t\t\t\"length\": 8,\n\t\t\t\"type\": \"n\"\n\t\t}\n\t}\n}","templates":{"SAS":"%global \n\tetls_logOutputPath etls_sasConnectContext etls_controlName \n\tetls_controlTable etls_statusTable etls_maxProcesses etls_UseOptimalProcesses;\n\n%let etls_logOutputPath=&_stepLogOutputPath;\n%let etls_sasConnectContext=&_stepConnectContext;\n%let etls_controlName = &_stepControlName;\n%let etls_controlTable = &_stepControlTable;\n%let etls_statusTable = &_stepStatusTable;\n%let etls_AbortOnError = &_stepAbortOnError;\n%let etls_maxProcesses = &_stepMaxProcesses;\n%let etls_FlowSourcePath = &_stepFlowSourcePath;\n\n%if %symexist(_stepUseOptimalProcesses) %then %do;\n\t%let etls_UseOptimalProcesses=&_stepUseOptimalProcesses;\n%end;\n\n%let etls_parameterTable = work.etls_parameterTable;\n\n/* region: Generation of code */\n%macro generateCodefromFlow(flowToCode=,outfileref=,errorabort=YES);\n\t/*\n\t\tDefine fileref used to store generated code\n\t*/\n\tfilename _gctemp temp;\n\t%put NOTE: Generating code from flow &flowToCode;\n\n\t/* Generate python code as this MUST be called using %inc when\n\t python is called inside a macro\n\t*/\n\n\t/* Creating request file */\n\t/* Generate following code \n\t\t{\n\t\t \"object\": {\n\t\t \"value\": \"string\",\n\t\t \"mediaType\": \"application/vnd.sas.dataflow\"\n\t\t },\n\t\t \"initCode\": true,\n\t\t \"wrapperCode\": true\n\t\t}\n\t*/\n\tfilename _phycode temp;\n\tdata _null_;\n\t\tfile _phycode lrecl=5000;\n\t\tput 'submit;'\n\t\t\t/\n\t\t\t/ 'import json'\n\t\t\t/\n\t\t\t/ 'flowToCode=SAS.symget(\"flowToCode\")'\n\t\t\t/ 'with open(flowToCode) as jsonFlowFile:'\n\t\t\t/ ' flow = jsonFlowFile.read()'\n\t\t\t/\n\t\t\t/ \"sourceObject = {'value': flow, 'mediaType': 'application/vnd.sas.dataflow'}\"\n\t\t\t/ \"codeGenerationRequest = {'object': sourceObject, 'initCode': False, 'wrapperCode': False}\"\n\t\t\t/\n\t\t\t/ 'reqFile=SAS.symget(\"jsonfile\")'\n\t\t\t/ \"with open(reqFile, 'w') as f:\"\n\t\t\t/ ' json.dump(codeGenerationRequest, f)'\n\t\t\t/\n\t\t\t/ 'endsubmit;';\n\trun;\n\n\t/* \n\tGenerate JSON-file and store in temp fileref _req\n\tUseses Python-code as this have greate support for JSON\n\t*/\n\tfilename _req temp;\n\n\tdata _null_;\n\t\tcall symput('jsonfile',strip(pathname('_req'))); /* file where the request is stored */\n\t\tcall symput('flowToCode',strip(\"&flowToCode\")); /* The flow located on the file system */\n\trun;\n\n\tproc python RESTART timeout=90;\n\t\t%inc _phycode;\n\trun;\n\tfilename _phycode clear;\n\n\t/* \n\tGenerate code using PROC HTTP to be able to reuse token and session.\n\tInput to the call (_req) is the JSON-file generated by the python call above\n\n\tThe generated code will be temporary placed in filename _gctemp.\n\n\tif the call is succesfull (SYS_PROCHTTP_STATUS_CODE = 201) the code is copied\n\t\tto the destination folder given by filename _gencode.\n\t*/\n\t\n\t/* SDKTJJ 04AUG2023: Perform code gen with some resilience */\n\t%local codegenRetrySleeptime codegenTryCount codegenRetries;\n\t%let codegenRetries=3 /* 3 */;\n\t%let codegenRetrySleeptime=120; /* In seconds */\n\t%let codegenTryCount=1;\n\t%global SYS_PROCHTTP_STATUS_CODE;\n\t%do %until(\n\t\t\t(&SYS_PROCHTTP_STATUS_CODE=200 or &SYS_PROCHTTP_STATUS_CODE=201) or \n\t\t\t(&codegenTryCount gt %eval(&codegenRetries+1))\n\t\t);\n\t\tproc http\n\t\t\tmethod=post FOLLOWLOC\n\t\t\turl = \"https://sas-studio-development/studioDevelopment/code\"\n\t\t\tout= _gctemp\n\t\t\tin=_req\n\t\t\toauth_bearer = sas_services\n\t\t\t;\t\n\t\t\theaders\n\t\t 'Accept'= 'application/vnd.sas.code.generation.request+json'\n\t\t\t\t'Content-Type' = 'application/json'\n\t\t\t;\n\n\t\t\tsslparms \"SSLREQCERT\"= \"ALLOW\";\n\t\t\t\n\t\t\t*debug level=2;\n\t\trun;\n\n\t\t%if (&SYS_PROCHTTP_STATUS_CODE>201) %then %do;\n\t\t\t/* We have a failure in code gen */\n\t\t\t%if &codegenTryCount lt %eval(&codegenRetries+1) %then %do;\n\t\t\t\t%put NOTE: Codegen attempt #&codegenTryCount failed. Retrying in &codegenRetrySleeptime seconds.;\n\t\t\t\t%let rc=%sysfunc(sleep(&codegenRetrySleeptime,1));\n\t\t\t%end;\n\t\t\t%else %if &codegenRetries gt 0 %then \n\t\t\t\t%put ERROR%QUOTE(:) Final Codegen attempt, #&codegenTryCount, failed.;\n\t\t\t%let codegenTryCount=%eval(&codegenTryCount+1);\n\t\t%end;\n\t%end;\n\n\t/*\n\t\tHandle response in fileref _gctemp\n\t*/\n\t%if &SYS_PROCHTTP_STATUS_CODE<=201 %then %do;\n\t\t/* \t\n\t\t\tWe use python code to store the content of the 'code' tag \n\t\t\tas this handles newlines (\\n), uft8 etc. nicely.\n\t\t*/\n\t\tfilename _jreq temp; \t\t/* The python code used to extract the 'code' tag */\n\t\tfilename _gencode temp;\t\t/* The generated code */\n\t\tdata _null_;\n\t\t\tfile _jreq lrecl=5000;\n\t\t\tput 'submit;'\n\t\t\t\t/\n\t\t\t\t/ 'import json'\n\t\t\t\t/ 'gctempfile=SAS.symget(\"_gctempfile\")'\n\t\t\t\t/ 'gencode=SAS.symget(\"_gencodefile\")'\n\t\t\t\t/\n\t\t\t\t/ 'with open(gctempfile) as jsonFlowFile:'\n\t\t\t\t/ ' flow = json.load(jsonFlowFile)'\n\t\t\t\t/\n\t\t\t\t/ \"with open(gencode, 'w') as the_file:\"\n\t\t \t/ \" the_file.writelines(flow['code'])\"\n\t\t\t\t/\n\t\t\t\t/ 'endsubmit;';\t\t\n\t\trun;\n\t\t\n\t\tdata _null_;\n\t\t\t/* The python code used to extract the 'code' tag */\n\t\t\tcall symput('_jreqfile',strip(pathname('_jreq')));\n\n\t\t\t/* The JSON gererated from the call to /studioDevelopment/code */\n\t\t\tcall symput('_gctempfile',strip(pathname('_gctemp')));\n\n\t\t\t/* The generated code must be stored in fileref _gencode */\n\t\t\tcall symput('_gencodefile',strip(pathname(\"_gencode\")));\n\t\trun;\n\t\tproc python RESTART;\n\t\t\t%inc \"&_jreqfile\";\n\t\trun;\n\t\tfilename _jreq clear;\n\n\t\t/* Copy from _gencode to target fileref */\n\t\t%let fcopyrc=%sysfunc(fcopy(_gencode,&outfileref));\n\t\tfilename _gencode clear;\n\t%end;\n\t%else %do;\n\t\t/* Handle response if not 201 */\n\t\tdata _null_;\n\t\t\t/* Print ERROR */\n\t\t\tif _n_=1 then do;\n\t\t\t\tput \"ERROR: Code generation failed for &flowToCode\";\n\t\t\tend;\n\n\t\t\t/* Print request */\n\t\t\tinfile _gctemp;\n\t\t\tinput;\n\t\t\tput _infile_;\n\n\t\t\tif \"&errorabort\"=\"YES\" then do;\n\t\t\t\tput \"ERROR: Failed Code Generation and errorabort=YES: Will abort.\";\n\t\t\t\tabort return 2 NOLIST;\n\t\t\tend;\n\t\trun;\n\t%end;\n\n\tfilename _req clear;\n\tfilename _gctemp clear;\n%mend;\n\n/* endregion */\n\n/* region: DI Studio Compability */\n%macro rcSet(error); \n %if (&error gt &trans_rc) %then \n %let trans_rc = &error;\n %if (&error gt &job_rc) %then \n %let job_rc = &error;\n%mend rcSet; \n\n%if %symexist(etls_jobName)=0 %then %do;\n\t%global etls_jobName;\n \t%let etls_jobName=GENERIC_JOBNAME;\t\t\n%end;\n\n/*==========================================================================* \n * Step: Parallel Loop A5V7UIZU.BJ0001L5 * \n *==========================================================================*/ \n%let transformID = %quote(A5V7UIZU.BJ0001L5);\n%let trans_rc = 0;\n%let job_rc = 0;\n%let etls_stepStartTime = %sysfunc(datetime(), datetime20.); \n/* endregion */\n\n/* region: Loop Macros */\n%macro etls_tsLevel(macroName=, featureName=);\n\n\tdata _null_;\n\t\tcall symput(\"¯oName\",tslvl(\"&featureName\"));\n\trun;\n\n%mend etls_tsLevel;\n\n%macro etls_signon(handleName=etlsRemote,\n\t\t\tcmacvar=etls_signonStatus,\n\t\t\tgridRC=rc,\n\t\t\tuseGrid=1,\n\t\t\tmachineIdMacroVariable=etls_machineId,\n\t\t\tgridOptionSet=,\n\t\t\tworkloadMacroVariable=,\n\t\t\tlog=,\n\t\t\toutput=,\n\t\t\tadditionalSignonOptions=,\n\t\t\tsignonRetries=3,\n\t\t\tgridJobName=&&etls_jobName&row);\n\t%let etls_gridInstalled =;\n\n\t%etls_tsLevel(macroName=etls_gridInstalled, featureName=uwugrdsv);\n\n\t%if (&useGrid = 1) %then %do;\n\t\t%if (\"&etls_gridInstalled\" ne \"\") %then %do;\n\t\t\t%if (&gridRC eq ) %then\n\t\t\t\t%let gridRC = &handleName.RC;\n\t\t\t%global &gridRC;\n\t\t\t%let workload =;\n\n\t\t\t%if (\"&workloadMacroVariable\" ne \"\") %then %do;\n\n\t\t\t\tdata _null_;\n\t\t\t\t\tcall symput('workload','; workload = '||symget(\"&workloadMacroVariable\"));\n\t\t\t\trun;\n\n\t\t\t%end;\n\n\t\t\t%let gridoptset =;\n\n\t\t\t%if (\"&gridOptionSet\" ne \"\") %then %do;\n\n\t\t\t\tdata _null_;\n\t\t\t\t\tcall symput('gridoptset',\"; gridoptset = '\"||\"&gridOptionSet\"||\"'\");\n\t\t\t\trun;\n\n\t\t\t%end;\n\n\t\t\t%let gridJobOpts =;\n\t\t\t%let gridSuppOpts =;\n\n\t\t\t%if (\"&etls_gridSuppJobOptions\" ne \"\") %then %do;\n\n\t\t\t\tdata _null_;\n\t\t\t\t\tcall symput('gridJobOpts', \"&etls_gridSuppJobOptions\");\n\t\t\t\trun;\n\n\t\t\t\tdata _null_;\n\t\t\t\t\tcall symput('gridSuppOpts', '; JOBOPTS=gridJobOpts');\n\t\t\t\trun;\n\n\t\t\t%end;\n\n\t\t\tdata _null_;\n\t\t\t\tcall symput('gridJbName', \"; jobname = gridJobName\");\n\t\t\trun;\n\n\t\t\t%let &gridRC=2; /* Reports that one or all server sessions were disabled from grid execution */\n\t\t\t%if %str(&etls_sasConnectContext) ne %str() %then %do;\n\t\t\t\t%let &gridRC = %sysfunc(grdsvc_enable(\"&handleName\", \"server = &etls_sasConnectContext &gridoptset &workload &gridSuppOpts &gridJbName\"));\n\t\t\t%end;\n\t\t%end;\n\t\t%else %put %str(NOTE: Grid service functions are not installed.);\n\t%end;\n\n\t%local sleeptime sleepIncreaseAmount tryCount;\n\t%let sleepIncreaseAmount=5;\n\t%let sleeptime=5;\n\t%let tryCount=1;\n\t%let &cmacvar=1;\n\n\t/* Store original return code values to reset them if a signon succeeds */\n\t%local etls_sysrc;\n\t%local etls_syscc;\n\t%let etls_sysrc = &sysrc;\n\t%let etls_syscc = &syscc;\n\t%put NOTE: Condition codes before signon: &=syserr &=syscc &=etls_syscc &=handleName;\n\n\t%do %until((&&&cmacvar ne 1) or (&tryCount gt %eval(&signonRetries+1)));\n\n\t\tdata _null_;\n\t\t\tsignon &handleName cmacvar=&cmacvar \n\t\t\t%if (\"&log\" ne \"\") %then \n\t\t\t\tlog=&log;\n\n\t\t\t%if (\"&output\" ne \"\") %then\n\t\t\t\toutput=&output;\n\n\t\t\t%if %str(&etls_sasConnectContext) ne %str() %then %do;\n\t\t\t\tinheritlib=(work=swork);\n\t\t\t%end;\n\n\t\t\t%if (\"&additionalSignonOptions\" ne \"\") %then\n\t\t\t\t&additionalSignonOptions;\n\t\t\t;\n\t\t\n\t\t\t\n\t\trun;\n\n\t\t%if \"&&&cmacvar\" eq \"1\" %then %do;\n\t\t\t%if &tryCount lt %eval(&signonRetries+1) %then %do;\n\t\t\t\t%put NOTE: Signon attempt #&tryCount failed. Retrying in &sleeptime seconds.;\n\t\t\t\t%let rc=%sysfunc(sleep(&sleeptime,1));\n\t\t\t%end;\n\t\t\t%else %if &signonRetries gt 0 %then %put ERROR%QUOTE(:) Final signon attempt, #&tryCount, failed.;\n\t\t\t%let sleeptime=%sysevalf(&sleeptime+&sleepIncreaseAmount,int); /* increase sleeptime by factor */\n\t\t\t%let tryCount=%eval(&tryCount+1);\n\t\t%end;\n\t\t%else %do;\n\n\t\t\t/* Signon succeeded make sure return codes are reset to original values */\n\t\t\t%let sysrc = &etls_sysrc;\n\t\t\t%let syscc = &etls_syscc;\n\n\t\t\t/* Initialize syserr to 0 */\n\t\t\tdata _null_;\n\t\t\trun;\n\n\t\t\t%put NOTE: Condition codes after succesful signon: &=syserr &=syscc &=etls_syscc &=handleName;\n\t\t%end;\n\t%end;\n\n\t%put NOTE: Condition codes after all signon attempts: &=syserr &=syscc &=handleName;\n\n\t%rcSet(&syserr);\n\n\t/* On successful signon, fetch and report machine name if grid was requested. */\n\t%if (\"&machineIdMacroVariable\" ne \"\") %then\n\t\t%let &machineIdMacroVariable =;\n\n\t%if (\"&&&cmacvar\" ne \"1\" and \"&machineIdMacroVariable\" ne \"\" and \n\t\t&useGrid = 1 and \"&etls_gridInstalled\" ne \"\") %then %do;\n\t\t%put;\n\n\t\t%if (\"&&&gridRC\" eq \"0\") %then %do;\n\t\t\t%let &machineIdmacroVariable = %sysfunc(grdsvc_getname(&handleName));\n\t\t\t%put %str(NOTE: Signed on to grid machine &&&machineIdmacroVariable..);\n\t\t%end;\n\t\t%else %put %str(NOTE: GRID SERVICES WERE NOT ENABLED. Signed on to LOCAL machine.);\n\t\t%put;\n\t%end;\n%mend etls_signon;\n\n%macro etls_serverConnected(serverId=,macvarname=isConnected);\n\t%let &macvarname=0;\n\t%local search_str none_str savenotes;\n\n\t/* setup information for listtask output */\n\t/* listtask output on OS/390 differs */\n\t%if %bquote(&sysscp) eq OS %then %do;\n\t\t%let search_str=Remote submit for;\n\t\t%let token=5;\n\t%end;\n\t%else %do;\n\t\t%let search_str=--------------;\n\t\t%let token=1;\n\t%end;\n\n\t/* Issue listtask within PROC PRINTTO - send output to file */\n\t%let __old_printtolog=%trim(%left(&sysprinttolog));\n\tfilename __tmptxt TEMP;\n\t%let savenotes=%sysfunc(getoption(notes));\n\toptions nonotes;\n\n\tproc printto log=__tmptxt new;\n\trun;\n\n\toptions notes;\n\tlisttask;\n\n\tproc printto log=\n\t\t%if %str(&__old_printtolog) ne %str() %then %do;\n\t\t\t&__old_printtolog\n\t\t%end;\n\t\t%else %do;\n\t\t\tlog\n\t\t%end;\n\t\t;\n\trun;\n\n\toptions nonotes;\n\n\t/* read in the output from listtask */\n\t%let dsname=_null_;\n\n\tdata _null_(keep=Session);\n\t\tlength line $256 sessionId $60;\n\t\tinfile __tmptxt length=len;\n\t\tinput @1 line $varying. len;\n\n\t\tif line=:'There are no' then\n\t\t\tstop;\n\n\t\t/* look for the search string and scan for server name */\n\t\tif index(line,\"&search_str\") then do;\n\t\t\tsessionId=kscan(line,&token,\" \");\n\t\t\tsessionId=kupcase(ktrim(kleft(tranwrd(sessionId,'\"',' '))));\n\n\t\t\tif sessionId=kupcase(symget(\"serverId\")) then do;\n\t\t\t\tcall symput(\"&macvarname\",\"1\");\n\t\t\t\tstop;\n\t\t\tend;\n\t\tend;\n\trun;\n\n\tfilename __tmptxt clear;\n\toptions &savenotes;\n%mend etls_serverConnected;\n\n%macro etls_signoff(handleName=rmt);\n\t%local savemprint;\n\t%let savemprint=%sysfunc(getoption(mprint));\n\t%put %str(NOTE: Testing connection with etls_serverConnected macro %(mprint off%).);\n\toptions nomprint;\n\t%local isConnected;\n\n\t%etls_serverConnected(serverId=&handleName);\n\toptions &savemprint;\n\n\t%if &isConnected %then\n\t\tsignoff &handleName ;;\n%mend etls_signoff;\n\n%macro etls_getHandle(statusTable=, handleVariable=, row=);\n\t%let etls_dsid = %sysfunc(open(&statusTable));\n\n\t%if (&etls_dsid = 0) %then\n\t\t%put %sysfunc(sysmsg());\n\t%else %do;\n\t\t%let rc = %sysfunc(fetchobs(&etls_dsid, &row));\n\n\t\t%if (&rc ne 0) %then\n\t\t\t%put %sysfunc(sysmsg());\n\t\t%else %do;\n\t\t\t%let etls_varnum = %sysfunc(varnum(&etls_dsid,&handleVariable));\n\n\t\t\t%if (&etls_varnum > 0) %then\n\t\t\t\t%sysfunc(getvarc(&etls_dsid,&etls_varnum));\n\t\t\t%else %put %sysfunc(sysmsg());\n\t\t%end;\n\n\t\t%let rc = %sysfunc(close(&etls_dsid));\n\t%end;\n%mend etls_getHandle;\n\n%macro etls_freeHandle(statusTable=, statusVariable=, handleVariable=, \n\t\t\thandleName=, statusSetting=\"Finished\", \n\t\t\tendTimeVariable=endTime, startTimeVariable=startTime, signoff=1, \n\t\t\treturnCodeVariable=, returnCodeMacroVariable=, setMainJobRC=1, \n\t\t\tstatusUnknownReturnCode=., startTimeMacroVariable=, endTimeMacroVariable= );\n\t%if (&statusTable ne ) %then %do;\n\t\t%local etls_rcMacroVarExisted;\n\t\t%let etls_rcMacroVarExisted = 0;\n\n\t\t%if (\"&returnCodeMacroVariable\" ne \"\") %then %do;\n\n\t\t\tproc sql noprint;\n\t\t\t\tselect '1' into: etls_rcMacroVarExisted from dictionary.macros \n\t\t\t\t\twhere name=kupcase(\"&returnCodeMacroVariable\");\n\t\t\tquit;\n\n\t\t\t%rcSet(&sqlrc);\n\n\t\t\t%if (&etls_rcMacroVarExisted = 0) %then %do;\n\t\t\t\t%put WARNING%QUOTE(:) Return code from inner job not found. Setting status to Unknown.;\n\t\t\t\t%let &returnCodeMacroVariable=&statusUnknownReturnCode;\n\t\t\t%end;\n\n\t\t\t%if (&setMainJobRC eq 1) %then\n\t\t\t\t%rcSet(&&&returnCodeMacroVariable);\n\t\t%end;\n\t\t%else %do;\n\t\t\t%let returnCodeMacroVariable=etls_rcmacvar;\n\t\t\t%let &returnCodeMacroVariable=&statusUnknownReturnCode;\n\t\t%end;\n\n\t\t%local etls_startTimeMacroVarExisted;\n\t\t%let etls_startTimeMacroVarExisted = 0;\n\n\t\t%if (\"&startTimeMacroVariable\" ne \"\") %then %do;\n\n\t\t\tproc sql noprint;\n\t\t\t\tselect '1' into: etls_startTimeMacroVarExisted from dictionary.macros \n\t\t\t\t\twhere name=kupcase(\"&startTimeMacroVariable\");\n\t\t\tquit;\n\n\t\t\t%rcSet(&sqlrc);\n\n\t\t\t%if (&etls_startTimeMacroVarExisted = 0) %then %do;\n\t\t\t\t%put WARNING%QUOTE(:) Start time from inner job not found. No value will be set.;\n\t\t\t\t%let &startTimeMacroVariable=;\n\t\t\t%end;\n\t\t%end;\n\n\t\t%local etls_endTimeMacroVarExisted;\n\t\t%let etls_endTimeMacroVarExisted = 0;\n\n\t\t%if (\"&endTimeMacroVariable\" ne \"\") %then %do;\n\n\t\t\tproc sql noprint;\n\t\t\t\tselect '1' into: etls_endTimeMacroVarExisted from dictionary.macros \n\t\t\t\t\twhere name=kupcase(\"&endTimeMacroVariable\");\n\t\t\tquit;\n\n\t\t\t%rcSet(&sqlrc);\n\n\t\t\t%if (&etls_endTimeMacroVarExisted = 0) %then %do;\n\t\t\t\t%put WARNING%QUOTE(:) End time from inner job not found. Setting end time to current\n\t\t\t\t\ttime.;\n\t\t\t\t%let &endTimeMacroVariable=%sysfunc(datetime());\n\t\t\t%end;\n\t\t%end;\n\n\t\tdata &statusTable;\n\t\t\tmodify &statusTable(where=(&handleVariable = &handleName));\n\n\t\t\t%if (\"&startTimeMacroVariable\" ne \"\") %then %do;\n\t\t\t\t&startTimeVariable = input(symget(\"&startTimeMacroVariable\"),32.);;\n\t\t\t%end;\n\n\t\t\t%if (\"&endTimeVariable\" ne \"\") %then %do;\n\t\t\t\t%if (\"&endTimeMacroVariable\" ne \"\") %then %do;\n\t\t\t\t\t&endTimeVariable = input(symget(\"&endTimeMacroVariable\"),32.);;\n\t\t\t\t%end;\n\t\t\t\t%else &endTimeVariable = datetime();;\n\t\t\t%end;\n\n\t\t\t%if (\"&returnCodeVariable\" ne \"\") %then\n\t\t\t\t&returnCodeVariable = input(symget(\"&returnCodeMacroVariable\"),32.);;\n\n\t\t\t%if (\"&statusVariable\" ne \"\") %then %do;\n\t\t\t\tif (symget(\"etls_rcMacroVarExisted\") eq \"0\") then\n\t\t\t\t\t&statusVariable = \"Unknown Status\";\n\t\t\t\telse &statusVariable = &statusSetting;\n\t\t\t%end;\n\n\t\t\tcall symput('handle',&handleVariable);\n\t\t\treplace;\n\t\t\tstop;\n\t\trun;\n\n\t\t%rcSet(&syserr);\n\n\t\t%if (&signoff eq 1) %then\n\t\t\t%etls_signoff(handleName=&handle);\n\n\t\t%if (&&&returnCodeMacroVariable > 4 or &&&returnCodeMacroVariable = .) and &etls_AbortOnError=1 %then %do;\n\t\t\t%put ERROR: Error code returned from %trim(&handle) is &&&returnCodeMacroVariable. Will abort.;\n\t\t\t%abort return 2;\n\t\t%end;\n\t\t\n\t%end;\n%mend etls_freeHandle;\n\n%macro etls_createHandle(statusTable=,\n\t\t\tstatusVariable=,\n\t\t\thandleVariable=,\n\t\t\thandlePrefix=rmt,\n\t\t\tgridOptionSet=,\n\t\t\tworkloadMacroVariable=,\n\t\t\trow=,\n\t\t\tmachineVariable=,\n\t\t\tstatusSetting=\"Running\", \n\t\t\tstartTimeVariable=startTime,\n\t\t\tsignon=1,\n\t\t\tuseGrid=1,\n\t\t\tlog=,\n\t\t\toutput=,\n\t\t\tgridRC=,\n\t\t\tcmacvar=etls_signonStatus,\n\t\t\tadditionalSignonOptions=,\n\t\t\tsignonRetries= );\n\t%local remoteSessionId;\n\t%let remoteSessionId = &handlePrefix.&row;\n\t%let &cmacvar = 1;\n\t%local etls_machineId;\n\n\t%if (&signon eq 1) %then\n\t\t%etls_signon(handleName=&remoteSessionId,\n\t\tuseGrid=&useGrid,\n\t\tmachineIdMacroVariable=etls_machineId,\n\t\tgridOptionSet=&etls_gridOptionSet,\n\t\tworkloadMacroVariable=&workloadMacroVariable,\n\t\tlog=&log,\n\t\toutput=&output,\n\t\tcmacvar=&cmacvar,\n\t\tgridRC=&gridRC,\n\t\tadditionalSignonOptions=&additionalSignonOptions,\n\t\tsignonRetries=&signonRetries,\n\t\tgridJobName=DIS_&etls_jobName._&row);\n\t%else %let &cmacvar=0;\n\n\tdata &statusTable;\n\t\tretain ptr &row;\n\t\tmodify &statusTable point = ptr;\n\t\t&handleVariable = \"&remoteSessionId\";\n\n\t\t%if (&signon eq 1) %then\n\t\t\t&machineVariable = \"&etls_machineId\";;\n\n\t\t%if (&&&cmacvar ne 0) %then %do;\n\t\t\t&statusVariable = \"Failed Signon\";\n\t\t%end;\n\t\t%else &statusVariable = &statusSetting;;\n\n\t\t%if (&startTimeVariable ne ) %then\n\t\t\t&startTimeVariable = datetime();;\n\t\treplace;\n\t\tstop;\n\trun;\n\n\t%rcSet(&syserr);\n%mend etls_createHandle;\n\n%macro etls_waitfor(statusTable=, statusVariable=, runningStatusSetting=\"Running\", handleVariable=, \n\t\t\tcompleteStatusSetting=\"Finished\", endTimeVariable=endTime, \n\t\t\tstartTimeVariable=startTime, waitType=_ANY_, signoff=1, \n\t\t\treturnCodeVariable=, returnCodeMacroVariable=, statusUnknownReturnCode=.);\n\n\tproc sql noprint;\n\t\tselect count(*) into :etls_rows \n\t\t\tfrom &statusTable where &statusVariable = &runningStatusSetting;\n\t\t%let etls_rows = &etls_rows;\n\n\t\t%if (&etls_rows gt 0) %then %do;\n\t\t\tselect &handleVariable into :etlsHandles1 - :etlsHandles&etls_rows \n\t\t\t\tfrom &statusTable where &statusVariable = &runningStatusSetting;\n\t\t%end;\n\tquit;\n\n\t%rcSet(&sqlrc);\n\n\t%if (&etls_rows gt 0) %then %do;\n\t\twaitfor &waitType \n\t\t\t%do i=1 %to &etls_rows;\n\t\t\t\t&&etlsHandles&i\n\t\t\t%end;\n\t\t;\n\t%end;\n\n\t%local useDefaultRCMacVar;\n\n\t%if (\"&returnCodeMacroVariable\" eq \"\") %then\n\t\t%let useDefaultRCMacVar=Y;\n\n\t%do i=1 %to &etls_rows;\n\t\t%if (\"&useDefaultRCMacVar\"=\"Y\") %then\n\t\t\t%let returnCodeMacroVariable=job_rc&&etlsHandles&i;\n\t\twaitfor &&etlsHandles&i timeout=1;\n\n\t\t%if (&SYSRC eq 0) or (&SYSRC eq -2) %then /* process complete */\n\n\t\t\t%do;\n\t\t\t\t%etls_freeHandle(statusTable=&statusTable, statusVariable=&statusVariable,\n\t\t\t\t\thandleVariable=&handleVariable, handleName=\"&&etlsHandles&i\", \n\t\t\t\t\tstatusSetting=&completeStatusSetting, endTimeVariable=&endTimeVariable, \n\t\t\t\t\tstartTimeVariable=&startTimeVariable, signoff=&signoff, \n\t\t\t\t\treturnCodeVariable=&returnCodeVariable, returnCodeMacroVariable=&returnCodeMacroVariable, \n\t\t\t\t\tstatusUnknownReturnCode=&statusUnknownReturnCode, \n\t\t\t\t\tstartTimeMacroVariable=etls_startTime_&&etlsHandles&i., \n\t\t\t\t\tendTimeMacroVariable=etls_endTime_&&etlsHandles&i. );\n\t\t\t\t%put NOTE: Process &&etlsHandles&i. has completed.;\n\t\t\t%end;\n\t\t\t%else %if (&SYSRC ne -1) %then /* process status unknown */\n\t\t\t\t%do;\n\t\t\t\t\t%etls_freeHandle(statusTable=&statusTable, statusVariable=&statusVariable,\n\t\t\t\t\t\thandleVariable=&handleVariable, handleName=\"&&etlsHandles&i\", \n\t\t\t\t\t\tstatusSetting=\"Unknown Status\", endTimeVariable=&endTimeVariable, \n\t\t\t\t\t\tstartTimeVariable=&startTimeVariable, signoff=&signoff, \n\t\t\t\t\t\treturnCodeVariable=&returnCodeVariable, returnCodeMacroVariable=&returnCodeMacroVariable, \n\t\t\t\t\t\tstatusUnknownReturnCode=&statusUnknownReturnCode, startTimeMacroVariable=etls_startTime_&&etlsHandles&i );\n\t\t\t\t\t%put NOTE: Cannot retrieve status from process &&etlsHandles&i..;\n\t\t\t\t%end;\n\t%end;\n%mend etls_waitfor;\n\n%macro etls_getProcessesRunning(statusTable=, statusVariable=, processCountMacro=, \n\t\t\tstatusSetting=\"Running\");\n\n\tproc sql noprint;\n\t\tselect count(*) into: &processCountMacro \n\t\t\tfrom &statusTable where &statusVariable = &statusSetting;\n\tquit;\n\n\t%rcSet(&sqlrc);\n%mend etls_getProcessesRunning;\n\n%macro etls_getParameterNames(parameterTable=, parameterVariableMacro=, startingColumnNumber=1);\n\t%let ¶meterVariableMacro =;\n\t%let dsid = %sysfunc(open(¶meterTable));\n\n\t%if (&dsid gt 0) %then %do;\n\n\t\t%do i=&startingColumnNumber %to %sysfunc(attrn(&dsid,nvars));\n\t\t\t%let ¶meterVariableMacro = &&¶meterVariableMacro %sysfunc(varname(&dsid,&i));\n\t\t%end;\n\n\t\t%let dsid = %sysfunc(close(&dsid));\n\t%end;\n\t%else %put %sysfunc(sysmsg());\n\t%rcSet(&syserr);\n%mend etls_getParameterNames;\n\n%macro etls_getParameters(parameterTable=, row=, startingColumnNumber=1 , handleName=);\n\n\tdata _null_;\n\t\tlength vname $256 vtype $1 value $32767;\n\t\tdsid = open(\"¶meterTable\");\n\n\t\tif (dsid > 0) then do;\n\n\t\t\tdo _i = 1 to &row;\n\t\t\t\tfetchrc = fetch(dsid);\n\t\t\tend;\n\n\t\t\tdo _i=&startingColumnNumber to attrn(dsid,'nvars');\n\t\t\t\tvname = varname(dsid,_i);\n\t\t\t\tvtype = vartype(dsid,_i);\n\n\t\t\t\tif (fetchrc = 0) then do;\n\t\t\t\t\tif (vtype = 'C') then do;\n\t\t\t\t\t\tvalue = getvarc(dsid,_i);\n\t\t\t\t\t\tvalue = tranwrd(value,\"%\",\"%%\");\n\t\t\t\t\t\tvalue = tranwrd(value,\"(\",\"%(\");\n\t\t\t\t\t\tvalue = tranwrd(value,\")\",\"%)\");\n\t\t\t\t\t\tvalue = tranwrd(value,'\"','%\"');\n\t\t\t\t\t\tvalue = tranwrd(value,\"'\",\"%'\");\n\t\t\t\t\tend;\n\t\t\t\t\telse value = left(put(getvarn(dsid,_i),best32.));\n\t\t\t\tend; /* fetchrc = 0 */\n\n\t\t\t\tput;\n\n\t\t\t\tif kindexc(trimn(value),\"+-*/<>=^~;, '()&%\",'\"')>0 then do;\n\t\t\t\t\tvalue='%nrstr('||ktrim(value)||')';\n\t\t\t\t\tput \"ETLS_DIAG%QUOTE(:) Special characters encountered; References may require: %nrbquote(%)UNQUOTE(&\" vname+(-1)').';\n\t\t\t\t\tput \"NOTE: Special characters encountered; References may require: %nrbquote(%)UNQUOTE(&\" vname+(-1)').';\n\t\t\t\tend;\n\n\t\t\t\t%if %str(&handlename) ne %str() %then %do;\n\t\t\t\t\tvalue = '%syslput '||ktrim(vname)||'= '||ktrim(value)||\" / remote = &handleName;\";\n\t\t\t\t%end;\n\t\t\t\t%else %do;\n\t\t\t\t\tvalue = '%let '||ktrim(vname)||'= '||ktrim(value)||';';\n\t\t\t\t%end;\n\n\t\t\t\tput \"NOTE: Setting macro variable \" vname \"with statement:\" value;\n\t\t\t\tcall execute(value);\n\t\t\tend; /* do i= */\n\n\t\t\tdsid = close(dsid);\n\t\tend; /* dsid > 0 */\n\t\telse do;\n\t\t\tput \"ERROR%QUOTE(:) Parameter table, ¶meterTable., could not be\"\n\t\t\t\t\" opened.\";\n\t\t\tabort;\n\t\tend;\n\n\t\tstop;\n\trun;\n\n\t%rcSet(&syserr);\n%mend etls_getParameters;\n\n\n/* endregion */\n\n/*==========================================================================* \n * Step: Main \t\t\t\t * \n *==========================================================================*/\n%macro etls_mainloop;\n\t%local etls_filePrefix;\n\t%let etls_filePrefix =;\n\n\t/* Compability change introduced by 2023.10 */\n\toptions set=SAS_LOCAL_MPCONNECT=true;\n\n\t/* \tLocal Connect and UseOptimalProcesses=1: we will select an optimal Maximum number of sub-process \n\t\tthat is defined in parall_loop_local_numsubprocs defined in autoexec.\n\t*/\n\t%if %str(&etls_sasConnectContext) = %str() and &etls_UseOptimalProcesses=1 and %symexist(PARALL_LOOP_LOCAL_NUMSUBPROCS) %then %do;\n\t\t%put NOTE: Local Signon configured and use environment maximum is checked. Will use value ¶ll_loop_local_numsubprocs defined in macrovar: parall_loop_local_numsubprocs;\n\t\t%let etls_maxProcesses=¶ll_loop_local_numsubprocs;\n\t%end;\n\n\t/* \n\t Generate Flow Code and store in catalog work.codegen.code.source.\n\t This will later be uploaded to session and executed\n\t*/\n\tfilename rmtcode catalog 'work.codegen.code.source' lrecl=32767;\n\t%generateCodefromFlow(\n\t\tflowToCode=&etls_FlowSourcePath,\n\t\toutfileref=rmtcode);\n\tfilename rmtcode clear;\n\n\t%macro etls_processToLoop(parameterTable=, row=, handleName=rmt);\n\t\t%etls_getParameters(parameterTable=¶meterTable, row=&row, \n\t\t\tstartingColumnNumber=1, handleName=&handleName);\n\t\t%let etls_previousFilePrefix = &etls_filePrefix;\n\t\t%local etls_filePrefix;\n\t\t%let etls_filePrefix = &etls_previousFilePrefix.&handleName;\n\t\t%let __workdir=%sysfunc(pathname(work));\n\n\t\t%syslput &etls_controlName = &row / remote = &handleName;\n\t\t%syslput handleName = &handleName / remote = &handleName;\n\n\t\t%if %symexist(jobID) %then %do;\n\t\t\t%syslput jobID = &jobID / remote = &handlename;\n\t\t%end;\n\t\t\n\t\t%if %symexist(etls_userID) %then %do;\n\t\t\t%syslput etls_userID = &etls_userID / remote = &handlename;\n\t\t%end;\n\t\t%if %symexist(AUTOEXEC_FILE) %then %do;\n\t\t\t%syslput AUTOEXEC_FILE = &AUTOEXEC_FILE / remote = &handlename;\n\t\t%end;\n\t\t\n\t\t%syslput etls_jobName = &etls_jobName / remote = &handlename;\n\t\t%syslput __workdir = &__workdir / remote = &handlename;\n\t\t%syslput etls_sasConnectContext = &etls_sasConnectContext / remote = &handlename;\n\n\t\t/* Setup to pass macros to remote session */\n\t\tdata _null_;\n\t\t\trmtname = \"/ remote = &handleName\";\n\t\t\tcall execute('%syslput etls_filePrefix = &etls_filePrefix '||rmtname||';');\n\t\t\tcall execute('run;');\n\t\trun;\n\n\t\trsubmit &handleName wait = no sysrputsync = yes persist = no\n\t\t\t\t%if %str(&etls_logOutputPath) ne %str() %then %do;\n\t\t\t\t\tlog = \"&etls_logOutputPath./&etls_filePrefix..log\" \n\t\t\t\t\toutput = \"&etls_logOutputPath./&etls_filePrefix..lst\"\n\t\t\t\t%end; \n\t\t\t;\n\t\t\t\n\t\t\t/* Define CASHOST as this is wrongly configured in SAS Connect Sessions */\n\t\t\toptions CASHOST='sas-cas-server-default-client';\n\n\t\t\t/* Upload generated code to this session in catalog work.codegen.code.source */\n\t\t\tproc upload incat=work.codegen outcat=work.codegen;\n\t\t\trun;\n\n\t\t\t/* Execute an alternative autoexec if specified in &AUTOEXEC_FILE */\n\t\t\t%macro execUserDefinedAutoexec;\n\t\t\t\t%if %str(%quote(&AUTOEXEC_FILE)) ne %str() %then %do;\n\t\t\t\t\t%put NOTE: Will execute user specified autoexec specified in macro variable AUTOEXEC_FILE;\n\t\t\t\t\t%inc \"&AUTOEXEC_FILE\";\t\t\n\t\t\t\t%end;\n\t\t\t\t%else %do;\n\t\t\t\t\t\n\t\t\t\t%end;\n\t\t\t%mend execUserDefinedAutoexec;\n\t\t\t%if %symexist(AUTOEXEC_FILE) %then %do;\n\t\t\t\t%execUserDefinedAutoexec;\n\t\t\t%end;\n\t\t\t\n\n\t\t\t/* Allocate SWORK: SASWORK of calling SAS-process */\n\t\t\t%if %str(%quote(&etls_sasConnectContext)) = %str() %then %do;\n\t\t\t\tlibname swork \"&__workdir\"; \n\t\t\t%end;\n\t\t\t\n\t\t\t/* Setup to capture return codes */\n\t\t\t%macro etls_setClientSideStatus;\n\t\t\t\t%sysrput job_rc&handleName = &syscc;\n\t\t\t\t%sysrput etls_endTime_&handleName = %sysfunc(datetime());\n\t\t\t%mend etls_setClientSideStatus;\n\t\t\t%nrstr(%sysrput etls_startTime_&handleName = %sysfunc(datetime()));\n\t\t\t\n\t\t\t/* Submit code */\n\t\t\tfilename execcode catalog 'work.codegen.code.source';\n\t\t\t%inc execcode;\n\n\t\t\t%etls_setClientSideStatus;\n\t\t\t\n\t\tendrsubmit;\n\t\t\n\t%mend etls_processToLoop;\n\n\t%local \n\t\tetls_processesRunning \n\t\tetls_additionalSignonOptions etls_signonRetries;\n\t%let etls_gridOptionSet = %nrstr();\n\t%let etls_workload =;\n\t%let etls_additionalSignonOptions =;\n\t%let etls_signonRetries = 3;\n\t%let etls_gridSuppJobOptions = %nrstr();\n\t%let &etls_controlName = 0;\n\t%put %str(NOTE: Creating status table...);\n\n\tdata &etls_statusTable;\n\t\tattrib etls_handleName length = $32\n\t\t\tlabel = 'Name of handle to remote session';\n\t\tattrib etls_machineId length = $32\n\t\t\tlabel = 'Name of machine executing the task';\n\t\tattrib etls_startTime length = 8\n\t\t\tformat = nldatmap.\n\t\t\tlabel = 'Start time of task';\n\t\tattrib etls_endTime length = 8\n\t\t\tformat = nldatmap.\n\t\t\tlabel = 'End time of task';\n\t\tattrib etls_status length = $32\n\t\t\tlabel = 'Current status of task';\n\t\tattrib etls_jobRC length = 8\n\t\t\tlabel = 'Return code of task';\n\t\tset &etls_controlTable;\n\trun;\n\n\t%rcSet(&syserr);\n\t%put %str(NOTE: Creating parameter table...);\n\n\tproc sql;\n\t\tcreate table &etls_parameterTable as \n\t\t\tselect *\n\t\t\t\tfrom &etls_controlTable;\n\tquit;\n\n\t%rcSet(&sqlrc);\n\n\t/* Get the number of times to iterate from the number of rows in the source */\n\t/* table */\n\tproc sql noprint;\n\t\tselect count(*) into :&etls_controlName._max from &etls_statusTable;\n\t\t%let &etls_controlName._max = &&&etls_controlName._max;\n\tquit;\n\n\t%rcSet(&sqlrc);\n\n\t%if (&etls_maxProcesses > 0) %then %do;\n\n\t\t%do %until (&&&etls_controlName ge &&&etls_controlName._max);\n\t\t\t%let etls_lastLoopPtr = &&&etls_controlName;\n\n\t\t\t%etls_getProcessesRunning(statusTable=&etls_statusTable, statusVariable=etls_status, \n\t\t\t\tprocessCountMacro=etls_processesRunning, statusSetting=\"Running\");\n\n\t\t\t%do %while(&etls_processesRunning lt &&&etls_controlName._max \n\t\t\t\tand &etls_processesRunning lt &etls_maxProcesses \n\t\t\t\tand &&&etls_controlName lt &&&etls_controlName._max);\n\t\t\t\t%let &etls_controlName = %eval(&&&etls_controlName+1);\n\t\t\t\t%let etls_signonMacVar = etls_signonStatus&&&etls_controlName;\n\t\t\t\t%global &etls_signonMacVar;\n\n\t\t\t\t%etls_createHandle(statusTable=&etls_statusTable,\n\t\t\t\t\tstatusVariable=etls_status,\n\t\t\t\t\thandleVariable=etls_handleName,\n\t\t\t\t\thandlePrefix=&etls_controlName,\n\t\t\t\t\tstatusSetting=\"Running\",\n\t\t\t\t\trow=&&&etls_controlName,\n\t\t\t\t\tmachineVariable=etls_machineId,\n\t\t\t\t\tstartTimeVariable=etls_startTime,\n\t\t\t\t\tcmacvar=&etls_signonMacVar,\n\t\t\t\t\tsignon=1,\n\t\t\t\t\tuseGrid=1,\n\t\t\t\t\tadditionalSignonOptions=&etls_additionalSignonOptions,\n\t\t\t\t\tsignonRetries=&etls_signonRetries);\n\t\t\t\t%etls_getProcessesRunning(statusTable=&etls_statusTable, statusVariable=etls_status, \n\t\t\t\t\tprocessCountMacro=etls_processesRunning, statusSetting=\"Running\");\n\n\t\t\t\t%if &etls_processesRunning > 0 and &&&etls_signonMacVar ne 1 %then\n\t\t\t\t\t%etls_processToLoop(parameterTable=&etls_parameterTable, row=&&&etls_controlName,\n\t\t\t\t\thandleName=%etls_getHandle(statusTable=&etls_statusTable,\n\t\t\t\t\thandleVariable=etls_handleName, row=&&&etls_controlName));\n\t\t\t\t%else %do;\n\n\t\t\t\t\t/* if signon error, set iterator to max to force loop to stop. */\n\t\t\t\t\t%let &etls_controlName = &&&etls_controlName._max;\n\t\t\t\t\t%put %str(ERROR%QUOTE(:) A parallel process did not start. Exiting loop.);\n\t\t\t\t%end;\n\t\t\t%end;\n\n\t\t\t%etls_waitFor(statusTable=&etls_statusTable, statusVariable=etls_status, \n\t\t\t\trunningStatusSetting=\"Running\", handleVariable=etls_handleName, \n\t\t\t\tcompleteStatusSetting=\"Finished\", endTimeVariable=etls_endTime, \n\t\t\t\tstartTimeVariable=etls_startTime, waitType=_ANY_, signoff=1, \n\t\t\t\treturnCodeVariable=etls_jobRC, \n\t\t\t\tstatusUnknownReturnCode=.);\n\t\t%end;\n\n\t\t%etls_waitFor(statusTable=&etls_statusTable, statusVariable=etls_status, \n\t\t\trunningStatusSetting=\"Running\", handleVariable=etls_handleName, \n\t\t\tcompleteStatusSetting=\"Finished\", endTimeVariable=etls_endTime, \n\t\t\tstartTimeVariable=etls_startTime, waitType=_ALL_, signoff=1, \n\t\t\treturnCodeVariable=etls_jobRC, \n\t\t\tstatusUnknownReturnCode=.);\n\t%end;\n\n\t/* Delete parameter table */\n\tproc datasets lib = %scan(&etls_parameterTable,1,.) nolist nowarn memtype = (data view);\n\t\tdelete %scan(&etls_parameterTable,2,.);\n\tquit;\n\n\t/* Delete generated code */\n\tproc catalog cat=work.codegen kill;\n\tquit;\n\n%mend etls_mainloop;\n\n%etls_mainloop;\n\n/* region: Generation of code */\n%symdel ETLS_ABORTONERROR /NOWARN;\n%symdel ETLS_CONTROLTABLE /NOWARN;\n%symdel ETLS_FLOWSOURCEPATH /NOWARN;\n%symdel ETLS_JOBNAME /NOWARN;\n%symdel ETLS_LOGOUTPUTPATH /NOWARN;\n%symdel ETLS_MAXPROCESSES /NOWARN;\n%symdel ETLS_PARAMETERTABLE /NOWARN;\n%symdel ETLS_SASCONNECTCONTEXT /NOWARN;\n%symdel ETLS_STATUSTABLE /NOWARN;\n%symdel ETLS_USEOPTIMALPROCESSES /NOWARN;\n%symdel ETLS_CONTROLNAME /NOWARN;\n%symdel ETLS_STEPSTARTTIME /NOWARN;\n%symdel transformID /NOWARN;\n%symdel trans_rc /NOWARN;\n%symdel job_rc /NOWARN;\n%symdel etls_stepStartTime /NOWARN;\n/* endregion */\n"}}