Merge pull request #255 from openclimatefix/issue/217-data-source-configs

Issue/217 data source configs

Author: peterdudfield

gcp.yaml

@@ -0,0 +1,66 @@
+general:
+  cloud: gcp
+  description: example configuration
+  name: example
+git: null
+input_data:
+  default_forecast_minutes: 60
+  default_history_minutes: 30
+  gsp:
+    forecast_minutes: 60
+    gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/GSP/v0/pv_gsp.zarr
+    history_minutes: 30
+  nwp:
+    forecast_minutes: 60
+    history_minutes: 30
+    nwp_channels:
+      - t
+      - dswrf
+      - prate
+      - r
+      - sde
+      - si10
+      - vis
+      - lcc
+      - mcc
+      - hcc
+    nwp_image_size_pixels: 64
+    nwp_zarr_path: gs://solar-pv-nowcasting-data/NWP/UK_Met_Office/UKV__2018-01_to_2019-12__chunks__variable10__init_time1__step1__x548__y704__.zarr
+  pv:
+    forecast_minutes: 60
+    history_minutes: 30
+    solar_pv_data_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_timeseries_batch.nc
+    solar_pv_metadata_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_metadata.csv
+  satellite:
+    forecast_minutes: 60
+    history_minutes: 30
+    sat_channels:
+      - HRV
+      - IR_016
+      - IR_039
+      - IR_087
+      - IR_097
+      - IR_108
+      - IR_120
+      - IR_134
+      - VIS006
+      - VIS008
+      - WV_062
+      - WV_073
+    satellite_image_size_pixels: 64
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr
+  sun:
+    forecast_minutes: 60
+    history_minutes: 30
+    sun_zarr_path: gs://solar-pv-nowcasting-data/Sun/v0/sun.zarr/
+  topographic:
+    forecast_minutes: 60
+    history_minutes: 30
+    topographic_filename: gs://solar-pv-nowcasting-data/Topographic/europe_dem_1km_osgb.tif
+output_data:
+  filepath: gs://solar-pv-nowcasting-data/prepared_ML_training_data/v5/
+process:
+  batch_size: 32
+  local_temp_path: ~/temp/
+  seed: 1234
+  upload_every_n_batches: 16

nowcasting_dataset/config/gcp.yaml

@@ -1,46 +1,65 @@
 general:
-  name: gcp
-  description: Configuration for Google Cloud
+  cloud: gcp
+  description: example configuration
+  name: example
 input_data:
-  nwp_zarr_path: gs://solar-pv-nowcasting-data/NWP/UK_Met_Office/UKV__2018-01_to_2019-12__chunks__variable10__init_time1__step1__x548__y704__.zarr
-  satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr
-  solar_pv_data_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_timeseries_batch.nc
-  solar_pv_metadata_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_metadata.csv
-  gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/GSP/v2/pv_gsp.zarr
-  topographic_filename: gs://solar-pv-nowcasting-data/Topographic/europe_dem_1km_osgb.tif
-  sun_zarr_path: gs://solar-pv-nowcasting-data/Sun/v0/sun.zarr
+  default_forecast_minutes: 60
+  default_history_minutes: 30
+  gsp:
+    forecast_minutes: 60
+    gsp_zarr_path: gs://solar-pv-nowcasting-data/PV/GSP/v2/pv_gsp.zarr
+    history_minutes: 30
+  nwp:
+    forecast_minutes: 60
+    history_minutes: 30
+    nwp_channels:
+      - t
+      - dswrf
+      - prate
+      - r
+      - sde
+      - si10
+      - vis
+      - lcc
+      - mcc
+      - hcc
+    nwp_image_size_pixels: 64
+    nwp_zarr_path: gs://solar-pv-nowcasting-data/NWP/UK_Met_Office/UKV__2018-01_to_2019-12__chunks__variable10__init_time1__step1__x548__y704__.zarr
+  pv:
+    forecast_minutes: 60
+    history_minutes: 30
+    solar_pv_data_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_timeseries_batch.nc
+    solar_pv_metadata_filename: gs://solar-pv-nowcasting-data/PV/PVOutput.org/UK_PV_metadata.csv
+  satellite:
+    forecast_minutes: 60
+    history_minutes: 30
+    sat_channels:
+      - HRV
+      - IR_016
+      - IR_039
+      - IR_087
+      - IR_097
+      - IR_108
+      - IR_120
+      - IR_134
+      - VIS006
+      - VIS008
+      - WV_062
+      - WV_073
+    satellite_image_size_pixels: 64
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr
+  sun:
+    forecast_minutes: 60
+    history_minutes: 30
+    sun_zarr_path: gs://solar-pv-nowcasting-data/Sun/v0/sun.zarr/
+  topographic:
+    forecast_minutes: 60
+    history_minutes: 30
+    topographic_filename: gs://solar-pv-nowcasting-data/Topographic/europe_dem_1km_osgb.tif
 output_data:
   filepath: gs://solar-pv-nowcasting-data/prepared_ML_training_data/v7/
 process:
+  batch_size: 32
   local_temp_path: ~/temp/
   seed: 1234
-  batch_size: 32
   upload_every_n_batches: 16
-  forecast_minutes: 60
-  history_minutes: 30
-  satellite_image_size_pixels: 64
-  nwp_image_size_pixels: 64
-  nwp_channels:
-    - t
-    - dswrf
-    - prate
-    - r
-    - sde
-    - si10
-    - vis
-    - lcc
-    - mcc
-    - hcc
-  sat_channels:
-    - HRV
-    - IR_016
-    - IR_039
-    - IR_087
-    - IR_097
-    - IR_108
-    - IR_120
-    - IR_134
-    - VIS006
-    - VIS008
-    - WV_062
-    - WV_073