Update to reusable workflows

.github/workflows/linters.yaml

@@ -3,22 +3,7 @@ name: Lint Python
 on: [push]
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.9]
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pydocstyle
-      - name: Docstyle linting
-        run: |
-          pydocstyle --convention=google --add-ignore=D200,D202,D210,D212,D415 nowcasting_dataset
+  call-run-python-linters:
+    uses: openclimatefix/.github/.github/workflows/python-lint.yml@main
+    with:
+      folder: "nowcasting_dataset"

.github/workflows/release.yaml

@@ -4,57 +4,7 @@ on:
     branches:
       - main
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    outputs:
-      new_tag: ${{ steps.bumper.outputs.new_ver }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Bump version
-        id: bumper
-        uses: jaumann/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Push changes
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          tags: true
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build
-      - name: Build package
-        run: python -m build
-      - name: Publish package
-        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-
-  release:
-    needs: [build]
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo ${{needs.build.outputs.new_tag}}
-      - name: Changelog
-        uses: scottbrenner/generate-changelog-action@master
-        id: Changelog
-        env:
-          REPO: ${{ github.repository }}
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@latest
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: v${{ needs.build.outputs.new_tag }}
-          release_name: v${{ needs.build.outputs.new_tag }}
-          body: |
-            ${{ steps.Changelog.outputs.changelog }}
-          draft: false
-          prerelease: false
+  call-run-python-release:
+    uses: openclimatefix/.github/.github/workflows/python-release.yml@main
+    secrets:
+      token: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/workflows.yaml

@@ -0,0 +1,6 @@
+name: Python package
+
+on: [push]
+jobs:
+  call-run-python-tests:
+    uses: openclimatefix/.github/.github/workflows/python-test.yml@main

.pre-commit-config.yaml

@@ -12,7 +12,27 @@ repos:
       - id: debug-statements
       - id: detect-private-key
 
-  # python code formatting
+  # python code formatting/linting
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.1.1
+    hooks:
+      - id: pydocstyle
+        args:
+          [
+            --convention=google,
+            "--add-ignore=D200,D202,D210,D212,D415",
+            "nowcasting_dataset",
+          ]
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        args: [--max-line-length, "100", --extend-ignore=E203]
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.9.3
+    hooks:
+      - id: isort
+        args: [--profile, black, --line-length, "100"]
   - repo: https://github.com/psf/black
     rev: 20.8b1
     hooks:

notebooks/2021-08/2021-08-25/video.py

@@ -1,11 +1,12 @@
 import os
-from nowcasting_dataset.dataset import NetCDFDataset, SAT_MEAN, SAT_STD
-import plotly.graph_objects as go
-import plotly
-import pandas as pd
+
+import cv2
 import numpy as np
+import pandas as pd
+import plotly
 import plotly.graph_objects as go
-import cv2
+
+from nowcasting_dataset.dataset import SAT_MEAN, SAT_STD, NetCDFDataset
 
 DATA_PATH = "gs://solar-pv-nowcasting-data/prepared_ML_training_data/v4/"
 TEMP_PATH = ""
@@ -40,6 +41,7 @@
 
 # get satellite image, currently from https://github.com/openclimatefix/py-staticmaps
 import staticmaps
+
 from nowcasting_dataset.geospatial import osgb_to_lat_lon
 
 bottom_left = osgb_to_lat_lon(x=x_min, y=y_min)

notebooks/2021-08/2021-08-26/video.py

@@ -1,18 +1,18 @@
 import os
-from nowcasting_dataset.dataset import NetCDFDataset, SAT_MEAN, SAT_STD
-import plotly.graph_objects as go
-import plotly
-import pandas as pd
+
+import cv2
 import numpy as np
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
+import pandas as pd
+import plotly
 import plotly.express as px
-import cv2
+import plotly.graph_objects as go
 
 # get satellite image, currently from https://github.com/openclimatefix/py-staticmaps
 import staticmaps
-from nowcasting_dataset.geospatial import osgb_to_lat_lon
+from plotly.subplots import make_subplots
 
+from nowcasting_dataset.dataset import SAT_MEAN, SAT_STD, NetCDFDataset
+from nowcasting_dataset.geospatial import osgb_to_lat_lon
 
 ##############
 # Setup data generator

notebooks/2021-09/2021-09-06/gsp.py

@@ -1,5 +1,6 @@
 import json
 from urllib.request import urlopen
+
 import geopandas as gpd
 import plotly.graph_objects as go
 

notebooks/2021-09/2021-09-07/gsp.py

@@ -1,9 +1,10 @@
+import itertools
 import json
 from urllib.request import urlopen
+
 import geopandas as gpd
-import plotly.graph_objects as go
 import numpy as np
-import itertools
+import plotly.graph_objects as go
 
 WGS84_CRS = "EPSG:4326"
 

notebooks/2021-09/2021-09-08/dan.py

@@ -1,5 +1,6 @@
 import gcsfs
 import xarray as xr
-filename = 'gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr/'
+
+filename = "gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/OSGB36/all_zarr_int16_single_timestep.zarr/"
 # filename2 = 'gs://solar-pv-nowcasting-data/prepared_ML_training_data/testing.zarr'
-ds = xr.open_dataset(filename, engine='zarr')
+ds = xr.open_dataset(filename, engine="zarr")

notebooks/2021-09/2021-09-13/remove_hash.py

@@ -1,6 +1,6 @@
-from nowcasting_dataset.filesystem.gcp import get_all_filenames_in_path, rename_file
 import random
 
+from nowcasting_dataset.filesystem.gcp import get_all_filenames_in_path, rename_file
 
 # batch files are save with hash id and then batch and we want to remove the hashing
 # {xxxxxx}_{batch_idx}.nc is the format of the file

notebooks/2021-09/2021-09-14/gsp_centroid.py

@@ -1,12 +1,13 @@
+import json
+
+import plotly.graph_objects as go
+
 from nowcasting_dataset.data_sources.gsp.eso import (
     get_gsp_metadata_from_eso,
     get_gsp_shape_from_eso,
 )
-import plotly.graph_objects as go
-import json
 from nowcasting_dataset.geospatial import WGS84_CRS
 
-
 # load data
 shape_data_raw = get_gsp_metadata_from_eso()
 shape_data_raw = shape_data_raw.to_crs(WGS84_CRS)

notebooks/2021-09/2021-09-14/gsp_duplicated.py

@@ -1,22 +1,23 @@
+import json
+
+import plotly.graph_objects as go
+
 from nowcasting_dataset.data_sources.gsp.eso import (
     get_gsp_metadata_from_eso,
     get_gsp_shape_from_eso,
 )
-import plotly.graph_objects as go
-import json
-
 
 # Seem to have 2 different GSP shape files, #Hams Hall, Melksham, Iron Acton, Axminster
 s = get_gsp_shape_from_eso(join_duplicates=False)
-duplicated_raw = s[s['RegionID'].duplicated(keep=False)]
+duplicated_raw = s[s["RegionID"].duplicated(keep=False)]
 duplicated_raw["Amount"] = range(0, len(duplicated_raw))
 
-for i in range(0,8,2):
+for i in range(0, 8, 2):
 
     # just select the first one
-    duplicated = duplicated_raw.iloc[i: i + 2]
+    duplicated = duplicated_raw.iloc[i : i + 2]
     shapes_dict = json.loads(duplicated["geometry"].to_json())
-    region_name = duplicated['RegionName'].iloc[0]
+    region_name = duplicated["RegionName"].iloc[0]
 
     # plot to check it looks right
     fig = go.Figure()
@@ -37,5 +38,3 @@
     # fig.show(renderer="browser")
     fig.write_html(f"images/duplicated_{region_name}.html")
     fig.write_image(f"images/duplicated_{region_name}.png")
-
-

notebooks/2021-09/2021-09-28/get_raw_sun_data.py

@@ -12,16 +12,18 @@
 logging.getLogger().setLevel(logging.DEBUG)
 logging.getLogger("urllib3").setLevel(logging.WARNING)
 
+import os
 from datetime import datetime
 from pathlib import Path
-import pandas as pd
+
 import numpy as np
-import os
+import pandas as pd
+
 import nowcasting_dataset
 from nowcasting_dataset.data_sources.gsp.eso import get_gsp_metadata_from_eso
 from nowcasting_dataset.data_sources.sun.raw_data_load_save import (
-    save_to_zarr,
     get_azimuth_and_elevation,
+    save_to_zarr,
 )
 
 # set up

notebooks/2021-09/2021-09-29/gsp_duplicated.py

@@ -1,13 +1,13 @@
+import json
+
+import plotly.graph_objects as go
+
 from nowcasting_dataset.data_sources.gsp.eso import (
     get_gsp_metadata_from_eso,
     get_gsp_shape_from_eso,
 )
-import plotly.graph_objects as go
-import json
-
 from nowcasting_dataset.geospatial import WGS84_CRS
 
-
 # Seem to have 2 different GSP shape files, #Hams Hall, Melksham, Iron Acton, Axminster
 s = get_gsp_shape_from_eso(join_duplicates=False)
 s = s.to_crs(WGS84_CRS)

notebooks/2021-09/2021-09-29/video.py

@@ -1,13 +1,14 @@
 # Idea is to make a GSP video using Sheffield solar data for one day
+import json
+import logging
+from datetime import datetime
+
 import pandas as pd
+import plotly.graph_objects as go
 
-from nowcasting_dataset.data_sources.gsp.pvlive import load_pv_gsp_raw_data_from_pvlive
 from nowcasting_dataset.data_sources.gsp.eso import get_gsp_metadata_from_eso
-from datetime import datetime
+from nowcasting_dataset.data_sources.gsp.pvlive import load_pv_gsp_raw_data_from_pvlive
 from nowcasting_dataset.geospatial import WGS84_CRS
-import logging
-import plotly.graph_objects as go
-import json
 
 logging.basicConfig()
 logging.getLogger().setLevel(logging.DEBUG)

notebooks/2021-10/2021-10-01/pydantic.py

@@ -1,10 +1,11 @@
-from pydantic import BaseModel, Field, validator
 from typing import Union
+
 import numpy as np
-import xarray as xr
 import torch
-from nowcasting_dataset.config.model import Configuration
+import xarray as xr
+from pydantic import BaseModel, Field, validator
 
+from nowcasting_dataset.config.model import Configuration
 
 Array = Union[xr.DataArray, np.ndarray, torch.Tensor]
 

notebooks/2021-10/2021-10-08/xr_compression.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import xarray as xr
+
 from nowcasting_dataset.utils import coord_to_range