Merge branch 'master' into testing

Author: seisman

.github/workflows/ci_tests_dev.yaml

@@ -1,7 +1,7 @@
-# This workflow installs PyGMT dependencies, builds documentation and runs tests on GMT master
+# This workflow installs PyGMT dependencies, builds documentation and runs tests on GMT latest
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: GMT Master Tests
+name: GMT Latest Tests
 
 on:
   # push:
@@ -14,13 +14,14 @@ on:
 
 jobs:
   test_gmt_master:
-    name: ${{ matrix.os }} - Python ${{ matrix.python-version }}
+    name: ${{ matrix.os }} - GMT ${{ matrix.gmt_git_ref }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         python-version: [3.8]
         os: [ubuntu-20.04, macOS-10.15]
+        gmt_git_ref: [6.1, master]
     env:
       # LD_LIBRARY_PATH: ${{ github.workspace }}/gmt/lib:$LD_LIBRARY_PATH
       GMT_INSTALL_DIR: ${{ github.workspace }}/gmt-install-dir
@@ -54,9 +55,11 @@ jobs:
       - name: Install build dependencies
         run: conda install cmake libblas libcblas liblapack fftw gdal ghostscript libnetcdf hdf5 zlib curl pcre ipython pytest pytest-cov pytest-mpl
 
-      # Install GMT master branch
-      - name: Install GMT from master
+      # Build and install latest GMT from GitHub
+      - name: Install GMT ${{ matrix.gmt_git_ref }} branch
         run: curl https://raw.githubusercontent.com/GenericMappingTools/gmt/master/ci/build-gmt.sh | bash
+        env:
+          GMT_GIT_REF: ${{ matrix.gmt_git_ref }}
 
       # Download cached remote files (artifacts) from Github
       - name: Download remote data from Github

AUTHORS.md

@@ -9,6 +9,7 @@ The following people have contributed code to the project (alphabetical by last
 and are considered the "PyGMT Developers":
 
 * [Wei Ji Leong](https://github.com/weiji14)
+* [Tyler Newton](http://www.tnewton.com/)
 * [Dongdong Tian](https://seisman.info/)
 * [Liam Toney](https://liam.earth/)
 * [Leonardo Uieda](http://www.leouieda.com/)

MAINTENANCE.md

@@ -54,7 +54,7 @@ There are 3 configuration files located in `.github/workflows`:
 This is ran on every commit on the *master* and Pull Request branches.
 It is also scheduled to run daily on the *master* branch.
 
-2. `ci_tests_dev.yaml` (GMT Master Tests on Linux/macOS).
+2. `ci_tests_dev.yaml` (GMT Latest Tests on Linux/macOS).
 
 This is only triggered when a review is requested or re-requested on a PR.
 It is also scheduled to run daily on the *master* branch.

README.rst

@@ -15,9 +15,9 @@ PyGMT
 .. image:: https://github.com/GenericMappingTools/pygmt/workflows/Tests/badge.svg
     :alt: GitHub Actions Tests status
     :target: https://github.com/GenericMappingTools/pygmt/actions?query=workflow%3ATests
-.. image:: https://github.com/GenericMappingTools/pygmt/workflows/GMT%20Master%20Tests/badge.svg
-    :alt: GitHub Actions GMT Master Tests status
-    :target: https://github.com/GenericMappingTools/pygmt/actions?query=workflow%3A"GMT+Master+Tests"
+.. image:: https://github.com/GenericMappingTools/pygmt/workflows/GMT%20Latest%20Tests/badge.svg
+    :alt: GitHub Actions GMT Latest Tests status
+    :target: https://github.com/GenericMappingTools/pygmt/actions?query=workflow%3A%22GMT+Latest+Tests%22
 .. image:: https://img.shields.io/codecov/c/github/GenericMappingTools/pygmt/master.svg?style=flat-square
     :alt: Test coverage status
     :target: https://codecov.io/gh/GenericMappingTools/pygmt

pygmt/clib/session.py

@@ -44,6 +44,8 @@
     "GMT_IS_SURFACE",
 ]
 
+METHODS = ["GMT_IS_DUPLICATE", "GMT_IS_REFERENCE"]
+
 MODES = ["GMT_CONTAINER_ONLY", "GMT_IS_OUTPUT"]
 
 REGISTRATIONS = ["GMT_GRID_PIXEL_REG", "GMT_GRID_NODE_REG"]
@@ -235,7 +237,7 @@ def __getitem__(self, name):
         value = c_get_enum(session, name.encode())
 
         if value is None or value == -99999:
-            raise GMTCLibError("Constant '{}' doesn't exits in libgmt.".format(name))
+            raise GMTCLibError(f"Constant '{name}' doesn't exist in libgmt.")
 
         return value
 
@@ -733,7 +735,7 @@ def put_vector(self, dataset, column, vector):
         """
         Attach a numpy 1D array as a column on a GMT dataset.
 
-        Use this functions to attach numpy array data to a GMT dataset and pass
+        Use this function to attach numpy array data to a GMT dataset and pass
         it to GMT modules. Wraps ``GMT_Put_Vector``.
 
         The dataset must be created by :meth:`~gmt.clib.Session.create_data`
@@ -793,11 +795,72 @@ def put_vector(self, dataset, column, vector):
                 )
             )
 
+    def put_strings(self, dataset, family, strings):
+        """
+        Attach a numpy 1D array of dtype str as a column on a GMT dataset.
+
+        Use this function to attach string type numpy array data to a GMT
+        dataset and pass it to GMT modules. Wraps ``GMT_Put_Strings``.
+
+        The dataset must be created by :meth:`~gmt.clib.Session.create_data`
+        first.
+
+        .. warning::
+            The numpy array must be C contiguous in memory. If it comes from a
+            column slice of a 2d array, for example, you will have to make a
+            copy. Use :func:`numpy.ascontiguousarray` to make sure your vector
+            is contiguous (it won't copy if it already is).
+
+        Parameters
+        ----------
+        dataset : :class:`ctypes.c_void_p`
+            The ctypes void pointer to a ``GMT_Dataset``. Create it with
+            :meth:`~gmt.clib.Session.create_data`.
+        family : str
+            The family type of the dataset. Can be either ``GMT_IS_VECTOR`` or
+            ``GMT_IS_MATRIX``.
+        strings : numpy 1d-array
+            The array that will be attached to the dataset. Must be a 1d C
+            contiguous array.
+
+        Raises
+        ------
+        GMTCLibError
+            If given invalid input or ``GMT_Put_Strings`` exits with status !=
+            0.
+
+        """
+        c_put_strings = self.get_libgmt_func(
+            "GMT_Put_Strings",
+            argtypes=[
+                ctp.c_void_p,
+                ctp.c_uint,
+                ctp.c_void_p,
+                ctp.POINTER(ctp.c_char_p),
+            ],
+            restype=ctp.c_int,
+        )
+
+        strings_pointer = (ctp.c_char_p * len(strings))()
+        strings_pointer[:] = np.char.encode(strings)
+
+        family_int = self._parse_constant(
+            family, valid=FAMILIES, valid_modifiers=METHODS
+        )
+
+        status = c_put_strings(
+            self.session_pointer, family_int, dataset, strings_pointer
+        )
+        if status != 0:
+            raise GMTCLibError(
+                f"Failed to put strings of type {strings.dtype} into dataset"
+            )
+
     def put_matrix(self, dataset, matrix, pad=0):
         """
         Attach a numpy 2D array to a GMT dataset.
 
-        Use this functions to attach numpy array data to a GMT dataset and pass
+        Use this function to attach numpy array data to a GMT dataset and pass
         it to GMT modules. Wraps ``GMT_Put_Matrix``.
 
         The dataset must be created by :meth:`~gmt.clib.Session.create_data`
@@ -1002,9 +1065,7 @@ def open_virtual_file(self, family, geometry, direction, data):
         family_int = self._parse_constant(family, valid=FAMILIES, valid_modifiers=VIAS)
         geometry_int = self._parse_constant(geometry, valid=GEOMETRIES)
         direction_int = self._parse_constant(
-            direction,
-            valid=["GMT_IN", "GMT_OUT"],
-            valid_modifiers=["GMT_IS_REFERENCE", "GMT_IS_DUPLICATE"],
+            direction, valid=["GMT_IN", "GMT_OUT"], valid_modifiers=METHODS,
         )
 
         buff = ctp.create_string_buffer(self["GMT_VF_LEN"])
@@ -1079,14 +1140,23 @@ def virtualfile_from_vectors(self, *vectors):
 
         """
         # Conversion to a C-contiguous array needs to be done here and not in
-        # put_matrix because we need to maintain a reference to the copy while
-        # it is being used by the C API. Otherwise, the array would be garbage
-        # collected and the memory freed. Creating it in this context manager
-        # guarantees that the copy will be around until the virtual file is
-        # closed. The conversion is implicit in vectors_to_arrays.
+        # put_vector or put_strings because we need to maintain a reference to
+        # the copy while it is being used by the C API. Otherwise, the array
+        # would be garbage collected and the memory freed. Creating it in this
+        # context manager guarantees that the copy will be around until the
+        # virtual file is closed. The conversion is implicit in
+        # vectors_to_arrays.
         arrays = vectors_to_arrays(vectors)
 
         columns = len(arrays)
+        # Find arrays that are of string dtype from column 3 onwards
+        # Assumes that first 2 columns contains coordinates like longitude
+        # latitude, or datetime string types.
+        for col, array in enumerate(arrays[2:]):
+            if np.issubdtype(array.dtype, np.str_):
+                columns = col + 2
+                break
+
         rows = len(arrays[0])
         if not all(len(i) == rows for i in arrays):
             raise GMTInvalidInput("All arrays must have same size.")
@@ -1098,9 +1168,24 @@ def virtualfile_from_vectors(self, *vectors):
             family, geometry, mode="GMT_CONTAINER_ONLY", dim=[columns, rows, 1, 0]
         )
 
-        for col, array in enumerate(arrays):
+        # Use put_vector for columns with numerical type data
+        for col, array in enumerate(arrays[:columns]):
             self.put_vector(dataset, column=col, vector=array)
 
+        # Use put_strings for last column(s) with string type data
+        # Have to use modifier "GMT_IS_DUPLICATE" to duplicate the strings
+        string_arrays = arrays[columns:]
+        if string_arrays:
+            if len(string_arrays) == 1:
+                strings = string_arrays[0]
+            elif len(string_arrays) > 1:
+                strings = np.apply_along_axis(
+                    func1d=" ".join, axis=0, arr=string_arrays
+                )
+            self.put_strings(
+                dataset, family="GMT_IS_VECTOR|GMT_IS_DUPLICATE", strings=strings
+            )
+
         with self.open_virtual_file(
             family, geometry, "GMT_IN|GMT_IS_REFERENCE", dataset
         ) as vfile:

pygmt/helpers/decorators.py

@@ -8,6 +8,8 @@
 import textwrap
 import functools
 
+import numpy as np
+
 from .utils import is_nonstr_iter
 from ..exceptions import GMTInvalidInput
 
@@ -302,6 +304,23 @@ def kwargs_to_strings(convert_bools=True, **conversions):
     >>> module(123, bla=(1, 2, 3), foo=True, A=False, i=(5, 6))
     {'bla': (1, 2, 3), 'foo': '', 'i': '5,6'}
     args: 123
+    >>> import datetime
+    >>> module(
+    ...     R=[
+    ...         np.datetime64("2010-01-01T16:00:00"),
+    ...         datetime.datetime(2020, 1, 1, 12, 23, 45),
+    ...     ]
+    ... )
+    {'R': '2010-01-01T16:00:00/2020-01-01T12:23:45.000000'}
+    >>> import pandas as pd
+    >>> import xarray as xr
+    >>> module(
+    ...     R=[
+    ...         xr.DataArray(data=np.datetime64("2005-01-01T08:00:00")),
+    ...         pd.Timestamp("2015-01-01T12:00:00.123456789"),
+    ...     ]
+    ... )
+    {'R': '2005-01-01T08:00:00.000000000/2015-01-01T12:00:00.123456'}
 
     """
     valid_conversions = [
@@ -338,9 +357,19 @@ def new_module(*args, **kwargs):
                     value = kwargs[arg]
                     issequence = fmt in separators
                     if issequence and is_nonstr_iter(value):
-                        kwargs[arg] = separators[fmt].join(
-                            "{}".format(item) for item in value
-                        )
+                        for index, item in enumerate(value):
+                            try:
+                                # check if there is a space " " when converting
+                                # a pandas.Timestamp/xr.DataArray to a string.
+                                # If so, use np.datetime_as_string instead.
+                                assert " " not in str(item)
+                            except AssertionError:
+                                # convert datetime-like item to ISO 8601
+                                # string format like YYYY-MM-DDThh:mm:ss.ffffff
+                                value[index] = np.datetime_as_string(
+                                    np.asarray(item, dtype=np.datetime64)
+                                )
+                        kwargs[arg] = separators[fmt].join(f"{item}" for item in value)
             # Execute the original function and return its output
             return module_func(*args, **kwargs)
 

pygmt/tests/test_clib.py

@@ -27,6 +27,9 @@
 
 TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
 
+with clib.Session() as _lib:
+    gmt_version = Version(_lib.info["version"])
+
 
 @contextmanager
 def mock(session, func, returns=None, mock_func=None):
@@ -399,6 +402,47 @@ def test_virtualfile_from_vectors():
             assert output == expected
 
 
+@pytest.mark.xfail(
+    condition=gmt_version < Version("6.1.1"),
+    reason="GMT_Put_Strings only works for GMT 6.1.1 and above",
+)
+def test_virtualfile_from_vectors_one_string_column():
+    "Test passing in one column with string dtype into virtual file dataset"
+    size = 5
+    x = np.arange(size, dtype=np.int32)
+    y = np.arange(size, size * 2, 1, dtype=np.int32)
+    strings = np.array(["a", "bc", "defg", "hijklmn", "opqrst"], dtype=np.str)
+    with clib.Session() as lib:
+        with lib.virtualfile_from_vectors(x, y, strings) as vfile:
+            with GMTTempFile() as outfile:
+                lib.call_module("convert", f"{vfile} ->{outfile.name}")
+                output = outfile.read(keep_tabs=True)
+        expected = "".join(f"{i}\t{j}\t{k}\n" for i, j, k in zip(x, y, strings))
+        assert output == expected
+
+
+@pytest.mark.xfail(
+    condition=gmt_version < Version("6.1.1"),
+    reason="GMT_Put_Strings only works for GMT 6.1.1 and above",
+)
+def test_virtualfile_from_vectors_two_string_columns():
+    "Test passing in two columns of string dtype into virtual file dataset"
+    size = 5
+    x = np.arange(size, dtype=np.int32)
+    y = np.arange(size, size * 2, 1, dtype=np.int32)
+    strings1 = np.array(["a", "bc", "def", "ghij", "klmno"], dtype=np.str)
+    strings2 = np.array(["pqrst", "uvwx", "yz!", "@#", "$"], dtype=np.str)
+    with clib.Session() as lib:
+        with lib.virtualfile_from_vectors(x, y, strings1, strings2) as vfile:
+            with GMTTempFile() as outfile:
+                lib.call_module("convert", f"{vfile} ->{outfile.name}")
+                output = outfile.read(keep_tabs=True)
+        expected = "".join(
+            f"{h}\t{i}\t{j} {k}\n" for h, i, j, k in zip(x, y, strings1, strings2)
+        )
+        assert output == expected
+
+
 def test_virtualfile_from_vectors_transpose():
     "Test transforming matrix columns to virtual file dataset"
     dtypes = "float32 float64 int32 int64 uint32 uint64".split()

pygmt/tests/test_clib_put_matrix.py

@@ -76,7 +76,7 @@ def test_put_matrix_grid():
             with GMTTempFile() as tmp_file:
                 lib.write_data(
                     "GMT_IS_MATRIX",
-                    "GMT_IS_SURFACE",
+                    "GMT_IS_POINT",
                     "GMT_CONTAINER_AND_DATA",
                     wesn,
                     tmp_file.name,