update stop times grouped and fix some yaml column names

Author: lauriemerrell

warehouse/models/docs/gtfs schedule/stop_time_enum_aggregations.md warehouse/models/docs/gtfs schedule/stop_times_grouped.md

@@ -120,3 +120,32 @@ Count of stop events on this trip where `departure_time` is populated.
 
 See https://gtfs.org/reference/static#stop_timestxt for the raw data definitions.
 {% enddocs %}
+
+{% docs column_flexible_trip %}
+Boolean indicator for whether this is a flexible trip defined using GTFS-Flex.
+GTFS-Flex flexible trips are indicated by using the fields `start_pickup_drop_off_window`and `end_pickup_drop_off_window` for at least some of their stop events.
+This means there is some flexible service without fixed schedules and this is not an entirely fixed-route trip.
+
+Note that there may be some demand-responsive behavior on trips where this column is `false`; for example, there can be stops where a phone call to the agency is required to arrange pickup/drop off, which will be indicated in pickup/drop off type columns even if they are not included as "flexible" in this column.
+{% enddocs %}
+
+{% docs column_fully_flexible_trip %}
+This is the same as `flexible_trip` except that the boolean here indicates that all (rather than just some) of the stop events had `start_pickup_drop_off_window` and `end_pickup_drop_off_window` populated, indicating that this is an entirely flexible/demand-responsive trip.
+{% enddocs %}
+
+{% docs column_num_flexible_stop_events %}
+Count of stop events for this trip with  `start_pickup_drop_off_window` and `end_pickup_drop_off_window` populated indicating flexible behavior.
+See `flexible_trip` for more information.
+{% enddocs %}
+
+{% docs column_first_start_pickup_drop_off_window_sec %}
+Earliest value of `start_pickup_drop_off_window_sec` for this trip. Only populated for flexible trips.
+
+Represents the number of seconds after 12 hours before noon (usually midnight) at which flexible pickups/drop offs begin for the earliest flexible stop on this trip.
+{% enddocs %}
+
+{% docs column_last_end_pickup_drop_off_window_sec %}
+Latest value of `end_pickup_drop_off_window_sec` for this trip. Only populated for flexible trips.
+
+Represents the number of seconds after 12 hours before noon (usually midnight) at which flexible pickups/drop offs end for final flexible stop on this trip.
+{% enddocs %}

warehouse/models/intermediate/gtfs/_int_gtfs.yaml

@@ -293,45 +293,55 @@ models:
           The time zone of the final stop in this trip, pulled from dim_stops.
           For time of day categorization purposes, it is recommended to use `trip_start_timezone`
           rather than this.
-      - name: ct_regularly_scheduled_pickup_stops
+      - name: flexible_trip
+        description: '{{ doc("column_flexible_trip") }}'
+      - name: fully_flexible_trip
+        description: '{{ doc("column_fully_flexible_trip") }}'
+      - name: num_flexible_stop_events
+        description: '{{ doc("column_num_flexible_stop_events") }}'
+      - name: first_start_pickup_drop_off_window_sec
+        description: '{{ doc("column_first_start_pickup_drop_off_window_sec") }}'
+      - name: last_end_pickup_drop_off_window_sec
+        description: '{{ doc("column_last_end_pickup_drop_off_window_sec") }}'
+      - name: num_regularly_scheduled_pickup_stops
         description: '{{ doc("column_num_regularly_scheduled_pickup_stop_events") }}'
-      - name: ct_no_pickup_stops
+      - name: num_no_pickup_stops
         description: '{{ doc("column_num_no_pickup_stop_events") }}'
-      - name: ct_phone_call_required_for_pickup_stops
+      - name: num_phone_call_required_for_pickup_stops
         description: '{{ doc("column_num_phone_call_required_for_pickup_stop_events") }}'
-      - name: ct_coordinate_pickup_with_driver_stops
+      - name: num_coordinate_pickup_with_driver_stops
         description: '{{ doc("column_num_coordinate_pickup_with_driver_stop_events") }}'
-      - name: ct_regularly_scheduled_drop_off_stops
+      - name: num_regularly_scheduled_drop_off_stops
         description: '{{ doc("column_num_regularly_scheduled_drop_off_stop_events") }}'
-      - name: ct_no_drop_off_stops
+      - name: num_no_drop_off_stops
         description: '{{ doc("column_num_no_drop_off_stop_events") }}'
-      - name: ct_phone_call_required_for_drop_off_stops
+      - name: num_phone_call_required_for_drop_off_stops
         description: '{{ doc("column_num_phone_call_required_for_drop_off_stop_events") }}'
-      - name: ct_coordinate_drop_off_with_driver_stops
+      - name: num_coordinate_drop_off_with_driver_stops
         description: '{{ doc("column_num_coordinate_drop_off_with_driver_stop_events") }}'
-      - name: ct_continuous_pickup_stops
+      - name: num_continuous_pickup_stops
         description: '{{ doc("column_num_continuous_pickup_stop_events") }}'
-      - name: ct_no_continuous_pickup_stops
+      - name: num_no_continuous_pickup_stops
         description: '{{ doc("column_num_no_continuous_pickup_stop_events") }}'
-      - name: ct_phone_call_required_for_continuous_pickup_stops
+      - name: num_phone_call_required_for_continuous_pickup_stops
         description: '{{ doc("column_num_phone_call_required_for_continuous_pickup_stop_events") }}'
-      - name: ct_coordinate_continuous_pickup_with_driver_stops
+      - name: num_coordinate_continuous_pickup_with_driver_stops
         description: '{{ doc("column_num_coordinate_continuous_pickup_with_driver_stop_events") }}'
-      - name: ct_continuous_drop_off_stops
+      - name: num_continuous_drop_off_stops
         description: '{{ doc("column_num_continuous_drop_off_stop_events") }}'
-      - name: ct_no_drop_off_stops
+      - name: num_no_drop_off_stops
         description: '{{ doc("column_num_no_drop_off_stop_events") }}'
-      - name: ct_phone_call_required_for_continuous_drop_off_stops
+      - name: num_phone_call_required_for_continuous_drop_off_stops
         description: '{{ doc("column_num_phone_call_required_for_continuous_drop_off_stop_events") }}'
-      - name: ct_coordinate_continuous_drop_off_with_driver_stops
+      - name: num_coordinate_continuous_drop_off_with_driver_stops
         description: '{{ doc("column_num_coordinate_continuous_drop_off_with_driver_stop_events") }}'
-      - name: ct_approximate_timepoint_stops
+      - name: num_approximate_timepoint_stops
         description: '{{ doc("column_num_approximate_timepoint_stop_events") }}'
-      - name: ct_exact_timepoint_stops
+      - name: num_exact_timepoint_stops
         description: '{{ doc("column_num_exact_timepoint_stop_events") }}'
-      - name: ct_arrival_times_populated
+      - name: num_arrival_times_populated
         description: '{{ doc("column_num_arrival_times_populated_stop_events") }}'
-      - name: ct_departure_times_populated
+      - name: num_departure_times_populated
         description: '{{ doc("column_num_departure_times_populated_stop_events") }}'
   - name: int_gtfs_rt__distinct_download_configs
     description: |

warehouse/models/intermediate/gtfs/int_gtfs_schedule__stop_times_grouped.sql

@@ -76,6 +76,18 @@ int_gtfs_schedule__stop_times_grouped AS (
         LOGICAL_OR(
             frequencies_defined_trip
         ) AS frequencies_defined_trip,
+        -- per: https://docs.google.com/spreadsheets/d/1iqvzJV_YWmFyYGtpbO2dqGMbf4XEjvar3rt9SxHU-xY/edit#gid=0
+        -- determine flex usage by presence of these two fields for any row in stop times
+        LOGICAL_OR(
+            start_pickup_drop_off_window IS NOT NULL
+            AND end_pickup_drop_off_window IS NOT NULL) AS flexible_trip,
+        LOGICAL_AND(
+            start_pickup_drop_off_window IS NOT NULL
+            AND end_pickup_drop_off_window IS NOT NULL) AS fully_flexible_trip,
+        COUNTIF(start_pickup_drop_off_window IS NOT NULL
+            AND end_pickup_drop_off_window IS NOT NULL) AS num_flexible_stop_events,
+        MIN(start_pickup_drop_off_window_sec) AS first_start_pickup_drop_off_window_sec,
+        MAX(end_pickup_drop_off_window_sec) AS last_end_pickup_drop_off_window_sec,
 
         -- see: https://gtfs.org/schedule/reference/#stop_timestxt for the enum definitions on the following fields
         -- including default value definitions