feat(elt-pipelines) - Add transform method to get times spent in status per issue

elt-pipelines/facility_ops/ingest/computing/jira/jira.py

@@ -64,7 +64,7 @@ def extract_resource_properties(self) -> Iterator[tuple[str, ResourceProperties]
             ),
         )
         yield (
-            "issue_status_changelog",
+            "issue_status_changelogs",
             ResourceProperties(
                 extractor=self.extract_issue_status_changelogs,
                 write_properties=ResourceWriteProperties(write_mode="replace"),

elt-pipelines/facility_ops/transform/models/intermediate/computing/int_times_in_status.sql

@@ -0,0 +1,43 @@
+-- Get the differences between the status. No need for final status, so this query is perfectly suitable
+with status_to_from as (
+  select
+    issue_key,
+    from_status as status,
+    lag(changed_at) over (partition by issue_key order by changed_at) as status_from,
+    changed_at as status_to
+    from {{ ref('stg_jira_issue_status_changelogs') }} as changelogs
+),
+
+-- Populate null values of status from with the issue creation date. Join required
+nn_status_to_from as (
+  select
+    status_to_from.issue_key,
+    status_to_from.status,
+    COALESCE(status_to_from.status_from, issues.created) as status_from,
+    status_to_from.status_to
+    from status_to_from
+    inner join facility_ops_landing.computing_jira.isis_jira_issues as issues
+    on status_to_from.issue_key = issues.issue_key
+),
+
+-- Subtract to and from date
+status_durations as (
+  select
+    issue_key,
+    status,
+    date_diff('second', status_from, status_to) as status_duration
+    from nn_status_to_from
+),
+
+-- Aggregate similar statuses and add their durations
+times_in_status as (
+  select
+    issue_key,
+    status,
+    sum(status_duration) as time_in_status
+    from status_durations
+    group by issue_key,
+    status
+)
+
+select * from times_in_status

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.sql

@@ -0,0 +1,78 @@
+with times_in_status as (
+  select * from {{ ref('int_times_in_status') }} where issue_key like 'CI-%'
+),
+
+time_in_status_data_driven_facility as (
+    select
+    issue_key,
+    MAX(
+        case
+        when status = 'analyzing' then time_in_status
+        else null
+        end
+    ) as time_in_analysing_secs,
+    MAX(
+        case
+        when status = 'backlog' then time_in_status
+        else null
+        end
+    ) as time_in_backlog_secs,
+    MAX(
+        case
+        when status = 'done' then time_in_status
+        else null
+        end
+    ) as time_in_done_secs,
+    MAX(
+        case
+        when status = 'funnel' then time_in_status
+        else null
+        end
+    ) as time_in_funnel_secs,
+    MAX(
+        case
+        when status = 'implementing' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_secs,
+    MAX(
+        case
+        when status = 'implementing (mvp)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_mvp_secs,
+    MAX(
+        case
+        when status = 'implementing (persevere)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_persevere_secs,
+    MAX(
+        case
+        when status = 'in progress' then time_in_status
+        else null
+        end
+    ) as time_in_in_progress_secs,
+    MAX(
+        case
+        when status = 'portfolio backlog' then time_in_status
+        else null
+        end
+    ) as time_in_portfolio_backlog_secs,
+    MAX(
+        case
+        when status = 'ready' then time_in_status
+        else null
+        end
+    ) as time_in_ready_secs,
+    MAX(
+        case
+        when status = 'reviewing' then time_in_status
+        else null
+        end
+    ) as time_in_reviewing_secs
+    from times_in_status
+    group by issue_key
+
+)
+select * from time_in_status_data_driven_facility

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.yml

@@ -0,0 +1,19 @@
+models:
+  - name: time_spent_in_status
+    description: >
+      Get length of time an issue spends in each status for user software issues.
+    columns:
+      - name: issue_key
+        data_tests:
+        - not_null
+      - name: time_in_analysing_secs
+      - name: time_in_backlog_secs
+      - name: time_in_done_secs
+      - name: time_in_funnel_secs
+      - name: time_in_implementing_secs
+      - name: time_in_implementing_mvp_secs
+      - name: time_in_implementing_persevere_secs
+      - name: time_in_in_progress_secs
+      - name: time_in_portfolio_backlog_secs
+      - name: time_in_ready_secs
+      - name: time_in_reviewing_secs

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.sql

@@ -0,0 +1,84 @@
+with times_in_status as (
+  select * from {{ ref('int_times_in_status') }} where issue_key like 'DD-%'
+),
+
+time_in_status_data_driven_facility as (
+    select
+    issue_key,
+    MAX(
+        case
+        when status = 'analyzing' then time_in_status
+        else null
+        end
+    ) as time_in_analysing_secs,
+    MAX(
+        case
+        when status = 'backlog' then time_in_status
+        else null
+        end
+    ) as time_in_backlog_secs,
+    MAX(
+        case
+        when status = 'done' then time_in_status
+        else null
+        end
+    ) as time_in_done_secs,
+    MAX(
+        case
+        when status = 'funnel' then time_in_status
+        else null
+        end
+    ) as time_in_funnel_secs,
+    MAX(
+        case
+        when status = 'implementing' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_secs,
+    MAX(
+        case
+        when status = 'implementing (mvp)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_mvp_secs,
+    MAX(
+        case
+        when status = 'implementing (persevere)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_persevere_secs,
+    MAX(
+        case
+        when status = 'in progress' then time_in_status
+        else null
+        end
+    ) as time_in_in_progress_secs,
+    MAX(
+        case
+        when status = 'portfolio backlog' then time_in_status
+        else null
+        end
+    ) as time_in_portfolio_backlog_secs,
+    MAX(
+        case
+        when status = 'ready' then time_in_status
+        else null
+        end
+    ) as time_in_ready_secs,
+    MAX(
+        case
+        when status = 'reviewing' then time_in_status
+        else null
+        end
+    ) as time_in_reviewing_secs,
+    MAX(
+        case
+        when status = 'selected for development' then time_in_status
+        else null
+        end
+    ) as time_in_selected_for_development_secs
+    from times_in_status
+    group by issue_key
+
+)
+select * from time_in_status_data_driven_facility

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.yml

@@ -0,0 +1,20 @@
+models:
+  - name: time_spent_in_status
+    description: >
+      Get length of time an issue spends in each status for user software issues.
+    columns:
+      - name: issue_key
+        data_tests:
+        - not_null
+      - name: time_in_analysing_secs
+      - name: time_in_backlog_secs
+      - name: time_in_done_secs
+      - name: time_in_funnel_secs
+      - name: time_in_implementing_secs
+      - name: time_in_implementing_mvp_secs
+      - name: time_in_implementing_persevere_secs
+      - name: time_in_in_progress_secs
+      - name: time_in_portfolio_backlog_secs
+      - name: time_in_ready_secs
+      - name: time_in_reviewing_secs
+      - name: time_in_selected_for_development_secs

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.sql

@@ -0,0 +1,78 @@
+with times_in_status as (
+  select * from {{ ref('int_times_in_status') }} where issue_key like 'SS-%'
+),
+
+time_in_status_scientific_software as (
+    select
+    issue_key,
+    MAX(
+        case
+        when status = 'analyzing' then time_in_status
+        else null
+        end
+    ) as time_in_analysing_secs,
+    MAX(
+        case
+        when status = 'backlog' then time_in_status
+        else null
+        end
+    ) as time_in_backlog_secs,
+    MAX(
+        case
+        when status = 'done' then time_in_status
+        else null
+        end
+    ) as time_in_done_secs,
+    MAX(
+        case
+        when status = 'funnel' then time_in_status
+        else null
+        end
+    ) as time_in_funnel_secs,
+    MAX(
+        case
+        when status = 'implementing' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_secs,
+    MAX(
+        case
+        when status = 'implementing (mvp)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_mvp_secs,
+    MAX(
+        case
+        when status = 'implementing (persevere)' then time_in_status
+        else null
+        end
+    ) as time_in_implementing_persevere_secs,
+    MAX(
+        case
+        when status = 'in progress' then time_in_status
+        else null
+        end
+    ) as time_in_in_progress_secs,
+    MAX(
+        case
+        when status = 'portfolio backlog' then time_in_status
+        else null
+        end
+    ) as time_in_portfolio_backlog_secs,
+    MAX(
+        case
+        when status = 'ready' then time_in_status
+        else null
+        end
+    ) as time_in_ready_secs,
+    MAX(
+        case
+        when status = 'reviewing' then time_in_status
+        else null
+        end
+    ) as time_in_reviewing_secs
+    from times_in_status
+    group by issue_key
+
+)
+select * from time_in_status_scientific_software

elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.yml

@@ -0,0 +1,19 @@
+models:
+  - name: time_spent_in_status
+    description: >
+      Get length of time an issue spends in each status
+    columns:
+      - name: issue_key
+        data_tests:
+        - not_null
+      - name: time_in_analysing_secs
+      - name: time_in_backlog_secs
+      - name: time_in_done_secs
+      - name: time_in_funnel_secs
+      - name: time_in_implementing_secs
+      - name: time_in_implementing_mvp_secs
+      - name: time_in_implementing_persevere_secs
+      - name: time_in_in_progress_secs
+      - name: time_in_portfolio_backlog_secs
+      - name: time_in_ready_secs
+      - name: time_in_reviewing_secs