diff --git a/elt-pipelines/facility_ops/ingest/computing/jira/jira.py b/elt-pipelines/facility_ops/ingest/computing/jira/jira.py index 5005bf2e..cf3ccf8c 100644 --- a/elt-pipelines/facility_ops/ingest/computing/jira/jira.py +++ b/elt-pipelines/facility_ops/ingest/computing/jira/jira.py @@ -64,7 +64,7 @@ def extract_resource_properties(self) -> Iterator[tuple[str, ResourceProperties] ), ) yield ( - "issue_status_changelog", + "issue_status_changelogs", ResourceProperties( extractor=self.extract_issue_status_changelogs, write_properties=ResourceWriteProperties(write_mode="replace"), diff --git a/elt-pipelines/facility_ops/transform/models/intermediate/computing/int_times_in_status.sql b/elt-pipelines/facility_ops/transform/models/intermediate/computing/int_times_in_status.sql new file mode 100644 index 00000000..925b9ba2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/intermediate/computing/int_times_in_status.sql @@ -0,0 +1,43 @@ +-- Get the differences between the status. No need for final status, so this query is perfectly suitable +with status_to_from as ( + select + issue_key, + from_status as status, + lag(changed_at) over (partition by issue_key order by changed_at) as status_from, + changed_at as status_to + from {{ ref('stg_jira_issue_status_changelogs') }} as changelogs +), + +-- Populate null values of status from with the issue creation date. Join required +nn_status_to_from as ( + select + status_to_from.issue_key, + status_to_from.status, + COALESCE(status_to_from.status_from, issues.created) as status_from, + status_to_from.status_to + from status_to_from + inner join facility_ops_landing.computing_jira.isis_jira_issues as issues + on status_to_from.issue_key = issues.issue_key +), + +-- Subtract to and from date +status_durations as ( + select + issue_key, + status, + date_diff('second', status_from, status_to) as status_duration + from nn_status_to_from +), + +-- Aggregate similar statuses and add their durations +times_in_status as ( + select + issue_key, + status, + sum(status_duration) as time_in_status + from status_durations + group by issue_key, + status +) + +select * from times_in_status diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.sql b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.sql new file mode 100644 index 00000000..d48d0cf3 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.sql @@ -0,0 +1,78 @@ +with times_in_status as ( + select * from {{ ref('int_times_in_status') }} where issue_key like 'CI-%' +), + +time_in_status_data_driven_facility as ( + select + issue_key, + MAX( + case + when status = 'analyzing' then time_in_status + else null + end + ) as time_in_analysing_secs, + MAX( + case + when status = 'backlog' then time_in_status + else null + end + ) as time_in_backlog_secs, + MAX( + case + when status = 'done' then time_in_status + else null + end + ) as time_in_done_secs, + MAX( + case + when status = 'funnel' then time_in_status + else null + end + ) as time_in_funnel_secs, + MAX( + case + when status = 'implementing' then time_in_status + else null + end + ) as time_in_implementing_secs, + MAX( + case + when status = 'implementing (mvp)' then time_in_status + else null + end + ) as time_in_implementing_mvp_secs, + MAX( + case + when status = 'implementing (persevere)' then time_in_status + else null + end + ) as time_in_implementing_persevere_secs, + MAX( + case + when status = 'in progress' then time_in_status + else null + end + ) as time_in_in_progress_secs, + MAX( + case + when status = 'portfolio backlog' then time_in_status + else null + end + ) as time_in_portfolio_backlog_secs, + MAX( + case + when status = 'ready' then time_in_status + else null + end + ) as time_in_ready_secs, + MAX( + case + when status = 'reviewing' then time_in_status + else null + end + ) as time_in_reviewing_secs + from times_in_status + group by issue_key + +) +select * from time_in_status_data_driven_facility diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.yml b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.yml new file mode 100644 index 00000000..7bac9733 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_computing_infrastructure.yml @@ -0,0 +1,19 @@ +models: + - name: time_spent_in_status + description: > + Get length of time an issue spends in each status for user software issues. + columns: + - name: issue_key + data_tests: + - not_null + - name: time_in_analysing_secs + - name: time_in_backlog_secs + - name: time_in_done_secs + - name: time_in_funnel_secs + - name: time_in_implementing_secs + - name: time_in_implementing_mvp_secs + - name: time_in_implementing_persevere_secs + - name: time_in_in_progress_secs + - name: time_in_portfolio_backlog_secs + - name: time_in_ready_secs + - name: time_in_reviewing_secs diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.sql b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.sql new file mode 100644 index 00000000..3d443aec --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.sql @@ -0,0 +1,84 @@ +with times_in_status as ( + select * from {{ ref('int_times_in_status') }} where issue_key like 'DD-%' +), + +time_in_status_data_driven_facility as ( + select + issue_key, + MAX( + case + when status = 'analyzing' then time_in_status + else null + end + ) as time_in_analysing_secs, + MAX( + case + when status = 'backlog' then time_in_status + else null + end + ) as time_in_backlog_secs, + MAX( + case + when status = 'done' then time_in_status + else null + end + ) as time_in_done_secs, + MAX( + case + when status = 'funnel' then time_in_status + else null + end + ) as time_in_funnel_secs, + MAX( + case + when status = 'implementing' then time_in_status + else null + end + ) as time_in_implementing_secs, + MAX( + case + when status = 'implementing (mvp)' then time_in_status + else null + end + ) as time_in_implementing_mvp_secs, + MAX( + case + when status = 'implementing (persevere)' then time_in_status + else null + end + ) as time_in_implementing_persevere_secs, + MAX( + case + when status = 'in progress' then time_in_status + else null + end + ) as time_in_in_progress_secs, + MAX( + case + when status = 'portfolio backlog' then time_in_status + else null + end + ) as time_in_portfolio_backlog_secs, + MAX( + case + when status = 'ready' then time_in_status + else null + end + ) as time_in_ready_secs, + MAX( + case + when status = 'reviewing' then time_in_status + else null + end + ) as time_in_reviewing_secs, + MAX( + case + when status = 'selected for development' then time_in_status + else null + end + ) as time_in_selected_for_development_secs + from times_in_status + group by issue_key + +) +select * from time_in_status_data_driven_facility diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.yml b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.yml new file mode 100644 index 00000000..2a3f8ba2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_data_driven_facility.yml @@ -0,0 +1,20 @@ +models: + - name: time_spent_in_status + description: > + Get length of time an issue spends in each status for user software issues. + columns: + - name: issue_key + data_tests: + - not_null + - name: time_in_analysing_secs + - name: time_in_backlog_secs + - name: time_in_done_secs + - name: time_in_funnel_secs + - name: time_in_implementing_secs + - name: time_in_implementing_mvp_secs + - name: time_in_implementing_persevere_secs + - name: time_in_in_progress_secs + - name: time_in_portfolio_backlog_secs + - name: time_in_ready_secs + - name: time_in_reviewing_secs + - name: time_in_selected_for_development_secs diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.sql b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.sql new file mode 100644 index 00000000..b2ae7833 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.sql @@ -0,0 +1,78 @@ +with times_in_status as ( + select * from {{ ref('int_times_in_status') }} where issue_key like 'SS-%' +), + +time_in_status_scientific_software as ( + select + issue_key, + MAX( + case + when status = 'analyzing' then time_in_status + else null + end + ) as time_in_analysing_secs, + MAX( + case + when status = 'backlog' then time_in_status + else null + end + ) as time_in_backlog_secs, + MAX( + case + when status = 'done' then time_in_status + else null + end + ) as time_in_done_secs, + MAX( + case + when status = 'funnel' then time_in_status + else null + end + ) as time_in_funnel_secs, + MAX( + case + when status = 'implementing' then time_in_status + else null + end + ) as time_in_implementing_secs, + MAX( + case + when status = 'implementing (mvp)' then time_in_status + else null + end + ) as time_in_implementing_mvp_secs, + MAX( + case + when status = 'implementing (persevere)' then time_in_status + else null + end + ) as time_in_implementing_persevere_secs, + MAX( + case + when status = 'in progress' then time_in_status + else null + end + ) as time_in_in_progress_secs, + MAX( + case + when status = 'portfolio backlog' then time_in_status + else null + end + ) as time_in_portfolio_backlog_secs, + MAX( + case + when status = 'ready' then time_in_status + else null + end + ) as time_in_ready_secs, + MAX( + case + when status = 'reviewing' then time_in_status + else null + end + ) as time_in_reviewing_secs + from times_in_status + group by issue_key + +) +select * from time_in_status_scientific_software diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.yml b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.yml new file mode 100644 index 00000000..f84a289d --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_scientific_software.yml @@ -0,0 +1,19 @@ +models: + - name: time_spent_in_status + description: > + Get length of time an issue spends in each status + columns: + - name: issue_key + data_tests: + - not_null + - name: time_in_analysing_secs + - name: time_in_backlog_secs + - name: time_in_done_secs + - name: time_in_funnel_secs + - name: time_in_implementing_secs + - name: time_in_implementing_mvp_secs + - name: time_in_implementing_persevere_secs + - name: time_in_in_progress_secs + - name: time_in_portfolio_backlog_secs + - name: time_in_ready_secs + - name: time_in_reviewing_secs diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.sql b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.sql new file mode 100644 index 00000000..dd0b1269 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.sql @@ -0,0 +1,96 @@ +with times_in_status as ( + select * from {{ ref('int_times_in_status') }} where issue_key like 'US-%' +), + +time_in_status_user_software as ( + select + issue_key, + MAX( + case + when status = 'analyzing' then time_in_status + else null + end + ) as time_in_analysing_secs, + MAX( + case + when status = 'backlog' then time_in_status + else null + end + ) as time_in_backlog_secs, + MAX( + case + when status = 'blocked' then time_in_status + else null + end + ) as time_in_blocked_secs, + MAX( + case + when status = 'closed' then time_in_status + else null + end + ) as time_in_closed_secs, + MAX( + case + when status = 'done' then time_in_status + else null + end + ) as time_in_done_secs, + MAX( + case + when status = 'funnel' then time_in_status + else null + end + ) as time_in_funnel_secs, + MAX( + case + when status = 'implementing' then time_in_status + else null + end + ) as time_in_implementing_secs, + MAX( + case + when status = 'implementing (mvp)' then time_in_status + else null + end + ) as time_in_implementing_mvp_secs, + MAX( + case + when status = 'implementing (persevere)' then time_in_status + else null + end + ) as time_in_implementing_persevere_secs, + MAX( + case + when status = 'in progress' then time_in_status + else null + end + ) as time_in_in_progress_secs, + MAX( + case + when status = 'portfolio backlog' then time_in_status + else null + end + ) as time_in_portfolio_backlog_secs, + MAX( + case + when status = 'ready' then time_in_status + else null + end + ) as time_in_ready_secs, + MAX( + case + when status = 'reviewing' then time_in_status + else null + end + ) as time_in_reviewing_secs, + MAX( + case + when status = 'selected for development' then time_in_status + else null + end + ) as time_in_selected_for_development_secs, + from times_in_status + group by issue_key + +) +select * from time_in_status_user_software diff --git a/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.yml b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.yml new file mode 100644 index 00000000..2230d211 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/computing/time_in_status_user_software.yml @@ -0,0 +1,22 @@ +models: + - name: time_spent_in_status + description: > + Get length of time an issue spends in each status for user software issues. + columns: + - name: issue_key + data_tests: + - not_null + - name: time_in_analysing_secs + - name: time_in_backlog_secs + - name: time_in_blocked_secs + - name: time_in_closed_secs + - name: time_in_done_secs + - name: time_in_funnel_secs + - name: time_in_implementing_secs + - name: time_in_implementing_mvp_secs + - name: time_in_implementing_persevere_secs + - name: time_in_in_progress_secs + - name: time_in_portfolio_backlog_secs + - name: time_in_ready_secs + - name: time_in_reviewing_secs + - name: time_in_selected_for_development_secs diff --git a/elt-pipelines/facility_ops/transform/models/staging/computing/_computing__sources.yml b/elt-pipelines/facility_ops/transform/models/staging/computing/_computing__sources.yml index a8e912f2..3ec40230 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/computing/_computing__sources.yml +++ b/elt-pipelines/facility_ops/transform/models/staging/computing/_computing__sources.yml @@ -5,3 +5,4 @@ sources: database: facility_ops_landing tables: - name: isis_jira_issues + - name: issue_status_changelogs diff --git a/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.sql b/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.sql new file mode 100644 index 00000000..5a4891d2 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.sql @@ -0,0 +1,14 @@ +with source as ( +select * from {{source('computing_jira', 'issue_status_changelogs')}} +), + +cleaned as ( +select + issue_key as issue_key, + TRIM(LOWER(from_status)) as from_status, + TRIM(LOWER(to_status)) as to_status, + changed_at as changed_at +from source +) + +select * from cleaned diff --git a/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.yml b/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.yml new file mode 100644 index 00000000..d6230403 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/staging/computing/stg_jira_issue_status_changelogs.yml @@ -0,0 +1,17 @@ +models: + - name: stg_jira_issue_status_changelogs + description: > + Get length of time an issue spends in each status + columns: + - name: issue_key + data_tests: + - not_null + - name: from_status + data_tests: + - not_null + - name: to_status + data_tests: + - not_null + - name: changed_at + data_tests: + - not_null