Ivs826 statistics work part2

.github/workflows/ci.yml

@@ -121,5 +121,7 @@ jobs:
           MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
           MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_status_combine --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
           MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_management_commands --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures DJANGO_DB=sqlite python3 manage.py test apps.ifc_validation.tests_statistics_query apps.ifc_validation.tests.test_statistics_tasks --debug-mode --verbosity 3
+          python3 -m pytest apps/ifc_validation/checks/statistics/tests -v
           MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_report_view --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
           MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_step_lines --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3

backend/Makefile

@@ -4,7 +4,7 @@ VIRTUAL_ENV = .dev/venv
 PYTHON = $(VIRTUAL_ENV)/bin/python
 PIP = $(VIRTUAL_ENV)/bin/pip 
 IFCOPENSHELL_SITE_PACKAGES = $(VIRTUAL_ENV)/lib/python3.11/site-packages
-PYTHON_MVDXML_REF = support-4.x-graphviz-format-2
+PYTHON_MVDXML_REF = master
 
 none:
 	@echo "MAKE: Enter at least one target (venv, install, install-dev, start-backend, start-worker, clean)"

backend/apps/ifc_validation/admin.py

@@ -86,7 +86,7 @@ class ValidationRequestAdmin(BaseAdmin, NonAdminAddable):
     ]
 
     list_display = ["id", "public_id", "file_name", "file_size_text", "authoring_tool_link", "model_link", "status", "progress", "queue_time_text", "duration_text", "is_vendor", "is_vendor_self_declared", "is_deleted", "channel_text", "created", "created_by_link", "updated", "updated_by"]
-    readonly_fields = ["id", "public_id", "model", "deleted", "file_name", "file", "file_size_text", "duration_text", "started", "completed", "channel", "created", "created_by", "updated", "updated_by", "file_removed"] 
+    readonly_fields = ["id", "public_id", "model", "deleted", "file_name", "file", "file_size_text", "duration_text", "started", "completed", "channel", "created", "created_by", "updated", "updated_by", "file_removed"]
     date_hierarchy = "created"
 
     list_filter = [
@@ -471,7 +471,7 @@ def queue_time_text(self, obj):
 class ValidationOutcomeAdmin(BaseAdmin, NonAdminAddable):
 
     list_display = ["id", "public_id", "model_text", "instance_id", "type_text", "feature", "feature_version", "outcome_code", "severity", "is_whitelisted", "expected", "observed", "created", "updated"]
-    readonly_fields = ["id", "public_id", "created", "updated"]
+    readonly_fields = ["id", "public_id", "instance", "created", "updated"]
 
     list_filter = ['validation_task__type', 'severity_in_db', 'outcome_code', ('created', AdvancedDateFilter)]
     search_fields = ('validation_task__request__file_name', 'feature', 'feature_version', 'outcome_code', 'severity_in_db', 'expected', 'observed')
@@ -665,6 +665,7 @@ def authoring_tool_link(self, obj):
 class ModelInstanceAdmin(BaseAdmin, NonAdminAddable):
 
     list_display = ["id", "public_id", "model", "stepfile_id", "ifc_type", "created", "updated"]
+    readonly_fields = ["model"]
     search_fields = ('stepfile_id', 'model__file_name', 'ifc_type')
     list_filter = ["ifc_type", "model_id", ('created', AdvancedDateFilter)]
 
@@ -673,21 +674,40 @@ class ModelInstanceAdmin(BaseAdmin, NonAdminAddable):
 
 
 class EntityCountHistogramAdmin(admin.ModelAdmin):
-    readonly_fields = ["entity_name"]
+    readonly_fields = ["model", "entity_name"]
+    list_select_related = ["model"]  # __str__ reads model.schema
+
+    paginator = utils.LargeTablePaginator
+    show_full_result_count = False # do not use COUNT(*) twice
 
     @admin.display(description="Entity name")
     def entity_name(self, obj):
         return obj.entity_name
 
 
 class PsetCountHistogramAdmin(admin.ModelAdmin):
-    readonly_fields = ["entity_name"]
+    readonly_fields = ["model", "entity_name"]
+    list_select_related = ["model"]  # __str__ reads model.schema
+
+    paginator = utils.LargeTablePaginator
+    show_full_result_count = False # do not use COUNT(*) twice
 
     @admin.display(description="Entity name")
     def entity_name(self, obj):
         return obj.entity_name
 
 
+class TemplateStatisticAdmin(admin.ModelAdmin):
+    readonly_fields = ["model", "focus_instance"]
+
+    list_display = ["id", "model_id", "template_name", "focus_instance_id"]
+    list_filter = ["template_name"]
+    list_select_related = ["focus_instance"]  # __str__ reads focus_instance
+
+    paginator = utils.LargeTablePaginator
+    show_full_result_count = False # do not use COUNT(*) twice
+
+
 class CompanyAdmin(BaseAdmin):
 
     fieldsets = [
@@ -1054,7 +1074,7 @@ class WhiteListTestForm(forms.Form):
 admin.site.register(Model, ModelAdmin)
 admin.site.register(EntityCountHistogram, EntityCountHistogramAdmin)
 admin.site.register(PsetCountHistogram, PsetCountHistogramAdmin)
-admin.site.register(TemplateStatistic)
+admin.site.register(TemplateStatistic, TemplateStatisticAdmin)
 admin.site.register(ModelInstance, ModelInstanceAdmin)
 admin.site.register(Company, CompanyAdmin)
 admin.site.register(AuthoringTool, AuthoringToolAdmin)

backend/apps/ifc_validation/checks/statistics/apply_mvd.py

@@ -51,19 +51,17 @@ def extract_template_statistics(
 
         for focus in focus_instances:
             rows = concept.extract(focus)
-            if not rows:
-                continue
-            graph = {
-                concept.binding_for(key) or key.attribute: json_value(value)
-                for row in rows
-                for key, value in row.items()
-            }
-            results.append({
-                "template": markdown.name,
-                "focus_step_id": focus.id(),
-                "focus_ifc_type": focus.is_a(),
-                "graph": graph,
-            })
+            for row in rows:
+                graph = {
+                    concept.binding_for(key) or key.attribute: json_value(value)
+                    for key, value in row.items()
+                }
+                results.append({
+                    "template": markdown.name,
+                    "focus_step_id": focus.id(),
+                    "focus_ifc_type": focus.is_a(),
+                    "graph": graph,
+                })
 
     return results
 

backend/apps/ifc_validation/checks/statistics/tests/test_template_statistics.py

@@ -13,27 +13,57 @@
 def test_column_property_set_statistics():
     results = extract_template_statistics(FIXTURES / "ColumnPSetsOfSets.ifc")
 
-    assert len(results) == 17
-    assert {result["template"] for result in results} == {"Use_of_property_types.md"}
-    assert {result["focus_ifc_type"] for result in results} == {"IfcPropertySet"}
-    assert {
-        result["graph"]["PropertyType"] for result in results
-    } == {"IfcPropertySingleValue"}
-    assert Counter(
-        result["graph"]["PropertySetName"] for result in results
-    ) == Counter({
-        "Pset_SpaceCommon": 4,
-        "Pset_BuildingStoreyCommon": 3,
-        "Pset_ColumnCommon": 2,
+    property_type_results = [
+        result
+        for result in results
+        if result["template"] == "Use_of_property_types.md"
+    ]
+    definition_set_results = [
+        result
+        for result in results
+        if result["template"] == "Usage_of_IfcPropertySetDefinitionSet.md"
+    ]
+    expected_property_projection_names = Counter({
+        "Pset_SpaceCommon": 8,
+        "Pset_BuildingStoreyCommon": 6,
+        "Pset_ColumnCommon": 4,
         "Pset_SiteCommon": 1,
         "Pset_EnvironmentalImpactIndicators": 1,
         "Pset_ReinforcementBarPitchOfColumn": 1,
-        "Pset_BuildingCommon": 1,
-        "Pset_BuildingElementProxyCommon": 1,
+        "Pset_BuildingCommon": 3,
+        "Pset_BuildingElementProxyCommon": 2,
         "Pset_BuildingSystemCommon": 1,
         "PSet_1": 1,
         "PSet_2": 1,
     })
+    expected_definition_set_results = [{
+        "template": "Usage_of_IfcPropertySetDefinitionSet.md",
+        "focus_step_id": 139,
+        "focus_ifc_type": "IfcRelDefinesByProperties",
+        "graph": {
+            "IfcPropertySetDefinitionSet": "IfcPropertySetDefinitionSet",
+        },
+    }]
+
+    assert Counter(result["template"] for result in results) == Counter({
+        "Use_of_property_types.md": sum(
+            expected_property_projection_names.values()
+        ),
+        "Usage_of_IfcPropertySetDefinitionSet.md": len(
+            expected_definition_set_results
+        ),
+    })
+    assert {
+        result["focus_ifc_type"] for result in property_type_results
+    } == {"IfcPropertySet"}
+    assert {
+        result["graph"]["PropertyType"] for result in property_type_results
+    } == {"IfcPropertySingleValue"}
+    assert Counter(
+        result["graph"]["PropertySetName"]
+        for result in property_type_results
+    ) == expected_property_projection_names
+    assert definition_set_results == expected_definition_set_results
 
 
 def test_ifc2x3_curve_style_statistics():