9058 Fix broken analytics tables in funnel (#557)

migrations/000.627.sql

@@ -0,0 +1,54 @@
+-- Restore original inserted timestamps in product_category from product_category_backup.
+-- The category migration flattened many timestamps to nearly the same datetime.
+-- This causes problems in the funnel import that batches things by inserted timestamp.
+-- With this query run the biggest group of timestamps is <10k which funnel can handle.
+
+UPDATE product_category AS pc
+SET inserted = pcb.inserted
+FROM product_category_backup AS pcb
+WHERE pc.product_id = pcb.product_id
+  AND pc.category_id = pcb.category_id
+  AND pc.inserted IS DISTINCT FROM pcb.inserted;
+
+-- Additionally in the category migration we missed that funnel was the consumer of the
+-- v_analytics_product_categories view and the v2 view that actually has the correct data since
+-- the migration has been unused. As a first cleanup step after the category migration we
+-- now drop the v_analytics_product_categories view and recreate it with the v2 definition.
+-- v2 structure is directly from the v2 view definition but has a changed definition of last_updated
+-- it previously was GREATEST(product_category.inserted, category_texts.updated) AS last_updated
+-- but that becomes problematic for funnel (the 10k limit noted above) when we do bulk updates of category_texts columns that
+-- didn't even effect name and path columns used by the view. The name and path should not change.
+
+CREATE OR REPLACE VIEW v_analytics_product_categories AS
+ SELECT tree.id AS category_id,
+   safe_xml_path.safe_xml_path AS category_path,
+    category_texts.name AS category_name,
+    product_category.product_id,
+    MAX(product_category.inserted) AS last_updated
+   FROM product_category
+     JOIN categories_v2_lft_rgt categories_v2 ON categories_v2.id = product_category.category_id
+     JOIN v_esales_categorytree_i18n_v2 tree ON (tree.lft < categories_v2.lft AND tree.rgt > categories_v2.rgt OR categories_v2.id = tree.id) AND tree.id <> 1
+     JOIN category_texts_v2 category_texts ON tree.id = category_texts.category_id AND tree.locale_id = category_texts.locale_id
+     JOIN mv_category_locale_safexmlpath_v2 safe_xml_path ON tree.id = safe_xml_path.category_id AND tree.locale_id = safe_xml_path.locale_id
+  WHERE tree.locale_id = 3
+  GROUP BY tree.id, safe_xml_path.safe_xml_path, category_texts.name, product_category.product_id;
+
+GRANT SELECT ON TABLE v_analytics_product_categories TO funnel;
+GRANT SELECT ON TABLE v_analytics_product_categories TO datastudio;
+
+DROP VIEW IF EXISTS v_analytics_product_categories_v2;
+
+-- Finally an ugly adjust of inserted timestamp for half of the products with a specific timestamp
+-- in product_category that still causes problems
+UPDATE product_category
+SET inserted = inserted - INTERVAL '1 day'
+WHERE inserted = '2022-04-21 10:40:29.975707+02'::timestamp
+AND product_id IN (
+    SELECT product_id
+    FROM product_category
+    WHERE inserted = '2022-04-21 10:40:29.975707+02'::timestamp
+    LIMIT (SELECT COUNT(*) FROM product_category WHERE inserted = '2022-04-21 10:40:29.975707+02'::timestamp) / 2
+);
+
+
+

migrations/000.628.sql

@@ -0,0 +1,30 @@
+-- Fix v_funnel_collections to use editorial_collection_products after metadata migration
+
+DROP VIEW IF EXISTS public.v_funnel_collections;
+
+CREATE VIEW public.v_funnel_collections AS
+SELECT
+    ep.id AS collection_id,
+    ep.name AS collection_name,
+    cp.product_id,
+    ep.inserted,
+    ep.published,
+    ep.updated,
+    ep.description
+FROM editorial_pages ep
+JOIN (
+    -- Keep output equivalent to the former metadata array (one row per collection/product)
+    SELECT DISTINCT
+        editorial_page_id,
+        product_id
+    FROM editorial_collection_products
+) cp ON cp.editorial_page_id = ep.id
+WHERE ep.type = 'collection'
+  AND EXISTS (
+      SELECT 1
+      FROM editorial_content ec
+      WHERE ec.editorial_page_id = ep.id
+        AND ec.visible = TRUE
+  );
+
+GRANT SELECT ON TABLE public.v_funnel_collections TO funnel;