From 7d64ffcde447af64ce3d5a611495578f5c3c94d6 Mon Sep 17 00:00:00 2001
From: Abhinav Sarkar <abhinav@abhinavsarkar.net>
Date: Sun, 20 Dec 2015 18:25:14 +0530
Subject: [PATCH] Adds incremental refresh sql generation for fact tables.

---
 app/Main.hs            | 20 +++++++++++---------
 src/Ringo.hs           |  7 ++++---
 src/Ringo/Generator.hs | 11 ++++++++---
 3 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/app/Main.hs b/app/Main.hs
index 1039b8a..d62ff15 100644
--- a/app/Main.hs
+++ b/app/Main.hs
@@ -47,19 +47,21 @@ writeSQLFiles outputDir env@Env{..} = forM_ sqls $ \(sqlType, table, sql) -> do
     factTableDefnSQLs   = [ (Create , tableName table, unlines . map sqlStr $ factTableDefnSQL env fact table)
                             | (fact, table) <- factTables ]
 
-    dimTablePopulateSQLs typ gen = [ (typ , tableName table, sqlStr $ gen env fact (tableName table))
-                                    | (fact, tabs) <- dimTables
-                                    , table        <- tabs
-                                    , table `notElem` envTables ]
+    dimTablePopulateSQLs typ gen  =
+      [ (typ , tableName table, sqlStr $ gen env fact (tableName table))
+        | (fact, tabs) <- dimTables
+        , table        <- tabs
+        , table `notElem` envTables ]
 
-    factTableInsertSQLs = [ (FullRefresh, tableName table, sqlStr $ factTableInsertSQL env fact)
-                            | (fact, table) <- factTables ]
+    factTablePopulateSQLs typ gen = [ (typ, tableName table, sqlStr $ gen env fact)
+                                      | (fact, table) <- factTables ]
 
     sqls = concat [ dimTableDefnSQLs
                   , factTableDefnSQLs
-                  , dimTablePopulateSQLs FullRefresh $ dimensionTablePopulateSQL FullPopulation
-                  , dimTablePopulateSQLs IncRefresh $ dimensionTablePopulateSQL IncrementalPopulation
-                  , factTableInsertSQLs
+                  , dimTablePopulateSQLs FullRefresh  $ dimensionTablePopulateSQL FullPopulation
+                  , dimTablePopulateSQLs IncRefresh   $ dimensionTablePopulateSQL IncrementalPopulation
+                  , factTablePopulateSQLs FullRefresh $ factTablePopulateSQL FullPopulation
+                  , factTablePopulateSQLs IncRefresh  $ factTablePopulateSQL IncrementalPopulation
                   ]
 
     sqlStr s = Text.unpack $ s <> ";\n"
diff --git a/src/Ringo.hs b/src/Ringo.hs
index 532e13b..3d47832 100644
--- a/src/Ringo.hs
+++ b/src/Ringo.hs
@@ -5,7 +5,7 @@ module Ringo
        , G.tableDefnSQL
        , factTableDefnSQL
        , dimensionTablePopulateSQL
-       , factTableInsertSQL
+       , factTablePopulateSQL
        , validateTable
        , validateFact
        ) where
@@ -31,8 +31,9 @@ dimensionTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> TableName ->
 dimensionTablePopulateSQL popMode env fact =
   flip runReader env . G.dimensionTablePopulateSQL popMode fact
 
-factTableInsertSQL :: Env -> Fact -> Text
-factTableInsertSQL env = flip runReader env . G.factTableInsertSQL
+factTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> Text
+factTablePopulateSQL popMode env =
+  flip runReader env . G.factTablePopulateSQL popMode
 
 validateTable :: Env -> Table -> [ValidationError]
 validateTable env = flip runReader env . V.validateTable
diff --git a/src/Ringo/Generator.hs b/src/Ringo/Generator.hs
index d2fc140..fa7e411 100644
--- a/src/Ringo/Generator.hs
+++ b/src/Ringo/Generator.hs
@@ -2,7 +2,7 @@ module Ringo.Generator
        ( tableDefnSQL
        , factTableDefnSQL
        , dimensionTablePopulateSQL
-       , factTableInsertSQL
+       , factTablePopulateSQL
        ) where
 
 import qualified Data.Text as Text
@@ -120,8 +120,8 @@ dimensionTablePopulateSQL popMode fact dimTableName = do
         <> "\nWHERE " <> Text.intercalate " \nAND "
                            [ fullColName dimTableName c <> " IS NULL" | (c, _) <- colMapping ]
 
-factTableInsertSQL :: Fact -> Reader Env Text
-factTableInsertSQL fact = do
+factTablePopulateSQL :: TablePopulationMode -> Fact -> Reader Env Text
+factTablePopulateSQL popMode fact = do
   Settings {..}    <- asks envSettings
   allDims          <- extractAllDimensionTables fact
   tables           <- asks envTables
@@ -176,11 +176,16 @@ factTableInsertSQL fact = do
         . map (factTableName . fst)
         $ allDims
 
+      timeCol     = fullColName fTableName $ head [ cName | DimTime cName <- factColumns fact ]
+
   return $ "INSERT INTO "
              <> extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
              <> " (\n" <> unlineCols (map fst3 colMap) <> "\n)"
              <> "\nSELECT \n" <> unlineCols (map snd3 colMap)
              <> "\nFROM " <> fTableName <> "\n" <> Text.intercalate"\n" joinClauses
+             <> (if popMode == IncrementalPopulation
+                   then "\nWHERE " <> timeCol <> " > ? AND " <> timeCol <> " <= ?"
+                   else "")
              <> "\nGROUP BY \n"
              <> unlineCols (map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap)
   where