abhin4v
/
ringo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adds incremental refresh sql generation for fact tables.

pull/1/head
Abhinav Sarkar 2015-12-20 18:25:14 +05:30
parent 7dc6db944f
commit 7d64ffcde4
3 changed files with 23 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/Main.hs
View File

@ -47,19 +47,21 @@ writeSQLFiles outputDir env@Env{..} = forM_ sqls $ \(sqlType, table, sql) -> do
factTableDefnSQLs = [ (Create , tableName table, unlines . map sqlStr $ factTableDefnSQL env fact table) factTableDefnSQLs = [ (Create , tableName table, unlines . map sqlStr $ factTableDefnSQL env fact table)
| (fact, table) <- factTables ] | (fact, table) <- factTables ]
dimTablePopulateSQLs typ gen = [ (typ , tableName table, sqlStr $ gen env fact (tableName table)) dimTablePopulateSQLs typ gen =
| (fact, tabs) <- dimTables [ (typ , tableName table, sqlStr $ gen env fact (tableName table))
, table <- tabs | (fact, tabs) <- dimTables
, table `notElem` envTables ] , table <- tabs
, table `notElem` envTables ]
factTableInsertSQLs = [ (FullRefresh, tableName table, sqlStr $ factTableInsertSQL env fact) factTablePopulateSQLs typ gen = [ (typ, tableName table, sqlStr $ gen env fact)
| (fact, table) <- factTables ] | (fact, table) <- factTables ]
sqls = concat [ dimTableDefnSQLs sqls = concat [ dimTableDefnSQLs
, factTableDefnSQLs , factTableDefnSQLs
, dimTablePopulateSQLs FullRefresh $ dimensionTablePopulateSQL FullPopulation , dimTablePopulateSQLs FullRefresh $ dimensionTablePopulateSQL FullPopulation
, dimTablePopulateSQLs IncRefresh $ dimensionTablePopulateSQL IncrementalPopulation , dimTablePopulateSQLs IncRefresh $ dimensionTablePopulateSQL IncrementalPopulation
, factTableInsertSQLs , factTablePopulateSQLs FullRefresh $ factTablePopulateSQL FullPopulation
, factTablePopulateSQLs IncRefresh $ factTablePopulateSQL IncrementalPopulation
] ]
sqlStr s = Text.unpack $ s <> ";\n" sqlStr s = Text.unpack $ s <> ";\n"

7
src/Ringo.hs
View File

@ -5,7 +5,7 @@ module Ringo
, G.tableDefnSQL , G.tableDefnSQL
, factTableDefnSQL , factTableDefnSQL
, dimensionTablePopulateSQL , dimensionTablePopulateSQL
, factTableInsertSQL , factTablePopulateSQL
, validateTable , validateTable
, validateFact , validateFact
) where ) where
@ -31,8 +31,9 @@ dimensionTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> TableName ->
dimensionTablePopulateSQL popMode env fact = dimensionTablePopulateSQL popMode env fact =
flip runReader env . G.dimensionTablePopulateSQL popMode fact flip runReader env . G.dimensionTablePopulateSQL popMode fact
factTableInsertSQL :: Env -> Fact -> Text factTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> Text
factTableInsertSQL env = flip runReader env . G.factTableInsertSQL factTablePopulateSQL popMode env =
flip runReader env . G.factTablePopulateSQL popMode
validateTable :: Env -> Table -> [ValidationError] validateTable :: Env -> Table -> [ValidationError]
validateTable env = flip runReader env . V.validateTable validateTable env = flip runReader env . V.validateTable

11
src/Ringo/Generator.hs
View File

@ -2,7 +2,7 @@ module Ringo.Generator
( tableDefnSQL ( tableDefnSQL
, factTableDefnSQL , factTableDefnSQL
, dimensionTablePopulateSQL , dimensionTablePopulateSQL
, factTableInsertSQL , factTablePopulateSQL
) where ) where
import qualified Data.Text as Text import qualified Data.Text as Text
@ -120,8 +120,8 @@ dimensionTablePopulateSQL popMode fact dimTableName = do
<> "\nWHERE " <> Text.intercalate " \nAND " <> "\nWHERE " <> Text.intercalate " \nAND "
[ fullColName dimTableName c <> " IS NULL" | (c, _) <- colMapping ] [ fullColName dimTableName c <> " IS NULL" | (c, _) <- colMapping ]
factTableInsertSQL :: Fact -> Reader Env Text factTablePopulateSQL :: TablePopulationMode -> Fact -> Reader Env Text
factTableInsertSQL fact = do factTablePopulateSQL popMode fact = do
Settings {..} <- asks envSettings Settings {..} <- asks envSettings
allDims <- extractAllDimensionTables fact allDims <- extractAllDimensionTables fact
tables <- asks envTables tables <- asks envTables
@ -176,11 +176,16 @@ factTableInsertSQL fact = do
. map (factTableName . fst) . map (factTableName . fst)
$ allDims $ allDims
timeCol = fullColName fTableName $ head [ cName | DimTime cName <- factColumns fact ]
return $ "INSERT INTO " return $ "INSERT INTO "
<> extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit <> extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
<> " (\n" <> unlineCols (map fst3 colMap) <> "\n)" <> " (\n" <> unlineCols (map fst3 colMap) <> "\n)"
<> "\nSELECT \n" <> unlineCols (map snd3 colMap) <> "\nSELECT \n" <> unlineCols (map snd3 colMap)
<> "\nFROM " <> fTableName <> "\n" <> Text.intercalate"\n" joinClauses <> "\nFROM " <> fTableName <> "\n" <> Text.intercalate"\n" joinClauses
<> (if popMode == IncrementalPopulation
then "\nWHERE " <> timeCol <> " > ? AND " <> timeCol <> " <= ?"
else "")
<> "\nGROUP BY \n" <> "\nGROUP BY \n"
<> unlineCols (map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap) <> unlineCols (map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap)
where where