abhin4v/ringo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adds incremental refresh sql generation for fact tables.

Browse Source
This commit is contained in:
Abhinav Sarkar 2015-12-20 18:25:14 +05:30
parent 7dc6db944f
commit 7d64ffcde4
3 changed files with 23 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/Main.hs
View File

@ -47,19 +47,21 @@ writeSQLFiles outputDir env@Env{..} = forM_ sqls $ \(sqlType, table, sql) -> do
factTableDefnSQLs = [ (Create , tableName table, unlines . map sqlStr $ factTableDefnSQL env fact table)
| (fact, table) <- factTables ]
dimTablePopulateSQLs typ gen = [ (typ , tableName table, sqlStr $ gen env fact (tableName table))
| (fact, tabs) <- dimTables
, table <- tabs
, table `notElem` envTables ]
dimTablePopulateSQLs typ gen =
[ (typ , tableName table, sqlStr $ gen env fact (tableName table))
| (fact, tabs) <- dimTables
, table <- tabs
, table `notElem` envTables ]
factTableInsertSQLs = [ (FullRefresh, tableName table, sqlStr $ factTableInsertSQL env fact)
| (fact, table) <- factTables ]
factTablePopulateSQLs typ gen = [ (typ, tableName table, sqlStr $ gen env fact)
| (fact, table) <- factTables ]
sqls = concat [ dimTableDefnSQLs
, factTableDefnSQLs
, dimTablePopulateSQLs FullRefresh $ dimensionTablePopulateSQL FullPopulation
, dimTablePopulateSQLs IncRefresh $ dimensionTablePopulateSQL IncrementalPopulation
, factTableInsertSQLs
, dimTablePopulateSQLs FullRefresh $ dimensionTablePopulateSQL FullPopulation
, dimTablePopulateSQLs IncRefresh $ dimensionTablePopulateSQL IncrementalPopulation
, factTablePopulateSQLs FullRefresh $ factTablePopulateSQL FullPopulation
, factTablePopulateSQLs IncRefresh $ factTablePopulateSQL IncrementalPopulation
]
sqlStr s = Text.unpack $ s <> ";\n"

7
src/Ringo.hs
View File

@ -5,7 +5,7 @@ module Ringo
, G.tableDefnSQL
, factTableDefnSQL
, dimensionTablePopulateSQL
, factTableInsertSQL
, factTablePopulateSQL
, validateTable
, validateFact
) where
@ -31,8 +31,9 @@ dimensionTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> TableName ->
dimensionTablePopulateSQL popMode env fact =
flip runReader env . G.dimensionTablePopulateSQL popMode fact
factTableInsertSQL :: Env -> Fact -> Text
factTableInsertSQL env = flip runReader env . G.factTableInsertSQL
factTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> Text
factTablePopulateSQL popMode env =
flip runReader env . G.factTablePopulateSQL popMode
validateTable :: Env -> Table -> [ValidationError]
validateTable env = flip runReader env . V.validateTable

11
src/Ringo/Generator.hs
View File

@ -2,7 +2,7 @@ module Ringo.Generator
( tableDefnSQL
, factTableDefnSQL
, dimensionTablePopulateSQL
, factTableInsertSQL
, factTablePopulateSQL
) where
import qualified Data.Text as Text
@ -120,8 +120,8 @@ dimensionTablePopulateSQL popMode fact dimTableName = do
<> "\nWHERE " <> Text.intercalate " \nAND "
[ fullColName dimTableName c <> " IS NULL" | (c, _) <- colMapping ]
factTableInsertSQL :: Fact -> Reader Env Text
factTableInsertSQL fact = do
factTablePopulateSQL :: TablePopulationMode -> Fact -> Reader Env Text
factTablePopulateSQL popMode fact = do
Settings {..} <- asks envSettings
allDims <- extractAllDimensionTables fact
tables <- asks envTables
@ -176,11 +176,16 @@ factTableInsertSQL fact = do
. map (factTableName . fst)
$ allDims
timeCol = fullColName fTableName $ head [ cName | DimTime cName <- factColumns fact ]
return $ "INSERT INTO "
<> extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
<> " (\n" <> unlineCols (map fst3 colMap) <> "\n)"
<> "\nSELECT \n" <> unlineCols (map snd3 colMap)
<> "\nFROM " <> fTableName <> "\n" <> Text.intercalate"\n" joinClauses
<> (if popMode == IncrementalPopulation
then "\nWHERE " <> timeCol <> " > ? AND " <> timeCol <> " <= ?"
else "")
<> "\nGROUP BY \n"
<> unlineCols (map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap)
where