Adds support for generated table suffixes and time where clause on source table in full refresh mode.

pull/1/head
Abhinav Sarkar 2015-12-30 19:57:38 +05:30
parent 644acc7ee9
commit 4e4d13b94f
8 changed files with 78 additions and 51 deletions

View File

@ -55,7 +55,7 @@ writeFiles outputDir env@Env{..} = do
dimTables = [ (fact, extractDimensionTables env fact) | fact <- envFacts ] dimTables = [ (fact, extractDimensionTables env fact) | fact <- envFacts ]
factTables = [ (fact, extractFactTable env fact) | fact <- envFacts ] factTables = [ (fact, extractFactTable env fact) | fact <- envFacts ]
dimTableDefnSQLs = [ (Create, tableName table, unlines . map sqlStr . tableDefnSQL $ table) dimTableDefnSQLs = [ (Create, tableName table, unlines . map sqlStr $ tableDefnSQL env table)
| (_, tabs) <- dimTables | (_, tabs) <- dimTables
, table <- tabs , table <- tabs
, table `notElem` envTables ] , table `notElem` envTables ]

View File

@ -71,6 +71,9 @@ settingsParser = let Settings {..} = defSettings
<> value settingForeignKeyIdCoalesceValue <> value settingForeignKeyIdCoalesceValue
<> showDefault <> showDefault
<> help "Value to coalease missing foriegn key ids to, in fact tables") <> help "Value to coalease missing foriegn key ids to, in fact tables")
<*> minorOption "tablename-suffix-template"
settingTableNameSuffixTemplate
"Suffix template for table names in SQL"
where where
minorOption longDesc defValue helpTxt = minorOption longDesc defValue helpTxt =
Text.pack <$> strOption (long longDesc Text.pack <$> strOption (long longDesc

View File

@ -3,7 +3,7 @@ module Ringo
, extractFactTable , extractFactTable
, extractDimensionTables , extractDimensionTables
, extractDependencies , extractDependencies
, G.tableDefnSQL , tableDefnSQL
, factTableDefnSQL , factTableDefnSQL
, dimensionTablePopulateSQL , dimensionTablePopulateSQL
, factTablePopulateSQL , factTablePopulateSQL
@ -28,6 +28,9 @@ extractDimensionTables env = flip runReader env . E.extractDimensionTables
extractDependencies :: Env -> Fact -> Dependencies extractDependencies :: Env -> Fact -> Dependencies
extractDependencies env = flip runReader env . E.extractDependencies extractDependencies env = flip runReader env . E.extractDependencies
tableDefnSQL :: Env -> Table -> [Text]
tableDefnSQL env = flip runReader env . G.tableDefnSQL
factTableDefnSQL :: Env -> Fact -> Table -> [Text] factTableDefnSQL :: Env -> Fact -> Table -> [Text]
factTableDefnSQL env fact = flip runReader env . G.factTableDefnSQL fact factTableDefnSQL env fact = flip runReader env . G.factTableDefnSQL fact

View File

@ -1,5 +1,10 @@
module Ringo.Generator.Create (tableDefnSQL, factTableDefnSQL) where module Ringo.Generator.Create (tableDefnSQL, factTableDefnSQL) where
#if MIN_VERSION_base(4,8,0)
#else
import Control.Applicative ((<$>))
#endif
import Control.Monad.Reader (Reader, asks) import Control.Monad.Reader (Reader, asks)
import Data.Monoid ((<>)) import Data.Monoid ((<>))
import Data.Text (Text) import Data.Text (Text)
@ -9,29 +14,31 @@ import Ringo.Generator.Internal
import Ringo.Types import Ringo.Types
import Ringo.Utils import Ringo.Utils
tableDefnSQL :: Table -> [Text] tableDefnSQL :: Table -> Reader Env [Text]
tableDefnSQL Table {..} = tableDefnSQL Table {..} = do
tableSQL : concatMap constraintDefnSQL tableConstraints Settings {..} <- asks envSettings
where let tabName = tableName <> settingTableNameSuffixTemplate
tableSQL = "CREATE TABLE " <> tableName <> " (\n"
<> (joinColumnNames . map columnDefnSQL $ tableColumns)
<> "\n)"
columnDefnSQL Column {..} = tableSQL = "CREATE TABLE " <> tabName <> " (\n"
columnName <> " " <> columnType <> " " <> nullableDefnSQL columnNullable <> (joinColumnNames . map columnDefnSQL $ tableColumns)
<> "\n)"
nullableDefnSQL Null = "NULL" columnDefnSQL Column {..} =
nullableDefnSQL NotNull = "NOT NULL" columnName <> " " <> columnType <> " " <> nullableDefnSQL columnNullable
constraintDefnSQL constraint = nullableDefnSQL Null = "NULL"
let alterTableSQL = "ALTER TABLE ONLY " <> tableName <> " ADD " nullableDefnSQL NotNull = "NOT NULL"
in case constraint of
PrimaryKey cName -> [ alterTableSQL <> "PRIMARY KEY (" <> cName <> ")" ]
ForeignKey oTableName cNamePairs ->
[ alterTableSQL <> "FOREIGN KEY (" <> joinColumnNames (map fst cNamePairs) <> ") REFERENCES "
<> oTableName <> " (" <> joinColumnNames (map snd cNamePairs) <> ")" ]
UniqueKey cNames -> ["CREATE UNIQUE INDEX ON " <> tableName <> " (" <> joinColumnNames cNames <> ")"]
constraintDefnSQL constraint =
let alterTableSQL = "ALTER TABLE ONLY " <> tabName <> " ADD "
in case constraint of
PrimaryKey cName -> [ alterTableSQL <> "PRIMARY KEY (" <> cName <> ")" ]
ForeignKey oTableName cNamePairs ->
[ alterTableSQL <> "FOREIGN KEY (" <> joinColumnNames (map fst cNamePairs) <> ") REFERENCES "
<> oTableName <> " (" <> joinColumnNames (map snd cNamePairs) <> ")" ]
UniqueKey cNames -> ["CREATE UNIQUE INDEX ON " <> tabName <> " (" <> joinColumnNames cNames <> ")"]
return $ tableSQL : concatMap constraintDefnSQL tableConstraints
factTableDefnSQL :: Fact -> Table -> Reader Env [Text] factTableDefnSQL :: Fact -> Table -> Reader Env [Text]
factTableDefnSQL fact table = do factTableDefnSQL fact table = do
@ -46,7 +53,8 @@ factTableDefnSQL fact table = do
dimCols = [ factDimFKIdColumnName settingDimPrefix settingDimTableIdColumnName tableName dimCols = [ factDimFKIdColumnName settingDimPrefix settingDimTableIdColumnName tableName
| (_, Table {..}) <- allDims ] | (_, Table {..}) <- allDims ]
indexSQLs = [ "CREATE INDEX ON " <> tableName table <> " USING btree (" <> col <> ")" indexSQLs = [ "CREATE INDEX ON " <> tableName table <> settingTableNameSuffixTemplate
<> " USING btree (" <> col <> ")"
| col <- factCols ++ dimCols ] | col <- factCols ++ dimCols ]
return $ tableDefnSQL table ++ indexSQLs (++ indexSQLs) <$> tableDefnSQL table

View File

@ -35,3 +35,8 @@ coalesceColumn defaults tName Column{..} =
. find (\(k, _) -> k `Text.isPrefixOf` colType) . find (\(k, _) -> k `Text.isPrefixOf` colType)
. Map.toList . Map.toList
$ defaults $ defaults
suffixTableName :: TablePopulationMode -> Text -> TableName -> TableName
suffixTableName popMode suffix tableName = case popMode of
FullPopulation -> tableName <> suffix
IncrementalPopulation -> tableName

View File

@ -18,30 +18,35 @@ import Ringo.Types
dimensionTablePopulateSQL :: TablePopulationMode -> Fact -> TableName -> Reader Env Text dimensionTablePopulateSQL :: TablePopulationMode -> Fact -> TableName -> Reader Env Text
dimensionTablePopulateSQL popMode fact dimTableName = do dimensionTablePopulateSQL popMode fact dimTableName = do
dimPrefix <- settingDimPrefix <$> asks envSettings Settings {..} <- asks envSettings
tables <- asks envTables tables <- asks envTables
defaults <- asks envTypeDefaults defaults <- asks envTypeDefaults
let factTable = fromJust $ findTable (factTableName fact) tables let factTable = fromJust $ findTable (factTableName fact) tables
colMapping = dimColumnMapping dimPrefix fact dimTableName colMapping = dimColumnMapping settingDimPrefix fact dimTableName
selectCols = [ coalesceColumn defaults (factTableName fact) col <> " AS " <> cName selectCols = [ coalesceColumn defaults (factTableName fact) col <> " AS " <> cName
| (_, cName) <- colMapping | (_, cName) <- colMapping
, let col = fromJust . findColumn cName $ tableColumns factTable ] , let col = fromJust . findColumn cName $ tableColumns factTable ]
baseSelectC = "SELECT DISTINCT\n" <> joinColumnNames selectCols timeCol = head [ cName | DimTime cName <- factColumns fact ]
<> "\nFROM " <> factTableName fact baseSelectC = "SELECT DISTINCT\n" <> joinColumnNames selectCols
baseWhereC = "(\n" <> "\nFROM " <> factTableName fact
<> Text.intercalate "\nOR " [ c <> " IS NOT NULL" | (_, c) <- colMapping ] baseWhereCs = [ "(\n"
<> "\n)" <> Text.intercalate "\nOR " [ c <> " IS NOT NULL" | (_, c) <- colMapping ]
<> "\n)"
, timeCol <> " <= ?"
]
insertC selectC whereCs = insertC selectC whereCs =
"INSERT INTO " <> dimTableName "INSERT INTO "
<> suffixTableName popMode settingTableNameSuffixTemplate dimTableName
<> " (\n" <> joinColumnNames (map fst colMapping) <> "\n) " <> " (\n" <> joinColumnNames (map fst colMapping) <> "\n) "
<> "SELECT x.* FROM (\n" <> "SELECT x.* FROM (\n"
<> selectC <> "\nWHERE " <> Text.intercalate " AND\n" whereCs <> selectC <> "\nWHERE " <> Text.intercalate " AND\n" whereCs
<> ") x" <> ") x"
timeCol = head [ cName | DimTime cName <- factColumns fact ]
return $ case popMode of return $ case popMode of
FullPopulation -> insertC baseSelectC [baseWhereC] FullPopulation -> insertC baseSelectC baseWhereCs
IncrementalPopulation -> IncrementalPopulation ->
insertC baseSelectC [baseWhereC, timeCol <> " > ?", timeCol <> " <= ?"] insertC baseSelectC (baseWhereCs ++ [ timeCol <> " > ?" ])
<> "\nLEFT JOIN " <> dimTableName <> " ON\n" <> "\nLEFT JOIN " <> dimTableName <> " ON\n"
<> Text.intercalate " \nAND " <> Text.intercalate " \nAND "
[ fullColumnName dimTableName c1 <> " = " <> fullColumnName "x" c2 [ fullColumnName dimTableName c1 <> " = " <> fullColumnName "x" c2

View File

@ -53,8 +53,8 @@ data FactTablePopulateSelectSQL = FactTablePopulateSelectSQL
, ftpsGroupByCols :: ![Text] , ftpsGroupByCols :: ![Text]
} deriving (Show, Eq) } deriving (Show, Eq)
factTableUpdateSQL :: Fact -> Text -> FactTablePopulateSelectSQL -> Reader Env [Text] factTableUpdateSQL :: TablePopulationMode -> Fact -> Text -> FactTablePopulateSelectSQL -> Reader Env [Text]
factTableUpdateSQL fact groupByColPrefix populateSelectSQL@FactTablePopulateSelectSQL {..} = do factTableUpdateSQL popMode fact groupByColPrefix populateSelectSQL@FactTablePopulateSelectSQL {..} = do
Settings {..} <- asks envSettings Settings {..} <- asks envSettings
tables <- asks envTables tables <- asks envTables
let countDistinctCols = [ col | col@(FactCountDistinct _ _) <- factColumns fact] let countDistinctCols = [ col | col@(FactCountDistinct _ _) <- factColumns fact]
@ -62,7 +62,8 @@ factTableUpdateSQL fact groupByColPrefix populateSelectSQL@FactTablePopulateSele
fTable = fromJust . findTable fTableName $ tables fTable = fromJust . findTable fTableName $ tables
tablePKColName = head [ cName | PrimaryKey cName <- tableConstraints fTable ] tablePKColName = head [ cName | PrimaryKey cName <- tableConstraints fTable ]
extFactTableName = extFactTableName =
extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit suffixTableName popMode settingTableNameSuffixTemplate
$ extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
return . (\xs -> if null xs then xs else ilog2FunctionString : xs) return . (\xs -> if null xs then xs else ilog2FunctionString : xs)
$ for countDistinctCols $ \(FactCountDistinct scName cName) -> $ for countDistinctCols $ \(FactCountDistinct scName cName) ->
@ -163,8 +164,9 @@ factTablePopulateSQL popMode fact = do
insertSQL = if factTable `elem` tables -- existing dimension table insertSQL = if factTable `elem` tables -- existing dimension table
then (if columnNullable dimFKIdColumn == Null then coalesceFKId else id) then (if columnNullable dimFKIdColumn == Null then coalesceFKId else id)
$ fullColumnName factSourceTableName dimFKIdColName $ fullColumnName factSourceTableName dimFKIdColName
else "SELECT " <> dimIdColName <> " FROM " <> tableName <> "\nWHERE " else "SELECT " <> dimIdColName <> " FROM "
<> Text.intercalate "\n AND " dimLookupWhereClauses <> suffixTableName popMode settingTableNameSuffixTemplate tableName <> " " <> tableName
<> "\nWHERE " <> Text.intercalate "\n AND " dimLookupWhereClauses
in (dimFKIdColName, coalesceFKId insertSQL, True) in (dimFKIdColName, coalesceFKId insertSQL, True)
colMap = [ (cName, (sql, groupByColPrefix <> cName), addToGroupBy) colMap = [ (cName, (sql, groupByColPrefix <> cName), addToGroupBy)
@ -178,17 +180,16 @@ factTablePopulateSQL popMode fact = do
timeCol = fullColumnName fTableName $ head [ cName | DimTime cName <- factColumns fact ] timeCol = fullColumnName fTableName $ head [ cName | DimTime cName <- factColumns fact ]
extFactTableName = extFactTableName = suffixTableName popMode settingTableNameSuffixTemplate
extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit $ extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
populateSelectSQL = populateSelectSQL =
FactTablePopulateSelectSQL FactTablePopulateSelectSQL
{ ftpsSelectCols = map snd3 colMap { ftpsSelectCols = map snd3 colMap
, ftpsSelectTable = fTableName , ftpsSelectTable = fTableName
, ftpsJoinClauses = joinClauses , ftpsJoinClauses = joinClauses
, ftpsWhereClauses = if popMode == IncrementalPopulation , ftpsWhereClauses =
then [ timeCol <> " > ?", timeCol <> " <= ?" ] timeCol <> " <= ?" : [ timeCol <> " > ?" | popMode == IncrementalPopulation ]
else []
, ftpsGroupByCols = map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap , ftpsGroupByCols = map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap
} }
@ -196,7 +197,7 @@ factTablePopulateSQL popMode fact = do
<> " (\n" <> Text.intercalate ",\n " (map fst3 colMap) <> "\n)\n" <> " (\n" <> Text.intercalate ",\n " (map fst3 colMap) <> "\n)\n"
<> toSelectSQL populateSelectSQL <> toSelectSQL populateSelectSQL
updateSQLs <- factTableUpdateSQL fact groupByColPrefix populateSelectSQL updateSQLs <- factTableUpdateSQL popMode fact groupByColPrefix populateSelectSQL
return $ insertIntoSQL : updateSQLs return $ insertIntoSQL : updateSQLs
where where

View File

@ -83,6 +83,7 @@ data Settings = Settings
, settingFactsJSONFileName :: !Text , settingFactsJSONFileName :: !Text
, settingDimensionJSONFileName :: !Text , settingDimensionJSONFileName :: !Text
, settingForeignKeyIdCoalesceValue :: !Int , settingForeignKeyIdCoalesceValue :: !Int
, settingTableNameSuffixTemplate :: !Text
} deriving (Eq, Show) } deriving (Eq, Show)
defSettings :: Settings defSettings :: Settings
@ -101,6 +102,7 @@ defSettings = Settings
, settingFactsJSONFileName = "facts.json" , settingFactsJSONFileName = "facts.json"
, settingDimensionJSONFileName = "dimensions.json" , settingDimensionJSONFileName = "dimensions.json"
, settingForeignKeyIdCoalesceValue = -1 , settingForeignKeyIdCoalesceValue = -1
, settingTableNameSuffixTemplate = "{{suff}}"
} }
data ValidationError = MissingTable !TableName data ValidationError = MissingTable !TableName