abhin4v/ringo
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adds support for generated table suffixes and time where clause on source table in full refresh mode.

Browse Source
This commit is contained in:
Abhinav Sarkar 2015-12-30 19:57:38 +05:30
parent 644acc7ee9
commit 4e4d13b94f
8 changed files with 78 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Main.hs
View File

@ -55,7 +55,7 @@ writeFiles outputDir env@Env{..} = do
dimTables = [ (fact, extractDimensionTables env fact) | fact <- envFacts ]
factTables = [ (fact, extractFactTable env fact) | fact <- envFacts ]
dimTableDefnSQLs = [ (Create, tableName table, unlines . map sqlStr . tableDefnSQL $ table)
dimTableDefnSQLs = [ (Create, tableName table, unlines . map sqlStr $ tableDefnSQL env table)
| (_, tabs) <- dimTables
, table <- tabs
, table `notElem` envTables ]

3
app/Ringo/ArgParser.hs
View File

@ -71,6 +71,9 @@ settingsParser = let Settings {..} = defSettings
<> value settingForeignKeyIdCoalesceValue
<> showDefault
<> help "Value to coalease missing foriegn key ids to, in fact tables")
<*> minorOption "tablename-suffix-template"
settingTableNameSuffixTemplate
"Suffix template for table names in SQL"
where
minorOption longDesc defValue helpTxt =
Text.pack <$> strOption (long longDesc

5
src/Ringo.hs
View File

@ -3,7 +3,7 @@ module Ringo
, extractFactTable
, extractDimensionTables
, extractDependencies
, G.tableDefnSQL
, tableDefnSQL
, factTableDefnSQL
, dimensionTablePopulateSQL
, factTablePopulateSQL
@ -28,6 +28,9 @@ extractDimensionTables env = flip runReader env . E.extractDimensionTables
extractDependencies :: Env -> Fact -> Dependencies
extractDependencies env = flip runReader env . E.extractDependencies
tableDefnSQL :: Env -> Table -> [Text]
tableDefnSQL env = flip runReader env . G.tableDefnSQL
factTableDefnSQL :: Env -> Fact -> Table -> [Text]
factTableDefnSQL env fact = flip runReader env . G.factTableDefnSQL fact

50
src/Ringo/Generator/Create.hs
View File

@ -1,5 +1,10 @@
module Ringo.Generator.Create (tableDefnSQL, factTableDefnSQL) where
#if MIN_VERSION_base(4,8,0)
#else
import Control.Applicative ((<$>))
#endif
import Control.Monad.Reader (Reader, asks)
import Data.Monoid ((<>))
import Data.Text (Text)
@ -9,29 +14,31 @@ import Ringo.Generator.Internal
import Ringo.Types
import Ringo.Utils
tableDefnSQL :: Table -> [Text]
tableDefnSQL Table {..} =
tableSQL : concatMap constraintDefnSQL tableConstraints
where
tableSQL = "CREATE TABLE " <> tableName <> " (\n"
<> (joinColumnNames . map columnDefnSQL $ tableColumns)
<> "\n)"
tableDefnSQL :: Table -> Reader Env [Text]
tableDefnSQL Table {..} = do
Settings {..} <- asks envSettings
let tabName = tableName <> settingTableNameSuffixTemplate
columnDefnSQL Column {..} =
columnName <> " " <> columnType <> " " <> nullableDefnSQL columnNullable
tableSQL = "CREATE TABLE " <> tabName <> " (\n"
<> (joinColumnNames . map columnDefnSQL $ tableColumns)
<> "\n)"
nullableDefnSQL Null = "NULL"
nullableDefnSQL NotNull = "NOT NULL"
columnDefnSQL Column {..} =
columnName <> " " <> columnType <> " " <> nullableDefnSQL columnNullable
constraintDefnSQL constraint =
let alterTableSQL = "ALTER TABLE ONLY " <> tableName <> " ADD "
in case constraint of
PrimaryKey cName -> [ alterTableSQL <> "PRIMARY KEY (" <> cName <> ")" ]
ForeignKey oTableName cNamePairs ->
[ alterTableSQL <> "FOREIGN KEY (" <> joinColumnNames (map fst cNamePairs) <> ") REFERENCES "
<> oTableName <> " (" <> joinColumnNames (map snd cNamePairs) <> ")" ]
UniqueKey cNames -> ["CREATE UNIQUE INDEX ON " <> tableName <> " (" <> joinColumnNames cNames <> ")"]
nullableDefnSQL Null = "NULL"
nullableDefnSQL NotNull = "NOT NULL"
constraintDefnSQL constraint =
let alterTableSQL = "ALTER TABLE ONLY " <> tabName <> " ADD "
in case constraint of
PrimaryKey cName -> [ alterTableSQL <> "PRIMARY KEY (" <> cName <> ")" ]
ForeignKey oTableName cNamePairs ->
[ alterTableSQL <> "FOREIGN KEY (" <> joinColumnNames (map fst cNamePairs) <> ") REFERENCES "
<> oTableName <> " (" <> joinColumnNames (map snd cNamePairs) <> ")" ]
UniqueKey cNames -> ["CREATE UNIQUE INDEX ON " <> tabName <> " (" <> joinColumnNames cNames <> ")"]
return $ tableSQL : concatMap constraintDefnSQL tableConstraints
factTableDefnSQL :: Fact -> Table -> Reader Env [Text]
factTableDefnSQL fact table = do
@ -46,7 +53,8 @@ factTableDefnSQL fact table = do
dimCols = [ factDimFKIdColumnName settingDimPrefix settingDimTableIdColumnName tableName
| (_, Table {..}) <- allDims ]
indexSQLs = [ "CREATE INDEX ON " <> tableName table <> " USING btree (" <> col <> ")"
indexSQLs = [ "CREATE INDEX ON " <> tableName table <> settingTableNameSuffixTemplate
<> " USING btree (" <> col <> ")"
| col <- factCols ++ dimCols ]
return $ tableDefnSQL table ++ indexSQLs
(++ indexSQLs) <$> tableDefnSQL table

5
src/Ringo/Generator/Internal.hs
View File

@ -35,3 +35,8 @@ coalesceColumn defaults tName Column{..} =
. find (\(k, _) -> k `Text.isPrefixOf` colType)
. Map.toList
$ defaults
suffixTableName :: TablePopulationMode -> Text -> TableName -> TableName
suffixTableName popMode suffix tableName = case popMode of
FullPopulation -> tableName <> suffix
IncrementalPopulation -> tableName

39
src/Ringo/Generator/Populate/Dimension.hs
View File

@ -18,30 +18,35 @@ import Ringo.Types
dimensionTablePopulateSQL :: TablePopulationMode -> Fact -> TableName -> Reader Env Text
dimensionTablePopulateSQL popMode fact dimTableName = do
dimPrefix <- settingDimPrefix <$> asks envSettings
tables <- asks envTables
defaults <- asks envTypeDefaults
let factTable = fromJust $ findTable (factTableName fact) tables
colMapping = dimColumnMapping dimPrefix fact dimTableName
selectCols = [ coalesceColumn defaults (factTableName fact) col <> " AS " <> cName
| (_, cName) <- colMapping
, let col = fromJust . findColumn cName $ tableColumns factTable ]
baseSelectC = "SELECT DISTINCT\n" <> joinColumnNames selectCols
<> "\nFROM " <> factTableName fact
baseWhereC = "(\n"
<> Text.intercalate "\nOR " [ c <> " IS NOT NULL" | (_, c) <- colMapping ]
<> "\n)"
Settings {..} <- asks envSettings
tables <- asks envTables
defaults <- asks envTypeDefaults
let factTable = fromJust $ findTable (factTableName fact) tables
colMapping = dimColumnMapping settingDimPrefix fact dimTableName
selectCols = [ coalesceColumn defaults (factTableName fact) col <> " AS " <> cName
| (_, cName) <- colMapping
, let col = fromJust . findColumn cName $ tableColumns factTable ]
timeCol = head [ cName | DimTime cName <- factColumns fact ]
baseSelectC = "SELECT DISTINCT\n" <> joinColumnNames selectCols
<> "\nFROM " <> factTableName fact
baseWhereCs = [ "(\n"
<> Text.intercalate "\nOR " [ c <> " IS NOT NULL" | (_, c) <- colMapping ]
<> "\n)"
, timeCol <> " <= ?"
]
insertC selectC whereCs =
"INSERT INTO " <> dimTableName
"INSERT INTO "
<> suffixTableName popMode settingTableNameSuffixTemplate dimTableName
<> " (\n" <> joinColumnNames (map fst colMapping) <> "\n) "
<> "SELECT x.* FROM (\n"
<> selectC <> "\nWHERE " <> Text.intercalate " AND\n" whereCs
<> ") x"
timeCol = head [ cName | DimTime cName <- factColumns fact ]
return $ case popMode of
FullPopulation -> insertC baseSelectC [baseWhereC]
FullPopulation -> insertC baseSelectC baseWhereCs
IncrementalPopulation ->
insertC baseSelectC [baseWhereC, timeCol <> " > ?", timeCol <> " <= ?"]
insertC baseSelectC (baseWhereCs ++ [ timeCol <> " > ?" ])
<> "\nLEFT JOIN " <> dimTableName <> " ON\n"
<> Text.intercalate " \nAND "
[ fullColumnName dimTableName c1 <> " = " <> fullColumnName "x" c2

23
src/Ringo/Generator/Populate/Fact.hs
View File

@ -53,8 +53,8 @@ data FactTablePopulateSelectSQL = FactTablePopulateSelectSQL
, ftpsGroupByCols :: ![Text]
} deriving (Show, Eq)
factTableUpdateSQL :: Fact -> Text -> FactTablePopulateSelectSQL -> Reader Env [Text]
factTableUpdateSQL fact groupByColPrefix populateSelectSQL@FactTablePopulateSelectSQL {..} = do
factTableUpdateSQL :: TablePopulationMode -> Fact -> Text -> FactTablePopulateSelectSQL -> Reader Env [Text]
factTableUpdateSQL popMode fact groupByColPrefix populateSelectSQL@FactTablePopulateSelectSQL {..} = do
Settings {..} <- asks envSettings
tables <- asks envTables
let countDistinctCols = [ col | col@(FactCountDistinct _ _) <- factColumns fact]
@ -62,7 +62,8 @@ factTableUpdateSQL fact groupByColPrefix populateSelectSQL@FactTablePopulateSele
fTable = fromJust . findTable fTableName $ tables
tablePKColName = head [ cName | PrimaryKey cName <- tableConstraints fTable ]
extFactTableName =
extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
suffixTableName popMode settingTableNameSuffixTemplate
$ extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
return . (\xs -> if null xs then xs else ilog2FunctionString : xs)
$ for countDistinctCols $ \(FactCountDistinct scName cName) ->
@ -163,8 +164,9 @@ factTablePopulateSQL popMode fact = do
insertSQL = if factTable `elem` tables -- existing dimension table
then (if columnNullable dimFKIdColumn == Null then coalesceFKId else id)
$ fullColumnName factSourceTableName dimFKIdColName
else "SELECT " <> dimIdColName <> " FROM " <> tableName <> "\nWHERE "
<> Text.intercalate "\n AND " dimLookupWhereClauses
else "SELECT " <> dimIdColName <> " FROM "
<> suffixTableName popMode settingTableNameSuffixTemplate tableName <> " " <> tableName
<> "\nWHERE " <> Text.intercalate "\n AND " dimLookupWhereClauses
in (dimFKIdColName, coalesceFKId insertSQL, True)
colMap = [ (cName, (sql, groupByColPrefix <> cName), addToGroupBy)
@ -178,17 +180,16 @@ factTablePopulateSQL popMode fact = do
timeCol = fullColumnName fTableName $ head [ cName | DimTime cName <- factColumns fact ]
extFactTableName =
extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
extFactTableName = suffixTableName popMode settingTableNameSuffixTemplate
$ extractedFactTableName settingFactPrefix settingFactInfix (factName fact) settingTimeUnit
populateSelectSQL =
FactTablePopulateSelectSQL
{ ftpsSelectCols = map snd3 colMap
, ftpsSelectTable = fTableName
, ftpsJoinClauses = joinClauses
, ftpsWhereClauses = if popMode == IncrementalPopulation
then [ timeCol <> " > ?", timeCol <> " <= ?" ]
else []
, ftpsWhereClauses =
timeCol <> " <= ?" : [ timeCol <> " > ?" | popMode == IncrementalPopulation ]
, ftpsGroupByCols = map ((groupByColPrefix <>) . fst3) . filter thd3 $ colMap
}
@ -196,7 +197,7 @@ factTablePopulateSQL popMode fact = do
<> " (\n" <> Text.intercalate ",\n " (map fst3 colMap) <> "\n)\n"
<> toSelectSQL populateSelectSQL
updateSQLs <- factTableUpdateSQL fact groupByColPrefix populateSelectSQL
updateSQLs <- factTableUpdateSQL popMode fact groupByColPrefix populateSelectSQL
return $ insertIntoSQL : updateSQLs
where

2
src/Ringo/Types.hs
View File

@ -83,6 +83,7 @@ data Settings = Settings
, settingFactsJSONFileName :: !Text
, settingDimensionJSONFileName :: !Text
, settingForeignKeyIdCoalesceValue :: !Int
, settingTableNameSuffixTemplate :: !Text
} deriving (Eq, Show)
defSettings :: Settings
@ -101,6 +102,7 @@ defSettings = Settings
, settingFactsJSONFileName = "facts.json"
, settingDimensionJSONFileName = "dimensions.json"
, settingForeignKeyIdCoalesceValue = -1
, settingTableNameSuffixTemplate = "{{suff}}"
}
data ValidationError = MissingTable !TableName