Adds a smart constructor for Env for better type safety.
- Env is now created through a smart constructor which does validations.pull/1/head
parent
b58e022b0e
commit
ade5c388d8
13
app/Main.hs
13
app/Main.hs
|
@ -1,5 +1,7 @@
|
|||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
{-# LANGUAGE ViewPatterns #-}
|
||||
|
||||
module Main where
|
||||
|
||||
import qualified Data.ByteString.Lazy as BS
|
||||
|
@ -8,7 +10,6 @@ import qualified Data.Text as Text
|
|||
|
||||
import Data.Aeson (encode)
|
||||
import Data.Char (toLower)
|
||||
import Data.List (nub)
|
||||
import Control.Monad (forM_)
|
||||
import System.Directory (createDirectoryIfMissing)
|
||||
import System.FilePath ((</>), (<.>))
|
||||
|
@ -27,14 +28,12 @@ main = do
|
|||
case result of
|
||||
Left err -> putStrLn err >> exitFailure
|
||||
Right (tables, facts, defaults) -> do
|
||||
let env = Env tables facts progSettings defaults
|
||||
let errors = nub $ concatMap (validateTable env) tables ++ concatMap (validateFact env) facts
|
||||
if not $ null errors
|
||||
then mapM_ print errors >> exitFailure
|
||||
else writeFiles progOutputDir env >> exitSuccess
|
||||
case makeEnv tables facts progSettings defaults of
|
||||
Left errors -> mapM_ print errors >> exitFailure
|
||||
Right env -> writeFiles progOutputDir env >> exitSuccess
|
||||
|
||||
writeFiles :: FilePath -> Env -> IO ()
|
||||
writeFiles outputDir env@Env{..} = do
|
||||
writeFiles outputDir env@(envView -> EnvV{..}) = do
|
||||
let Settings{..} = envSettings
|
||||
forM_ sqls $ \(sqlType, table, sql) -> do
|
||||
let dirName = outputDir </> map toLower (show sqlType)
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
{-# LANGUAGE CPP #-}
|
||||
|
||||
module Ringo.InputParser (parseInput) where
|
||||
|
||||
import qualified Data.Text as Text
|
||||
|
|
|
@ -26,6 +26,7 @@ library
|
|||
Ringo.Generator.Create,
|
||||
Ringo.Generator.Populate.Dimension,
|
||||
Ringo.Generator.Populate.Fact,
|
||||
Ringo.Types.Internal,
|
||||
Ringo.Utils
|
||||
build-depends: base >=4.7 && <5,
|
||||
text >=1.2 && <1.3,
|
||||
|
|
24
src/Ringo.hs
24
src/Ringo.hs
|
@ -12,8 +12,7 @@ module Ringo
|
|||
, factTableDefnSQL
|
||||
, dimensionTablePopulateSQL
|
||||
, factTablePopulateSQL
|
||||
, validateTable
|
||||
, validateFact
|
||||
, makeEnv
|
||||
) where
|
||||
|
||||
import Control.Monad.Reader (runReader)
|
||||
|
@ -135,9 +134,14 @@ import qualified Ringo.Validator as V
|
|||
-- , ("text", "'__UNKNOWN_VAL__'")
|
||||
-- ]
|
||||
-- settings = defSettings { settingTableNameSuffixTemplate = "" }
|
||||
-- env = Env tables facts settings typeDefaults
|
||||
-- env = case makeEnv tables facts settings typeDefaults of
|
||||
-- Left errors -> error . unlines . map show $ errors
|
||||
-- Right env -> env
|
||||
-- :}
|
||||
|
||||
makeEnv :: [Table] -> [Fact] -> Settings -> TypeDefaults -> Either [ValidationError] Env
|
||||
makeEnv = V.validateEnv
|
||||
|
||||
-- |
|
||||
--
|
||||
-- >>> print $ extractFactTable env sessionFact
|
||||
|
@ -615,17 +619,3 @@ dimensionTablePopulateSQL popMode env fact =
|
|||
factTablePopulateSQL :: TablePopulationMode -> Env -> Fact -> [Text]
|
||||
factTablePopulateSQL popMode env =
|
||||
flip runReader env . G.factTablePopulateSQL popMode
|
||||
|
||||
-- |
|
||||
--
|
||||
-- >>> concatMap (validateTable env) tables
|
||||
-- []
|
||||
validateTable :: Env -> Table -> [ValidationError]
|
||||
validateTable env = flip runReader env . V.validateTable
|
||||
|
||||
-- |
|
||||
--
|
||||
-- >>> concatMap (validateFact env) facts
|
||||
-- []
|
||||
validateFact :: Env -> Fact -> [ValidationError]
|
||||
validateFact env = flip runReader env . V.validateFact
|
||||
|
|
|
@ -11,7 +11,7 @@ module Ringo.Extractor
|
|||
import qualified Data.Map as Map
|
||||
import qualified Data.Tree as Tree
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, asks, withReader)
|
||||
import Data.Maybe (fromJust)
|
||||
import Data.Monoid ((<>))
|
||||
import Data.List (nub)
|
||||
|
@ -22,8 +22,9 @@ import Ringo.Utils
|
|||
|
||||
extractFactTable :: Fact -> Reader Env Table
|
||||
extractFactTable fact = do
|
||||
Settings {..} <- asks envSettings
|
||||
allDims <- extractAllDimensionTables fact
|
||||
withReader envView $ do
|
||||
Settings {..} <- asks envSettings
|
||||
tables <- asks envTables
|
||||
let table = fromJust . findTable (factTableName fact) $ tables
|
||||
|
||||
|
@ -72,7 +73,7 @@ extractFactTable fact = do
|
|||
}
|
||||
|
||||
extractDependencies :: Fact -> Reader Env Dependencies
|
||||
extractDependencies fact = do
|
||||
extractDependencies fact = withReader envView $ do
|
||||
settings@Settings{..} <- asks envSettings
|
||||
facts <- asks envFacts
|
||||
let factSourceDeps =
|
||||
|
|
|
@ -13,7 +13,7 @@ import qualified Data.Text as Text
|
|||
import Control.Applicative ((<$>))
|
||||
#endif
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, asks, withReader)
|
||||
import Data.Function (on)
|
||||
import Data.Maybe (mapMaybe, fromMaybe, fromJust, catMaybes)
|
||||
import Data.Monoid ((<>))
|
||||
|
@ -59,7 +59,7 @@ idColTypeToFKIdColType typ = case Text.toLower typ of
|
|||
_ -> typ
|
||||
|
||||
extractDimensionTables :: Fact -> Reader Env [Table]
|
||||
extractDimensionTables fact = do
|
||||
extractDimensionTables fact = withReader envView $ do
|
||||
settings <- asks envSettings
|
||||
tables <- asks envTables
|
||||
let table = fromJust . findTable (factTableName fact) $ tables
|
||||
|
@ -99,4 +99,5 @@ extractAllDimensionTables fact = do
|
|||
parentDims <- concat <$> mapM extract (factParentNames fact)
|
||||
return . nubBy ((==) `on` snd) $ myDims ++ parentDims
|
||||
where
|
||||
extract fName = asks envFacts >>= extractAllDimensionTables . fromJust . findFact fName
|
||||
extract fName =
|
||||
asks (envFacts . envView) >>= extractAllDimensionTables . fromJust . findFact fName
|
||||
|
|
|
@ -11,7 +11,7 @@ module Ringo.Generator.Create (dimensionTableDefnSQL, factTableDefnSQL) where
|
|||
import Control.Applicative ((<$>))
|
||||
#endif
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, asks, withReader)
|
||||
import Database.HsSqlPpp.Syntax ( Statement(..), RowConstraint(..), AlterTableAction(..)
|
||||
, AlterTableOperation(..), Constraint(..), Cascade(..) )
|
||||
import Data.Maybe (listToMaybe, maybeToList)
|
||||
|
@ -24,7 +24,7 @@ import Ringo.Types
|
|||
import Ringo.Utils
|
||||
|
||||
tableDefnStmts :: Table -> Reader Env [Statement]
|
||||
tableDefnStmts Table {..} = do
|
||||
tableDefnStmts Table {..} = withReader envView $ do
|
||||
Settings {..} <- asks envSettings
|
||||
let tabName = tableName <> settingTableNameSuffixTemplate
|
||||
|
||||
|
@ -58,7 +58,7 @@ dimensionTableDefnSQL :: Table -> Reader Env [Text]
|
|||
dimensionTableDefnSQL table = tableDefnSQL table dimensionTableIndexStmts
|
||||
|
||||
dimensionTableIndexStmts :: Table -> Reader Env [Statement]
|
||||
dimensionTableIndexStmts Table {..} = do
|
||||
dimensionTableIndexStmts Table {..} = withReader envView $do
|
||||
Settings {..} <- asks envSettings
|
||||
let tabName = tableName <> settingTableNameSuffixTemplate
|
||||
tablePKColName = head [ cName | PrimaryKey cName <- tableConstraints ]
|
||||
|
@ -72,9 +72,10 @@ factTableDefnSQL fact table = tableDefnSQL table (factTableIndexStmts fact)
|
|||
|
||||
factTableIndexStmts :: Fact -> Table -> Reader Env [Statement]
|
||||
factTableIndexStmts fact table = do
|
||||
allDims <- extractAllDimensionTables fact
|
||||
withReader envView $ do
|
||||
Settings {..} <- asks envSettings
|
||||
tables <- asks envTables
|
||||
allDims <- extractAllDimensionTables fact
|
||||
|
||||
let dimTimeCol = head [ cName | DimTimeV cName <- factColumns fact ]
|
||||
tenantIdCol = listToMaybe [ cName | TenantIdV cName <- factColumns fact ]
|
||||
|
|
|
@ -10,7 +10,7 @@ module Ringo.Generator.Populate.Dimension (dimensionTablePopulateSQL) where
|
|||
import Control.Applicative ((<$>))
|
||||
#endif
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, asks, withReader)
|
||||
import Database.HsSqlPpp.Syntax (Statement, QueryExpr(..), Distinct(..), makeSelect, JoinType(..))
|
||||
import Data.Maybe (fromJust)
|
||||
import Data.Text (Text)
|
||||
|
@ -25,7 +25,7 @@ dimensionTablePopulateSQL popMode fact dimTableName =
|
|||
ppStatement <$> dimensionTablePopulateStmt popMode fact dimTableName
|
||||
|
||||
dimensionTablePopulateStmt :: TablePopulationMode -> Fact -> TableName -> Reader Env Statement
|
||||
dimensionTablePopulateStmt popMode fact dimTableName = do
|
||||
dimensionTablePopulateStmt popMode fact dimTableName = withReader envView $ do
|
||||
Settings {..} <- asks envSettings
|
||||
tables <- asks envTables
|
||||
defaults <- asks envTypeDefaults
|
||||
|
|
|
@ -16,7 +16,7 @@ import qualified Data.Text as Text
|
|||
import Control.Applicative ((<$>))
|
||||
#endif
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, asks, withReader)
|
||||
import Database.HsSqlPpp.Syntax ( QueryExpr(..), Statement, makeSelect
|
||||
, SelectList(..), SelectItem(..), JoinType(..) )
|
||||
import Data.List (nub)
|
||||
|
@ -57,7 +57,7 @@ $$
|
|||
LANGUAGE 'plpgsql' IMMUTABLE;
|
||||
|]
|
||||
|
||||
factCountDistinctUpdateStmts :: TablePopulationMode -> Fact -> Text -> QueryExpr -> Reader Env [Statement]
|
||||
factCountDistinctUpdateStmts :: TablePopulationMode -> Fact -> Text -> QueryExpr -> Reader EnvV [Statement]
|
||||
factCountDistinctUpdateStmts popMode fact groupByColPrefix expr = case expr of
|
||||
Select {selSelectList = SelectList _ origSelectItems, ..} -> do
|
||||
Settings {..} <- asks envSettings
|
||||
|
@ -126,8 +126,9 @@ factCountDistinctUpdateStmts popMode fact groupByColPrefix expr = case expr of
|
|||
|
||||
factTablePopulateStmts :: TablePopulationMode -> Fact -> Reader Env [Statement]
|
||||
factTablePopulateStmts popMode fact = do
|
||||
Settings {..} <- asks envSettings
|
||||
allDims <- extractAllDimensionTables fact
|
||||
withReader envView $ do
|
||||
Settings {..} <- asks envSettings
|
||||
tables <- asks envTables
|
||||
defaults <- asks envTypeDefaults
|
||||
let fTableName = factTableName fact
|
||||
|
|
|
@ -5,186 +5,25 @@
|
|||
{-# LANGUAGE StandaloneDeriving #-}
|
||||
{-# LANGUAGE PatternSynonyms #-}
|
||||
|
||||
module Ringo.Types where
|
||||
module Ringo.Types
|
||||
( ColumnName, ColumnType, TableName
|
||||
, Nullable(..), Column(..), TableConstraint(..), Table(..)
|
||||
, TimeUnit(..), timeUnitName, timeUnitToSeconds
|
||||
, Fact(..), FactColumnType(..), FactColumn(..), factSourceColumnName
|
||||
, pattern DimTimeV
|
||||
, pattern NoDimIdV
|
||||
, pattern TenantIdV
|
||||
, pattern DimIdV
|
||||
, pattern DimValV
|
||||
, pattern FactCountV
|
||||
, pattern FactCountDistinctV
|
||||
, pattern FactSumV
|
||||
, pattern FactAverageV
|
||||
, pattern FactMaxV
|
||||
, pattern FactMinV
|
||||
, Settings(..), defSettings
|
||||
, ValidationError(..), TypeDefaults
|
||||
, Env, EnvV(..), envView
|
||||
, TablePopulationMode(..), Dependencies) where
|
||||
|
||||
import qualified Data.Text as Text
|
||||
|
||||
import Data.Map (Map)
|
||||
import Data.Monoid ((<>))
|
||||
import Data.Text (Text)
|
||||
|
||||
showColNames :: [Text] -> String
|
||||
showColNames cols = Text.unpack $ "(" <> Text.intercalate ", " cols <> ")"
|
||||
|
||||
type ColumnName = Text
|
||||
type ColumnType = Text
|
||||
type TableName = Text
|
||||
|
||||
data Nullable = Null | NotNull deriving (Eq, Enum)
|
||||
|
||||
instance Show Nullable where
|
||||
show Null = "NULL"
|
||||
show NotNull = "NOT NULL"
|
||||
|
||||
data Column = Column
|
||||
{ columnName :: !ColumnName
|
||||
, columnType :: !ColumnType
|
||||
, columnNullable :: !Nullable
|
||||
} deriving (Eq)
|
||||
|
||||
instance Show Column where
|
||||
show Column {..} = "Column "
|
||||
++ Text.unpack columnName ++ " "
|
||||
++ Text.unpack columnType ++ " "
|
||||
++ show columnNullable
|
||||
|
||||
data TableConstraint = PrimaryKey !ColumnName
|
||||
| UniqueKey ![ColumnName]
|
||||
| ForeignKey !TableName ![(ColumnName, ColumnName)]
|
||||
deriving (Eq)
|
||||
|
||||
instance Show TableConstraint where
|
||||
show (PrimaryKey col) = "PrimaryKey " ++ Text.unpack col
|
||||
show (UniqueKey cols) = "UniqueKey " ++ showColNames cols
|
||||
show (ForeignKey tName colMap) = "ForeignKey " ++ showColNames (map fst colMap) ++ " "
|
||||
++ Text.unpack tName ++ " " ++ showColNames (map snd colMap)
|
||||
data Table = Table
|
||||
{ tableName :: !TableName
|
||||
, tableColumns :: ![Column]
|
||||
, tableConstraints :: ![TableConstraint]
|
||||
} deriving (Eq)
|
||||
|
||||
instance Show Table where
|
||||
show Table {..} =
|
||||
unlines $ ("Table " ++ Text.unpack tableName) : map show tableColumns ++ map show tableConstraints
|
||||
|
||||
data TimeUnit = Second | Minute | Hour | Day | Week
|
||||
deriving (Eq, Enum, Show, Read)
|
||||
|
||||
timeUnitName :: TimeUnit -> Text
|
||||
timeUnitName = Text.toLower . Text.pack . show
|
||||
|
||||
timeUnitToSeconds :: TimeUnit -> Int
|
||||
timeUnitToSeconds Second = 1
|
||||
timeUnitToSeconds Minute = 60 * timeUnitToSeconds Second
|
||||
timeUnitToSeconds Hour = 60 * timeUnitToSeconds Minute
|
||||
timeUnitToSeconds Day = 24 * timeUnitToSeconds Hour
|
||||
timeUnitToSeconds Week = 7 * timeUnitToSeconds Day
|
||||
|
||||
data Fact = Fact
|
||||
{ factName :: !TableName
|
||||
, factTableName :: !TableName
|
||||
, factTablePersistent :: !Bool
|
||||
, factParentNames :: ![TableName]
|
||||
, factColumns :: ![FactColumn]
|
||||
} deriving (Show)
|
||||
|
||||
data FCTNone
|
||||
data FCTTargetTable
|
||||
data FCTMaybeSourceColumn
|
||||
data FCTSourceColumn
|
||||
|
||||
data FactColumnType a where
|
||||
DimTime :: FactColumnType FCTNone
|
||||
NoDimId :: FactColumnType FCTNone
|
||||
TenantId :: FactColumnType FCTNone
|
||||
DimId :: { factColTargetTable :: !TableName } -> FactColumnType FCTTargetTable
|
||||
DimVal :: { factColTargetTable :: !TableName } -> FactColumnType FCTTargetTable
|
||||
FactCount :: { factColMaybeSourceColumn :: !(Maybe ColumnName) } -> FactColumnType FCTMaybeSourceColumn
|
||||
FactCountDistinct :: { factColMaybeSourceColumn :: !(Maybe ColumnName) } -> FactColumnType FCTMaybeSourceColumn
|
||||
FactSum :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactAverage :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactMax :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactMin :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
|
||||
deriving instance Show (FactColumnType a)
|
||||
|
||||
pattern DimTimeV col <- FactColumn col DimTime
|
||||
pattern NoDimIdV col <- FactColumn col NoDimId
|
||||
pattern TenantIdV col <- FactColumn col TenantId
|
||||
pattern DimIdV col <- FactColumn col DimId {..}
|
||||
pattern DimValV col <- FactColumn col DimVal {..}
|
||||
pattern FactCountV col <- FactColumn col FactCount {..}
|
||||
pattern FactCountDistinctV col <- FactColumn col FactCountDistinct {..}
|
||||
pattern FactSumV col <- FactColumn col FactSum {..}
|
||||
pattern FactAverageV col <- FactColumn col FactAverage {..}
|
||||
pattern FactMaxV col <- FactColumn col FactMax {..}
|
||||
pattern FactMinV col <- FactColumn col FactMin {..}
|
||||
|
||||
data FactColumn = forall a. FactColumn
|
||||
{ factColTargetColumn :: !ColumnName
|
||||
, factColType :: FactColumnType a }
|
||||
|
||||
deriving instance Show FactColumn
|
||||
|
||||
factSourceColumnName :: FactColumn -> Maybe ColumnName
|
||||
factSourceColumnName FactColumn {..} = case factColType of
|
||||
DimTime -> Just factColTargetColumn
|
||||
NoDimId -> Just factColTargetColumn
|
||||
TenantId -> Just factColTargetColumn
|
||||
DimId {..} -> Just factColTargetColumn
|
||||
DimVal {..} -> Just factColTargetColumn
|
||||
FactCount {..} -> factColMaybeSourceColumn
|
||||
FactCountDistinct {..} -> factColMaybeSourceColumn
|
||||
FactSum {..} -> Just factColSourceColumn
|
||||
FactAverage {..} -> Just factColSourceColumn
|
||||
FactMax {..} -> Just factColSourceColumn
|
||||
FactMin {..} -> Just factColSourceColumn
|
||||
|
||||
data Settings = Settings
|
||||
{ settingDimPrefix :: !Text
|
||||
, settingFactPrefix :: !Text
|
||||
, settingTimeUnit :: !TimeUnit
|
||||
, settingAvgCountColumSuffix :: !Text
|
||||
, settingAvgSumColumnSuffix :: !Text
|
||||
, settingDimTableIdColumnName :: !Text
|
||||
, settingDimTableIdColumnType :: !Text
|
||||
, settingFactCountColumnType :: !Text
|
||||
, settingFactCountDistinctErrorRate :: !Double
|
||||
, settingFactInfix :: !Text
|
||||
, settingDependenciesJSONFileName :: !Text
|
||||
, settingFactsJSONFileName :: !Text
|
||||
, settingDimensionJSONFileName :: !Text
|
||||
, settingForeignKeyIdCoalesceValue :: !Int
|
||||
, settingTableNameSuffixTemplate :: !Text
|
||||
} deriving (Eq, Show)
|
||||
|
||||
defSettings :: Settings
|
||||
defSettings = Settings
|
||||
{ settingDimPrefix = "dim_"
|
||||
, settingFactPrefix = "fact_"
|
||||
, settingTimeUnit = Minute
|
||||
, settingAvgCountColumSuffix = "_count"
|
||||
, settingAvgSumColumnSuffix = "_sum"
|
||||
, settingDimTableIdColumnName = "id"
|
||||
, settingDimTableIdColumnType = "serial"
|
||||
, settingFactCountColumnType = "integer"
|
||||
, settingFactCountDistinctErrorRate = 0.05
|
||||
, settingFactInfix = "_by_"
|
||||
, settingDependenciesJSONFileName = "dependencies.json"
|
||||
, settingFactsJSONFileName = "facts.json"
|
||||
, settingDimensionJSONFileName = "dimensions.json"
|
||||
, settingForeignKeyIdCoalesceValue = -1
|
||||
, settingTableNameSuffixTemplate = "{{suff}}"
|
||||
}
|
||||
|
||||
data ValidationError = MissingTable !TableName
|
||||
| MissingFact !TableName
|
||||
| MissingColumn !TableName !ColumnName
|
||||
| MissingTimeColumn !TableName
|
||||
| MissingNotNullConstraint !TableName !ColumnName
|
||||
| MissingTypeDefault !Text
|
||||
deriving (Eq, Show)
|
||||
|
||||
type TypeDefaults = Map Text Text
|
||||
|
||||
data Env = Env
|
||||
{ envTables :: ![Table]
|
||||
, envFacts :: ![Fact]
|
||||
, envSettings :: !Settings
|
||||
, envTypeDefaults :: !TypeDefaults
|
||||
} deriving (Show)
|
||||
|
||||
data TablePopulationMode = FullPopulation | IncrementalPopulation deriving (Eq, Show)
|
||||
|
||||
type Dependencies = Map TableName [TableName]
|
||||
import Ringo.Types.Internal
|
||||
|
|
|
@ -0,0 +1,198 @@
|
|||
{-# LANGUAGE OverloadedStrings #-}
|
||||
{-# LANGUAGE RecordWildCards #-}
|
||||
{-# LANGUAGE GADTs #-}
|
||||
{-# LANGUAGE Rank2Types #-}
|
||||
{-# LANGUAGE StandaloneDeriving #-}
|
||||
{-# LANGUAGE PatternSynonyms #-}
|
||||
|
||||
module Ringo.Types.Internal where
|
||||
|
||||
import qualified Data.Text as Text
|
||||
|
||||
import Data.Map (Map)
|
||||
import Data.Monoid ((<>))
|
||||
import Data.Text (Text)
|
||||
|
||||
showColNames :: [Text] -> String
|
||||
showColNames cols = Text.unpack $ "(" <> Text.intercalate ", " cols <> ")"
|
||||
|
||||
type ColumnName = Text
|
||||
type ColumnType = Text
|
||||
type TableName = Text
|
||||
|
||||
data Nullable = Null | NotNull deriving (Eq, Enum)
|
||||
|
||||
instance Show Nullable where
|
||||
show Null = "NULL"
|
||||
show NotNull = "NOT NULL"
|
||||
|
||||
data Column = Column
|
||||
{ columnName :: !ColumnName
|
||||
, columnType :: !ColumnType
|
||||
, columnNullable :: !Nullable
|
||||
} deriving (Eq)
|
||||
|
||||
instance Show Column where
|
||||
show Column {..} = "Column "
|
||||
++ Text.unpack columnName ++ " "
|
||||
++ Text.unpack columnType ++ " "
|
||||
++ show columnNullable
|
||||
|
||||
data TableConstraint = PrimaryKey !ColumnName
|
||||
| UniqueKey ![ColumnName]
|
||||
| ForeignKey !TableName ![(ColumnName, ColumnName)]
|
||||
deriving (Eq)
|
||||
|
||||
instance Show TableConstraint where
|
||||
show (PrimaryKey col) = "PrimaryKey " ++ Text.unpack col
|
||||
show (UniqueKey cols) = "UniqueKey " ++ showColNames cols
|
||||
show (ForeignKey tName colMap) = "ForeignKey " ++ showColNames (map fst colMap) ++ " "
|
||||
++ Text.unpack tName ++ " " ++ showColNames (map snd colMap)
|
||||
data Table = Table
|
||||
{ tableName :: !TableName
|
||||
, tableColumns :: ![Column]
|
||||
, tableConstraints :: ![TableConstraint]
|
||||
} deriving (Eq)
|
||||
|
||||
instance Show Table where
|
||||
show Table {..} =
|
||||
unlines $ ("Table " ++ Text.unpack tableName) : map show tableColumns ++ map show tableConstraints
|
||||
|
||||
data TimeUnit = Second | Minute | Hour | Day | Week
|
||||
deriving (Eq, Enum, Show, Read)
|
||||
|
||||
timeUnitName :: TimeUnit -> Text
|
||||
timeUnitName = Text.toLower . Text.pack . show
|
||||
|
||||
timeUnitToSeconds :: TimeUnit -> Int
|
||||
timeUnitToSeconds Second = 1
|
||||
timeUnitToSeconds Minute = 60 * timeUnitToSeconds Second
|
||||
timeUnitToSeconds Hour = 60 * timeUnitToSeconds Minute
|
||||
timeUnitToSeconds Day = 24 * timeUnitToSeconds Hour
|
||||
timeUnitToSeconds Week = 7 * timeUnitToSeconds Day
|
||||
|
||||
data Fact = Fact
|
||||
{ factName :: !TableName
|
||||
, factTableName :: !TableName
|
||||
, factTablePersistent :: !Bool
|
||||
, factParentNames :: ![TableName]
|
||||
, factColumns :: ![FactColumn]
|
||||
} deriving (Show)
|
||||
|
||||
data FCTNone
|
||||
data FCTTargetTable
|
||||
data FCTMaybeSourceColumn
|
||||
data FCTSourceColumn
|
||||
|
||||
data FactColumnType a where
|
||||
DimTime :: FactColumnType FCTNone
|
||||
NoDimId :: FactColumnType FCTNone
|
||||
TenantId :: FactColumnType FCTNone
|
||||
DimId :: { factColTargetTable :: !TableName } -> FactColumnType FCTTargetTable
|
||||
DimVal :: { factColTargetTable :: !TableName } -> FactColumnType FCTTargetTable
|
||||
FactCount :: { factColMaybeSourceColumn :: !(Maybe ColumnName) } -> FactColumnType FCTMaybeSourceColumn
|
||||
FactCountDistinct :: { factColMaybeSourceColumn :: !(Maybe ColumnName) } -> FactColumnType FCTMaybeSourceColumn
|
||||
FactSum :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactAverage :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactMax :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
FactMin :: { factColSourceColumn :: !ColumnName } -> FactColumnType FCTSourceColumn
|
||||
|
||||
deriving instance Show (FactColumnType a)
|
||||
|
||||
pattern DimTimeV col <- FactColumn col DimTime
|
||||
pattern NoDimIdV col <- FactColumn col NoDimId
|
||||
pattern TenantIdV col <- FactColumn col TenantId
|
||||
pattern DimIdV col <- FactColumn col DimId {..}
|
||||
pattern DimValV col <- FactColumn col DimVal {..}
|
||||
pattern FactCountV col <- FactColumn col FactCount {..}
|
||||
pattern FactCountDistinctV col <- FactColumn col FactCountDistinct {..}
|
||||
pattern FactSumV col <- FactColumn col FactSum {..}
|
||||
pattern FactAverageV col <- FactColumn col FactAverage {..}
|
||||
pattern FactMaxV col <- FactColumn col FactMax {..}
|
||||
pattern FactMinV col <- FactColumn col FactMin {..}
|
||||
|
||||
data FactColumn = forall a. FactColumn
|
||||
{ factColTargetColumn :: !ColumnName
|
||||
, factColType :: FactColumnType a }
|
||||
|
||||
deriving instance Show FactColumn
|
||||
|
||||
factSourceColumnName :: FactColumn -> Maybe ColumnName
|
||||
factSourceColumnName FactColumn {..} = case factColType of
|
||||
DimTime -> Just factColTargetColumn
|
||||
NoDimId -> Just factColTargetColumn
|
||||
TenantId -> Just factColTargetColumn
|
||||
DimId {..} -> Just factColTargetColumn
|
||||
DimVal {..} -> Just factColTargetColumn
|
||||
FactCount {..} -> factColMaybeSourceColumn
|
||||
FactCountDistinct {..} -> factColMaybeSourceColumn
|
||||
FactSum {..} -> Just factColSourceColumn
|
||||
FactAverage {..} -> Just factColSourceColumn
|
||||
FactMax {..} -> Just factColSourceColumn
|
||||
FactMin {..} -> Just factColSourceColumn
|
||||
|
||||
data Settings = Settings
|
||||
{ settingDimPrefix :: !Text
|
||||
, settingFactPrefix :: !Text
|
||||
, settingTimeUnit :: !TimeUnit
|
||||
, settingAvgCountColumSuffix :: !Text
|
||||
, settingAvgSumColumnSuffix :: !Text
|
||||
, settingDimTableIdColumnName :: !Text
|
||||
, settingDimTableIdColumnType :: !Text
|
||||
, settingFactCountColumnType :: !Text
|
||||
, settingFactCountDistinctErrorRate :: !Double
|
||||
, settingFactInfix :: !Text
|
||||
, settingDependenciesJSONFileName :: !Text
|
||||
, settingFactsJSONFileName :: !Text
|
||||
, settingDimensionJSONFileName :: !Text
|
||||
, settingForeignKeyIdCoalesceValue :: !Int
|
||||
, settingTableNameSuffixTemplate :: !Text
|
||||
} deriving (Eq, Show)
|
||||
|
||||
defSettings :: Settings
|
||||
defSettings = Settings
|
||||
{ settingDimPrefix = "dim_"
|
||||
, settingFactPrefix = "fact_"
|
||||
, settingTimeUnit = Minute
|
||||
, settingAvgCountColumSuffix = "_count"
|
||||
, settingAvgSumColumnSuffix = "_sum"
|
||||
, settingDimTableIdColumnName = "id"
|
||||
, settingDimTableIdColumnType = "serial"
|
||||
, settingFactCountColumnType = "integer"
|
||||
, settingFactCountDistinctErrorRate = 0.05
|
||||
, settingFactInfix = "_by_"
|
||||
, settingDependenciesJSONFileName = "dependencies.json"
|
||||
, settingFactsJSONFileName = "facts.json"
|
||||
, settingDimensionJSONFileName = "dimensions.json"
|
||||
, settingForeignKeyIdCoalesceValue = -1
|
||||
, settingTableNameSuffixTemplate = "{{suff}}"
|
||||
}
|
||||
|
||||
data ValidationError = MissingTable !TableName
|
||||
| DuplicateTable !TableName
|
||||
| MissingFact !TableName
|
||||
| DuplicateFact !TableName
|
||||
| MissingColumn !TableName !ColumnName
|
||||
| DuplicateColumn !TableName !ColumnName
|
||||
| MissingTimeColumn !TableName
|
||||
| MissingNotNullConstraint !TableName !ColumnName
|
||||
| MissingTypeDefault !Text
|
||||
deriving (Eq, Show)
|
||||
|
||||
type TypeDefaults = Map Text Text
|
||||
|
||||
data Env = Env ![Table] ![Fact] !Settings !TypeDefaults
|
||||
|
||||
data EnvV = EnvV
|
||||
{ envTables :: ![Table]
|
||||
, envFacts :: ![Fact]
|
||||
, envSettings :: !Settings
|
||||
, envTypeDefaults :: !TypeDefaults
|
||||
} deriving (Show)
|
||||
|
||||
envView :: Env -> EnvV
|
||||
envView (Env tables facts settings typeDefaults) = EnvV tables facts settings typeDefaults
|
||||
|
||||
data TablePopulationMode = FullPopulation | IncrementalPopulation deriving (Eq, Show)
|
||||
|
||||
type Dependencies = Map TableName [TableName]
|
|
@ -34,6 +34,14 @@ second = Arrow.second
|
|||
(&&&) :: (a -> b) -> (a -> c) -> a -> (b, c)
|
||||
(&&&) = (Arrow.&&&)
|
||||
|
||||
(>>>) :: (a -> b) -> (b -> c) -> (a -> c)
|
||||
(>>>) = (Arrow.>>>)
|
||||
|
||||
(>>-) :: a -> (a -> b) -> b
|
||||
(>>-) v f = f v
|
||||
|
||||
infixr 1 >>-
|
||||
|
||||
dupe :: a -> (a,a)
|
||||
dupe x = (x, x)
|
||||
|
||||
|
|
|
@ -4,10 +4,7 @@
|
|||
{-# LANGUAGE GADTs #-}
|
||||
{-# LANGUAGE PatternSynonyms #-}
|
||||
|
||||
module Ringo.Validator
|
||||
( validateTable
|
||||
, validateFact
|
||||
) where
|
||||
module Ringo.Validator (validateEnv) where
|
||||
|
||||
import qualified Data.Map as Map
|
||||
import qualified Data.Text as Text
|
||||
|
@ -17,20 +14,25 @@ import qualified Data.Text as Text
|
|||
import Control.Applicative ((<$>))
|
||||
#endif
|
||||
|
||||
import Control.Monad.Reader (Reader, asks)
|
||||
import Control.Monad.Reader (Reader, ask, runReader)
|
||||
import Data.Maybe (isJust, fromJust)
|
||||
import Data.List (nub, group, sort)
|
||||
|
||||
import Ringo.Extractor.Internal
|
||||
import Ringo.Types
|
||||
import Ringo.Types.Internal
|
||||
import Ringo.Utils
|
||||
|
||||
data RawEnv = RawEnv ![Table] ![Fact] !Settings !TypeDefaults deriving (Show)
|
||||
|
||||
checkTableForCol :: Table -> ColumnName -> [ValidationError]
|
||||
checkTableForCol tab colName =
|
||||
[ MissingColumn (tableName tab) colName |
|
||||
not . any ((colName ==) . columnName) . tableColumns $ tab ]
|
||||
|
||||
validateTable :: Table -> Reader Env [ValidationError]
|
||||
validateTable :: Table -> Reader RawEnv [ValidationError]
|
||||
validateTable table = do
|
||||
tables <- asks envTables
|
||||
RawEnv tables _ _ _ <- ask
|
||||
return . concatMap (checkConstraint tables) . tableConstraints $ table
|
||||
where
|
||||
checkConstraint _ (PrimaryKey colName) = checkTableForCol table colName
|
||||
|
@ -43,10 +45,10 @@ validateTable table = do
|
|||
|
||||
checkTableForColRefs tab = concatMap (checkTableForCol tab)
|
||||
|
||||
validateFact :: Fact -> Reader Env [ValidationError]
|
||||
validateFact :: Fact -> Reader RawEnv [ValidationError]
|
||||
validateFact Fact {..} = do
|
||||
tables <- asks envTables
|
||||
defaults <- Map.keys <$> asks envTypeDefaults
|
||||
RawEnv tables _ _ typeDefaults <- ask
|
||||
let defaults = Map.keys typeDefaults
|
||||
case findTable factTableName tables of
|
||||
Nothing -> return [ MissingTable factTableName ]
|
||||
Just table -> do
|
||||
|
@ -75,7 +77,7 @@ validateFact Fact {..} = do
|
|||
return $ tableVs ++ parentVs ++ colVs ++ timeVs ++ notNullVs ++ typeDefaultVs
|
||||
where
|
||||
checkFactParents fName = do
|
||||
facts <- asks envFacts
|
||||
RawEnv _ facts _ _ <- ask
|
||||
case findFact fName facts of
|
||||
Nothing -> return [ MissingFact fName ]
|
||||
Just pFact -> validateFact pFact
|
||||
|
@ -88,3 +90,21 @@ validateFact Fact {..} = do
|
|||
checkColumnTable tables FactColumn {..} = case factColType of
|
||||
DimId {factColTargetTable = tName} -> maybe [ MissingTable tName ] (const []) $ findTable tName tables
|
||||
_ -> []
|
||||
|
||||
validateEnv :: [Table] -> [Fact] -> Settings -> TypeDefaults -> Either [ValidationError] Env
|
||||
validateEnv tables facts settings typeDefaults =
|
||||
flip runReader (RawEnv tables facts settings typeDefaults) $ do
|
||||
tableVs <- concat <$> mapM validateTable tables
|
||||
factVs <- concat <$> mapM validateFact facts
|
||||
let dupTableVs = [ DuplicateTable table | table <- findDups . map tableName $ tables ]
|
||||
let dupFactVs = [ DuplicateFact fact | fact <- findDups . map factName $ facts ]
|
||||
let dupColVs = [ DuplicateColumn tableName col
|
||||
| Table{..} <- tables
|
||||
, col <- findDups . map columnName $ tableColumns ]
|
||||
let vs = nub $ tableVs ++ factVs ++ dupTableVs ++ dupFactVs ++ dupColVs
|
||||
if null vs
|
||||
then return . Right $ Env tables facts settings typeDefaults
|
||||
else return . Left $ vs
|
||||
where
|
||||
findDups =
|
||||
sort >>> group >>> map (head &&& length) >>> filter (snd >>> (> 1)) >>> map fst
|
||||
|
|
Loading…
Reference in New Issue