Skip to content

Add provider specific column types #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Nov 2, 2021
Merged

Add provider specific column types #10

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
c656d78
Add column type attribute in schema generator
Oct 18, 2021
cf557ea
Fix schema generation with attribute
Oct 18, 2021
a0f01b3
Add npgsql db column type mapping
Oct 18, 2021
ba702b7
Add command parameter type to box db column type value
Oct 18, 2021
525a05c
Set db type when building command in query context
Oct 19, 2021
b2aecb9
Add open sqlhydra in generated file
Oct 19, 2021
bc80625
Create domain project
Oct 20, 2021
c08a55d
Fix schema generation, add sqldbtype handling
Oct 21, 2021
268bb58
Cleanup
Oct 21, 2021
3b32f80
Use full name for provider db type attribute
Oct 23, 2021
602d507
Add integration tests for npgsql provider specific db types
Oct 23, 2021
e646095
Fix updates for provider specific values
Oct 23, 2021
4e4db59
Fix test to not depend on json formatting in postgres
Oct 23, 2021
a88f445
Add tests for provider type attribute generation
Oct 24, 2021
f38df21
Sort columns in ordinal position for postgres
Oct 24, 2021
f1d2d95
Fix number of table npgsql test
Oct 24, 2021
fd0e9a6
Remove unused open
Oct 24, 2021
feaf2ef
Add provider db test to postgres install script in dev container
Oct 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ nupkg
**/.idea/**/tasks.xml
**/.idea/shelf/*
**/.idea/dictionaries
.idea

# Sensitive or high-churn files
**/.idea/**/dataSources/
Expand Down
31 changes: 1 addition & 30 deletions src/SqlHydra/Domain.fs → src/SqlHydra.Domain/Domain.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type TypeMapping =
{
ClrType: string
DbType: DbType
ProviderDbType: string option
ColumnTypeAlias: string
ReaderMethod: string
}
Expand Down Expand Up @@ -74,33 +75,3 @@ type Config =
Filters: Filters
Readers: ReadersConfig option
}

open GlobExpressions

let applyFilters (filters: Filters) (tables: Table list) =
match filters with
| { Includes = []; Excludes = [] } ->
tables
| _ ->
let getPath tbl = $"{tbl.Schema}/{tbl.Name}"
let tablesByPath = tables |> List.map (fun t -> getPath t, t) |> Map.ofList
let paths = tablesByPath |> Map.toList |> List.map fst

let includePatterns = filters.Includes |> List.map Glob
let excludePatterns = filters.Excludes |> List.map Glob

let includedPaths =
includePatterns
|> List.collect (fun pattern -> paths |> List.filter pattern.IsMatch)
|> List.distinct
|> Set.ofList

let excludedPaths =
excludePatterns
|> List.collect (fun pattern -> paths |> List.filter pattern.IsMatch)
|> List.distinct
|> Set.ofList

let filteredPaths = includedPaths - excludedPaths
let filteredTables = filteredPaths |> Seq.map (fun path -> tablesByPath.[path]) |> Seq.toList
filteredTables
10 changes: 10 additions & 0 deletions src/SqlHydra.Domain/ProviderDbTypeAttribute.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
namespace SqlHydra

open System

[<AttributeUsage(AttributeTargets.Property
||| AttributeTargets.Field)>]
type ProviderDbTypeAttribute(providerDbTypeName: string) =
inherit Attribute()

member this.ProviderDbTypeName = providerDbTypeName
14 changes: 14 additions & 0 deletions src/SqlHydra.Domain/SqlHydra.Domain.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<WarnOn>3390;$(WarnOn)</WarnOn>
</PropertyGroup>

<ItemGroup>
<Compile Include="Domain.fs" />
<Compile Include="ProviderDbTypeAttribute.fs" />
</ItemGroup>

</Project>
58 changes: 30 additions & 28 deletions src/SqlHydra.Npgsql/NpgsqlDataTypes.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,55 +1,57 @@
module SqlHydra.Npgsql.NpgsqlDataTypes

open System.Data
open NpgsqlTypes
open SqlHydra.Domain

let private r : Npgsql.NpgsqlDataReader = null

/// A list of supported column type mappings
let supportedTypeMappings =
[
"boolean", "bool", DbType.Boolean, nameof r.GetBoolean
"smallint", "int16", DbType.Int16, nameof r.GetInt16
"integer", "int", DbType.Int32, nameof r.GetInt32
"bigint", "int64", DbType.Int64, nameof r.GetInt64
"real", "double", DbType.Double, nameof r.GetDouble
"double precision", "double", DbType.Double, nameof r.GetDouble
"numeric", "decimal", DbType.Decimal, nameof r.GetDecimal
"money", "decimal", DbType.Decimal, nameof r.GetDecimal
"text", "string", DbType.String, nameof r.GetString
"character varying", "string", DbType.String, nameof r.GetString
"character", "string", DbType.String, nameof r.GetString
"citext", "string", DbType.String, nameof r.GetString
"json", "string", DbType.String, nameof r.GetString
"jsonb", "string", DbType.String, nameof r.GetString
"xml", "string", DbType.String, nameof r.GetString
"boolean", "bool", DbType.Boolean, None, nameof r.GetBoolean
"smallint", "int16", DbType.Int16, None, nameof r.GetInt16
"integer", "int", DbType.Int32, None, nameof r.GetInt32
"bigint", "int64", DbType.Int64, None, nameof r.GetInt64
"real", "double", DbType.Double, None, nameof r.GetDouble
"double precision", "double", DbType.Double, None, nameof r.GetDouble
"numeric", "decimal", DbType.Decimal, None, nameof r.GetDecimal
"money", "decimal", DbType.Decimal, None, nameof r.GetDecimal
"text", "string", DbType.String, None, nameof r.GetString
"character varying", "string", DbType.String, None, nameof r.GetString
"character", "string", DbType.String, None, nameof r.GetString
"citext", "string", DbType.String, None, nameof r.GetString
"json", "string", DbType.String, Some (nameof NpgsqlDbType.Json), nameof r.GetString
"jsonb", "string", DbType.String, Some (nameof NpgsqlDbType.Jsonb), nameof r.GetString
"xml", "string", DbType.String, None, nameof r.GetString
// skipped unsupported types
"bit(1)", "bool", DbType.Boolean, nameof r.GetBoolean
"bit(1)", "bool", DbType.Boolean, None, nameof r.GetBoolean
// skipped unsupported types
"uuid", "System.Guid", DbType.Guid, nameof r.GetGuid
"uuid", "System.Guid", DbType.Guid, None, nameof r.GetGuid
// skipped unsupported types
"date", "System.DateTime", DbType.DateTime, nameof r.GetDateTime
"interval", "System.TimeSpan", DbType.Time, nameof r.GetTimeSpan
"timestamp without time zone", "System.DateTime", DbType.DateTime, nameof r.GetDateTime
"timestamp with time zone", "System.DateTime", DbType.DateTime, nameof r.GetDateTime
"time without time zone", "System.TimeSpan", DbType.Time, nameof r.GetTimeSpan
"time with time zone", "System.DateTime", DbType.DateTime, nameof r.GetDateTime
"bytea", "byte[]", DbType.Binary, nameof r.GetValue
"date", "System.DateTime", DbType.DateTime, None, nameof r.GetDateTime
"interval", "System.TimeSpan", DbType.Time, None, nameof r.GetTimeSpan
"timestamp without time zone", "System.DateTime", DbType.DateTime, None, nameof r.GetDateTime
"timestamp with time zone", "System.DateTime", DbType.DateTime, None, nameof r.GetDateTime
"time without time zone", "System.TimeSpan", DbType.Time, None, nameof r.GetTimeSpan
"time with time zone", "System.DateTime", DbType.DateTime, None, nameof r.GetDateTime
"bytea", "byte[]", DbType.Binary, None, nameof r.GetValue
// skipped unsupported types
"name", "string", DbType.String, nameof r.GetString
"(internal) char", "char", DbType.String, nameof r.GetChar
"name", "string", DbType.String, None, nameof r.GetString
"(internal) char", "char", DbType.String, None, nameof r.GetChar
// skipped unsupported types
]

let typeMappingsByName =
supportedTypeMappings
|> List.map (fun (columnTypeAlias, clrType, dbType, readerMethod) ->
|> List.map (fun (columnTypeAlias, clrType, dbType, providerDbType, readerMethod) ->
columnTypeAlias,
{
TypeMapping.ColumnTypeAlias = columnTypeAlias
TypeMapping.ClrType = clrType
TypeMapping.DbType = dbType
TypeMapping.ReaderMethod = readerMethod
TypeMapping.ProviderDbType = providerDbType
}
)
|> Map.ofList
Expand All @@ -59,7 +61,7 @@ let tryFindTypeMapping (providerTypeName: string) =

let primitiveTypeReaders =
supportedTypeMappings
|> List.map(fun (_, clrType, _, readerMethod) ->
|> List.map(fun (_, clrType, _, _, readerMethod) ->
{ PrimitiveTypeReader.ClrType = clrType; PrimitiveTypeReader.ReaderMethod = readerMethod }
)
|> List.distinctBy (fun ptr -> ptr.ClrType)
4 changes: 3 additions & 1 deletion src/SqlHydra.Npgsql/NpgsqlSchemaProvider.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module SqlHydra.Npgsql.NpgsqlSchemaProvider

open System.Data
open Npgsql
open NpgsqlTypes
open SqlHydra.Domain

let getSchema (cfg: Config) : Schema =
Expand Down Expand Up @@ -52,12 +52,14 @@ let getSchema (cfg: Config) : Schema =
TableName = col.["TABLE_NAME"] :?> string
ColumnName = col.["COLUMN_NAME"] :?> string
ProviderTypeName = col.["DATA_TYPE"] :?> string
OrdinalPosition = col.["ORDINAL_POSITION"] :?> int
Copy link
Author

@MargaretKrutikova MargaretKrutikova Oct 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I noticed that every time you run the generator the order of the fields changes, and then one of the tests for insert query failed because the order of the fields has changed (see the test here) so I thought I would add sorting by ordinal position (I assume it is the order columns were added to the table) to avoid random order after every re-generation.

Copy link
Owner

@JordanMarr JordanMarr Oct 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a good idea, but I'm not sure that "ORDINAL_POSITION" is given for the other database providers.
If not, then I think a reasonable alternative would be to sort by column name.

IsNullable =
match col.["IS_NULLABLE"] :?> string with
| "YES" -> true
| _ -> false
|}
)
|> Seq.sortBy (fun column -> column.OrdinalPosition)

let tables =
sTables.Rows
Expand Down
34 changes: 28 additions & 6 deletions src/SqlHydra.Query/Kata.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
namespace SqlHydra.Query

open System.Reflection
open SqlKata
open System.Collections.Generic
open System
Expand All @@ -25,6 +26,12 @@ module FQ =
| Some schema -> $"{schema}.{tbl.Name}"
| None -> tbl.Name

type QueryParameter =
{
Value: obj
ProviderDbType: string option
}

type InsertQuerySpec<'T, 'Identity> =
{
Table: string
Expand Down Expand Up @@ -68,7 +75,7 @@ type QuerySource<'T, 'Query>(query, tableMappings) =
module private KataUtils =

/// Boxes values (and option values)
let boxValueOrOption (value: obj) =
let private boxValueOrOption (value: obj) =
if isNull value then
box System.DBNull.Value
else
Expand All @@ -80,28 +87,43 @@ module private KataUtils =
| null -> box System.DBNull.Value
| o -> o

let private getProviderDbTypeName (p: PropertyInfo) =
let attrs = p.GetCustomAttributes(true)
(attrs
|> Seq.choose (function
| :? SqlHydra.ProviderDbTypeAttribute as attr -> Some attr.ProviderDbTypeName
| _ -> None))
|> Seq.tryHead

let getQueryParameterForValue (p: PropertyInfo) (value: obj) =
{ Value = value |> boxValueOrOption; ProviderDbType = getProviderDbTypeName p }

let getQueryParameterForEntity (entity: 'T) (p: PropertyInfo) =
p.GetValue(entity)
|> getQueryParameterForValue p

let fromUpdate (spec: UpdateQuerySpec<'T>) =
let kvps =
match spec.Entity, spec.SetValues with
| Some entity, [] ->
match spec.Fields with
| [] ->
FSharp.Reflection.FSharpType.GetRecordFields(typeof<'T>)
|> Array.map (fun p -> p.Name, p.GetValue(entity))
|> Array.map (fun p -> p.Name, getQueryParameterForEntity entity p :> obj)

| fields ->
let included = fields |> Set.ofList
FSharp.Reflection.FSharpType.GetRecordFields(typeof<'T>)
|> Array.filter (fun p -> included.Contains(p.Name))
|> Array.map (fun p -> p.Name, p.GetValue(entity))
|> Array.map (fun p -> p.Name, getQueryParameterForEntity entity p :> obj)

| Some _, _ -> failwith "Cannot have both `entity` and `set` operations in an `update` expression."
| None, [] -> failwith "Either an `entity` or `set` operations must be present in an `update` expression."
| None, setValues -> setValues |> List.toArray

let preparedKvps =
kvps
|> Seq.map (fun (key,value) -> key, boxValueOrOption value)
|> Seq.map (fun (key,value) -> key, value)
|> dict
|> Seq.map id

Expand Down Expand Up @@ -129,7 +151,7 @@ module private KataUtils =
| [ entity ] ->
let keyValuePairs =
includedProperties
|> Array.map (fun p -> KeyValuePair(p.Name, p.GetValue(entity) |> boxValueOrOption))
|> Array.map (fun p -> KeyValuePair(p.Name, getQueryParameterForEntity entity p :> obj))
|> Array.toList
Query(spec.Table).AsInsert(keyValuePairs, returnId = spec.IdentityField.IsSome)

Expand All @@ -141,7 +163,7 @@ module private KataUtils =
entities
|> List.map (fun entity ->
includedProperties
|> Array.map (fun p -> p.GetValue(entity) |> boxValueOrOption)
|> Array.map (fun p -> getQueryParameterForEntity entity p :> obj)
|> Array.toSeq
)
Query(spec.Table).AsInsert(columns, rowsValues)
Expand Down
4 changes: 3 additions & 1 deletion src/SqlHydra.Query/KataBuilders.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,8 +370,10 @@ type UpdateExpressionBuilder<'T>() =
member this.Set (state: QuerySource<'T>, [<ProjectionParameter>] propertySelector: Expression<Func<'T, 'Prop>>, value: 'Prop) =
let query = state |> getQueryOrDefault
let prop = LinqExpressionVisitors.visitPropertySelector<'T, 'Prop> propertySelector :?> Reflection.PropertyInfo

let value = KataUtils.getQueryParameterForValue prop value :> obj
QuerySource<'T, UpdateQuerySpec<'T>>(
{ query with SetValues = query.SetValues @ [ prop.Name, box value ] }
{ query with SetValues = query.SetValues @ [ prop.Name, value ] }
, state.TableMappings)

/// Includes a column in the insert query.
Expand Down
24 changes: 22 additions & 2 deletions src/SqlHydra.Query/QueryContext.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,21 @@ open SqlKata

/// Contains methods that compile and read a query.
type QueryContext(conn: DbConnection, compiler: SqlKata.Compilers.Compiler) =

let setProviderDbType (param: DbParameter) (propertyName: string) (providerDbType: string) =
let property = param.GetType().GetProperty(propertyName)
let dbTypeSetter = property.GetSetMethod()

let value = System.Enum.Parse(property.PropertyType, providerDbType)
dbTypeSetter.Invoke(param, [|value|]) |> ignore

let setParameterDbType (param: DbParameter) (qp: QueryParameter) =
match qp.ProviderDbType, compiler with
| Some type', :? SqlKata.Compilers.PostgresCompiler ->
setProviderDbType param "NpgsqlDbType" type'
| Some type', :? SqlKata.Compilers.SqlServerCompiler ->
setProviderDbType param "SqlDbType" type'
| _ -> ()

Copy link
Owner

@JordanMarr JordanMarr Oct 20, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to add a handler here for SQL Server as well:

| Some type', :? SqlKata.Compilers.SqlServerCompiler when type'.TypeName = "SqlDbType" ->
            let property = param.GetType().GetProperty("SqlDbType")
 ...

Copy link
Author

@MargaretKrutikova MargaretKrutikova Oct 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had TypeName and TypeValue in the attribute before, but I figured you meant to just have a string there so I stored only the enum value of the provider type without the type name (like NpgsqlDbType.Json became just Json). So now I can't really do the check on type'.TypeName = "SqlDbType". What are you thoughts on this? Do we need to store the full type like NpgsqlDbType.Json or SqlDbType.Whatever?

Copy link
Owner

@JordanMarr JordanMarr Oct 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What you have in NpgsqlDataTypes.fs and the attribute looks perfect. I think that it will be safe enough to assume that if they are using a given SqlKata compiler that they should be passing in types with the appropriate attributes.

So this should work:

    let setParameterDbType (param: DbParameter) (qp: QueryParameter) =
      match qp.ProviderDbType, compiler with
      | Some dbType, :? SqlKata.Compilers.PostgresCompiler ->
          let property = param.GetType().GetProperty("NpgsqlDbType")
          let dbTypeSetter = property.GetSetMethod()            
          let value = System.Enum.Parse(property.PropertyType, dbType)
          dbTypeSetter.Invoke(param, [|value|]) |> ignore

      | Some dbType, :? SqlKata.Compilers.SqlServerCompiler ->
          let property = param.GetType().GetProperty("SqlDbType")
          let dbTypeSetter = property.GetSetMethod()            
          let value = System.Enum.Parse(property.PropertyType, dbType)
          dbTypeSetter.Invoke(param, [|value|]) |> ignore
      | _ -> ()

Copy link
Author

@MargaretKrutikova MargaretKrutikova Oct 24, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, the code now looks almost as yours, just with a helper function to hide the reflection ugliness 🙂

interface System.IDisposable with
member this.Dispose() =
conn.Dispose()
Expand Down Expand Up @@ -44,7 +58,13 @@ type QueryContext(conn: DbConnection, compiler: SqlKata.Compilers.Compiler) =
for kvp in compiledQuery.NamedBindings do
let p = cmd.CreateParameter()
p.ParameterName <- kvp.Key
p.Value <- kvp.Value

match kvp.Value with
| :? QueryParameter as qp ->
do setParameterDbType p qp
p.Value <- qp.Value
| _ ->
p.Value <- kvp.Value
cmd.Parameters.Add(p) |> ignore
cmd

Expand Down
5 changes: 5 additions & 0 deletions src/SqlHydra.Query/SqlHydra.Query.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
<Authors>Jordan Marr</Authors>
<PackageTags>F# fsharp data database orm sql</PackageTags>
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
<PackageReleaseNotes>
SqlHydra.Query:
Added support for multiple inserts via the `entities` operation.
Expand All @@ -32,4 +33,8 @@
<PackageReference Include="SqlKata" Version="2.3.7" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\SqlHydra.Domain\SqlHydra.Domain.fsproj" />
</ItemGroup>

</Project>
1 change: 1 addition & 0 deletions src/SqlHydra.SqlServer/SqlHydra.SqlServer.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\SqlHydra.Domain\SqlHydra.Domain.fsproj" />
<ProjectReference Include="..\SqlHydra\SqlHydra.fsproj">
<ReferenceOutputAssembly>true</ReferenceOutputAssembly>
<IncludeAssets>SqlHydra.dll</IncludeAssets>
Expand Down
1 change: 1 addition & 0 deletions src/SqlHydra.SqlServer/SqlServerDataTypes.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ let typeMappingsByName =
TypeMapping.ClrType = clrType
TypeMapping.DbType = dbType
TypeMapping.ReaderMethod = readerMethod
TypeMapping.ProviderDbType = None
}
)
|> Map.ofList
Expand Down
2 changes: 1 addition & 1 deletion src/SqlHydra.SqlServer/SqlServerSchemaProvider.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ let getSchema (cfg: Config) : Schema =
|> Seq.choose (fun col ->
SqlServerDataTypes.tryFindTypeMapping(col.ProviderTypeName)
|> Option.map (fun typeMapping ->
{
{
Column.Name = col.ColumnName
Column.IsNullable = col.IsNullable
Column.TypeMapping = typeMapping
Expand Down
1 change: 1 addition & 0 deletions src/SqlHydra.Sqlite/SqlHydra.Sqlite.fsproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\SqlHydra.Domain\SqlHydra.Domain.fsproj" />
<ProjectReference Include="..\SqlHydra\SqlHydra.fsproj">
<ReferenceOutputAssembly>true</ReferenceOutputAssembly>
<IncludeAssets>SqlHydra.dll</IncludeAssets>
Expand Down
1 change: 1 addition & 0 deletions src/SqlHydra.Sqlite/SqliteDataTypes.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ let typeMappingsByName =
TypeMapping.ClrType = clrType
TypeMapping.DbType = dbType
TypeMapping.ReaderMethod = readerMethod
TypeMapping.ProviderDbType = None
}
)
|> Map.ofList
Expand Down
Loading