From 71b93a657667faf87417368a4977e6228bd0428a Mon Sep 17 00:00:00 2001 From: "Oleg V. Kozlyuk" Date: Sun, 3 Nov 2024 17:55:37 +0100 Subject: [PATCH] WIP: support for JSON type #531 --- .../workflows/analysis.yml | 0 .github/workflows/reusable.yml | 2 +- .../ClickHouse.Client.Benchmark.csproj | 2 +- .../ClickHouse.Client.Tests.csproj | 2 +- .../SQL/SqlSimpleSelectTests.cs | 2 +- ClickHouse.Client.Tests/TestUtilities.cs | 13 +++ ClickHouse.Client/ADO/Feature.cs | 3 + .../ClickHouseServerException.cs | 5 - .../Formats/HttpParameterFormatter.cs | 9 ++ .../Numerics/ClickHouseDecimal.cs | 2 +- .../Types/AggregateFunctionType.cs | 2 - ClickHouse.Client/Types/JsonType.cs | 104 ++++++++++++++++++ ClickHouse.Client/Types/ParameterizedType.cs | 1 + ClickHouse.Client/Types/TypeConverter.cs | 4 + 14 files changed, 139 insertions(+), 12 deletions(-) rename analysis.yml => .github/workflows/analysis.yml (100%) create mode 100644 ClickHouse.Client/Types/JsonType.cs diff --git a/analysis.yml b/.github/workflows/analysis.yml similarity index 100% rename from analysis.yml rename to .github/workflows/analysis.yml diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index 8f018c3c..36fbf2af 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: framework: - default: net6.0 + default: net8.0 required: false type: string clickhouse-version: diff --git a/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj b/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj index beb24430..bc027ae9 100644 --- a/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj +++ b/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 true true snupkg diff --git a/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj b/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj index 2485dd75..574011a9 100644 --- a/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj +++ b/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 false latest diff --git a/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs b/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs index 85080fcc..cd37a437 100644 --- a/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs +++ b/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs @@ -252,7 +252,7 @@ public async Task ShouldGetValueDecimal() [TestCaseSource(typeof(SqlSimpleSelectTests), nameof(SimpleSelectTypes))] public async Task ShouldExecuteRandomDataSelectQuery(string type) { - if (type.StartsWith("Nested") || type == "Nothing" || type.StartsWith("Variant")) + if (type.StartsWith("Nested") || type == "Nothing" || type.StartsWith("Variant") || type.StartsWith("Json")) Assert.Ignore($"Type {type} not supported by generateRandom"); using var reader = await connection.ExecuteReaderAsync($"SELECT * FROM generateRandom('value {type.Replace("'", "\\'")}', 10, 10, 10) LIMIT 100"); diff --git a/ClickHouse.Client.Tests/TestUtilities.cs b/ClickHouse.Client.Tests/TestUtilities.cs index a10de84c..1554cf9d 100644 --- a/ClickHouse.Client.Tests/TestUtilities.cs +++ b/ClickHouse.Client.Tests/TestUtilities.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Numerics; +using System.Text.Json; using System.Threading.Tasks; using ClickHouse.Client.ADO; using ClickHouse.Client.Numerics; @@ -62,6 +63,10 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression { builder["set_allow_experimental_variant_type"] = 1; } + if (SupportedFeatures.HasFlag(Feature.Json)) + { + builder["set_allow_experimental_json_type"] = 1; + } return new ClickHouseConnection(builder.ConnectionString); } @@ -221,6 +226,14 @@ public static IEnumerable GetDataTypeSamples() { yield return new DataTypeSample("Variant(UInt64, String, Array(UInt64))", typeof(string), "'Hello, World!'::Variant(UInt64, String, Array(UInt64))", "Hello, World!"); } + + if (SupportedFeatures.HasFlag(Feature.Json)) + { + yield return new DataTypeSample("Json", typeof(string), "'{\"i1\":1,\"i2\":2}'::Json", "{\"i1\":1,\"i2\":2}"); + yield return new DataTypeSample("Json", typeof(string), "'{\"str\":\"val\"}'::Json", "{\"str\":\"val\"}"); + yield return new DataTypeSample("Json", typeof(string), "'{\"flt\":0.0}'::Json", "{\"flt\":0}"); + // yield return new DataTypeSample("Json", typeof(string), "'{\"arr\":[1,2,3]}'::Json", "{\"arr\":[1,2,3]}"); // TODO + } } public static object[] GetEnsureSingleRow(this DbDataReader reader) diff --git a/ClickHouse.Client/ADO/Feature.cs b/ClickHouse.Client/ADO/Feature.cs index 9205319b..c4c3b505 100644 --- a/ClickHouse.Client/ADO/Feature.cs +++ b/ClickHouse.Client/ADO/Feature.cs @@ -39,5 +39,8 @@ public enum Feature [SinceVersion("22.3")] ParamsInMultipartFormData = 32768, + [SinceVersion("24.1")] + Json = 65536, + All = ~None, // Special value } diff --git a/ClickHouse.Client/ClickHouseServerException.cs b/ClickHouse.Client/ClickHouseServerException.cs index 30a41012..47ffeb5b 100644 --- a/ClickHouse.Client/ClickHouseServerException.cs +++ b/ClickHouse.Client/ClickHouseServerException.cs @@ -20,11 +20,6 @@ public ClickHouseServerException(string error, string query, int errorCode) Query = query; } - protected ClickHouseServerException(SerializationInfo info, StreamingContext context) - : base(info, context) - { - } - public string Query { get; } public static ClickHouseServerException FromServerResponse(string error, string query) diff --git a/ClickHouse.Client/Formats/HttpParameterFormatter.cs b/ClickHouse.Client/Formats/HttpParameterFormatter.cs index 2372c4a0..6dd69cab 100644 --- a/ClickHouse.Client/Formats/HttpParameterFormatter.cs +++ b/ClickHouse.Client/Formats/HttpParameterFormatter.cs @@ -3,6 +3,9 @@ using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Xml.Linq; using ClickHouse.Client.ADO.Parameters; using ClickHouse.Client.Numerics; using ClickHouse.Client.Types; @@ -97,6 +100,12 @@ internal static string Format(ClickHouseType type, object value, bool quote) var (_,chType) = variantType.GetMatchingType(value); return Format(chType, value, quote); + case JsonType jsonType: + if (value is string jsonString) + return jsonString; + else + return JsonSerializer.Serialize(value); + default: throw new ArgumentException($"Cannot convert {value} to {type}"); } diff --git a/ClickHouse.Client/Numerics/ClickHouseDecimal.cs b/ClickHouse.Client/Numerics/ClickHouseDecimal.cs index 667406f4..39b7fe2f 100644 --- a/ClickHouse.Client/Numerics/ClickHouseDecimal.cs +++ b/ClickHouse.Client/Numerics/ClickHouseDecimal.cs @@ -407,7 +407,7 @@ public object ToType(Type conversionType, IFormatProvider provider) Truncate(ref mantissa, ref scale, 0); return mantissa; } - return Convert.ChangeType(this, conversionType); + return Convert.ChangeType(this, conversionType, provider); } public int CompareTo(decimal other) => CompareTo((ClickHouseDecimal)other); diff --git a/ClickHouse.Client/Types/AggregateFunctionType.cs b/ClickHouse.Client/Types/AggregateFunctionType.cs index 34bc4301..0071793a 100644 --- a/ClickHouse.Client/Types/AggregateFunctionType.cs +++ b/ClickHouse.Client/Types/AggregateFunctionType.cs @@ -11,8 +11,6 @@ internal class AggregateFunctionType : ParameterizedType public override string Name => "AggregateFunction"; - - public override Type FrameworkType => throw new AggregateFunctionException(Function); public override ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings) diff --git a/ClickHouse.Client/Types/JsonType.cs b/ClickHouse.Client/Types/JsonType.cs new file mode 100644 index 00000000..c4dbf8d4 --- /dev/null +++ b/ClickHouse.Client/Types/JsonType.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using System.Text.Json; +using System.Text.Json.Nodes; +using ClickHouse.Client.Formats; +using ClickHouse.Client.Types.Grammar; + +namespace ClickHouse.Client.Types; + +internal class JsonType : ParameterizedType +{ + public override Type FrameworkType => typeof(string); + + public override string Name => "Json"; + + public override object Read(ExtendedBinaryReader reader) + { + var value = new JsonObject(); + var nfields = reader.Read7BitEncodedInt(); + + for (int i = 0; i < nfields; i++) + { + var fieldName = reader.ReadString(); + // See https://github.com/ClickHouse/ClickHouse/blob/b618fe03bf96e64bea1a1bdec01adc1c00cd61fb/src/DataTypes/DataTypesBinaryEncoding.cpp#L48 + // https://clickhouse.com/docs/en/sql-reference/data-types/data-types-binary-encoding + // TODO: add type codes within types themselves + var typeCode = reader.Read7BitEncodedInt(); + object fieldValue = typeCode switch + { + 0x07 => reader.ReadByte(), + 0x08 => reader.ReadInt16(), + 0x09 => reader.ReadInt32(), + 0x0a => reader.ReadInt64(), + 0x0e => reader.ReadDouble(), + 0x15 => reader.ReadString(), + _ => throw new ArgumentException(fieldName, $"Unknown type code: {typeCode:X}") + }; + value[fieldName] = JsonValue.Create(fieldValue); + } + return value.ToJsonString(); + } + + public override void Write(ExtendedBinaryWriter writer, object value) + { + JsonElement element; + if (value is string stringValue) + { + element = JsonDocument.Parse(stringValue).RootElement; + } + else + { + element = JsonSerializer.SerializeToElement(value); + } + + writer.Write7BitEncodedInt(element.EnumerateObject().Count()); + foreach (var field in element.EnumerateObject()) + { + writer.Write(field.Name); + switch (field.Value.ValueKind) + { + case JsonValueKind.Number: + if (field.Value.TryGetInt64(out var int64Value)) + { + writer.Write((byte)0x0a); + writer.Write(int64Value); + } + else + { + writer.Write((byte)0x0e); + writer.Write(field.Value.GetDouble()); + } + break; + case JsonValueKind.String: + writer.Write((byte)0x15); + writer.Write(field.Value.GetString()); + break; + case JsonValueKind.True: + writer.Write((byte)0x07); + writer.Write((byte)1); + break; + case JsonValueKind.False: + writer.Write((byte)0x07); + writer.Write((byte)0); + break; + default: + throw new ArgumentException(field.Name, $"Unknown JSON value kind: {field.Value.ValueKind}"); + } + } + } + + public override string ToString() => Name; + + public override ParameterizedType Parse(SyntaxTreeNode typeName, Func parseClickHouseTypeFunc, TypeSettings settings) + { + if (typeName.ChildNodes.Any()) + { + throw new SerializationException("JSON type does not accept parameters"); + } + + return this; + } +} diff --git a/ClickHouse.Client/Types/ParameterizedType.cs b/ClickHouse.Client/Types/ParameterizedType.cs index 30511069..bd1dfd99 100644 --- a/ClickHouse.Client/Types/ParameterizedType.cs +++ b/ClickHouse.Client/Types/ParameterizedType.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Runtime.Serialization; using System.Xml.Linq; diff --git a/ClickHouse.Client/Types/TypeConverter.cs b/ClickHouse.Client/Types/TypeConverter.cs index e42e10b8..6b8525d8 100644 --- a/ClickHouse.Client/Types/TypeConverter.cs +++ b/ClickHouse.Client/Types/TypeConverter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; using ClickHouse.Client.Numerics; using ClickHouse.Client.Types.Grammar; @@ -167,6 +168,7 @@ static TypeConverter() RegisterPlainType(); // JSON/Object + RegisterParameterizedType(); RegisterParameterizedType(); RegisterParameterizedType(); @@ -179,6 +181,8 @@ static TypeConverter() #endif ReverseMapping[typeof(DateTime)] = new DateTimeType(); ReverseMapping[typeof(DateTimeOffset)] = new DateTimeType(); + + ReverseMapping[typeof(JsonElement)] = new JsonType(); } private static void RegisterPlainType()