WIP: support for JSON type #531

DarkWanderer · Nov 3, 2024 · 71b93a6 · 71b93a6
1 parent 30b77b1
commit 71b93a6
Show file tree

Hide file tree

Showing 14 changed files with 139 additions and 12 deletions.
diff --git a/analysis.yml → .github/workflows/analysis.yml b/analysis.yml → .github/workflows/analysis.yml
diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml
@@ -4,7 +4,7 @@ on:
   workflow_call:
     inputs:
       framework:
-        default: net6.0
+        default: net8.0
         required: false
         type: string
       clickhouse-version:

diff --git a/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj b/ClickHouse.Client.Benchmark/ClickHouse.Client.Benchmark.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
     <IncludeSymbols>true</IncludeSymbols>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>

diff --git a/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj b/ClickHouse.Client.Tests/ClickHouse.Client.Tests.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net6.0</TargetFrameworks>
+    <TargetFrameworks>net8.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>

diff --git a/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs b/ClickHouse.Client.Tests/SQL/SqlSimpleSelectTests.cs
@@ -252,7 +252,7 @@ public async Task ShouldGetValueDecimal()
     [TestCaseSource(typeof(SqlSimpleSelectTests), nameof(SimpleSelectTypes))]
     public async Task ShouldExecuteRandomDataSelectQuery(string type)
     {
-        if (type.StartsWith("Nested") || type == "Nothing" || type.StartsWith("Variant"))
+        if (type.StartsWith("Nested") || type == "Nothing" || type.StartsWith("Variant") || type.StartsWith("Json"))
             Assert.Ignore($"Type {type} not supported by generateRandom");
 
         using var reader = await connection.ExecuteReaderAsync($"SELECT * FROM generateRandom('value {type.Replace("'", "\\'")}', 10, 10, 10) LIMIT 100");

diff --git a/ClickHouse.Client.Tests/TestUtilities.cs b/ClickHouse.Client.Tests/TestUtilities.cs
@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Net;
 using System.Numerics;
+using System.Text.Json;
 using System.Threading.Tasks;
 using ClickHouse.Client.ADO;
 using ClickHouse.Client.Numerics;
@@ -62,6 +63,10 @@ public static ClickHouseConnection GetTestClickHouseConnection(bool compression
         {
             builder["set_allow_experimental_variant_type"] = 1;
         }
+        if (SupportedFeatures.HasFlag(Feature.Json))
+        {
+            builder["set_allow_experimental_json_type"] = 1;
+        }
         return new ClickHouseConnection(builder.ConnectionString);
     }
 
@@ -221,6 +226,14 @@ public static IEnumerable<DataTypeSample> GetDataTypeSamples()
         {
             yield return new DataTypeSample("Variant(UInt64, String, Array(UInt64))", typeof(string), "'Hello, World!'::Variant(UInt64, String, Array(UInt64))", "Hello, World!");
         }
+
+        if (SupportedFeatures.HasFlag(Feature.Json))
+        {
+            yield return new DataTypeSample("Json", typeof(string), "'{\"i1\":1,\"i2\":2}'::Json", "{\"i1\":1,\"i2\":2}");
+            yield return new DataTypeSample("Json", typeof(string), "'{\"str\":\"val\"}'::Json", "{\"str\":\"val\"}");
+            yield return new DataTypeSample("Json", typeof(string), "'{\"flt\":0.0}'::Json", "{\"flt\":0}");
+            // yield return new DataTypeSample("Json", typeof(string), "'{\"arr\":[1,2,3]}'::Json", "{\"arr\":[1,2,3]}"); // TODO
+        }
     }
 
     public static object[] GetEnsureSingleRow(this DbDataReader reader)

diff --git a/ClickHouse.Client/ADO/Feature.cs b/ClickHouse.Client/ADO/Feature.cs
@@ -39,5 +39,8 @@ public enum Feature
     [SinceVersion("22.3")]
     ParamsInMultipartFormData = 32768,
 
+    [SinceVersion("24.1")]
+    Json = 65536,
+
     All = ~None, // Special value
 }
diff --git a/ClickHouse.Client/ClickHouseServerException.cs b/ClickHouse.Client/ClickHouseServerException.cs
@@ -20,11 +20,6 @@ public ClickHouseServerException(string error, string query, int errorCode)
         Query = query;
     }
 
-    protected ClickHouseServerException(SerializationInfo info, StreamingContext context)
-        : base(info, context)
-    {
-    }
-
     public string Query { get; }
 
     public static ClickHouseServerException FromServerResponse(string error, string query)

diff --git a/ClickHouse.Client/Formats/HttpParameterFormatter.cs b/ClickHouse.Client/Formats/HttpParameterFormatter.cs
@@ -3,6 +3,9 @@
 using System.Globalization;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Xml.Linq;
 using ClickHouse.Client.ADO.Parameters;
 using ClickHouse.Client.Numerics;
 using ClickHouse.Client.Types;
@@ -97,6 +100,12 @@ internal static string Format(ClickHouseType type, object value, bool quote)
                 var (_,chType) = variantType.GetMatchingType(value);
                 return Format(chType, value, quote);
 
+            case JsonType jsonType:
+                if (value is string jsonString)
+                    return jsonString;
+                else
+                    return JsonSerializer.Serialize(value);
+
             default:
                 throw new ArgumentException($"Cannot convert {value} to {type}");
         }

diff --git a/ClickHouse.Client/Numerics/ClickHouseDecimal.cs b/ClickHouse.Client/Numerics/ClickHouseDecimal.cs
@@ -407,7 +407,7 @@ public object ToType(Type conversionType, IFormatProvider provider)
             Truncate(ref mantissa, ref scale, 0);
             return mantissa;
         }
-        return Convert.ChangeType(this, conversionType);
+        return Convert.ChangeType(this, conversionType, provider);
     }
 
     public int CompareTo(decimal other) => CompareTo((ClickHouseDecimal)other);

diff --git a/ClickHouse.Client/Types/AggregateFunctionType.cs b/ClickHouse.Client/Types/AggregateFunctionType.cs
@@ -11,8 +11,6 @@ internal class AggregateFunctionType : ParameterizedType
 
     public override string Name => "AggregateFunction";
 
-
-
     public override Type FrameworkType => throw new AggregateFunctionException(Function);
 
     public override ParameterizedType Parse(SyntaxTreeNode typeName, Func<SyntaxTreeNode, ClickHouseType> parseClickHouseTypeFunc, TypeSettings settings)

diff --git a/ClickHouse.Client/Types/JsonType.cs b/ClickHouse.Client/Types/JsonType.cs
@@ -0,0 +1,104 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.Serialization;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using ClickHouse.Client.Formats;
+using ClickHouse.Client.Types.Grammar;
+
+namespace ClickHouse.Client.Types;
+
+internal class JsonType : ParameterizedType
+{
+    public override Type FrameworkType => typeof(string);
+
+    public override string Name => "Json";
+
+    public override object Read(ExtendedBinaryReader reader)
+    {
+        var value = new JsonObject();
+        var nfields = reader.Read7BitEncodedInt();
+
+        for (int i = 0; i < nfields; i++)
+        {
+            var fieldName = reader.ReadString();
+            // See https://github.com/ClickHouse/ClickHouse/blob/b618fe03bf96e64bea1a1bdec01adc1c00cd61fb/src/DataTypes/DataTypesBinaryEncoding.cpp#L48
+            // https://clickhouse.com/docs/en/sql-reference/data-types/data-types-binary-encoding
+            // TODO: add type codes within types themselves
+            var typeCode = reader.Read7BitEncodedInt();
+            object fieldValue = typeCode switch
+            {
+                0x07 => reader.ReadByte(),
+                0x08 => reader.ReadInt16(),
+                0x09 => reader.ReadInt32(),
+                0x0a => reader.ReadInt64(),
+                0x0e => reader.ReadDouble(),
+                0x15 => reader.ReadString(),
+                _ => throw new ArgumentException(fieldName, $"Unknown type code: {typeCode:X}")
+            };
+            value[fieldName] = JsonValue.Create(fieldValue);
+        }
+        return value.ToJsonString();
+    }
+
+    public override void Write(ExtendedBinaryWriter writer, object value)
+    {
+        JsonElement element;
+        if (value is string stringValue)
+        {
+            element = JsonDocument.Parse(stringValue).RootElement;
+        }
+        else
+        {
+            element = JsonSerializer.SerializeToElement(value);
+        }
+
+        writer.Write7BitEncodedInt(element.EnumerateObject().Count());
+        foreach (var field in element.EnumerateObject())
+        {
+            writer.Write(field.Name);
+            switch (field.Value.ValueKind)
+            {
+                case JsonValueKind.Number:
+                    if (field.Value.TryGetInt64(out var int64Value))
+                    {
+                        writer.Write((byte)0x0a);
+                        writer.Write(int64Value);
+                    }
+                    else
+                    {
+                        writer.Write((byte)0x0e);
+                        writer.Write(field.Value.GetDouble());
+                    }
+                    break;
+                case JsonValueKind.String:
+                    writer.Write((byte)0x15);
+                    writer.Write(field.Value.GetString());
+                    break;
+                case JsonValueKind.True:
+                    writer.Write((byte)0x07);
+                    writer.Write((byte)1);
+                    break;
+                case JsonValueKind.False:
+                    writer.Write((byte)0x07);
+                    writer.Write((byte)0);
+                    break;
+                default:
+                    throw new ArgumentException(field.Name, $"Unknown JSON value kind: {field.Value.ValueKind}");
+            }
+        }
+    }
+
+    public override string ToString() => Name;
+
+    public override ParameterizedType Parse(SyntaxTreeNode typeName, Func<SyntaxTreeNode, ClickHouseType> parseClickHouseTypeFunc, TypeSettings settings)
+    {
+        if (typeName.ChildNodes.Any())
+        {
+            throw new SerializationException("JSON type does not accept parameters");
+        }
+
+        return this;
+    }
+}
diff --git a/ClickHouse.Client/Types/ParameterizedType.cs b/ClickHouse.Client/Types/ParameterizedType.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.Serialization;
 using System.Xml.Linq;

diff --git a/ClickHouse.Client/Types/TypeConverter.cs b/ClickHouse.Client/Types/TypeConverter.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Text.Json;
 using ClickHouse.Client.Numerics;
 using ClickHouse.Client.Types.Grammar;
 
@@ -167,6 +168,7 @@ static TypeConverter()
         RegisterPlainType<MultiPolygonType>();
 
         // JSON/Object
+        RegisterParameterizedType<JsonType>();
         RegisterParameterizedType<ObjectType>();
 
         RegisterParameterizedType<AggregateFunctionType>();
@@ -179,6 +181,8 @@ static TypeConverter()
 #endif
         ReverseMapping[typeof(DateTime)] = new DateTimeType();
         ReverseMapping[typeof(DateTimeOffset)] = new DateTimeType();
+
+        ReverseMapping[typeof(JsonElement)] = new JsonType();
     }
 
     private static void RegisterPlainType<T>()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,8 +11,6 @@ internal class AggregateFunctionType : ParameterizedType

		public override string Name => "AggregateFunction";



		public override Type FrameworkType => throw new AggregateFunctionException(Function);

		public override ParameterizedType Parse(SyntaxTreeNode typeName, Func<SyntaxTreeNode, ClickHouseType> parseClickHouseTypeFunc, TypeSettings settings)
Expand Down