Skip to content

Commit

Permalink
Merge branch 'main' into yrajas/objectstorerecovery
Browse files Browse the repository at this point in the history
  • Loading branch information
badrishc authored May 9, 2024
2 parents dd22fd0 + c930108 commit e48ea03
Show file tree
Hide file tree
Showing 61 changed files with 8,778 additions and 445 deletions.
2 changes: 1 addition & 1 deletion .azure/pipelines/azure-pipelines-external-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# 1) update the name: string below (line 6) -- this is the version for the nuget package (e.g. 1.0.0)
# 2) update \libs\host\GarnetServer.cs readonly string version (~line 45) -- NOTE - these two values need to be the same
######################################
name: 1.0.7
name: 1.0.8
trigger: none
resources:
repositories:
Expand Down
11 changes: 11 additions & 0 deletions Garnet.sln
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Embedded.perftest", "playgr
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BDN.benchmark", "benchmark\BDN.benchmark\BDN.benchmark.csproj", "{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommandInfoUpdater", "playground\CommandInfoUpdater\CommandInfoUpdater.csproj", "{9BE474A2-1547-43AC-B4F2-FB48A01FA995}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -259,6 +261,14 @@ Global
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|Any CPU.Build.0 = Release|Any CPU
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|x64.ActiveCfg = Release|Any CPU
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD}.Release|x64.Build.0 = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|x64.ActiveCfg = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Debug|x64.Build.0 = Debug|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|Any CPU.Build.0 = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|x64.ActiveCfg = Release|Any CPU
{9BE474A2-1547-43AC-B4F2-FB48A01FA995}.Release|x64.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -286,6 +296,7 @@ Global
{8941A05C-099B-45AC-A7BF-F0E226BD59A8} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
{5BEDAC1F-6458-4EBA-8174-EC06B07F2132} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
{9F6E4734-6341-4A9C-A7FF-636A39D8BEAD} = {346A5A53-51E4-4A75-B7E6-491D950382CE}
{9BE474A2-1547-43AC-B4F2-FB48A01FA995} = {69A71E2C-00E3-42F3-854E-BE157A24834E}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {2C02C405-4798-41CA-AF98-61EDFEF6772E}
Expand Down
68 changes: 68 additions & 0 deletions benchmark/BDN.benchmark/SpanVsPointer.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System.Runtime.CompilerServices;
using BenchmarkDotNet.Attributes;

namespace BDN.benchmark
{
public unsafe class SpanVsPointer
{
const int Count = 1024;

byte[] bytes;
byte* bytesPtr;

[GlobalSetup]
public void GlobalSetup()
{
bytes = GC.AllocateArray<byte>(Count, true);
bytesPtr = (byte*)Unsafe.AsPointer(ref bytes[0]);
for (var ii = 0; ii < Count; ++ii)
bytes[ii] = (byte)ii;
}

[BenchmarkCategory("Swap"), Benchmark(Baseline = true)]
public void Pointer()
{
byte* pointer = bytesPtr;
byte* end = pointer + Count - 1;
while (pointer < end)
{
var tmp = *pointer;
*pointer = *++pointer;
*pointer = tmp;
}
}

[BenchmarkCategory("Swap"), Benchmark]
public void Span()
{
var span = new Span<byte>(bytesPtr, Count);
int i = 0;
while (i < Count - 1)
{
var tmp = span[i];
span[i] = span[++i];
span[i] = tmp;
}
}

[BenchmarkCategory("Swap"), Benchmark]
public void SpanToPointer()
{
var span = new Span<byte>(bytesPtr, Count);
fixed (byte* ptr = span)
{
byte* pointer = ptr;
byte* end = pointer + Count - 1;
while (pointer < end)
{
var tmp = *pointer;
*pointer = *++pointer;
*pointer = tmp;
}
}
}
}
}
7 changes: 6 additions & 1 deletion benchmark/Resp.benchmark/BenchUtils.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
Expand Down Expand Up @@ -31,10 +32,14 @@ public static SslClientAuthenticationOptions GetTlsOptions(string tlsHost, strin

public static ConfigurationOptions GetConfig(string address, int port = default, bool allowAdmin = false, bool useTLS = false, string tlsHost = null)
{
var commands = RespCommandsInfo.TryGetRespCommandNames(out var cmds)
? new HashSet<string>(cmds)
: new HashSet<string>();

var configOptions = new ConfigurationOptions
{
EndPoints = { { address, port }, },
CommandMap = CommandMap.Create(RespInfo.GetCommands()),
CommandMap = CommandMap.Create(commands),
ConnectTimeout = 100_000,
SyncTimeout = 100_000,
AllowAdmin = allowAdmin,
Expand Down
7 changes: 2 additions & 5 deletions benchmark/Resp.benchmark/Resp.benchmark.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,6 @@
<ServerGarbageCollection>true</ServerGarbageCollection>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\..\libs\server\Resp\RespInfo.cs" Link="RespInfo.cs" />
</ItemGroup>

<ItemGroup>
<None Include="..\..\test\testcerts\testcert.pfx" Link="testcert.pfx">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
Expand All @@ -25,7 +21,8 @@

<ItemGroup>
<ProjectReference Include="..\..\libs\client\Garnet.client.csproj" />
<ProjectReference Include="..\..\metrics\HdrHistogram\HdrHistogram.csproj" />
<ProjectReference Include="..\..\libs\server\Garnet.server.csproj" />
<ProjectReference Include="..\..\metrics\HdrHistogram\HdrHistogram.csproj" />
</ItemGroup>

</Project>
113 changes: 113 additions & 0 deletions libs/common/EnumUtils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;

namespace Garnet.common
{
/// <summary>
/// Utilities for enums
/// </summary>
public static class EnumUtils
{
private static readonly Dictionary<Type, IDictionary<string, string>> EnumNameToDescriptionCache = new();
private static readonly Dictionary<Type, IDictionary<string, List<string>>> EnumDescriptionToNameCache = new();

/// <summary>
/// Gets a mapping between an enum's string value to its description, for each of the enum's values
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <returns>A dictionary mapping between the enum's string value to its description</returns>
public static IDictionary<string, string> GetEnumNameToDescription<T>() where T : Enum
{
// Check if mapping is already in the cache. If not, add it to the cache.
if (!EnumNameToDescriptionCache.ContainsKey(typeof(T)))
AddTypeToCache<T>();

return EnumNameToDescriptionCache[typeof(T)];
}

/// <summary>
/// If enum does not have the 'Flags' attribute, gets an array of size 1 with the description of the enum's value.
/// If no description exists, returns the ToString() value of the input value.
/// If enum has the 'Flags' attribute, gets an array with all the descriptions of the flags which are turned on in the input value.
/// If no description exists, returns the ToString() value of the flag.
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="value">Enum value</param>
/// <returns>Array of descriptions</returns>
public static string[] GetEnumDescriptions<T>(T value) where T : Enum
{
var nameToDesc = GetEnumNameToDescription<T>();
return value.ToString().Split(',').Select(f => nameToDesc.ContainsKey(f.Trim()) ? nameToDesc[f.Trim()] : f).ToArray();
}

/// <summary>
/// Gets an enum's values based on the description attribute
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="strVal">Enum description</param>
/// <param name="vals">Enum values</param>
/// <returns>True if matched more than one value successfully</returns>
public static bool TryParseEnumsFromDescription<T>(string strVal, out IEnumerable<T> vals) where T : struct, Enum
{
vals = new List<T>();

if (!EnumDescriptionToNameCache.ContainsKey(typeof(T)))
AddTypeToCache<T>();

if (!EnumDescriptionToNameCache[typeof(T)].ContainsKey(strVal))
return false;

foreach (var enumName in EnumDescriptionToNameCache[typeof(T)][strVal])
{
if (Enum.TryParse(enumName, out T enumVal))
{
((List<T>)vals).Add(enumVal);
}
}

return ((List<T>)vals).Count > 0;
}

/// <summary>
/// Gets an enum's value based on its description attribute
/// If more than one values match the same description, returns the first one
/// </summary>
/// <typeparam name="T">Enum type</typeparam>
/// <param name="strVal">Enum description</param>
/// <param name="val">Enum value</param>
/// <returns>True if successful</returns>
public static bool TryParseEnumFromDescription<T>(string strVal, out T val) where T : struct, Enum
{
var isSuccessful = TryParseEnumsFromDescription(strVal, out IEnumerable<T> vals);
val = isSuccessful ? vals.First() : default;
return isSuccessful;
}


private static void AddTypeToCache<T>()
{
var valToDesc = new Dictionary<string, string>();
var descToVals = new Dictionary<string, List<string>>();

foreach (var flagFieldInfo in typeof(T).GetFields())
{
var descAttr = (DescriptionAttribute)flagFieldInfo.GetCustomAttributes(typeof(DescriptionAttribute), false).FirstOrDefault();
if (descAttr != null)
{
valToDesc.Add(flagFieldInfo.Name, descAttr.Description);
if (!descToVals.ContainsKey(descAttr.Description))
descToVals.Add(descAttr.Description, new List<string>());
descToVals[descAttr.Description].Add(flagFieldInfo.Name);
}
}

EnumNameToDescriptionCache.Add(typeof(T), valToDesc);
EnumDescriptionToNameCache.Add(typeof(T), descToVals);
}
}
}
5 changes: 5 additions & 0 deletions libs/common/Garnet.common.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,9 @@
<PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="All" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\storage\Tsavorite\cs\src\core\Tsavorite.core.csproj" />
<ProjectReference Include="..\storage\Tsavorite\cs\src\devices\AzureStorageDevice\Tsavorite.devices.AzureStorageDevice.csproj" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
using System;
using System.Net.Security;
using System.Text;
using Garnet.common;
using Garnet.networking;

namespace Garnet.test
namespace Garnet.common
{
public unsafe class LightClientRequest : IDisposable
{
Expand Down
32 changes: 20 additions & 12 deletions libs/host/StreamProvider.cs → libs/common/StreamProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
using Tsavorite.core;
using Tsavorite.devices;

namespace Garnet
namespace Garnet.common
{
internal enum FileLocationType
public enum FileLocationType
{
Local,
AzureStorage,
Expand All @@ -22,7 +22,7 @@ internal enum FileLocationType
/// <summary>
/// Interface for reading / writing into local / remote files
/// </summary>
internal interface IStreamProvider
public interface IStreamProvider
{
/// <summary>
/// Read data from file specified in path
Expand Down Expand Up @@ -112,15 +112,16 @@ private static void IOCallback(uint errorCode, uint numBytes, object context)
/// <summary>
/// Provides a StreamProvider instance
/// </summary>
internal class StreamProviderFactory
public class StreamProviderFactory
{
/// <summary>
/// Get a StreamProvider instance
/// </summary>
/// <param name="locationType">Type of location of files the stream provider reads from / writes to</param>
/// <param name="connectionString">Connection string to Azure Storage, if applicable</param>
/// <param name="resourceAssembly">Assembly from which to load the embedded resource, if applicable</param>
/// <returns>StreamProvider instance</returns>
internal static IStreamProvider GetStreamProvider(FileLocationType locationType, string connectionString = null)
public static IStreamProvider GetStreamProvider(FileLocationType locationType, string connectionString = null, Assembly resourceAssembly = null)
{
switch (locationType)
{
Expand All @@ -131,7 +132,7 @@ internal static IStreamProvider GetStreamProvider(FileLocationType locationType,
case FileLocationType.Local:
return new LocalFileStreamProvider();
case FileLocationType.EmbeddedResource:
return new EmbeddedResourceStreamProvider();
return new EmbeddedResourceStreamProvider(resourceAssembly);
default:
throw new NotImplementedException();
}
Expand Down Expand Up @@ -199,22 +200,29 @@ protected override long GetBytesToWrite(byte[] bytes, IDevice device)
/// </summary>
internal class EmbeddedResourceStreamProvider : IStreamProvider
{
private readonly Assembly assembly;

public EmbeddedResourceStreamProvider(Assembly assembly)
{
this.assembly = assembly;
}

public Stream Read(string path)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceName = assembly.GetManifestResourceNames().FirstOrDefault(rn => rn.EndsWith(path));
var resourceName = assembly.GetManifestResourceNames()
.FirstOrDefault(rn => rn.EndsWith($".{path}"));
if (resourceName == null) return null;

return Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName);
return assembly.GetManifestResourceStream(resourceName);
}

public void Write(string path, byte[] data)
{
var assembly = Assembly.GetExecutingAssembly();
var resourceName = assembly.GetManifestResourceNames().FirstOrDefault(rn => rn.EndsWith(path));
var resourceName = assembly.GetManifestResourceNames()
.FirstOrDefault(rn => rn.EndsWith($".{path}"));
if (resourceName == null) return;

using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName);
using var stream = assembly.GetManifestResourceStream(resourceName);
if (stream != null)
stream.Write(data, 0, data.Length);
}
Expand Down
1 change: 1 addition & 0 deletions libs/host/Configuration/ConfigProviders.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO;
using System.Text;
using System.Text.Json;
using Garnet.common;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using ErrorEventArgs = Newtonsoft.Json.Serialization.ErrorEventArgs;
Expand Down
2 changes: 1 addition & 1 deletion libs/host/GarnetServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public class GarnetServer : IDisposable
protected StoreWrapper storeWrapper;

// IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6.
readonly string version = "1.0.7";
readonly string version = "1.0.8";

/// <summary>
/// Resp protocol version
Expand Down
Loading

0 comments on commit e48ea03

Please sign in to comment.